From 4dd40ed201573bb31fe834fe6214f6e5635efbd6 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 12:51:36 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E8=BE=93=E5=87=BA=E9=A2=9D=E5=BA=A6?= =?UTF-8?q?=E7=9A=84=E9=80=9A=E7=94=A8=E9=80=BB=E8=BE=91=E6=8C=AA=E5=88=B0?= =?UTF-8?q?=20providers/budget.py=EF=BC=9A=E8=AE=A4=E6=88=AA=E6=96=AD?= =?UTF-8?q?=EF=BC=88finish=5Freason=3Dlength=E3=80=81stop=5Freason=3Dmax?= =?UTF-8?q?=5Ftokens=E3=80=81openai=20=E7=9A=84=20LengthFinishReasonError?= =?UTF-8?q?=EF=BC=89=E3=80=81=E8=AE=A4=E6=9C=8D=E5=8A=A1=E6=96=B9=E5=AB=8C?= =?UTF-8?q?=E9=A2=9D=E5=BA=A6=E5=A4=AA=E5=A4=A7=EF=BC=88400/422=20?= =?UTF-8?q?=E4=B8=94=E6=8A=A5=E9=94=99=E6=8F=90=E5=88=B0=20max=5Ftokens=20?= =?UTF-8?q?=E4=B8=80=E7=B1=BB=EF=BC=89=E3=80=81=E6=88=AA=E6=96=AD=E9=82=A3?= =?UTF-8?q?=E6=AC=A1=E7=9A=84=E7=94=A8=E9=87=8F=E3=80=81=E7=BB=99=E4=BA=BA?= =?UTF-8?q?=E7=9C=8B=E7=9A=84=E3=80=8C=E6=9F=90=E6=9F=90=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E7=9A=84=E8=BE=93=E5=87=BA=E8=BE=BE=E5=88=B0=E4=B8=8A=E9=99=90?= =?UTF-8?q?=EF=BC=88N=20token=EF=BC=89=E8=A2=AB=E6=88=AA=E6=96=AD=E3=80=8D?= =?UTF-8?q?=EF=BC=8C=E4=BB=A5=E5=8F=8A=E9=80=80=E5=9B=9E=E5=B0=8F=E9=A2=9D?= =?UTF-8?q?=E5=BA=A6=E7=9A=84=20BudgetFallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BudgetFallback 在助手原来的做法上补齐核查和裁判要用的:非流式 ainvoke 也认;bind_tools、with_structured_output 包出来的和原来的共用同一个开关,另一份配置建的模型(核查收尾时关掉思考的那个)经 shared_with 也共用,工具循环里退回过,收尾那次直接用小的;小额度的模型可以要退回时才建,平常一个都不多建。只在还没吐出任何东西时重来,和原来一样。 助手改为引用它,_cut_off、_BudgetFallback、_BUDGET_WORDS 留着老名字,行为不变,助手和配方助手的测试照样通过。 --- backend/app/api/copilot.py | 50 +----- backend/app/providers/budget.py | 168 ++++++++++++++++++++ backend/tests/test_model_budget.py | 244 +++++++++++++++++++++++++++++ 3 files changed, 418 insertions(+), 44 deletions(-) create mode 100644 backend/app/providers/budget.py create mode 100644 backend/tests/test_model_budget.py diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 33ea5d50..14e19738 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -27,6 +27,10 @@ from app.db.models import Workflow from app.engine.layout import CORRIDOR_MIN, MIN_ROW_GAP, NODE_W, _height, auto_layout from app.engine.schema import GraphSpec, NodeType, ValidationIssue, text_parse_issues, validate_graph from app.engine.state import message_text +# 额度用完的认法、退回小额度的做法挪到了 providers/budget.py(核查、证据裁判也用),这里留着老名字; +# _BUDGET_WORDS 另有 data/recipe_ai.py 的测试拿来比对 +from app.providers.budget import BUDGET_WORDS as _BUDGET_WORDS # noqa: F401 +from app.providers.budget import BudgetFallback as _BudgetFallback, cut_off as _cut_off from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model from app.tools.registry import all_specs @@ -1053,54 +1057,12 @@ class CopilotIncomplete(Exception): self.detail = detail -def _cut_off(chunk: Any) -> str: - """这一块是不是在说「额度用完了」:OpenAI 系是 finish_reason=length,Anthropic 是 stop_reason=max_tokens。""" - meta = getattr(chunk, "response_metadata", None) or {} - reason = meta.get("finish_reason") or meta.get("stop_reason") - return str(reason) if reason in ("length", "max_tokens") else "" - - #: 流式改图的输出额度。会思考的模型(DeepSeek、Claude 的 adaptive thinking)思考也算在里面: #: 8192 时复杂的改图常常想完就没额度了,正文一个字没写(2026-09 部署实例上同类请求 5 轮坏了 4 轮) COPILOT_MAX_TOKENS = 32768 -#: 服务方不收大额度时退回这个(原来的默认值) +#: 服务方不收大额度时退回这个(原来的默认值)。退回的做法(_BudgetFallback:只在还没吐出任何东西时重来, +#: 之后的调用即自查修正轮都用小的)和截断的认法(_cut_off)在 providers/budget.py,核查、证据裁判也用它 _FALLBACK_MAX_TOKENS = 8192 -#: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K), -#: 或者输入加额度超过上下文长度(本地 vLLM) -_BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") - - -def _budget_rejected(e: BaseException) -> bool: - status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None) - return status in (400, 422) and any(w in str(e).lower() for w in _BUDGET_WORDS) - - -class _BudgetFallback: - """先按大额度调;服务方一开口就嫌额度太大时按原来的额度重来,之后的调用(自查修正轮)都用小的。 - - 只在还没吐出任何东西时重来:这类拒绝都发生在请求刚发出去的时候,重来不会重复输出。 - """ - - def __init__(self, big: Any, small: Any) -> None: - self.model, self._small = big, small - - @property - def max_tokens(self) -> Any: - return getattr(self.model, "max_tokens", None) - - async def astream(self, messages: list[Any]): - started = False - try: - async for chunk in self.model.astream(messages): - started = True - yield chunk - return - except Exception as e: - if started or self.model is self._small or not _budget_rejected(e): - raise - self.model = self._small - async for chunk in self.model.astream(messages): - yield chunk async def _op_model(session: AsyncSession, payload: GenerateIn) -> tuple[Any, str]: diff --git a/backend/app/providers/budget.py b/backend/app/providers/budget.py new file mode 100644 index 00000000..a6da84e0 --- /dev/null +++ b/backend/app/providers/budget.py @@ -0,0 +1,168 @@ +"""模型调用的输出额度:认出「输出到了上限被截断」,服务方嫌额度太大时退回小额度重来。 + +助手改图(api/copilot.py)、核查(engine/verify_agent.py:双盲复算、反方审稿、审批质询共用)和证据裁判(engine/judge.py) +共用这一份。会思考的模型思考也算在输出额度里:经 OpenAI 兼容接入的推理模型(DeepSeek、GLM、Kimi 这一类)收不到 +thinking=off,Claude 开了 adaptive thinking 也一样。额度小了,想完就没额度写正文——回复是空的,或者 JSON 只写了一半。 +这时要照实说是输出额度不够,不能当成「模型没按格式交」「没查到」。 + +- cut_off:这一块(流式)或这条回复是不是因为额度用完停下的 +- truncated_error:这个异常是不是在说同一件事(openai 结构化输出解析时抛的 LengthFinishReasonError) +- budget_rejected:服务方一开口就嫌额度太大 +- error_usage:截断的那次调用花掉的用量 +- cut_words:给人看的「某某模型的输出达到上限(N token)被截断」 +- BudgetFallback:先按大额度调,被嫌大时按小额度重来,之后都用小的 + +engine/nodes/llm.py(cite_fields 的抽取)、data/recipe_ai.py 各有一份同样的判断,是在这里之前写的,行为相同。 +""" +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any + +#: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K), +#: 或者输入加额度超过上下文长度(本地 vLLM) +BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") +#: 截断类报错的原话:openai 的 LengthFinishReasonError(结构化输出解析时 finish_reason=length) +LENGTH_WORDS = ("length limit was reached", "finish_reason=length", "finish_reason='length'") + + +def cut_off(chunk: Any) -> str: + """这一块是不是在说「额度用完了」:OpenAI 系是 finish_reason=length,Anthropic 是 stop_reason=max_tokens。""" + meta = getattr(chunk, "response_metadata", None) or {} + reason = meta.get("finish_reason") or meta.get("stop_reason") + return str(reason) if reason in ("length", "max_tokens") else "" + + +def truncated_error(error: BaseException | None) -> bool: + """这个异常是不是在说「输出被截断」:openai 的 LengthFinishReasonError,或者 langchain、网关转述的同一件事。 + + 服务方嫌额度太大的 400(报错里也有 max_tokens)不算:那是请求被拒,模型一个字都没写。 + """ + if error is None: + return False + if type(error).__name__ == "LengthFinishReasonError": + return True + text = str(error).lower() + return any(word in text for word in LENGTH_WORDS) + + +def budget_rejected(e: BaseException) -> bool: + status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None) + return status in (400, 422) and any(w in str(e).lower() for w in BUDGET_WORDS) + + +def error_usage(error: BaseException | None) -> tuple[int, int]: + """失败的那次调用花掉的 (输入, 输出) token。截断时 openai 的异常带着 completion.usage:额度是真花出去了,要记账, + 不然每日上限会漏掉最贵的那几次。别的异常没有用量,是 (0, 0)。""" + usage = getattr(getattr(error, "completion", None), "usage", None) + return int(getattr(usage, "prompt_tokens", 0) or 0), int(getattr(usage, "completion_tokens", 0) or 0) + + +def cut_words(who: str, limit: int | None) -> str: + """「核查模型的输出达到上限(16384 token)被截断」。额度不知道(没给 max_tokens、按接入的默认值)就不写数。""" + return f"{who}的输出达到上限{f'({limit} token)' if limit else ''}被截断" + + +class _Switch: + """几次调用共用的「退回小额度了没有」:核查的工具循环和收尾、裁判的各批。""" + + __slots__ = ("small",) + + def __init__(self) -> None: + self.small = False + + +class BudgetFallback: + """先按大额度调;服务方一开口就嫌额度太大时按小额度重来,之后的调用都用小的。 + + 只在还没吐出任何东西时重来:这类拒绝都发生在请求刚发出去的时候,重来不会重复输出。流式(astream)、非流式 + (ainvoke)都认。bind_tools、with_structured_output 照样包一层,包出来的和原来的共用同一个开关;shared_with 给了, + 另一份配置建的模型(核查收尾时关掉思考的那个)也和它共用:工具循环里退回过,收尾那次直接用小的,不再先撞一次。 + + small 是现成的小额度模型;也可以不给,给 build(无参的协程函数,返回小额度的模型),要退回时才建,平常一个都不多建。 + limits 是 (大额度, 小额度):给了,max_tokens 就按它报(包过一层的模型身上不一定读得到额度);不给就读眼下那个模型的 + max_tokens。 + """ + + def __init__(self, big: Any, small: Any = None, *, build: Callable[[], Awaitable[Any]] | None = None, + limits: tuple[int | None, int | None] | None = None, + shared_with: BudgetFallback | None = None) -> None: + self._big = big + self._small = small + self._build = build + self._limits = limits + self._switch = shared_with._switch if shared_with is not None else _Switch() + + @property + def model(self) -> Any: + """眼下用的那个:退回过、小额度的也建好了就是小的,否则是大的。""" + return self._small if self._switch.small and self._small is not None else self._big + + @property + def max_tokens(self) -> Any: + if self._limits is not None: + return self._limits[1] if self._switch.small else self._limits[0] + return getattr(self.model, "max_tokens", None) + + def _can_fall_back(self, used: Any) -> bool: + """刚才那次用的是大额度、又有小额度可退。大小是同一个模型时重来也是同样的额度,不算可退。""" + if used is not self._big: + return False + return self._build is not None if self._small is None else self._small is not self._big + + async def _small_model(self) -> Any: + if self._small is None and self._build is not None: + built = await self._build() + if self._small is None: # 并发的几次调用同时要建时,留先建好的那个 + self._small = built + return self._small + + async def _current(self) -> Any: + if self._switch.small and (small := await self._small_model()) is not None: + return small + return self._big + + async def ainvoke(self, messages: Any, *args: Any, **kwargs: Any) -> Any: + used = await self._current() + try: + return await used.ainvoke(messages, *args, **kwargs) + except Exception as e: + if not self._can_fall_back(used) or not budget_rejected(e): + raise + self._switch.small = True + return await (await self._current()).ainvoke(messages, *args, **kwargs) + + async def astream(self, messages: Any, *args: Any, **kwargs: Any): + used = await self._current() + started = False + try: + async for chunk in used.astream(messages, *args, **kwargs): + started = True + yield chunk + return + except Exception as e: + if started or not self._can_fall_back(used) or not budget_rejected(e): + raise + self._switch.small = True + async for chunk in (await self._current()).astream(messages, *args, **kwargs): + yield chunk + + def _derived(self, make: Callable[[Any], Any]) -> BudgetFallback: + """对大小两个模型做同一件事(绑工具、要结构化输出),包成共用开关的一层。小的还没建时,建好再做。""" + small, build = None, None + if self._small is not None: + small = make(self._small) + elif self._build is not None: + async def build() -> Any: + return make(await self._small_model()) + return BudgetFallback(make(self._big), small, build=build, limits=self._limits, shared_with=self) + + def bind_tools(self, tools: Any, **kwargs: Any) -> BudgetFallback: + return self._derived(lambda m: m.bind_tools(tools, **kwargs)) + + def with_structured_output(self, schema: Any, **kwargs: Any) -> BudgetFallback: + return self._derived(lambda m: m.with_structured_output(schema, **kwargs)) + + +__all__ = ["BUDGET_WORDS", "LENGTH_WORDS", "BudgetFallback", "budget_rejected", "cut_off", "cut_words", "error_usage", + "truncated_error"] diff --git a/backend/tests/test_model_budget.py b/backend/tests/test_model_budget.py new file mode 100644 index 00000000..21ee3ecc --- /dev/null +++ b/backend/tests/test_model_budget.py @@ -0,0 +1,244 @@ +"""模型调用的输出额度(providers/budget.py):助手改图、核查、证据裁判共用的那几样。 + +推理模型的思考也算在输出额度里,额度小了,想完就没额度写正文:回复是空的,或者 JSON 只写了一半。这里守几件事: +- 认得出「输出到了上限被截断」:OpenAI 系 finish_reason=length、Anthropic stop_reason=max_tokens,以及 openai 结构化 + 输出解析时抛的 LengthFinishReasonError;服务方嫌额度太大的 400 不算截断(那是请求被拒,一个字都没写) +- 认得出「服务方嫌额度太大」:400 / 422,报错里有 max_tokens 一类字样;别的 400 不算 +- 截断的那次调用花掉的用量从异常里取得到(额度是真花出去了) +- BudgetFallback:先按大额度调,被嫌大时按小额度重来一次,之后的调用都用小的;流式、非流式都认;bind_tools、 + with_structured_output 包出来的和原来的共用同一个开关;只在还没吐出任何东西时重来;小额度的模型要退回时才建 + +模型都是剧本,不碰真接口。 +""" +from __future__ import annotations + +import pytest +from langchain_core.messages import AIMessage, AIMessageChunk +from openai import LengthFinishReasonError +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion + +from app.providers.budget import ( + BUDGET_WORDS, + BudgetFallback, + budget_rejected, + cut_off, + cut_words, + error_usage, + truncated_error, +) + + +def length_error(prompt: int = 9000, completion: int = 16384) -> LengthFinishReasonError: + """openai 的结构化输出在 finish_reason=length 时抛的那个错(原话和真实运行里一样)。""" + usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) + return LengthFinishReasonError(completion=ChatCompletion.model_construct( + id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage)) + + +class Rejected(Exception): + """仿 openai / anthropic SDK 的 400:一发请求就被拒,还没吐出任何东西。""" + + def __init__(self, message: str, status_code: int = 400) -> None: + super().__init__(message) + self.status_code = status_code + + +TOO_LARGE = "max_tokens: 16384 > 8192, which is the maximum allowed number of output tokens" + + +class Model: + """记下每次调用的剧本模型。fail:每次调用都抛它;stream:流式时吐的块,吐到 boom 处抛 fail。""" + + def __init__(self, name: str, *, fail: Exception | None = None, stream: list | None = None, + boom: int | None = None, max_tokens: int | None = None) -> None: + self.name = name + self.fail = fail + self.stream = stream if stream is not None else [AIMessageChunk(content=name)] + self.boom = boom + self.max_tokens = max_tokens + self.calls: list[tuple[str, object]] = [] + self.bound: list[tuple[str, object, dict]] = [] + + async def ainvoke(self, messages, *args, **kwargs): + self.calls.append(("ainvoke", messages)) + if self.fail is not None: + raise self.fail + return AIMessage(content=self.name) + + async def astream(self, messages, *args, **kwargs): + self.calls.append(("astream", messages)) + for i, chunk in enumerate(self.stream): + if self.boom is not None and i == self.boom: + raise self.fail + yield chunk + if self.fail is not None and self.boom is None: + raise self.fail + + def bind_tools(self, tools, **kwargs): + self.bound.append(("tools", tools, kwargs)) + return self + + def with_structured_output(self, schema, **kwargs): + self.bound.append(("structured", schema, kwargs)) + return self + + +# -------------------------------------------------------------------------- +# 认出截断、认出嫌额度太大 +# -------------------------------------------------------------------------- + + +def test_cut_off_replies_are_recognised_for_both_families(): + assert cut_off(AIMessage(content="", response_metadata={"finish_reason": "length"})) == "length" + assert cut_off(AIMessageChunk(content=[], response_metadata={"stop_reason": "max_tokens"})) == "max_tokens" + assert cut_off(AIMessage(content="好了", response_metadata={"finish_reason": "stop"})) == "" + assert cut_off(AIMessage(content="", response_metadata={"stop_reason": "end_turn"})) == "" + assert cut_off(None) == "" and cut_off({"finish_reason": "length"}) == "" + + +def test_length_errors_are_recognised_but_a_rejected_budget_is_not(): + assert truncated_error(length_error()) + assert truncated_error(RuntimeError("Could not parse response content as the length limit was reached")) + assert not truncated_error(None) + assert not truncated_error(RuntimeError("网关超时")) + assert not truncated_error(Rejected(TOO_LARGE)), "嫌额度太大是请求被拒,模型一个字都没写,不是截断" + + +@pytest.mark.parametrize("message", [ + "Invalid max_tokens value, the valid range of max_tokens is [1, 8192]", + TOO_LARGE, + "max_completion_tokens is too large: 16384. This model supports at most 8192 completion tokens", + "This model's maximum context length is 32768 tokens. However, you requested 40369 tokens " + "(7601 in the messages, 32768 in the completion).", +]) +def test_a_rejected_budget_is_recognised(message): + assert budget_rejected(Rejected(message)) + assert budget_rejected(Rejected(message, status_code=422)) + + +def test_other_rejections_are_not_a_budget_problem(): + assert not budget_rejected(Rejected("model 'no-such-model' does not exist")) + assert not budget_rejected(Rejected(TOO_LARGE, status_code=500)), "服务方自己出错不是嫌额度大" + assert not budget_rejected(RuntimeError(TOO_LARGE)), "没有状态码的不认" + assert "max_tokens" in BUDGET_WORDS + + +def test_the_tokens_spent_by_a_cut_off_call_come_from_the_error(): + assert error_usage(length_error(prompt=9000, completion=16384)) == (9000, 16384) + assert error_usage(RuntimeError("网关超时")) == (0, 0) and error_usage(None) == (0, 0) + + +def test_the_words_name_who_was_cut_off_and_the_limit(): + assert cut_words("核查模型", 16384) == "核查模型的输出达到上限(16384 token)被截断" + assert cut_words("裁判模型", None) == "裁判模型的输出达到上限被截断", "额度不知道就不写数" + + +# -------------------------------------------------------------------------- +# 退回小额度 +# -------------------------------------------------------------------------- + + +async def test_a_rejected_budget_is_retried_once_with_the_small_one_and_stays_small(): + big, small = Model("big", fail=Rejected(TOO_LARGE)), Model("small") + model = BudgetFallback(big, small) + assert (await model.ainvoke(["问"])).content == "small" + assert (await model.ainvoke(["再问"])).content == "small", "之后的调用都用小的,不再先撞一次" + assert len(big.calls) == 1 and len(small.calls) == 2 + assert model.model is small + + +async def test_other_errors_are_not_retried(): + big, small = Model("big", fail=Rejected("model 'no-such-model' does not exist")), Model("small") + with pytest.raises(Rejected): + await BudgetFallback(big, small).ainvoke(["问"]) + assert small.calls == [] + + +async def test_the_small_model_is_only_built_when_needed(): + built: list[str] = [] + small = Model("small") + + async def build(): + built.append("small") + return small + + fine = BudgetFallback(Model("big"), build=build, limits=(16384, 4096)) + assert (await fine.ainvoke(["问"])).content == "big" + assert built == [] and fine.max_tokens == 16384, "平常一个小额度的模型都不多建" + + rejecting = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=build, limits=(16384, 4096)) + assert (await rejecting.ainvoke(["问"])).content == "small" + assert (await rejecting.ainvoke(["再问"])).content == "small" + assert built == ["small"], "退回之后建一次,之后沿用" + assert rejecting.max_tokens == 4096 + + +async def test_without_a_small_model_the_rejection_is_raised(): + with pytest.raises(Rejected): + await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE))).ainvoke(["问"]) + same = Model("same", fail=Rejected(TOO_LARGE)) + with pytest.raises(Rejected): + await BudgetFallback(same, same).ainvoke(["问"]) + assert len(same.calls) == 1, "大小是同一个模型:重来也是同样的额度,不白撞第二次" + + +async def test_bound_and_structured_views_share_the_switch(): + """核查的工具循环里退回过小额度,收尾那次直接用小的;包出来的视图各自转给对应的小模型。""" + big, small = Model("big", fail=Rejected(TOO_LARGE)), Model("small") + model = BudgetFallback(big, small, limits=(16384, 4096)) + tools = model.bind_tools(["db_query"], parallel_tool_calls=False) + assert big.bound == [("tools", ["db_query"], {"parallel_tool_calls": False})] + assert small.bound == [("tools", ["db_query"], {"parallel_tool_calls": False})] + assert (await tools.ainvoke(["查"])).content == "small" + structured = model.with_structured_output({"type": "object"}, include_raw=True) + assert (await structured.ainvoke(["交结果"])).content == "small" + assert len(big.calls) == 1, "收尾那次不再先按大额度撞一次" + assert structured.max_tokens == 4096 and model.max_tokens == 4096 + + +async def test_a_sibling_shares_the_switch_and_builds_its_own_small_model(): + """收尾用的是另一份配置(关掉思考)建的模型:和工具循环的模型共用开关,要退回时按自己的配置建小的。""" + built: list[str] = [] + loop_small, settle_small = Model("loop-small"), Model("settle-small") + + async def build_loop(): + built.append("loop") + return loop_small + + async def build_settle(): + built.append("settle") + return settle_small + + loop = BudgetFallback(Model("loop-big", fail=Rejected(TOO_LARGE)), build=build_loop, limits=(16384, 4096)) + assert (await loop.ainvoke(["查"])).content == "loop-small" + settle_big = Model("settle-big") + settle = BudgetFallback(settle_big, build=build_settle, limits=(16384, 4096), shared_with=loop) + assert (await settle.with_structured_output({"type": "object"}).ainvoke(["交结果"])).content == "settle-small" + assert settle_big.calls == [] and built == ["loop", "settle"] + + +async def test_streaming_falls_back_only_before_anything_came_out(): + big = Model("big", fail=Rejected(TOO_LARGE), stream=[]) + small = Model("small", stream=[AIMessageChunk(content="a"), AIMessageChunk(content="b")]) + model = BudgetFallback(big, small) + assert [c.content async for c in model.astream(["问"])] == ["a", "b"] + assert len(big.calls) == 1 and len(small.calls) == 1 + + started = Model("big", fail=Rejected(TOO_LARGE), stream=[AIMessageChunk(content="x"), AIMessageChunk(content="y")], + boom=1) + unused = Model("small") + got: list[str] = [] + with pytest.raises(Rejected): + async for chunk in BudgetFallback(started, unused).astream(["问"]): + got.append(chunk.content) + assert got == ["x"] and unused.calls == [], "已经吐出东西了:重来会重复输出,原样抛出" + + +async def test_without_limits_max_tokens_reads_the_current_model(): + big = Model("big", fail=Rejected(TOO_LARGE), max_tokens=32768) + small = Model("small", max_tokens=8192) + model = BudgetFallback(big, small) + assert model.max_tokens == 32768 + await model.ainvoke(["问"]) + assert model.max_tokens == 8192 From a552f4feab2e582f65027f482881ee0451ee2d38 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 12:58:37 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E6=A0=B8=E6=9F=A5=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E7=9A=84=E8=BE=93=E5=87=BA=E9=A2=9D=E5=BA=A6=E6=94=B9=E4=B8=BA?= =?UTF-8?q?=2016384=E3=80=81=E6=9C=8D=E5=8A=A1=E6=96=B9=E5=AB=8C=E5=A4=A7?= =?UTF-8?q?=E6=97=B6=E9=80=80=E5=9B=9E=204096=EF=BC=8C=E6=88=AA=E6=96=AD?= =?UTF-8?q?=E7=85=A7=E5=AE=9E=E8=AF=B4=EF=BC=9A=E5=B7=A5=E5=85=B7=E5=BE=AA?= =?UTF-8?q?=E7=8E=AF=E3=80=81=E6=94=B6=E5=B0=BE=E7=9A=84=E6=AF=8F=E6=AC=A1?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E9=83=BD=E7=9C=8B=E6=9C=89=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E5=88=B0=E4=BA=86=E4=B8=8A=E9=99=90=E8=A2=AB=E6=88=AA=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 额度:judge.MAX_TOKENS 从 4096 改为 16384(核查的 verify_model_spec 沿用它)。经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考算在额度里,4096 常常想完就没额度发查询、交结果;16384 是常见网关接受的输出上限里偏大的一档,cite_fields 的抽取也是这个数。服务方嫌大(400/422 且报错提到 max_tokens 一类)时退回 judge.FALLBACK_MAX_TOKENS(4096)再试一次,工具循环和收尾共用一个开关,之后都用小的;小额度的模型要退回时才建。 截断:工具循环里没有工具调用、finish_reason=length(或 stop_reason=max_tokens)不再当成「模型说完了」,一条查询都没成功时不再收尾(收尾只能交「查不到」,双盲复算就会记成「盲算没有查到这个数」);查到过东西的照常收尾,取不到要的数时同样交回截断的原因,取得到的照常交、notes 里注明。收尾被截断(Anthropic 强制工具调用写到一半、openai 结构化输出抛 LengthFinishReasonError、普通调用只有思考)时交回截断的原因,不再退回普通调用多花一次额度;截断那次的用量从异常里取,照记账。给人看的原因写明「核查模型的输出达到上限(N token)被截断,核查未能完成。通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型」,经 error 交给双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错;节点里调用时另记一条 verify_truncated,llm.end 标出 truncated 和额度。没有截断时原有说法不变。 测试:新增 test_verify_truncation(假模型返回 finish_reason=length、只有思考、LengthFinishReasonError,覆盖工具循环、收尾结构化和普通调用、退回 4096、别的 400 不重试、三条渠道);三处钉着 4096 的断言改成 16384。 --- backend/app/api/settings.py | 4 +- backend/app/engine/judge.py | 14 +- backend/app/engine/verify_agent.py | 110 ++++++-- backend/tests/test_evidence_judge.py | 4 +- backend/tests/test_evidence_judge_api.py | 2 +- backend/tests/test_judge_settings.py | 3 +- backend/tests/test_verify_truncation.py | 334 +++++++++++++++++++++++ 7 files changed, 447 insertions(+), 24 deletions(-) create mode 100644 backend/tests/test_verify_truncation.py diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 566fae74..5c98ba86 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -549,7 +549,9 @@ async def verify_model_spec(session: AsyncSession, node_config: dict[str, Any] | """核查用哪个模型:节点配置里的 provider / model > 设置里的核查模型 > 证据裁判模型的取值链(裁判设置 > 助手模型 > 默认接入,见 judge.judge_model_spec)。哪一级写了就整组用哪一级,不跨级拼接入和模型。 - 和裁判一样关掉 thinking、max_tokens 4096:核查最后要按结构交结果(强制工具调用),和思考模式不兼容。 + 和裁判一样关掉 thinking:核查最后要按结构交结果(强制工具调用),和思考模式不兼容。额度也和裁判一样(judge.MAX_TOKENS, + 16384):经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考算在额度里,4096 常常想完就没额度查库、交结果了; + 服务方嫌大时核查 agent 退回 judge.FALLBACK_MAX_TOKENS(engine/verify_agent.py)。 建议和被核查的模型不同,否则等于自己核自己;比较、提醒由调用方做(它知道被核查的是哪个模型)。 """ node = node_config if isinstance(node_config, dict) else {} diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index 1dc0e67d..27ba26c0 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -12,7 +12,8 @@ (按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过。带期 4 文档标记的报告,上传表格的 查询在 SQL 之后另有来历行(来源、口径、已接受、区域外文字,_provenance_lines) 3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析, - 再失败就记未裁判。thinking 关掉,max_tokens 4096 + 再失败就记未裁判。thinking 关掉,输出额度 MAX_TOKENS(推理模型的思考也算在里面),服务方嫌大时退回 + FALLBACK_MAX_TOKENS;输出到了上限被截断的记未裁判(truncated),照实说是额度不够、不再退回纯文本多花一次 4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算, 超了按优先级截断;触顶就停,已判的保留,没判的记未裁判并说明是哪个上限——不报错 5. 每日计数落在设置表(SPEND_KEY),按本地日期滚动。目录里没有价格的模型估不出金额,金额上限对它 @@ -104,7 +105,13 @@ BATCH_SIZE = 8 #: 一波最多几批同时问:几批共用剩下的时长,句数上限内的句子一两波就判完。第一批单独问:接入调不通(鉴权、 #: 网关挂了)时只撞这一次,不让一整波一起失败 PARALLEL_BATCHES = 4 -MAX_TOKENS = 4096 +#: 裁判(和核查:api/settings.py 的 verify_model_spec、engine/verify_agent.py 沿用这一档)的输出额度。thinking 关掉只对 +#: Anthropic 有用:经 OpenAI 兼容接入的推理模型(DeepSeek、GLM、Kimi 这一类)收不到 thinking=off,思考算在额度里,原来的 +#: 4096 常常想完就没额度写判定了(cite_fields 的抽取在 2026-10 撞过同一堵墙,那边也改成了 16384)。16384 是常见网关、 +#: 模型接受的输出上限里偏大的一档(GPT-4o 的上限就是它);更小的服务方会拒收,那时退回 FALLBACK_MAX_TOKENS +MAX_TOKENS = 16384 +#: 服务方嫌 MAX_TOKENS 太大(400 / 422,报错里有 max_tokens 一类字样)时退回这个:原来的额度,各家都收。之后的调用都用它 +FALLBACK_MAX_TOKENS = 4096 #: 证据摘录:SQL 前几个字、每次查询最多几行几列、每段知识库片段最多几个字、整份引用的检索给几段。 #: 列数上限只管没被引用的列:句子引用的格子所在的列一律给(遮罩的除外),再用别的列补满到这么多列 SQL_CHARS = 300 @@ -897,7 +904,8 @@ def _text(value: Any) -> str | None: async def judge_model_spec(session: Any, node_judge: dict[str, Any] | None = None, *, settings: dict[str, Any] | None = None) -> ModelSpec: """裁判用哪个模型:节点上的 judge.model(连同 provider)> 设置里的证据裁判模型 > Copilot 模型 > - 默认 provider。哪一级写了就整组用哪一级,不跨级拼 provider 和模型。thinking 关、max_tokens 4096。 + 默认 provider。哪一级写了就整组用哪一级,不跨级拼 provider 和模型。thinking 关、额度 MAX_TOKENS(推理模型关不掉 + 思考,见 MAX_TOKENS 的说明;服务方嫌大时 run_request 退回 FALLBACK_MAX_TOKENS)。 建议和写作模型不同,否则等于自己审自己(报告节点发现相同会发 judge_same_model 警告)。 """ diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index ad0a39d2..ad5b59a7 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -25,6 +25,12 @@ query_artifact 和 role: verify——随运行封存、证据面板能下钻。 每日上限不传就按设置里的「核查每日金额上限」,明确传 None 才是不限。模型不在价格目录里时估不出金额,金额上限 对它无效,照裁判的说法记一条说明。每次调用都记进核查的每日账(engine/spend.py)。 +**输出额度。** 经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考也算在输出额度里:额度按 spec(缺省 judge.MAX_TOKENS), +服务方嫌大时退回 judge.FALLBACK_MAX_TOKENS 再试一次,之后工具循环、收尾都用小的(providers/budget.py)。每次调用都看 +有没有到了上限被截断:工具循环里被截断(话没说完、也没发查询)不算模型自己停下,一条查询都没成功时不再收尾;收尾被截断 +时不再退回普通调用多花一次额度。因为截断没交成结果的,error 照实说是输出额度(TRUNCATED_ERROR),不说成「没查到」 +「没按结构交」;节点里调用时另记一条 verify_truncated。 + 永远不抛:模型没配好、调用失败、格式不对、触顶、库写不进去,都落在返回值里(error / limit)。模型调用都经 get_chat_model,测试换成剧本。 """ @@ -49,11 +55,12 @@ from app.db.base import SessionLocal from app.engine import spend from app.engine.evidence import build_catalog, cell_eid, compose_doc, query_entry_fields, schema_ledger_entry from app.engine.expressions import CellError, column_kind, locate_cell, numeric_text -from app.engine.judge import _json_in, _tokens, _usd +from app.engine.judge import FALLBACK_MAX_TOKENS, MAX_TOKENS, _json_in, _tokens, _usd from app.engine.nodes.llm import _DIGEST_BUDGET, _digest, _split_structured, _typed, explain_model_error from app.engine.state import message_text from app.engine.toolcalls import ToolTimeout, limit_fields, limit_of, run_bounded from app.providers import catalog as pricing +from app.providers.budget import BudgetFallback, cut_off, cut_words, error_usage, truncated_error from app.providers.factory import ModelSpec, ProviderNotConfigured, bind_tools_safely, get_chat_model from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX, build_datasource_tools from app.tools.registry import ToolArgsError, ToolContext, Versions, args_model_of, prepare_args @@ -79,6 +86,12 @@ _REF_DEPTH = 8 RATIONALE = "rationale" #: 触顶原因 LIMITS = ("max_queries", "timeout_s", "max_cost_usd", "daily_max_usd", "max_steps") +#: 核查模型的输出到了上限被截断时给人看的话。TRUNCATED_ERROR 是没交成结果的原因:双盲复算的无法复算、反方审稿的「反查 +#: 未完成」、质询的报错都照它说,也是 verify_truncated 日志的原话。TRUNCATED_SETTLED 是截断前查到过东西、照常收尾了的, +#: 记进 notes 和日志。cut 是 budget.cut_words 的「核查模型的输出达到上限(N token)被截断」 +TRUNCATED_ERROR = "{cut},核查未能完成。{advice}" +TRUNCATED_SETTLED = "{cut},已按截断前查到的结果收尾,结果可能不完整。{advice}" +TRUNCATED_ADVICE = "通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型" VERIFY_SYSTEM = ( "你是数据核查员:只读地查库,用查询结果完成交给你的任务。\n" @@ -129,7 +142,8 @@ class VerifyOutcome: source, tool, call_id}。 rationale:理由组装成的报告文档(evidence.compose_doc 的形状,violations 里有裸数字)。usage:{input_tokens, output_tokens, total_tokens, cost_usd, calls, unpriced_calls}。limit:触顶原因(LIMITS 之一),没触顶为 None。 - answered:模型是不是自己停下来的(没被上限掐断)。 + answered:模型是不是自己停下来的(没被上限掐断,也没因为输出到了上限被截断)。因为截断没交成结果的,error 是 + TRUNCATED_ERROR;截断前查到过东西、照常交了结果的,error 为 None,notes 里记一句 TRUNCATED_SETTLED。 """ result: dict[str, Any] | None @@ -686,13 +700,16 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou def left() -> float | None: return None if deadline is None else deadline - time.monotonic() - # ---- 模型 + # ---- 模型。额度没写的按 MAX_TOKENS:推理模型的思考算在输出额度里,4096 常常想完就没额度发查询、交结果了 + spec = spec.model_copy(update={"max_tokens": spec.max_tokens or MAX_TOKENS}) + small = FALLBACK_MAX_TOKENS if spec.max_tokens > FALLBACK_MAX_TOKENS else None try: async with SessionLocal() as session: - base_model, model_id = await get_chat_model(session, spec) + first, model_id = await get_chat_model(session, spec) except Exception as e: # noqa: BLE001 - 核查模型没配好:这回核不了,照实交回 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) return outcome(error=f"核查模型不可用({why})") + base_model = _with_fallback(first, spec, small) state.model = model_id if daily_max_usd is Default.SETTINGS: from app.api.settings import verify_settings # 在这里导入:api.settings 导入了 engine 里的模块 @@ -743,7 +760,8 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou raw = got if isinstance(got, BaseMessage) else _split_structured(got)[0] except (asyncio.TimeoutError, Exception) as e: # noqa: BLE001 - 调用失败落在返回值里 failure = e - inp, out = _usage(raw) + # openai 的结构化输出被截断时抛异常,用量在异常里:额度是真花出去了 + inp, out = _usage(raw) if raw is not None else error_usage(failure) cost = await budget.settle(day, estimate, inp, out) usage["input_tokens"] += inp usage["output_tokens"] += out @@ -756,6 +774,10 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou if failure is not None: end["error"] = "超过核查的时长上限,已放弃这次调用" if isinstance(failure, asyncio.TimeoutError) \ else explain_model_error(failure, model_id) + if truncated_error(failure) or cut_off(raw): + # 到了上限被截断:不是「调用失败,请稍后重试」,重试解决不了额度问题 + limit = base_model.max_tokens + end.update(truncated=True, max_tokens=limit, error=cut_words("核查模型", limit)) await _announce(emit, EventType.LLM_END, **end) return got, raw, failure @@ -766,6 +788,7 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou said = "" attempts = 0 done: dict[str, str] = {} # 成功过的查询(工具名 + 参数)→ 编号:一模一样的不再执行 + cut: tuple[str, int | None] | None = None # 最近一次因为输出到了上限被截断的调用:(哪一步, 那次的额度) for _ in range(steps): remaining = left() if remaining is not None and remaining <= 0: @@ -781,11 +804,18 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou if isinstance(failure, asyncio.TimeoutError): stopped = "timeout_s" break + if truncated_error(failure): + cut = ("verify", base_model.max_tokens) + break if failure is not None or not isinstance(response, BaseMessage): return outcome(error=explain_model_error(failure, model_id) if failure else "核查模型没有给出回复") messages.append(response) calls = getattr(response, "tool_calls", None) or [] if not calls: + if cut_off(response): + # 话没说完、也没发查询就到了上限(推理模型想完了额度也用完了):不是模型自己停下来的,不能当成说完了去收尾 + cut, said = ("verify", base_model.max_tokens), message_text(response) or said + break answered, said = True, message_text(response) break said = message_text(response) or said @@ -850,12 +880,35 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou await _announce(emit, EventType.LOG, level="warn", code="verify_limit", limit=stopped, message=f"核查已达上限({words}),已停止查询,按已有结果收尾") - # ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容 + async def report_cut(delivered: bool) -> str: + """被截断过:记一条 verify_truncated,返回给人看的话(没交成结果的原因,或者交了结果时记进 notes 的说明)。""" + purpose, limit = cut or ("verify", None) + text = (TRUNCATED_SETTLED if delivered else TRUNCATED_ERROR).format(cut=cut_words("核查模型", limit), + advice=TRUNCATED_ADVICE) + await _announce(emit, EventType.LOG, level="warn", code="verify_truncated", purpose=purpose, max_tokens=limit, + delivered=delivered, message=text) + return text + + async def give_up(error: str, limit: str | None = None) -> VerifyOutcome: + """没交成结果,原因照 error 说。工具循环里被截断过的,时间线上照记一条:收尾失败可能就是它带出来的。""" + if cut is not None: + await report_cut(False) + return outcome(limit=limit or stopped, answered=answered, error=error) + + if cut is not None and not queries: + # 一条查询都没成功就被截断:收尾只能交「查不到」,双盲复算会记成「盲算没有查到这个数」,原因却是输出额度。 + # 收尾那次多半同样被推理占满,不再为它多花一次额度,直接交回截断的原因 + return outcome(limit=stopped, answered=answered, error=await report_cut(False)) + + # ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容(经 OpenAI 兼容接入的推理模型 + # 关不掉,所以额度和工具循环一样,退回小额度也和工具循环共用一个开关) + settle_spec = spec.model_copy(update={"thinking": "off"}) try: async with SessionLocal() as session: - settler, _ = await get_chat_model(session, spec.model_copy(update={"thinking": "off"})) + settle_model, _ = await get_chat_model(session, settle_spec) except Exception as e: # noqa: BLE001 - return outcome(limit=stopped, answered=answered, error=f"核查模型不可用({describe_exception(e)})") + return await give_up(f"核查模型不可用({describe_exception(e)})") + settler = _with_fallback(settle_model, settle_spec, small, shared_with=base_model) final = _final_messages(head, task, queries, said, words) try: structured = settler.with_structured_output(schema, include_raw=True) @@ -870,11 +923,12 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou parsed: dict[str, Any] | None = None failure: BaseException | None = None + settle_cut = False # 收尾那次到了上限被截断 if structured is not None: estimate = budget.estimate(_text_of(final) + schema_text, _OUT_FINAL) day, refused = await budget.reserve(estimate, kept=kept) if refused: - return outcome(limit=stopped or refused, answered=answered, error=f"已达{words_of(refused)},没能交结果") + return await give_up(f"已达{words_of(refused)},没能交结果", refused) got, raw, failure = await ask(final, structured.ainvoke, estimate, day or "", "verify_result", until_finish()) parsed = None if failure is not None else _parsed(_split_structured(got)[1]) if parsed is None and raw is not None: @@ -882,43 +936,67 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: if (parsed := _parsed(candidate)) is not None: break - if parsed is None and not isinstance(failure, asyncio.TimeoutError): + settle_cut = parsed is None and (truncated_error(failure) or bool(cut_off(raw))) + if parsed is None and not isinstance(failure, asyncio.TimeoutError) and not settle_cut: # 不支持结构化输出的模型:普通调用一次,结构写进消息,从回复里把 JSON 抠出来。结构化那次已经做过的, - # 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱 + # 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱。结构化那次是被截断的不补救: + # 推理占满额度时普通调用多半一样被截断,白花一次额度 plain = _plain_messages(final, schema) remaining = until_finish() if remaining is None or remaining > 0: estimate = budget.estimate(_text_of(plain), _OUT_FINAL) day, refused = await budget.reserve(estimate, kept=kept and structured is None) if refused and structured is None: - return outcome(limit=stopped or refused, answered=answered, - error=f"已达{words_of(refused)},没能交结果") + return await give_up(f"已达{words_of(refused)},没能交结果", refused) if not refused: got, _, failure = await ask(plain, settler.ainvoke, estimate, day or "", "verify_result", remaining) parsed = None if failure is not None else _parsed(message_text(got) if got is not None else None) + settle_cut = parsed is None and (truncated_error(failure) or bool(cut_off(got))) if parsed is None: + if settle_cut: + cut = ("verify_result", settler.max_tokens) + return outcome(limit=stopped, answered=answered, error=await report_cut(False)) why = "收尾时超过时长上限,没能交结果" if isinstance(failure, asyncio.TimeoutError) else \ f"核查模型没有按结构交结果({explain_model_error(failure, model_id)})" if failure is not None else \ "核查模型没有按结构交结果" - return outcome(limit=stopped, answered=answered, error=why) + return await give_up(why) declared = list((resolved.get("properties") or {}) if _is_object(resolved) else {}) if declared and not any(k in parsed for k in declared): # 一个要求的字段都没交:不是「查不到」,是没按结构交。不这么分,字段名对不上的回复会被当成什么都没查到 - return outcome(limit=stopped, answered=answered, - error="核查模型没有按结构交结果(交回的内容里没有要求的字段:" + return await give_up("核查模型没有按结构交结果(交回的内容里没有要求的字段:" + "、".join(declared[:8]) + ("等" if len(declared) > 8 else "") + ")") # ---- 取值、组装理由 cells = _Cells(queries) result = cells.walk(resolved, parsed) if _is_object(resolved) else {} + if cut is not None and any(c["status"] != "resolved" for c in cells.cells): + # 工具循环被截断,截断前查到的里面没有要的数(或者引用指不到):话没说完就停了,说「没查到」会让人以为数据里 + # 没有。不交这份残缺的结果,原因照截断说;查过的照记 + return outcome(cells=cells.cells, limit=stopped, answered=answered, error=await report_cut(False)) rationale = parsed.get(RATIONALE) doc = None if isinstance(rationale, str) and rationale.strip(): doc = compose_doc(rationale, build_catalog(nodes={}, ledger=ledger), node_id=owner, run_id=run_id or None) result[RATIONALE] = doc["markdown"] if doc else "" + if cut is not None: + # 截断前查到的够用:照常交结果,说明记进 notes,时间线上照记一条(结果可能不完整) + notes.append(await report_cut(True)) return outcome(result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered) +def _with_fallback(model: Any, spec: ModelSpec, small: int | None, *, shared_with: BudgetFallback | None = None + ) -> BudgetFallback: + """按 spec 建好的模型包一层:服务方嫌额度大时按 small 重建一个再调,要退回时才建,平常一个都不多建。small 为 None + 是额度本来就不比它大,没有可退的。经本模块的 get_chat_model 建,测试换成剧本时小额度的那个也是剧本。""" + + async def build() -> Any: + async with SessionLocal() as session: + return (await get_chat_model(session, spec.model_copy(update={"max_tokens": small})))[0] + + return BudgetFallback(model, build=build if small else None, limits=(spec.max_tokens, small or spec.max_tokens), + shared_with=shared_with) + + async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, remaining: float | None, emit: Emit | None, queries: list[dict[str, Any]], ledger: list[dict[str, Any]], owner: str) -> str | None: diff --git a/backend/tests/test_evidence_judge.py b/backend/tests/test_evidence_judge.py index 75a3b995..f46d6a62 100644 --- a/backend/tests/test_evidence_judge.py +++ b/backend/tests/test_evidence_judge.py @@ -6,7 +6,7 @@ - 一份报告合并成一次调用(超过每批的句数就分批),每句给 supported / partial / contradicted / insufficient / not_a_claim - 模型不支持结构化输出、或者给的不是合法结构时退回 JSON 解析,再失败就记「未裁判」,不编判定 -- 裁判模型关掉 thinking,max_tokens 4096 +- 裁判模型关掉 thinking,输出额度 16384(推理模型的思考也算在里面);服务方嫌大时退回 4096,截断照实说(见 test_judge_truncation) - 预算(每份报告的句数、金额、时长,全局每日金额)每一项都能写 null 表示不限;触顶就停、已判的保留、 没判的记未裁判并说明是哪个上限——不报错、不让报告失败。金额超了按优先级截断:先判挂了依据的, 再判有方向词、因果词的 @@ -207,7 +207,7 @@ async def test_the_judge_model_runs_without_thinking(monkeypatch, stored): max_tokens=64000), budget=Budget(max_claims=None, max_cost_usd=None, timeout_s=30, daily_max_usd=None)) [spec] = judge.specs - assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 4096 + assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 16384 assert spec.model == PRICED and spec.timeout and spec.timeout <= 30 diff --git a/backend/tests/test_evidence_judge_api.py b/backend/tests/test_evidence_judge_api.py index 598d5cad..923aec55 100644 --- a/backend/tests/test_evidence_judge_api.py +++ b/backend/tests/test_evidence_judge_api.py @@ -350,7 +350,7 @@ async def test_the_judge_model_follows_node_then_settings(client, monkeypatch): row = await finish(weekly(claims="judge")) await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]]) [spec] = models.specs - assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 4096 + assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 16384 row = await finish(weekly(claims="judge", judge={"model": "judge-from-node"})) body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json() diff --git a/backend/tests/test_judge_settings.py b/backend/tests/test_judge_settings.py index 88fd6de3..80934b2d 100644 --- a/backend/tests/test_judge_settings.py +++ b/backend/tests/test_judge_settings.py @@ -17,6 +17,7 @@ from app.db.base import SessionLocal from app.db.models import Setting from app.engine.judge import ( JUDGE_DEFAULTS, + MAX_TOKENS, SETTING_GROUP, SPEND_KEY, Budget, @@ -166,7 +167,7 @@ async def test_the_judge_model_priority(): async with SessionLocal() as session: spec = await judge_model_spec(session, None) assert (spec.provider, spec.model) == (None, None), "什么都没配:默认 provider" - assert spec.thinking == "off" and spec.max_tokens == 4096 + assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 16384 session.add(Setting(key="copilot", value={"provider": "", "model": "copilot-model"})) await session.commit() diff --git a/backend/tests/test_verify_truncation.py b/backend/tests/test_verify_truncation.py new file mode 100644 index 00000000..26cf8d7e --- /dev/null +++ b/backend/tests/test_verify_truncation.py @@ -0,0 +1,334 @@ +"""核查 agent(engine/verify_agent.py)的输出额度:截断照实说,服务方嫌额度大时退回小额度。 + +核查(双盲复算、反方审稿、审批质询共用)原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到 +thinking=off,思考就能把 4096 用光:这次调用 finish_reason=length,没有工具调用也没有 JSON。于是工具循环把它当成 +「模型说完了」去收尾,双盲复算记成「盲算没有查到这个数」;收尾被截断时记成「核查模型没有按结构交结果」;质询回答失败 +——都没说出真正的原因。这里守几件事: + +- 额度:默认 16384(judge.MAX_TOKENS,核查和裁判共用);服务方嫌大时退回 4096 再试一次,之后工具循环、收尾都用小的, + 只在还没有任何输出时重来;别的 400 不重试 +- 工具循环里被截断(没有工具调用、话没说完)不算模型自己停下:一条查询都没成功时不再收尾,直接交回截断的原因; + 查到过东西的照常按查到的收尾,取不到要的数时同样交回截断的原因 +- 收尾被截断(Anthropic 的 stop_reason=max_tokens、openai 结构化输出抛的 LengthFinishReasonError、普通调用的 + finish_reason=length):交回截断的原因,不再退回普通调用多花一次额度;截断那次的用量照记 +- 时间线上记一条 verify_truncated(节点里调用时),llm.end 标出这次被截断 +- 原因经现有的渠道交出去:双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错;没有截断时原有说法不变 + +模型是剧本,数据源是现造的 SQLite 文件(借 test_verify_agent 的夹具),表名、数字都是虚构的。 +""" +from __future__ import annotations + +import json + +from langchain_core.messages import AIMessage, BaseMessage +from openai import LengthFinishReasonError +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion + +from app.core.events import EventType +from app.engine import judge as judging +from app.engine import verify_agent +from app.engine.inquiry import _error_words +from app.engine.nodes.challenge import _claim_of +from app.engine.nodes.dual_check import _slim +from tests.test_verify_agent import VALID_SUM, clean_spend, q, run, shop # noqa: F401 - 夹具按名字引入 + +CUT_16K = "核查模型的输出达到上限(16384 token)被截断" +ADVICE = "通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型" +FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", + "rationale": "未作废订单的金额合计是 [[v:Q1.r0.total]]。"} + + +def cut_reply(output: int = 16384) -> AIMessage: + """推理模型把额度全用在思考上:正文为空、没有工具调用,finish_reason=length(DeepSeek 经 LiteLLM 的样子)。""" + return AIMessage(content="", additional_kwargs={"reasoning_content": "先想想该查哪张表……"}, + response_metadata={"finish_reason": "length"}, + usage_metadata={"input_tokens": 900, "output_tokens": output, "total_tokens": 900 + output}) + + +def anthropic_cut() -> dict: + """结构化输出走强制工具调用(Anthropic)时被截断:原始回复 stop_reason=max_tokens,解析不出结果。""" + raw = AIMessage(content=[], response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 700, "output_tokens": 16384, "total_tokens": 17084}) + return {"raw": raw, "parsed": None, "parsing_error": None} + + +def length_error(prompt: int = 800, completion: int = 16384) -> LengthFinishReasonError: + """openai 的结构化输出(json_schema)在 finish_reason=length 时抛的那个错。""" + usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) + return LengthFinishReasonError(completion=ChatCompletion.model_construct( + id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage)) + + +def query_step(name: str, sql: str = VALID_SUM, call_id: str = "c1") -> AIMessage: + return AIMessage(content="查一下。", tool_calls=[{"name": n, "args": a, "id": i} for n, a, i in q(name, sql, call_id)]) + + +class BudgetRejected(Exception): + """服务方嫌输出额度太大时的 400(DeepSeek chat 系只收 8K)。""" + + status_code = 400 + + +class Reasoner: + """核查模型的剧本。steps:工具循环里每一轮的回复,用完了回「查完了。」;structured / plain:收尾时结构化调用、 + 普通调用的回复(AIMessage、结构化结果,或者一个异常就抛它)。calls 按顺序记下每次调用是哪一种。""" + + def __init__(self, steps=(), *, structured=None, plain=None, unstructured: bool = False) -> None: + self.steps = list(steps) + self.structured = structured + self.plain = plain + self.unstructured = unstructured + self.calls: list[str] = [] + + def bind_tools(self, tools, **kwargs): + return _View(self, "step") + + def with_structured_output(self, schema, **kwargs): + if self.unstructured: + raise NotImplementedError("这个模型不支持结构化输出") + return _View(self, "final") + + async def ainvoke(self, messages, *args, **kwargs): + return self.answer("plain") + + def answer(self, kind: str): + self.calls.append(kind) + if kind == "step": + reply = self.steps.pop(0) if self.steps else AIMessage(content="查完了。") + else: + reply = self.structured if kind == "final" else self.plain + if isinstance(reply, BaseException): + raise reply + return reply + + +class _View: + def __init__(self, model: Reasoner, kind: str) -> None: + self.model, self.kind = model, kind + + async def ainvoke(self, messages, *args, **kwargs): + return self.model.answer(self.kind) + + +class Rejecting: + """一开口就被拒的模型:工具循环、收尾都一样。""" + + def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens"): + self.message = message + self.calls = 0 + + def bind_tools(self, tools, **kwargs): + return self + + def with_structured_output(self, schema, **kwargs): + return self + + async def ainvoke(self, messages, *args, **kwargs): + self.calls += 1 + raise BudgetRejected(self.message) + + +class Models: + """按额度给模型:rejects_above 给了,额度比它大的一律交 Rejecting。specs 记下每次建模型的配置。""" + + def __init__(self, monkeypatch, model, *, rejects_above: int | None = None, rejecting: Rejecting | None = None): + self.specs = [] + self.rejecting = rejecting or Rejecting() + + async def fake(session, spec): + self.specs.append(spec) + if rejects_above is not None and (spec.max_tokens or 0) > rejects_above: + return self.rejecting, "reasoner-x" + return model, "reasoner-x" + + monkeypatch.setattr(verify_agent, "get_chat_model", fake) + + +def collector(): + events: list[tuple] = [] + return events, (lambda kind, **data: events.append((kind, data))) + + +def truncated_logs(events) -> list[dict]: + return [d for k, d in events if k == EventType.LOG and d.get("code") == "verify_truncated"] + + +# -------------------------------------------------------------------------- +# 工具循环里被截断 +# -------------------------------------------------------------------------- + + +async def test_thinking_that_eats_the_budget_in_the_loop_is_reported_as_truncation(monkeypatch, shop): + name, _ = shop + model = Reasoner([cut_reply()], structured={"value": None, "verdict": "无法判断", "rationale": "没有查询结果"}) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert out.result is None and out.queries == [] + assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert model.calls == ["step"], "一条查询都没有:收尾只能交「查不到」,不再为它多花一次额度" + assert out.answered is False, "被截断不是模型自己停下来的" + assert out.usage["output_tokens"] == 16384, "截断那次的额度是真花出去了" + [log] = truncated_logs(events) + assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["delivered"] is False + assert log["purpose"] == "verify" and log["message"] == out.error + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["truncated"] is True and end["max_tokens"] == 16384 and CUT_16K in end["error"] + + +async def test_a_cut_after_a_query_still_settles_on_what_was_found(monkeypatch, shop): + name, _ = shop + model = Reasoner([query_step(name), cut_reply()], structured=FOUND) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step", "step", "final"] + assert out.error is None and out.result["value"] == 230, "查到的数照常交:截断前的查询是真查过的" + assert out.answered is False + assert any(CUT_16K in n and "已按截断前查到的结果收尾" in n for n in out.notes) + [log] = truncated_logs(events) + assert log["delivered"] is True and log["purpose"] == "verify" and log["max_tokens"] == 16384 + + +async def test_a_cut_that_leaves_the_number_unfound_is_reported_as_truncation(monkeypatch, shop): + """查过、可要的数没查到就被截断了:说「没查到」会让人以为数据里没有,原因要写截断。""" + name, _ = shop + model = Reasoner([query_step(name, "SELECT COUNT(*) AS n FROM orders"), cut_reply()], + structured={"value": None, "verdict": "无法判断", "rationale": "还没查到合计"}) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert [c["status"] for c in out.cells] == ["missing"] and len(out.queries) == 1, "查过的照记" + [log] = truncated_logs(events) + assert log["delivered"] is False and log["purpose"] == "verify" + + +# -------------------------------------------------------------------------- +# 收尾被截断 +# -------------------------------------------------------------------------- + + +async def test_a_cut_off_settle_is_reported_and_not_retried_as_plain(monkeypatch, shop): + """Anthropic 系:强制工具调用写到一半到了上限。退回普通调用也多半被推理占满,不再白花一次额度。""" + name, _ = shop + model = Reasoner([query_step(name)], structured=anthropic_cut(), plain=AIMessage(content=json.dumps(FOUND))) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step", "step", "final"], "没有再退回普通调用" + assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + [log] = truncated_logs(events) + assert log["purpose"] == "verify_result" and log["delivered"] is False + end = [d for k, d in events if k == EventType.LLM_END][-1] + assert end["purpose"] == "verify_result" and end["truncated"] is True and CUT_16K in end["error"] + + +async def test_an_openai_length_error_in_the_settle_is_truncation_not_a_failed_call(monkeypatch, shop): + """openai 的结构化输出解析时抛 LengthFinishReasonError:不是「调用失败,请稍后重试」,是额度被推理占满。""" + name, _ = shop + model = Reasoner([query_step(name)], structured=length_error(completion=16384)) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step", "step", "final"] + assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.usage["output_tokens"] == 16384 and out.usage["input_tokens"] >= 800, "异常里带着的用量照记" + end = [d for k, d in events if k == EventType.LLM_END][-1] + assert "请稍后重试" not in end["error"] and CUT_16K in end["error"] and end["output_tokens"] == 16384 + + +async def test_a_plain_settle_cut_off_is_reported(monkeypatch, shop): + """不支持结构化输出的模型:收尾直接走普通调用,回复只有思考、finish_reason=length。""" + name, _ = shop + model = Reasoner([query_step(name)], plain=cut_reply(), unstructured=True) + Models(monkeypatch, model) + out = await run(name) + assert model.calls == ["step", "step", "plain"] + assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + + +async def test_without_truncation_the_old_wording_stays(monkeypatch, shop): + name, _ = shop + model = Reasoner([query_step(name)], structured="不是 JSON", plain=AIMessage(content="还是不是 JSON")) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step", "step", "final", "plain"], "格式不对照旧退回普通调用补救一次" + assert out.error == "核查模型没有按结构交结果" + assert truncated_logs(events) == [] + assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END) + + +# -------------------------------------------------------------------------- +# 额度与退回 +# -------------------------------------------------------------------------- + + +async def test_the_default_budget_leaves_room_for_reasoning(monkeypatch, shop): + name, _ = shop + models = Models(monkeypatch, Reasoner([query_step(name)], structured=FOUND)) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [s.max_tokens for s in models.specs] == [16384, 16384], "工具循环、收尾都按 16384,不多建模型" + assert models.specs[1].thinking == "off" + + +async def test_a_rejected_budget_falls_back_to_4096_for_the_loop_and_the_settle(monkeypatch, shop): + name, _ = shop + model = Reasoner([query_step(name)], structured=FOUND) + models = Models(monkeypatch, model, rejects_above=4096) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096] + assert [s.thinking for s in models.specs][2:] == ["off", "off"], "收尾的小额度模型同样关掉思考" + assert models.rejecting.calls == 1, "退回过一次之后收尾直接用小的,不再先撞一次" + assert model.calls == ["step", "step", "final"] + + +async def test_a_cut_after_the_fallback_names_the_small_budget(monkeypatch, shop): + name, _ = shop + Models(monkeypatch, Reasoner([cut_reply(output=4096)]), rejects_above=4096) + out = await run(name) + assert out.error.startswith("核查模型的输出达到上限(4096 token)被截断") + + +async def test_other_rejections_are_not_retried_with_the_small_budget(monkeypatch, shop): + name, _ = shop + models = Models(monkeypatch, Reasoner(), rejects_above=4096, + rejecting=Rejecting("model 'reasoner-x' does not exist")) + out = await run(name) + assert out.result is None and "reasoner-x" in out.error + assert [s.max_tokens for s in models.specs] == [16384], "不是额度的问题:不拿小额度再撞一次" + + +async def test_the_verify_spec_asks_for_the_larger_budget(): + """核查模型的配置(api/settings.py 的 verify_model_spec 沿用证据裁判的额度):16384,思考照旧关掉。""" + assert judging.MAX_TOKENS == 16384 and judging.FALLBACK_MAX_TOKENS == 4096 + + +# -------------------------------------------------------------------------- +# 原因经现有的渠道交出去 +# -------------------------------------------------------------------------- + + +async def test_the_reason_reaches_dual_check_challenge_and_inquiry(monkeypatch, shop): + name, _ = shop + Models(monkeypatch, Reasoner([cut_reply()])) + out = await run(name) + expected = f"{CUT_16K},核查未能完成。{ADVICE}" + # 双盲复算:无法复算的原因 + slim = _slim(out) + assert slim["value"] is None and slim["reason"] == expected + # 反方审稿:这句记为无法判断,理由写「反查未完成」,没能反查的原因就是截断 + cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="", + aliases=("Q1",)) + claim, why = _claim_of(cand, out) + assert claim["verdict"] == "inconclusive" and claim["rationale"] == f"反查未完成:{expected}" and why == expected + # 审批质询:提问人看到的报错照原话 + assert _error_words(out, timeout_s=60, daily_max_usd=1.0) == expected + + +def test_the_reply_shapes_used_here_are_messages(): + assert isinstance(cut_reply(), BaseMessage) and isinstance(anthropic_cut()["raw"], BaseMessage) From 4a3de10d38431fa4a4bfa0e1f8695dc51aaf0372 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 13:02:24 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E8=AF=81=E6=8D=AE=E8=A3=81=E5=88=A4?= =?UTF-8?q?=E6=8C=89=2016384=20=E7=9A=84=E9=A2=9D=E5=BA=A6=E5=88=A4?= =?UTF-8?q?=E3=80=81=E6=9C=8D=E5=8A=A1=E6=96=B9=E5=AB=8C=E5=A4=A7=E6=97=B6?= =?UTF-8?q?=E9=80=80=E5=9B=9E=204096=EF=BC=8C=E5=88=A4=E5=AE=9A=E5=86=99?= =?UTF-8?q?=E5=88=B0=E4=B8=80=E5=8D=8A=E5=88=B0=E4=BA=86=E4=B8=8A=E9=99=90?= =?UTF-8?q?=E7=9A=84=E8=AE=B0=E6=9C=AA=E8=A3=81=E5=88=A4=EF=BC=88truncated?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E7=85=A7=E5=AE=9E=E8=AF=B4=E6=98=AF=E8=BE=93?= =?UTF-8?q?=E5=87=BA=E9=A2=9D=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 裁判原来固定 4096、thinking off;经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考能把 4096 用光,结构化调用写不出判定、退回纯文本也一样,最后记成「裁判返回的结果格式有误」。现在: - 额度 MAX_TOKENS(16384),服务方嫌大(400/422 且报错提到 max_tokens 一类)时退回 FALLBACK_MAX_TOKENS(4096)再问一次,之后各批都用小的、只撞一次;小额度的模型要退回时才建 - 每次调用看有没有被截断:Anthropic 的 stop_reason=max_tokens、OpenAI 系的 finish_reason=length、openai 结构化输出抛的 LengthFinishReasonError。截断的这一批记未裁判,原因是新的 truncated(和格式不对分开记:说成「格式有误」人会去查提示词,该换的是模型),理由写「裁判模型的输出达到上限(N token)被截断,这句未裁判。通常是推理占满了输出额度,可换用不带推理的裁判模型」,缺口、llm.end 的 error(带 truncated、max_tokens)、judge_failed 日志同样写明 - 结构化调用被截断的不再退回纯文本多花一次额度;截断那次的用量从异常里取,照记账 - 别的批照常问(同格式不对);没有截断时原有说法不变 测试:新增 test_judge_truncation(结构化调用截断、LengthFinishReasonError、纯文本只有思考、一批截断不影响别的批、退回 4096 只撞一次、退回后截断写 4096、格式不对的说法不变)。 --- backend/app/engine/judge.py | 62 ++++++- backend/tests/test_judge_truncation.py | 236 +++++++++++++++++++++++++ 2 files changed, 292 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_judge_truncation.py diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index 27ba26c0..b43d2f87 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -39,7 +39,7 @@ from dataclasses import asdict, dataclass from datetime import datetime from typing import Any -from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage from sqlalchemy import select from app.core.artifact_store import canonical_json, content_hash @@ -70,6 +70,7 @@ from app.engine.merge_query import MERGE_SOURCE from app.engine.nodes.llm import _split_structured, _usage_of from app.engine.state import message_text from app.providers import catalog as pricing +from app.providers.budget import BudgetFallback, cut_off, cut_words, error_usage, truncated_error from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model logger = logging.getLogger(__name__) @@ -131,9 +132,12 @@ SHORT_CHARS = 6 _OUT_PER_UNIT = 120 _OUT_BASE = 20 -#: 未裁判的原因。前四个是上限,报告里写「已到上限」 +#: 未裁判的原因。前四个是上限,报告里写「已到上限」;truncated 是裁判模型的输出到了上限被截断(推理模型的思考也算在 +#: 额度里),和格式不对分开记:说成「格式有误」,人会去查提示词,真正该换的是模型 LIMITS = ("max_claims", "max_cost_usd", "daily_max_usd", "timeout_s") -REASONS = (*LIMITS, "error", "format", "missing", "on_demand") +REASONS = (*LIMITS, "error", "format", "truncated", "missing", "on_demand") +#: 输出被截断时给人看的建议,接在 budget.cut_words 的「裁判模型的输出达到上限(N token)被截断」后面 +TRUNCATED_ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型" JUDGE_ROLE = "你是报告核查员:逐句判断报告里的话,能不能被它挂的证据支持。你不写报告,也不改报告。" @@ -1071,7 +1075,10 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non """问一次裁判:(判定清单, 模型的原始回复, 没成的原因, 调了几次模型)。 先走结构化输出;不成(模型不支持、给的不是合法结构)先从原始回复里找 JSON,再退回纯文本调用、 - 从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对。超时由调用方掐。 + 从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对,truncated 是输出到了上限被截断 + (Anthropic 的 stop_reason=max_tokens、OpenAI 系的 finish_reason=length,以及 openai 结构化输出抛的 + LengthFinishReasonError)。结构化调用被截断的不再退回纯文本:推理占满额度时纯文本调用多半一样被截断,白花一次 + 额度。超时由调用方掐。 """ raws: list[BaseMessage] = [] try: @@ -1086,17 +1093,47 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non for found in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: if (items := _items_of(found)) is not None: return items, raws, None, 1 - except Exception: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析 - pass + if cut_off(raw): + return None, raws, _TRUNCATED, 1 + except Exception as e: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析 + if truncated_error(e): + return None, [*raws, *_spent(e)], _TRUNCATED, 1 try: reply = await model.ainvoke(messages) except Exception as e: # noqa: BLE001 + if truncated_error(e): + return None, [*raws, *_spent(e)], _TRUNCATED, 2 return None, raws, f"error:{describe_exception(e)}", 2 raws.append(reply) items = _items_of(message_text(reply)) + if items is None and cut_off(reply): + return None, raws, _TRUNCATED, 2 return items, raws, None if items is not None else "format", 2 +#: _ask 交回的没成的原因:输出到了上限被截断 +_TRUNCATED = "truncated" + + +def _spent(error: BaseException) -> list[BaseMessage]: + """截断时 openai 的异常带着这次的用量:折成一条空回复交给记账(额度是真花出去了,每日上限不能漏掉它)。""" + inp, out = error_usage(error) + if not (inp or out): + return [] + return [AIMessage(content="", usage_metadata={"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out})] + + +def _with_fallback(model: Any, spec: ModelSpec) -> BudgetFallback: + """裁判模型包一层:服务方嫌 MAX_TOKENS 太大时按 FALLBACK_MAX_TOKENS 重建一个再问(要退回时才建,平常不多建),之后 + 各批都用小的。经本模块的 get_chat_model 建:测试换成剧本时,小额度的那个也是剧本。""" + + async def build() -> Any: + async with SessionLocal() as session: + return (await get_chat_model(session, spec.model_copy(update={"max_tokens": FALLBACK_MAX_TOKENS})))[0] + + return BudgetFallback(model, build=build, limits=(spec.max_tokens, FALLBACK_MAX_TOKENS)) + + def _unjudged(reason: str, rationale: str, *, judge: str | None, post_seal: bool) -> dict[str, Any]: return {"status": "unjudged", "rationale": rationale, "judge": judge, "post_seal": post_seal, "reason": reason} @@ -1124,6 +1161,8 @@ def _why_unjudged(reason: str, budget: Budget, detail: str = "", *, click: bool return f"裁判失败:{detail}"[:120] if reason == "format": return detail or "裁判返回的结果格式有误,这句未裁判" + if reason == "truncated": + return f"{detail or cut_words('裁判模型', None)},这句未裁判。{TRUNCATED_ADVICE}" if reason == "missing": return "裁判未对这句给出判定" return "探索运行按需裁判:点开这句并请模型判断后才会裁判" @@ -1157,6 +1196,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, try: async with SessionLocal() as session: model, model_id = await get_chat_model(session, spec) + model = _with_fallback(model, spec) except Exception as e: # noqa: BLE001 - 裁判模型没配好:这回判不了,报告照常 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) details["error"] = f"裁判模型不可用({why})" @@ -1269,6 +1309,14 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, end = {"model": model_id, "purpose": "judge", "duration_ms": int((time.perf_counter() - began) * 1000), "input_tokens": inp, "output_tokens": out, "total_tokens": inp + out, "cost_usd": round(actual, 6), "calls": calls, "units": len(batch)} + if problem == _TRUNCATED: + # 输出到了上限被截断:这一批记未裁判,原因照实说是额度,不说「格式有误」「调用失败」。别的批照常问 + # (同格式不对):截断取决于这一批要想多久,不像鉴权、网关出错那样后面几批多半一样 + limit = model.max_tokens + details["truncated"] = cut_words("裁判模型", limit) + emit(EventType.LLM_END, **end, error=details["truncated"], truncated=True, max_tokens=limit) + stop(batch, "truncated", details["truncated"]) + continue if problem is not None and problem != "format": reason = "timeout_s" if problem == "timeout_s" else "error" detail = problem.partition(":")[2] @@ -1402,6 +1450,8 @@ def _outcome(request: JudgeRequest, verdicts: dict[str, dict[str, Any]], budget: text = f"有 {n} 句结论未裁判:裁判调用失败({details.get('error') or '原因不明'})" elif reason == "format": text = f"有 {n} 句结论未裁判:裁判返回的结果格式有误" + elif reason == "truncated": + text = f"有 {n} 句结论未裁判:{details.get('truncated') or cut_words('裁判模型', None)},{TRUNCATED_ADVICE}" else: text = f"有 {n} 句结论未获得裁判结果" gaps.append((reason, text)) diff --git a/backend/tests/test_judge_truncation.py b/backend/tests/test_judge_truncation.py new file mode 100644 index 00000000..cf0af450 --- /dev/null +++ b/backend/tests/test_judge_truncation.py @@ -0,0 +1,236 @@ +"""结论句裁判(engine/judge.py)的输出额度:截断照实说,服务方嫌额度大时退回小额度。 + +裁判原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考就能把 4096 用光: +结构化调用写不出判定(openai 抛 LengthFinishReasonError,Anthropic 的强制工具调用写到一半),退回纯文本调用也一样, +最后记成「裁判返回的结果格式有误」——没说出真正的原因。这里守几件事: + +- 额度:16384(MAX_TOKENS);服务方嫌大时退回 4096(FALLBACK_MAX_TOKENS)再问一次,之后各批都用小的,只撞一次 +- 输出到了上限被截断(stop_reason=max_tokens、finish_reason=length、LengthFinishReasonError):这一批记未裁判,原因 + 是 truncated,理由、缺口、llm.end、judge_failed 日志都写明「裁判模型的输出达到上限(N token)被截断」;结构化调用 + 被截断的不再退回纯文本多花一次额度;截断那次的用量照记 +- 没有截断时原有说法不变(格式不对仍是 format) + +模型都是剧本;证据借 test_evidence_judge 的虚构夹具(口径卡、一次查询、一次检索)。 +""" +from __future__ import annotations + +import json +import re + +from langchain_core.messages import AIMessage +from openai import LengthFinishReasonError +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion + +from app.core.events import EventType +from app.engine import judge as judge_mod +from app.engine.evidence import compose_doc +from app.engine.judge import BATCH_SIZE, FALLBACK_MAX_TOKENS, MAX_TOKENS, REASONS, judge_doc +from tests.test_evidence_judge import PRICED, RAW, SPEC, UNLIMITED, clean_settings, make_catalog, stored # noqa: F401 + +CUT_16K = "裁判模型的输出达到上限(16384 token)被截断" +ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型" + + +def length_error(prompt: int = 3000, completion: int = 16384) -> LengthFinishReasonError: + usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) + return LengthFinishReasonError(completion=ChatCompletion.model_construct( + id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage)) + + +def anthropic_cut(units) -> dict: + """强制工具调用写到一半到了上限:原始回复 stop_reason=max_tokens,解析不出判定。""" + raw = AIMessage(content=[], response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384}) + return {"raw": raw, "parsed": None, "parsing_error": None} + + +def thinking_only(units) -> AIMessage: + """纯文本调用:额度全用在思考上,正文为空,finish_reason=length(推理模型经 OpenAI 兼容接入的样子)。""" + return AIMessage(content="", additional_kwargs={"reasoning_content": "逐句看证据……"}, + response_metadata={"finish_reason": "length"}, + usage_metadata={"input_tokens": 3000, "output_tokens": 4096, "total_tokens": 7096}) + + +def supported(units) -> dict: + items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] + return {"raw": AIMessage(content="", response_metadata={"finish_reason": "stop"}), "parsed": {"items": items}} + + +class BudgetRejected(Exception): + """服务方嫌输出额度太大时的 400。""" + + status_code = 400 + + +class Reasoner: + """裁判模型的剧本。structured(units) / plain(units) 给出结构化调用、退回纯文本调用的回复(返回异常就抛它); + unstructured 为真时不支持结构化输出。calls 记下每次调用的种类和判的句子。""" + + def __init__(self, *, structured=supported, plain=None, unstructured: bool = False) -> None: + self.structured, self.plain, self.unstructured = structured, plain, unstructured + self.calls: list[tuple[str, list[str]]] = [] + + def with_structured_output(self, schema, **kwargs): + if self.unstructured: + raise NotImplementedError("这个模型不支持结构化输出") + return _View(self) + + async def ainvoke(self, messages, *args, **kwargs): + return self.answer("plain", messages) + + def answer(self, kind: str, messages): + units = re.findall(r"^\[(u\d+)\]", str(messages[-1].content), re.M) + self.calls.append((kind, units)) + make = self.structured if kind == "structured" else self.plain + reply = make(units) + if isinstance(reply, BaseException): + raise reply + return reply + + +class _View: + def __init__(self, model: Reasoner) -> None: + self.model = model + + async def ainvoke(self, messages, *args, **kwargs): + return self.model.answer("structured", messages) + + +class Rejecting: + def __init__(self) -> None: + self.calls = 0 + + def with_structured_output(self, schema, **kwargs): + return self + + async def ainvoke(self, messages, *args, **kwargs): + self.calls += 1 + raise BudgetRejected("max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens") + + +class Models: + def __init__(self, monkeypatch, model, *, rejects_above: int | None = None) -> None: + self.specs = [] + self.rejecting = Rejecting() + + async def fake(session, spec): + self.specs.append(spec) + if rejects_above is not None and (spec.max_tokens or 0) > rejects_above: + return self.rejecting, PRICED + return model, PRICED + + monkeypatch.setattr(judge_mod, "get_chat_model", fake) + + +async def judged(stored, raw: str = RAW): + catalog = make_catalog(stored) + events: list[tuple] = [] + outcome = await judge_doc(compose_doc(raw, catalog), catalog, spec=SPEC, budget=UNLIMITED, + emit=lambda kind, **data: events.append((kind, data))) + return outcome, events + + +# -------------------------------------------------------------------------- +# 截断 +# -------------------------------------------------------------------------- + + +async def test_a_cut_off_structured_reply_is_unjudged_as_truncation_without_a_plain_retry(monkeypatch, stored): + model = Reasoner(structured=anthropic_cut, plain=lambda units: AIMessage(content=json.dumps({"items": []}))) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured"], "判定写到一半到了上限:不再退回纯文本多花一次额度" + verdicts = list(outcome["verdicts"].values()) + assert {(v["status"], v["reason"]) for v in verdicts} == {("unjudged", "truncated")} + assert verdicts[0]["rationale"] == f"{CUT_16K},这句未裁判。{ADVICE}" + assert outcome["unjudged"] == {"truncated": 4} and outcome["limits_hit"] == [] + assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"] + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["purpose"] == "judge" and end["error"] == CUT_16K and end["truncated"] is True + assert end["max_tokens"] == 16384 and end["output_tokens"] == 16384 + [failed] = [d for k, d in events if k == EventType.LOG and d.get("code") == "judge_failed"] + assert failed["message"] == f"结论句裁判未完成:有 4 句结论未裁判:{CUT_16K},{ADVICE}" + + +async def test_an_openai_length_error_is_truncation_and_its_tokens_are_counted(monkeypatch, stored): + model = Reasoner(structured=lambda units: length_error(completion=16384)) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured"] + assert outcome["unjudged"] == {"truncated": 4} + assert outcome["usage"]["output_tokens"] == 16384 and outcome["cost_usd"] > 0, "截断那次的额度是真花出去了" + end = [d for k, d in events if k == EventType.LLM_END][-1] + assert "调用" not in end["error"] and end["error"] == CUT_16K + + +async def test_a_plain_reply_with_only_thinking_is_truncation(monkeypatch, stored): + """不支持结构化输出的模型直接退回纯文本调用:回复只有思考、finish_reason=length。""" + model = Reasoner(plain=thinking_only, unstructured=True) + Models(monkeypatch, model) + outcome, _ = await judged(stored) + assert [kind for kind, _ in model.calls] == ["plain"] and outcome["calls"] == 2 + assert outcome["unjudged"] == {"truncated": 4} + assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"] + + +async def test_a_truncated_batch_does_not_stop_the_others(monkeypatch, stored): + """只有一批被截断:别的批照常判(和格式不对一样,不像调用失败那样后面的都不再问)。""" + first: list[bool] = [] + + def once(units): + if not first: + first.append(True) + return anthropic_cut(units) + return supported(units) + + Models(monkeypatch, Reasoner(structured=once)) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3)) + outcome, _ = await judged(stored, raw) + assert outcome["unjudged"] == {"truncated": BATCH_SIZE} + assert outcome["counts"]["supported"] == 3 + + +async def test_without_truncation_format_problems_keep_their_wording(monkeypatch, stored): + model = Reasoner(structured=lambda units: {"raw": AIMessage(content="不是 JSON", + response_metadata={"finish_reason": "stop"}), + "parsed": None}, + plain=lambda units: AIMessage(content="还是不是", response_metadata={"finish_reason": "stop"})) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured", "plain"] + assert outcome["unjudged"] == {"format": 4} + assert outcome["gaps"] == ["有 4 句结论未裁判:裁判返回的结果格式有误"] + assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END) + + +# -------------------------------------------------------------------------- +# 额度与退回 +# -------------------------------------------------------------------------- + + +async def test_the_judge_asks_for_the_larger_budget(monkeypatch, stored): + models = Models(monkeypatch, Reasoner()) + outcome, _ = await judged(stored) + assert outcome["counts"]["supported"] == 4 + assert [s.max_tokens for s in models.specs] == [MAX_TOKENS] == [16384], "平常只建一个模型" + assert "truncated" in REASONS and FALLBACK_MAX_TOKENS == 4096 + + +async def test_a_rejected_budget_falls_back_to_4096_once_for_all_batches(monkeypatch, stored): + model = Reasoner() + models = Models(monkeypatch, model, rejects_above=4096) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE * 2 + 3)) + outcome, _ = await judged(stored, raw) + assert outcome["counts"]["supported"] == BATCH_SIZE * 2 + 3 and outcome["gaps"] == [] + assert [s.max_tokens for s in models.specs] == [16384, 4096] + assert models.rejecting.calls == 1, "第一批退回过,后面几批直接用小的" + assert [kind for kind, _ in model.calls] == ["structured"] * 3 + + +async def test_a_truncation_after_the_fallback_names_the_small_budget(monkeypatch, stored): + Models(monkeypatch, Reasoner(plain=thinking_only, unstructured=True), rejects_above=4096) + outcome, _ = await judged(stored) + [verdict, *_] = outcome["verdicts"].values() + assert verdict["reason"] == "truncated" + assert verdict["rationale"].startswith("裁判模型的输出达到上限(4096 token)被截断") From 2c2672cc04f7f41a1c6e59ea847ee429bf2e3795 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 13:06:18 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E6=97=B6=E9=97=B4=E7=BA=BF=E8=AE=A4?= =?UTF-8?q?=E5=BE=97=E6=A0=B8=E6=9F=A5=E6=A8=A1=E5=9E=8B=E7=9A=84=E8=BE=93?= =?UTF-8?q?=E5=87=BA=E8=A2=AB=E6=88=AA=E6=96=AD=EF=BC=88verify=5Ftruncated?= =?UTF-8?q?=EF=BC=89=EF=BC=9A=E6=A0=87=E9=A2=98=E8=AF=B4=E3=80=8C=E6=A0=B8?= =?UTF-8?q?=E6=9F=A5=E6=A8=A1=E5=9E=8B=E7=9A=84=E8=BE=93=E5=87=BA=E8=BE=BE?= =?UTF-8?q?=E5=88=B0=E4=B8=8A=E9=99=90=EF=BC=88N=20token=EF=BC=89=E8=A2=AB?= =?UTF-8?q?=E6=88=AA=E6=96=AD=E3=80=8D=EF=BC=8C=E5=89=AF=E6=A0=87=E9=A2=98?= =?UTF-8?q?=E8=AF=B4=E8=BF=99=E6=AC=A1=E6=A0=B8=E6=9F=A5=E6=9C=AA=E8=83=BD?= =?UTF-8?q?=E5=AE=8C=E6=88=90=E6=88=96=E5=B7=B2=E6=8C=89=E6=88=AA=E6=96=AD?= =?UTF-8?q?=E5=89=8D=E6=9F=A5=E5=88=B0=E7=9A=84=E7=BB=93=E6=9E=9C=E6=94=B6?= =?UTF-8?q?=E5=B0=BE=EF=BC=8C=E4=B8=8B=E4=B8=80=E6=AD=A5=E6=8C=87=E5=88=B0?= =?UTF-8?q?=E6=8D=A2=E4=B8=8D=E5=B8=A6=E6=8E=A8=E7=90=86=E7=9A=84=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=EF=BC=8C=E5=85=A5=E5=8F=A3=E8=90=BD=E5=88=B0=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E7=9A=84=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decode.ts 按结构化字段(max_tokens、delivered)解读,不从原话里认,缺字段时只给不带数的标题;文案放在 terms.ts 新的 VERIFY_TRUNCATED_TEXT。check-decode 加一节:先核对 verify_agent.py 还是这么发的,再核对三种取值(没交出结果、退回 4096 后交了结果、缺字段)的标题、副标题、下一步、入口和原话。 跑完之后的复核层(engine/review.py 的 WARN_CODES)登记 verify_truncated:同 verify_limit 记成它自己的 kind、degraded,不落成笼统的 warn,也不触发「重跑一次大概率能好」。 --- backend/app/engine/review.py | 3 ++ backend/tests/test_verify_truncation.py | 11 ++++++ frontend/src/lib/terms.ts | 13 +++++++ frontend/src/run/decode.ts | 21 +++++++++-- scripts/check-decode.mjs | 46 +++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 3 deletions(-) diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 7b4cc70b..c497c95e 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -128,6 +128,9 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 核查 agent 自己触顶、按已有结果收尾:多半和节点自己的 *_limit 说的是同一次触顶,照样是缺口,记成它自己的 kind, # 不落成笼统的 warn——步数上限(max_steps)只有它说 "verify_limit": ("verify_limit", DEGRADED), + # 核查 agent 的输出到了上限被截断(推理模型的思考也算在额度里):没交成结果的由节点按无法复算、未能反查记缺口,交了的 + # 可能不完整。同 verify_limit,是缺口不是断裂,记成它自己的 kind:说得清是输出额度的问题,换个模型就好 + "verify_truncated": ("verify_truncated", DEGRADED), } # 这些 kind 重跑一次大概率能好:原因说得清,且不是「问题本身没问明白」。 diff --git a/backend/tests/test_verify_truncation.py b/backend/tests/test_verify_truncation.py index 26cf8d7e..09397ee3 100644 --- a/backend/tests/test_verify_truncation.py +++ b/backend/tests/test_verify_truncation.py @@ -27,6 +27,7 @@ from openai.types.chat import ChatCompletion from app.core.events import EventType from app.engine import judge as judging +from app.engine import review as rv from app.engine import verify_agent from app.engine.inquiry import _error_words from app.engine.nodes.challenge import _claim_of @@ -330,5 +331,15 @@ async def test_the_reason_reaches_dual_check_challenge_and_inquiry(monkeypatch, assert _error_words(out, timeout_s=60, daily_max_usd=1.0) == expected +def test_the_review_layer_knows_the_truncation_log(): + """跑完之后的复核层(engine/review.py)认得 verify_truncated:同 verify_limit,是缺口不是断裂,kind 是它自己, + 不落成笼统的 warn;不触发「重跑一次大概率能好」——同样的模型再跑一遍还是撞同一堵墙。""" + assert rv.WARN_CODES["verify_truncated"] == ("verify_truncated", rv.DEGRADED) + event = {"type": str(EventType.LOG), "node_id": "dc", + "data": {"level": "warn", "code": "verify_truncated", "message": f"{CUT_16K},核查未能完成。{ADVICE}"}} + [signal] = rv.scan([event], {"结论": "本周订单 120 单。"}) + assert (signal.kind, signal.severity) == ("verify_truncated", rv.DEGRADED) and not rv.should_retry([signal]) + + def test_the_reply_shapes_used_here_are_messages(): assert isinstance(cut_reply(), BaseMessage) and isinstance(anthropic_cut()["raw"], BaseMessage) diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 8ea7b9fb..cdd1af77 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1039,6 +1039,19 @@ export const AGENT_GUARD_TEXT = { unlimitedShort: '不限', } as const +/** + * 时间线上「核查模型的输出到了上限被截断」那一行(engine/verify_agent.py 的 verify_truncated 日志,双盲复算、反方审稿的 + * 核查 Agent 发)。推理模型的思考也算在输出额度里,额度用完时回复是空的或只写了一半:照实说是输出额度不够,不说成 + * 「没查到」「格式不对」,下一步指到换模型。delivered:截断发生在查询过程中、已按截断前查到的结果收尾(结果可能不完整); + * undelivered:这次核查没有给出结果,节点按无法复算、未能反查处理 + */ +export const VERIFY_TRUNCATED_TEXT = { + title: (maxTokens?: number) => `核查模型的输出达到上限${maxTokens ? `(${formatNumber(maxTokens)} token)` : ''}被截断`, + delivered: '已按截断前查到的结果收尾,结果可能不完整', + undelivered: '这次核查未能完成', + next: '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型', +} as const + /** * 上传表格:数据页「表格」标签的上传弹窗、导入回执和卡片。种类的说法和后端 data/tabular.py 的 * CONVERSION_KINDS、SHAPE_KINDS 对得上;后端以后多出来的种类,界面上只显示它给的那句话 diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index f8799259..8a3a92ce 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -3,7 +3,7 @@ import type { NodeState, RunPhase } from './trace' import { formatDuration, formatNumber } from '../lib/format' import { ABSENCE_CLAIM_TEXT, CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, SCHEMA_PARTIAL_TEXT, SQL_CHECK_TEXT, TYPE_LABEL, - CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, NODE_TYPE_LABEL, + CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, NODE_TYPE_LABEL, VERIFY_TRUNCATED_TEXT, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel, } from '../lib/terms' @@ -765,17 +765,19 @@ function gateTitle(tool: string, verdict: 'allow' | 'escalate' | 'team', reason: * 双盲复算、反方审稿的日志代码(期 3 前后端共用的约定)。这几条按 code 和结构化字段解读,不看原话;后端发成 info 的 * (一致、裁决、记入数据目录、审稿完成)也上时间线,不然这两种节点就是一个跑了几十秒、什么都没说的黑盒。 * 无法复算(dual_check_unchecked)、未能反查(challenge_unreviewed)、只查了部分数据源(challenge_sources_lost)是 - * 期 4 补上的:以前认不得,只能照原话显示 + * 期 4 补上的:以前认不得,只能照原话显示。核查模型的输出到了上限被截断(verify_truncated,两种节点共用的核查 Agent 发) + * 是期 5 验收前补上的 */ const VERIFY_LOG_CODES = new Set([ 'dual_check_skipped', 'dual_check_same_model', 'dual_check_agreed', 'dual_check_disagreed', 'dual_check_resolved', 'dual_check_catalog', 'dual_check_limit', 'dual_check_unchecked', 'challenge_skipped', 'challenge_same_model', 'challenge_done', 'challenge_refuted', 'challenge_limit', - 'challenge_unreviewed', 'challenge_sources_lost', + 'challenge_unreviewed', 'challenge_sources_lost', 'verify_truncated', ]) const DC = DUAL_CHECK_TEXT const CL = CHALLENGE_TEXT.log +const VT = VERIFY_TRUNCATED_TEXT /** * 出具缺口里哪几条是双盲复算、反方审稿报上来的:出具契约给上游节点报的缺口加了「节点类型「节点名」:」的前缀 @@ -1214,6 +1216,19 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.max_tokens : undefined + return { + title: VT.title(limit), + sub: d.delivered === true ? VT.delivered : d.delivered === false ? VT.undelivered : undefined, + next: VT.next, + fix: 'canvas', + fixField: 'model', + level: 'warn', + } + } default: return null } diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index 68f283cd..3804887b 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -2319,6 +2319,52 @@ await section('核查类日志与出具缺口(期 4):无法复算、未能 rows.map((r) => `${r?.title}|${r?.sub ?? ''}`).join(' ; ')) }) +await section('核查模型的输出到了上限被截断(verify_truncated):说清额度、交没交出结果,下一步指到换模型', async () => { + // 推理模型的思考也算在输出额度里:核查 Agent 的一次调用 finish_reason=length、没有工具调用也没有 JSON 时,以前双盲复算 + // 记成「盲算没有查到这个数」、反方审稿记成「核查模型没有按结构交结果」。现在核查 Agent 另发 verify_truncated(max_tokens、 + // delivered、purpose),时间线按结构化字段说人话,原话留在展开区。先核对后端源码还是这么发的 + const verifyPy = readFileSync(`${root}backend/app/engine/verify_agent.py`, 'utf8') + check('后端发截断日志的格式还是夹具里这一种', verifyPy.includes('code="verify_truncated"') && verifyPy.includes('max_tokens=limit') + && verifyPy.includes('delivered=delivered') && verifyPy.includes('purpose=purpose') + && verifyPy.includes('TRUNCATED_ERROR = "{cut},核查未能完成。{advice}"'), + '改了 verify_agent.py 的字段或说法就同步改 decode.ts 的 explainLog 和这里的夹具') + const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data }) + const ADVICE = '通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型' + const FAILED = `核查模型的输出达到上限(16384 token)被截断,核查未能完成。${ADVICE}` + const SETTLED = `核查模型的输出达到上限(16384 token)被截断,已按截断前查到的结果收尾,结果可能不完整。${ADVICE}` + const all = flatten(mod.decodeRun([ + E(1, 'run.started', null, { total: 2 }), + E(2, 'node.started', 'dc', { node_type: 'dual_check', label: '复核销售额' }), + E(3, 'log', 'dc', { level: 'warn', code: 'verify_truncated', purpose: 'verify', max_tokens: 16384, delivered: false, message: FAILED }), + E(4, 'node.finished', 'dc', { duration_ms: 20 }), + E(5, 'node.started', 'ch', { node_type: 'challenge', label: '审月报' }), + E(6, 'log', 'ch', { level: 'warn', code: 'verify_truncated', purpose: 'verify_result', max_tokens: 4096, delivered: true, + message: SETTLED }), + // 缺字段、字段类型不对(别的服务端版本):只给不带数的标题,不瞎填 + E(7, 'log', 'ch', { level: 'warn', code: 'verify_truncated', max_tokens: '16384', delivered: 'no', message: FAILED }), + E(8, 'node.finished', 'ch', { duration_ms: 30 }), + E(9, 'run.finished', null, { status: 'succeeded' }), + ])) + const rows = all.filter((s) => s.code === 'verify_truncated') + const nodes = all.filter((s) => s.kind === 'node') + check('每条一行,挂在自己的节点下', rows.length === 3 && nodes.find((s) => s.nodeId === 'dc')?.children?.filter((x) => x.code === 'verify_truncated').length === 1 + && nodes.find((s) => s.nodeId === 'ch')?.children?.filter((x) => x.code === 'verify_truncated').length === 2, `${rows.length} 行`) + const [failed, settled, bare] = rows + check('没交出结果:标题说输出到了上限被截断、带额度,副标题说核查未能完成', failed?.title === '核查模型的输出达到上限(16,384 token)被截断' + && failed.sub === '这次核查未能完成', `${failed?.title} / ${failed?.sub}`) + check('交了结果:额度照实写(退回小额度之后是 4,096),副标题说按截断前查到的结果收尾、可能不完整', + settled?.title === '核查模型的输出达到上限(4,096 token)被截断' && settled.sub === '已按截断前查到的结果收尾,结果可能不完整', + `${settled?.title} / ${settled?.sub}`) + check('……下一步指到换不带推理的模型,入口落到节点的模型,提醒色', rows.every((s) => !!s.next?.includes('不带推理的模型') + && s.next.includes('设置 → 偏好设置 → 核查') && s.fix === 'canvas' && s.fixField === 'model' && s.level === 'warn'), + `${failed?.next} / ${failed?.fix} / ${failed?.fixField} / ${failed?.level}`) + check('……原话留在展开区', failed?.detail === FAILED && settled?.detail === SETTLED) + check('缺字段、类型不对:标题不带数,不写副标题,不出 undefined、NaN', bare?.title === '核查模型的输出达到上限被截断' && bare.sub === undefined + && !/undefined|NaN/.test(`${bare?.title}${bare?.next ?? ''}`), `${bare?.title} / ${bare?.sub}`) + check('标题、副标题里没有内部代码', !rows.some((s) => /verify_truncated|max_tokens|delivered/.test(`${s.title}${s.sub ?? ''}${s.next ?? ''}`))) + check('节点照常完成', nodes.every((s) => s.status === 'done')) +}) + await section('审批质询 approval.inquiry(期 4):一行「审批人质询」,标题带问题前 40 字,副标题说回答是否完成、查了几次', async () => { // 前后端共用的约定:运行停在审批上时服务端追加 approval.inquiry,node_id 为空;data 里有问题、划选的原文、回答正文、 // 回答文档、成没成、触顶原因、查询(queries 和 inquiry_artifacts 两份)。以前这类事件落进「未识别的事件」 From 3bb06e32f17956093b229ee69e0bfe04ba4be89c Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 13:13:49 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E6=95=B4=E7=90=86=EF=BC=9A=E6=A0=B8?= =?UTF-8?q?=E6=9F=A5=E6=94=B6=E5=B0=BE=E6=97=B6=E4=B8=80=E6=9D=A1=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E7=9A=84=E7=BB=AD=E8=A1=8C=E5=AF=B9=E9=BD=90=EF=BC=8C?= =?UTF-8?q?=E8=A3=81=E5=88=A4=E7=9A=84=20=5FTRUNCATED=20=E6=8C=AA=E5=88=B0?= =?UTF-8?q?=E7=94=A8=E5=AE=83=E7=9A=84=20=5Fask=20=E5=89=8D=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/judge.py | 8 ++++---- backend/app/engine/verify_agent.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index b43d2f87..d4c3903d 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -1070,6 +1070,10 @@ def _items_of(value: Any) -> list[Any] | None: return value if isinstance(value, list) else None +#: _ask 交回的没成的原因:输出到了上限被截断 +_TRUNCATED = "truncated" + + async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | None, list[BaseMessage], str | None, int]: """问一次裁判:(判定清单, 模型的原始回复, 没成的原因, 调了几次模型)。 @@ -1111,10 +1115,6 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non return items, raws, None if items is not None else "format", 2 -#: _ask 交回的没成的原因:输出到了上限被截断 -_TRUNCATED = "truncated" - - def _spent(error: BaseException) -> list[BaseMessage]: """截断时 openai 的异常带着这次的用量:折成一条空回复交给记账(额度是真花出去了,每日上限不能漏掉它)。""" inp, out = error_usage(error) diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index ad5b59a7..022706c9 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -963,8 +963,8 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou declared = list((resolved.get("properties") or {}) if _is_object(resolved) else {}) if declared and not any(k in parsed for k in declared): # 一个要求的字段都没交:不是「查不到」,是没按结构交。不这么分,字段名对不上的回复会被当成什么都没查到 - return await give_up("核查模型没有按结构交结果(交回的内容里没有要求的字段:" - + "、".join(declared[:8]) + ("等" if len(declared) > 8 else "") + ")") + return await give_up("核查模型没有按结构交结果(交回的内容里没有要求的字段:" + "、".join(declared[:8]) + + ("等" if len(declared) > 8 else "") + ")") # ---- 取值、组装理由 cells = _Cells(queries) From ed6f5d064a077cd65608485fe36d5621ed63c6e9 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 13:17:19 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E6=88=AA=E6=96=AD=E5=92=8C=E9=80=80?= =?UTF-8?q?=E5=9B=9E=E5=B0=8F=E9=A2=9D=E5=BA=A6=E5=9C=A8=E7=9C=9F=E5=AE=9E?= =?UTF-8?q?=E7=9A=84=20ChatOpenAI=E3=80=81openai=20SDK=20=E4=B8=8A?= =?UTF-8?q?=E4=B9=9F=E6=A0=B8=E5=AF=B9=E4=B8=80=E9=81=8D=EF=BC=9A=E5=8F=AA?= =?UTF-8?q?=E6=9C=89=20HTTP=20=E5=B1=82=E6=98=AF=E5=81=87=E7=9A=84?= =?UTF-8?q?=EF=BC=88=E7=BB=8F=20OpenAI=20=E5=85=BC=E5=AE=B9=E7=BD=91?= =?UTF-8?q?=E5=85=B3=E6=8E=A5=E5=85=A5=E7=9A=84=E6=8E=A8=E7=90=86=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E5=B0=B1=E6=98=AF=E8=BF=99=E4=B8=80=E8=B7=AF=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 核查:工具循环那次只有 reasoning_content、finish_reason=length;收尾走 json_schema 时 SDK 抛 LengthFinishReasonError(用量从异常里取);网关嫌 16384 太大回 400 时退回 4096,之后工具循环和收尾都用小的。裁判:json_schema 那次被截断记未裁判(truncated)、不退回纯文本;网关回 400 时退回 4096。请求体里的额度(max_completion_tokens)逐个核对。 --- backend/tests/test_judge_truncation.py | 72 +++++++++++++++++++ backend/tests/test_verify_truncation.py | 91 +++++++++++++++++++++++++ 2 files changed, 163 insertions(+) diff --git a/backend/tests/test_judge_truncation.py b/backend/tests/test_judge_truncation.py index cf0af450..9686bd2a 100644 --- a/backend/tests/test_judge_truncation.py +++ b/backend/tests/test_judge_truncation.py @@ -16,6 +16,7 @@ from __future__ import annotations import json import re +import uuid from langchain_core.messages import AIMessage from openai import LengthFinishReasonError @@ -23,11 +24,18 @@ from openai.types import CompletionUsage from openai.types.chat import ChatCompletion from app.core.events import EventType +from app.db.models import Provider from app.engine import judge as judge_mod from app.engine.evidence import compose_doc from app.engine.judge import BATCH_SIZE, FALLBACK_MAX_TOKENS, MAX_TOKENS, REASONS, judge_doc +from app.providers.factory import build_chat_model from tests.test_evidence_judge import PRICED, RAW, SPEC, UNLIMITED, clean_settings, make_catalog, stored # noqa: F401 +try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上 + from langchain_openai._compat import httpx +except ImportError: # pragma: no cover - 老版本的 langchain_openai + import httpx + CUT_16K = "裁判模型的输出达到上限(16384 token)被截断" ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型" @@ -234,3 +242,67 @@ async def test_a_truncation_after_the_fallback_names_the_small_budget(monkeypatc [verdict, *_] = outcome["verdicts"].values() assert verdict["reason"] == "truncated" assert verdict["rationale"].startswith("裁判模型的输出达到上限(4096 token)被截断") + + +# -------------------------------------------------------------------------- +# 真实的 ChatOpenAI 和 openai SDK:只有 HTTP 层是假的(经 OpenAI 兼容网关接入的推理模型就是这一路) +# -------------------------------------------------------------------------- + + +class Gateway: + """OpenAI 兼容网关:reply(这一批的句子编号) 给出 (message, finish_reason, 输出 token 数)。cap 给了,请求的额度超过它 + 就回 400。bodies 记下每个请求体。""" + + def __init__(self, reply, *, cap: int | None = None) -> None: + self.reply, self.cap = reply, cap + self.bodies: list[dict] = [] + + async def handle(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content or b"{}") + self.bodies.append(body) + budget = body.get("max_completion_tokens") or body.get("max_tokens") + if self.cap and budget and budget > self.cap: + message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens" + return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}}, + request=request) + units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M) + message, finish, spent = self.reply(units) + return httpx.Response(200, request=request, json={ + "id": "c-1", "object": "chat.completion", "created": 1, "model": PRICED, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": {"prompt_tokens": 3000, "completion_tokens": spent, "total_tokens": 3000 + spent}}) + + +def gateway(monkeypatch, reply, *, cap: int | None = None) -> Gateway: + fake = Gateway(reply, cap=cap) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle) + provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1", + api_key=None, models=[{"id": PRICED}], default_model=PRICED, enabled=True, extra={}) + + async def build(session, spec): + return build_chat_model(provider, spec), PRICED + + monkeypatch.setattr(judge_mod, "get_chat_model", build) + return fake + + +async def test_a_real_client_cut_off_in_the_json_schema_call(monkeypatch, stored): + """json_schema 的结构化输出:finish_reason=length 时 openai SDK 抛 LengthFinishReasonError,不再退回纯文本。""" + fake = gateway(monkeypatch, lambda units: ({"role": "assistant", "content": '{"items": [{"unit": "u'}, "length", 16384)) + outcome, _ = await judged(stored) + assert outcome["unjudged"] == {"truncated": 4} + assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"] + assert len(fake.bodies) == 1 and "response_format" in fake.bodies[0] + assert fake.bodies[0]["max_completion_tokens"] == 16384 + assert outcome["usage"]["output_tokens"] == 16384, "异常里带着的那次用量照记" + + +async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, stored): + def answer(units): + items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] + return {"role": "assistant", "content": json.dumps({"items": items}, ensure_ascii=False)}, "stop", 80 + + fake = gateway(monkeypatch, answer, cap=8192) + outcome, _ = await judged(stored) + assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == [] + assert [b.get("max_completion_tokens") for b in fake.bodies] == [16384, 4096] diff --git a/backend/tests/test_verify_truncation.py b/backend/tests/test_verify_truncation.py index 09397ee3..2d237caf 100644 --- a/backend/tests/test_verify_truncation.py +++ b/backend/tests/test_verify_truncation.py @@ -19,6 +19,7 @@ thinking=off,思考就能把 4096 用光:这次调用 finish_reason=length from __future__ import annotations import json +import uuid from langchain_core.messages import AIMessage, BaseMessage from openai import LengthFinishReasonError @@ -26,14 +27,21 @@ from openai.types import CompletionUsage from openai.types.chat import ChatCompletion from app.core.events import EventType +from app.db.models import Provider from app.engine import judge as judging from app.engine import review as rv from app.engine import verify_agent from app.engine.inquiry import _error_words from app.engine.nodes.challenge import _claim_of from app.engine.nodes.dual_check import _slim +from app.providers.factory import build_chat_model from tests.test_verify_agent import VALID_SUM, clean_spend, q, run, shop # noqa: F401 - 夹具按名字引入 +try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上 + from langchain_openai._compat import httpx +except ImportError: # pragma: no cover - 老版本的 langchain_openai + import httpx + CUT_16K = "核查模型的输出达到上限(16384 token)被截断" ADVICE = "通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型" FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", @@ -343,3 +351,86 @@ def test_the_review_layer_knows_the_truncation_log(): def test_the_reply_shapes_used_here_are_messages(): assert isinstance(cut_reply(), BaseMessage) and isinstance(anthropic_cut()["raw"], BaseMessage) + + +# -------------------------------------------------------------------------- +# 真实的 ChatOpenAI 和 openai SDK:只有 HTTP 层是假的(经 OpenAI 兼容网关接入的推理模型就是这一路) +# -------------------------------------------------------------------------- + +GATEWAY_MODEL = "reasoner-v4" + + +class Gateway: + """OpenAI 兼容网关:按顺序回 chat.completion(最后一个重复用)。cap 给了,请求的额度超过它就回 400(DeepSeek chat 系 + 只收 8K 那样)。bodies 记下每个请求体。""" + + def __init__(self, replies: list, *, cap: int | None = None) -> None: + self.replies, self.cap = list(replies), cap + self.bodies: list[dict] = [] + + async def handle(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content or b"{}") + self.bodies.append(body) + budget = body.get("max_completion_tokens") or body.get("max_tokens") + if self.cap and budget and budget > self.cap: + message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens" + return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}}, + request=request) + message, finish, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0] + return httpx.Response(200, request=request, json={ + "id": "c-1", "object": "chat.completion", "created": 1, "model": GATEWAY_MODEL, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": {"prompt_tokens": 900, "completion_tokens": spent, "total_tokens": 900 + spent}}) + + +def gateway(monkeypatch, replies: list, *, cap: int | None = None) -> Gateway: + fake = Gateway(replies, cap=cap) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle) + provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1", + api_key=None, models=[{"id": GATEWAY_MODEL}], default_model=GATEWAY_MODEL, enabled=True, extra={}) + + async def build(session, spec): + return build_chat_model(provider, spec), GATEWAY_MODEL + + monkeypatch.setattr(verify_agent, "get_chat_model", build) + return fake + + +THINKING_ONLY = ({"role": "assistant", "content": "", "reasoning_content": "先看看有哪些表……"}, "length", 16384) + + +def tool_call(name: str) -> tuple: + return ({"role": "assistant", "content": None, "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": f"db_query__{name}", "arguments": json.dumps({"sql": VALID_SUM})}}]}, "tool_calls", 120) + + +async def test_a_real_client_cut_off_in_the_loop(monkeypatch, shop): + name, _ = shop + fake = gateway(monkeypatch, [THINKING_ONLY]) + out = await run(name) + assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert len(fake.bodies) == 1 and fake.bodies[0]["max_completion_tokens"] == 16384 + assert "tools" in fake.bodies[0], "工具循环那次带着只读的查询工具" + + +async def test_a_real_client_cut_off_in_the_json_schema_settle(monkeypatch, shop): + """收尾走 openai 的结构化输出(json_schema):finish_reason=length 时 SDK 抛 LengthFinishReasonError。""" + name, _ = shop + half = ({"role": "assistant", "content": '{"value": {"alias": "Q1", "row'}, "length", 16384) + fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), half]) + out = await run(name) + assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" and len(out.queries) == 1 + assert len(fake.bodies) == 3 and "response_format" in fake.bodies[2], "收尾被截断:没有再退回普通调用" + assert out.usage["output_tokens"] == 120 + 20 + 16384, "异常里带着的那次用量照记" + + +async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, shop): + name, _ = shop + done = ({"role": "assistant", "content": json.dumps(FOUND, ensure_ascii=False)}, "stop", 60) + fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), done], + cap=8192) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + budgets = [b.get("max_completion_tokens") for b in fake.bodies] + assert budgets == [16384, 4096, 4096, 4096], "第一次被拒后退回 4096,之后工具循环和收尾都用小的,不再先撞一次" From 5afa7a9a70f47a071d7057f2b3bf1f36591cfd8c Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 14:29:48 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E6=A0=B8=E6=9F=A5=E3=80=81=E8=AF=81?= =?UTF-8?q?=E6=8D=AE=E8=A3=81=E5=88=A4=E9=80=80=E5=9B=9E=E5=B0=8F=E9=A2=9D?= =?UTF-8?q?=E5=BA=A6=E6=97=B6=E5=A4=9A=E8=AE=A4=E4=B8=A4=E7=A7=8D=E6=8B=92?= =?UTF-8?q?=E7=BB=9D=EF=BC=8C=E8=A2=AB=E5=AB=8C=E5=A4=A7=E8=BF=87=E7=9A=84?= =?UTF-8?q?=E6=8E=A5=E5=85=A5=E8=AE=B0=E5=9C=A8=E8=BF=9B=E7=A8=8B=E9=87=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复核第 1 轮的发现: - 自建服务(text-generation-inference 一类)嫌额度大时回 422,原话说的是 max_new_tokens,不含 max_tokens,认不出就不退回;核查、裁判的额度从 4096 提到 16384 之后才容易撞上。新增 WIDER_BUDGET_WORDS 和 budget_refused,只给核查、裁判用;助手改图仍按 budget_rejected 认,BUDGET_WORDS 不动(data/recipe_ai.py 的测试拿它比对) - Bearer 鉴权的 Anthropic 接入(factory 换上的 SDK 客户端没设超时)调 Opus 4、4.1 这一代的 id 时,16384 的非流式请求 anthropic SDK 在本地就抛 ValueError(Streaming is required…),请求根本没发出去。budget_refused 把它也认成额度被挡下,退回小额度;重来不会重复输出,也没花钱 - 退回的开关原来只在一组调用里共用:双盲复算的几路盲算、反方审稿的每一句、每次质询、每次点击裁判各自先按 16384 撞一次。BudgetFallback 加 key:被嫌大一次就记进进程里的 _REJECTED,之后同一个 key 新建的一开始就用小的(同 factory 的 _NO_STREAM_USAGE,重启就忘);不给 key 的(助手改图)不记也不查,行为不变 --- backend/app/providers/budget.py | 72 +++++++++++++++------ backend/tests/test_model_budget.py | 100 +++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 19 deletions(-) diff --git a/backend/app/providers/budget.py b/backend/app/providers/budget.py index a6da84e0..3d52402a 100644 --- a/backend/app/providers/budget.py +++ b/backend/app/providers/budget.py @@ -7,21 +7,28 @@ thinking=off,Claude 开了 adaptive thinking 也一样。额度小了,想完 - cut_off:这一块(流式)或这条回复是不是因为额度用完停下的 - truncated_error:这个异常是不是在说同一件事(openai 结构化输出解析时抛的 LengthFinishReasonError) -- budget_rejected:服务方一开口就嫌额度太大 +- budget_rejected:服务方一开口就嫌额度太大(助手改图按它认) +- budget_refused:核查、证据裁判按它认,多认自建服务说的 max_new_tokens 和 anthropic SDK 在本地的拒绝 - error_usage:截断的那次调用花掉的用量 - cut_words:给人看的「某某模型的输出达到上限(N token)被截断」 -- BudgetFallback:先按大额度调,被嫌大时按小额度重来,之后都用小的 +- BudgetFallback:先按大额度调,被嫌大时按小额度重来,之后都用小的;给了 key 的,被嫌大过就记在进程里 engine/nodes/llm.py(cite_fields 的抽取)、data/recipe_ai.py 各有一份同样的判断,是在这里之前写的,行为相同。 """ from __future__ import annotations -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Hashable from typing import Any #: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K), -#: 或者输入加额度超过上下文长度(本地 vLLM) +#: 或者输入加额度超过上下文长度(本地 vLLM)。助手改图按这一份认,data/recipe_ai.py 的测试拿它比对 BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") +#: 核查、证据裁判另外认的:text-generation-inference 一类自建服务说的是 max_new_tokens(「`inputs` tokens + +#: `max_new_tokens` must be <= 8192」)。这两处的额度从 4096 提到 16384 之后才容易撞上;助手改图的认法不动 +WIDER_BUDGET_WORDS = (*BUDGET_WORDS, "max_new_tokens") +#: anthropic SDK 在本地拒绝时的原话:客户端没设超时(Bearer 鉴权的接入,factory 换上的就是这种)、非流式调用、额度 +#: 超过这个模型不流式能给的上限(Opus 4、4.1 这一代只到 8192)或估计要写十分钟以上,请求没发出去就抛 ValueError +LOCAL_REFUSAL = "streaming is required for operations that may take longer than 10 minutes" #: 截断类报错的原话:openai 的 LengthFinishReasonError(结构化输出解析时 finish_reason=length) LENGTH_WORDS = ("length limit was reached", "finish_reason=length", "finish_reason='length'") @@ -46,9 +53,18 @@ def truncated_error(error: BaseException | None) -> bool: return any(word in text for word in LENGTH_WORDS) -def budget_rejected(e: BaseException) -> bool: +def budget_rejected(e: BaseException, words: tuple[str, ...] = BUDGET_WORDS) -> bool: status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None) - return status in (400, 422) and any(w in str(e).lower() for w in BUDGET_WORDS) + return status in (400, 422) and any(w in str(e).lower() for w in words) + + +def budget_refused(e: BaseException) -> bool: + """核查、证据裁判按这个认「额度被挡下了,换小的重来」:服务方嫌额度太大(多认 max_new_tokens),或者 anthropic SDK + 在本地就拒了。两种都是一个字还没写、请求要么被拒要么根本没发出去,重来不会重复输出,也没花钱。助手改图走流式, + SDK 不在本地拦,仍按 budget_rejected 认。""" + if isinstance(e, ValueError) and LOCAL_REFUSAL in str(e).lower(): + return True + return budget_rejected(e, WIDER_BUDGET_WORDS) def error_usage(error: BaseException | None) -> tuple[int, int]: @@ -63,13 +79,27 @@ def cut_words(who: str, limit: int | None) -> str: return f"{who}的输出达到上限{f'({limit} token)' if limit else ''}被截断" +#: 嫌过大额度的接入:BudgetFallback 的 key(核查、证据裁判给的是 (接入, 模型, 大额度))。双盲复算的几路盲算、反方 +#: 审稿的每一句、每次质询、每次点击裁判各是一组新建的调用,开关只在一组里共用;记在这里,之后新建的直接用小的, +#: 不再每组先撞一次。只记被拒过的;进程内有效,重启后再按大的试(服务方可能调高了上限),同 factory 的 _NO_STREAM_USAGE +_REJECTED: set[Hashable] = set() + + class _Switch: - """几次调用共用的「退回小额度了没有」:核查的工具循环和收尾、裁判的各批。""" + """几次调用共用的「退回小额度了没有」:核查的工具循环和收尾、裁判的各批。rejected 是怎么认「额度被挡下了」; + key 给了,开关一开始就照 _REJECTED 定,退回时记进去。""" - __slots__ = ("small",) + __slots__ = ("key", "rejected", "small") - def __init__(self) -> None: - self.small = False + def __init__(self, key: Hashable | None, rejected: Callable[[BaseException], bool]) -> None: + self.key = key + self.rejected = rejected + self.small = key is not None and key in _REJECTED + + def fall_back(self) -> None: + self.small = True + if self.key is not None: + _REJECTED.add(self.key) class BudgetFallback: @@ -81,17 +111,21 @@ class BudgetFallback: small 是现成的小额度模型;也可以不给,给 build(无参的协程函数,返回小额度的模型),要退回时才建,平常一个都不多建。 limits 是 (大额度, 小额度):给了,max_tokens 就按它报(包过一层的模型身上不一定读得到额度);不给就读眼下那个模型的 - max_tokens。 + max_tokens。rejected 是怎么认「额度被挡下了」,缺省 budget_rejected(助手改图),核查、证据裁判给 budget_refused。 + key 给了(核查、证据裁判给 (接入, 模型, 大额度)),被嫌大一次就记在进程里,之后同一个 key 新建的一开始就用小的; + 不给(助手改图)不记也不查。shared_with 给了,rejected、key 跟着它。 """ def __init__(self, big: Any, small: Any = None, *, build: Callable[[], Awaitable[Any]] | None = None, limits: tuple[int | None, int | None] | None = None, - shared_with: BudgetFallback | None = None) -> None: + shared_with: BudgetFallback | None = None, + rejected: Callable[[BaseException], bool] = budget_rejected, + key: Hashable | None = None) -> None: self._big = big self._small = small self._build = build self._limits = limits - self._switch = shared_with._switch if shared_with is not None else _Switch() + self._switch = shared_with._switch if shared_with is not None else _Switch(key, rejected) @property def model(self) -> Any: @@ -127,9 +161,9 @@ class BudgetFallback: try: return await used.ainvoke(messages, *args, **kwargs) except Exception as e: - if not self._can_fall_back(used) or not budget_rejected(e): + if not self._can_fall_back(used) or not self._switch.rejected(e): raise - self._switch.small = True + self._switch.fall_back() return await (await self._current()).ainvoke(messages, *args, **kwargs) async def astream(self, messages: Any, *args: Any, **kwargs: Any): @@ -141,9 +175,9 @@ class BudgetFallback: yield chunk return except Exception as e: - if started or not self._can_fall_back(used) or not budget_rejected(e): + if started or not self._can_fall_back(used) or not self._switch.rejected(e): raise - self._switch.small = True + self._switch.fall_back() async for chunk in (await self._current()).astream(messages, *args, **kwargs): yield chunk @@ -164,5 +198,5 @@ class BudgetFallback: return self._derived(lambda m: m.with_structured_output(schema, **kwargs)) -__all__ = ["BUDGET_WORDS", "LENGTH_WORDS", "BudgetFallback", "budget_rejected", "cut_off", "cut_words", "error_usage", - "truncated_error"] +__all__ = ["BUDGET_WORDS", "LENGTH_WORDS", "LOCAL_REFUSAL", "WIDER_BUDGET_WORDS", "BudgetFallback", "budget_refused", + "budget_rejected", "cut_off", "cut_words", "error_usage", "truncated_error"] diff --git a/backend/tests/test_model_budget.py b/backend/tests/test_model_budget.py index 21ee3ecc..4ede095f 100644 --- a/backend/tests/test_model_budget.py +++ b/backend/tests/test_model_budget.py @@ -7,6 +7,10 @@ - 截断的那次调用花掉的用量从异常里取得到(额度是真花出去了) - BudgetFallback:先按大额度调,被嫌大时按小额度重来一次,之后的调用都用小的;流式、非流式都认;bind_tools、 with_structured_output 包出来的和原来的共用同一个开关;只在还没吐出任何东西时重来;小额度的模型要退回时才建 +- 核查、证据裁判另认两种「额度被挡下」(budget_refused):自建服务(text-generation-inference 一类)说的 + max_new_tokens,和 anthropic SDK 在本地就拒的额度(请求没发出去);助手改图的认法(budget_rejected)不变 +- 给了 key(接入、模型、大额度)的,被嫌大一次就记在进程里:之后新建的调用直接用小的,不再每次先撞一次;没给 key 的 + (助手改图)不记也不查 模型都是剧本,不碰真接口。 """ @@ -18,9 +22,11 @@ from openai import LengthFinishReasonError from openai.types import CompletionUsage from openai.types.chat import ChatCompletion +from app.providers import budget from app.providers.budget import ( BUDGET_WORDS, BudgetFallback, + budget_refused, budget_rejected, cut_off, cut_words, @@ -29,6 +35,14 @@ from app.providers.budget import ( ) +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里:每个测试从没记过开始,也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() + + def length_error(prompt: int = 9000, completion: int = 16384) -> LengthFinishReasonError: """openai 的结构化输出在 finish_reason=length 时抛的那个错(原话和真实运行里一样)。""" usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) @@ -45,6 +59,12 @@ class Rejected(Exception): TOO_LARGE = "max_tokens: 16384 > 8192, which is the maximum allowed number of output tokens" +#: text-generation-inference 的 OpenAI 兼容接口:输入加额度超过 max_total_tokens 时的 422(原话的样子) +TGI_TOO_LARGE = ("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 1200 `inputs` tokens and 16384 `max_new_tokens`") +#: anthropic SDK 在本地拒绝时的原话:没设超时的客户端非流式调用、额度超过这个模型不流式能给的上限 +SDK_REFUSED = ("Streaming is required for operations that may take longer than 10 minutes. See " + "https://platform.claude.com/docs/en/cli-sdks-libraries/sdks/python#long-requests for more details") class Model: @@ -124,6 +144,25 @@ def test_other_rejections_are_not_a_budget_problem(): assert "max_tokens" in BUDGET_WORDS +def test_verify_and_judge_also_recognise_tgi_and_a_local_sdk_refusal(): + """核查、证据裁判的额度从 4096 提到了 16384,才撞得上这两种:自建服务说的是 max_new_tokens;Bearer 鉴权的 Anthropic + 接入(客户端没设超时)调 Opus 4、4.1 这一代时,SDK 嫌 16384 太大、请求没发出去就抛 ValueError。""" + assert budget_refused(Rejected(TGI_TOO_LARGE, status_code=422)) + assert budget_refused(ValueError(SDK_REFUSED)) + assert budget_refused(Rejected(TOO_LARGE)), "原来认的照样认" + assert not budget_refused(ValueError("别的本地错误")) + assert not budget_refused(RuntimeError(SDK_REFUSED)), "只认 SDK 抛的 ValueError" + assert not budget_refused(Rejected("model 'no-such-model' does not exist")) + assert not budget_refused(Rejected(TGI_TOO_LARGE, status_code=500)) + + +def test_the_copilot_wording_list_stays_as_it_was(): + """助手改图按 budget_rejected 认(流式调用,SDK 不在本地拦):认法不变,data/recipe_ai.py 的测试拿 BUDGET_WORDS 比对。""" + assert BUDGET_WORDS == ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") + assert not budget_rejected(Rejected(TGI_TOO_LARGE, status_code=422)) + assert not budget_rejected(ValueError(SDK_REFUSED)) + + def test_the_tokens_spent_by_a_cut_off_call_come_from_the_error(): assert error_usage(length_error(prompt=9000, completion=16384)) == (9000, 16384) assert error_usage(RuntimeError("网关超时")) == (0, 0) and error_usage(None) == (0, 0) @@ -242,3 +281,64 @@ async def test_without_limits_max_tokens_reads_the_current_model(): assert model.max_tokens == 32768 await model.ainvoke(["问"]) assert model.max_tokens == 8192 + + +async def test_a_wider_check_can_be_given(): + """核查、证据裁判按 budget_refused 认:服务方说的是 max_new_tokens 也退回小额度。缺省(助手改图)不认。""" + tgi = Rejected(TGI_TOO_LARGE, status_code=422) + model = BudgetFallback(Model("big", fail=tgi), Model("small"), rejected=budget_refused) + assert (await model.ainvoke(["问"])).content == "small" + with pytest.raises(Rejected): + await BudgetFallback(Model("big", fail=tgi), Model("small")).ainvoke(["问"]) + local = BudgetFallback(Model("big", fail=ValueError(SDK_REFUSED)), Model("small"), rejected=budget_refused) + structured = local.with_structured_output({"type": "object"}) + assert (await structured.ainvoke(["交结果"])).content == "small", "包出来的视图照样按这个认" + + +async def test_a_rejection_is_remembered_for_later_calls_to_the_same_model(): + """双盲复算的几路盲算、反方审稿的每一句、每次质询、每次点击裁判,各是一组新建的调用:同一接入、模型、额度被嫌大 + 过一次,之后新建的直接用小的,不再先撞一次。别的模型、别的额度照旧先按大的试。""" + built: list[str] = [] + + def builder(name: str): + async def build(): + built.append(name) + return Model(name) + return build + + key = ("gw", "reasoner-x", 16384) + first = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=builder("small-1"), limits=(16384, 4096), + key=key) + assert first.max_tokens == 16384 + assert (await first.ainvoke(["问"])).content == "small-1" + later_big = Model("big", fail=Rejected(TOO_LARGE)) + later = BudgetFallback(later_big, build=builder("small-2"), limits=(16384, 4096), key=key) + assert later.max_tokens == 4096, "还没调用就知道要用小的:截断时报的额度对得上" + assert (await later.bind_tools(["db_query"]).ainvoke(["再问"])).content == "small-2" + assert later_big.calls == [], "不再先按大额度撞一次" + other = Model("other-big") + assert (await BudgetFallback(other, build=builder("x"), key=("gw", "other", 16384)).ainvoke(["问"])).content \ + == "other-big" + bigger = Model("bigger") + assert (await BudgetFallback(bigger, build=builder("y"), key=("gw", "reasoner-x", 32768)).ainvoke(["问"])).content \ + == "bigger" + assert built == ["small-1", "small-2"] + + +async def test_without_a_key_nothing_is_remembered(): + """助手改图不给 key:每次改图照旧先按大额度试(行为和挪过来之前一样)。""" + await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), Model("small")).ainvoke(["问"]) + assert budget._REJECTED == set() + fresh = Model("big") + assert (await BudgetFallback(fresh, Model("small")).ainvoke(["问"])).content == "big" + + +async def test_a_streaming_rejection_is_remembered_too(): + key = ("gw", "streamer", 16384) + big = Model("big", fail=Rejected(TOO_LARGE), stream=[]) + model = BudgetFallback(big, Model("small", stream=[AIMessageChunk(content="a")]), key=key) + assert [c.content async for c in model.astream(["问"])] == ["a"] + again = Model("big", stream=[AIMessageChunk(content="b")]) + assert [c.content async for c in BudgetFallback(again, Model("small", stream=[AIMessageChunk(content="c")]), + key=key).astream(["问"])] == ["c"] + assert again.calls == [] From 4905de728c4f8c47284022c77326b43dd90234dd Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 14:30:03 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E6=A0=B8=E6=9F=A5=E6=88=AA=E6=96=AD?= =?UTF-8?q?=E7=9A=84=E5=9B=9E=E5=A4=8D=E4=B8=80=E5=BE=8B=E4=B8=8D=E9=87=87?= =?UTF-8?q?=E4=BF=A1=EF=BC=8C=E8=A7=A6=E9=A1=B6=E5=8E=9F=E5=9B=A0=E7=9A=84?= =?UTF-8?q?=E5=85=88=E5=90=8E=E7=85=A7=E6=97=A7=EF=BC=8C=E6=88=AA=E6=96=AD?= =?UTF-8?q?=E7=9A=84=E5=BB=BA=E8=AE=AE=E6=8C=89=E6=A8=A1=E5=9E=8B=E4=BB=8E?= =?UTF-8?q?=E5=93=AA=E6=9D=A5=E7=9A=84=E8=AF=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复核第 1 轮的发现(主要一条):Anthropic 的工具调用写到一半到了上限时,回复照样带着 tool_use(input 是空的,有的网关给半截对象)。工具循环只在没有工具调用时才看截断,于是一轮轮执行空参数的查询,撞到步数上限,最后记成「没有按结构交结果」;收尾的强制工具调用截在半路时解析出空对象或半截对象,空对象记成没按结构交,反方审稿的半截对象(结论写完、理由写到一半)被当成「经受住」,质询把半句话当成回答。 - 工具循环:回复被截断就停,带着的工具调用不执行(半截的 SQL 查出来的数是错的),也不收尾 - 截断前查到过东西的也不再按查到的收尾:截断前查到的不一定是要的那个数,交出去的结果只在 notes 里写一句「可能不完整」,审批卡、反方结论、质询回答都看不到,残缺的结果还照常参与比对和处置。同时长到点一样算半路被掐断,原因照截断说,查过的照记;TRUNCATED_SETTLED 和日志里的 delivered 字段去掉 - 收尾:回复被截断时,解析出来的、从回复里抠出来的都不采信(结构化、普通调用都是),也不退回普通调用 - 上一轮的 give_up 把触顶原因的先后从「先 stopped」换成了「先 refused」:去掉 give_up,收尾被上限拦下时照旧报先撞上的那个(双盲复算按它写无法复算的原因,反方审稿按每日上限决定还派不派后面的句子) - 截断的原因开头只说「未能给出结果」(质询里提问人看到的就是这句,不说「核查未能完成」);建议在节点里说「在节点上指定不带推理的模型,或在设置里更换核查模型」(节点上的模型优先于设置,和时间线上的下一步一致),质询只说设置 - 退回小额度按 budget_refused 认,按 (接入, 模型, 额度) 记在进程里 - 时间线:verify_truncated 的副标题固定说「这次核查未能给出结果」,check-decode 跟上;review.py 里 verify_truncated 的注释跟上 测试补上真实 ChatAnthropic(只有 HTTP 层是假的)的循环截断、收尾截断(空对象、半截对象),真实 ChatOpenAI 写到一半的工具调用参数,Bearer 鉴权下 SDK 在本地拒绝时退回 4096,自建服务的 max_new_tokens,跨次核查记住被嫌大,触顶原因的先后 --- backend/app/engine/review.py | 4 +- backend/app/engine/verify_agent.py | 139 +++++---- backend/tests/test_verify_truncation.py | 360 ++++++++++++++++++++---- frontend/src/lib/terms.ts | 11 +- frontend/src/run/decode.ts | 6 +- scripts/check-decode.mjs | 49 ++-- 6 files changed, 405 insertions(+), 164 deletions(-) diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index c497c95e..bae1b65b 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -128,8 +128,8 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 核查 agent 自己触顶、按已有结果收尾:多半和节点自己的 *_limit 说的是同一次触顶,照样是缺口,记成它自己的 kind, # 不落成笼统的 warn——步数上限(max_steps)只有它说 "verify_limit": ("verify_limit", DEGRADED), - # 核查 agent 的输出到了上限被截断(推理模型的思考也算在额度里):没交成结果的由节点按无法复算、未能反查记缺口,交了的 - # 可能不完整。同 verify_limit,是缺口不是断裂,记成它自己的 kind:说得清是输出额度的问题,换个模型就好 + # 核查 agent 的输出到了上限被截断(推理模型的思考也算在额度里):没交成结果,由节点按无法复算、未能反查记缺口。 + # 同 verify_limit,是缺口不是断裂,记成它自己的 kind:说得清是输出额度的问题,换个模型就好 "verify_truncated": ("verify_truncated", DEGRADED), } diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index 022706c9..b127eaef 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -26,10 +26,13 @@ query_artifact 和 role: verify——随运行封存、证据面板能下钻。 对它无效,照裁判的说法记一条说明。每次调用都记进核查的每日账(engine/spend.py)。 **输出额度。** 经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考也算在输出额度里:额度按 spec(缺省 judge.MAX_TOKENS), -服务方嫌大时退回 judge.FALLBACK_MAX_TOKENS 再试一次,之后工具循环、收尾都用小的(providers/budget.py)。每次调用都看 -有没有到了上限被截断:工具循环里被截断(话没说完、也没发查询)不算模型自己停下,一条查询都没成功时不再收尾;收尾被截断 -时不再退回普通调用多花一次额度。因为截断没交成结果的,error 照实说是输出额度(TRUNCATED_ERROR),不说成「没查到」 -「没按结构交」;节点里调用时另记一条 verify_truncated。 +服务方嫌大时(或者 anthropic SDK 在本地就拒了)退回 judge.FALLBACK_MAX_TOKENS 再试一次,之后工具循环、收尾都用小的; +同一接入、模型被嫌大过,之后的核查一开始就用小的(providers/budget.py)。每次调用都看有没有到了上限被截断,截断的回复 +一律不采信:工具循环里截断的回复不算模型自己停下,带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input +是空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾——同时长到点一样算半路被掐断,按截断前查到的收尾 +交出去的结果说不清完整不完整,还会照常参与比对和处置。收尾的强制工具调用被截断时,解析出来的空对象、半截对象都不算 +交了结果,也不再退回普通调用多花一次额度。因为截断没交成结果的,error 照实说是输出额度(TRUNCATED_ERROR),不说成 +「没查到」「没按结构交」;节点里调用时另记一条 verify_truncated。 永远不抛:模型没配好、调用失败、格式不对、触顶、库写不进去,都落在返回值里(error / limit)。模型调用都经 get_chat_model,测试换成剧本。 @@ -60,7 +63,7 @@ from app.engine.nodes.llm import _DIGEST_BUDGET, _digest, _split_structured, _ty from app.engine.state import message_text from app.engine.toolcalls import ToolTimeout, limit_fields, limit_of, run_bounded from app.providers import catalog as pricing -from app.providers.budget import BudgetFallback, cut_off, cut_words, error_usage, truncated_error +from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error from app.providers.factory import ModelSpec, ProviderNotConfigured, bind_tools_safely, get_chat_model from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX, build_datasource_tools from app.tools.registry import ToolArgsError, ToolContext, Versions, args_model_of, prepare_args @@ -86,12 +89,14 @@ _REF_DEPTH = 8 RATIONALE = "rationale" #: 触顶原因 LIMITS = ("max_queries", "timeout_s", "max_cost_usd", "daily_max_usd", "max_steps") -#: 核查模型的输出到了上限被截断时给人看的话。TRUNCATED_ERROR 是没交成结果的原因:双盲复算的无法复算、反方审稿的「反查 -#: 未完成」、质询的报错都照它说,也是 verify_truncated 日志的原话。TRUNCATED_SETTLED 是截断前查到过东西、照常收尾了的, -#: 记进 notes 和日志。cut 是 budget.cut_words 的「核查模型的输出达到上限(N token)被截断」 -TRUNCATED_ERROR = "{cut},核查未能完成。{advice}" -TRUNCATED_SETTLED = "{cut},已按截断前查到的结果收尾,结果可能不完整。{advice}" -TRUNCATED_ADVICE = "通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型" +#: 核查模型的输出到了上限被截断、没交成结果时给人看的话:双盲复算的无法复算、反方审稿的「反查未完成」、质询的报错都照 +#: 它说,也是 verify_truncated 日志的原话。cut 是 budget.cut_words 的「核查模型的输出达到上限(N token)被截断」。开头只说 +#: 「未能给出结果」:质询里提问人看到的就是这句,它问的是问题,不是在核查 +TRUNCATED_ERROR = "{cut},未能给出结果。{advice}" +#: 截断的建议:该换的是模型。节点里(双盲复算、反方审稿)节点上指定的模型优先于设置(api/settings.py 的 +#: verify_model_spec),两处都说,和时间线上的下一步一致;不在节点里(审批质询)核查模型只看设置 +TRUNCATED_ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型" +TRUNCATED_ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型" VERIFY_SYSTEM = ( "你是数据核查员:只读地查库,用查询结果完成交给你的任务。\n" @@ -142,8 +147,8 @@ class VerifyOutcome: source, tool, call_id}。 rationale:理由组装成的报告文档(evidence.compose_doc 的形状,violations 里有裸数字)。usage:{input_tokens, output_tokens, total_tokens, cost_usd, calls, unpriced_calls}。limit:触顶原因(LIMITS 之一),没触顶为 None。 - answered:模型是不是自己停下来的(没被上限掐断,也没因为输出到了上限被截断)。因为截断没交成结果的,error 是 - TRUNCATED_ERROR;截断前查到过东西、照常交了结果的,error 为 None,notes 里记一句 TRUNCATED_SETTLED。 + answered:模型是不是自己停下来的(没被上限掐断,也没因为输出到了上限被截断)。被截断的一律没有结果,error 是 + TRUNCATED_ERROR。 """ result: dict[str, Any] | None @@ -709,7 +714,7 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou except Exception as e: # noqa: BLE001 - 核查模型没配好:这回核不了,照实交回 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) return outcome(error=f"核查模型不可用({why})") - base_model = _with_fallback(first, spec, small) + base_model = _with_fallback(first, spec, small, model_id) state.model = model_id if daily_max_usd is Default.SETTINGS: from app.api.settings import verify_settings # 在这里导入:api.settings 导入了 engine 里的模块 @@ -781,6 +786,16 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou await _announce(emit, EventType.LLM_END, **end) return got, raw, failure + # 截断的建议按模型从哪来的说:节点里节点上指定的优先,不在节点里(质询)只看设置 + advice = TRUNCATED_ADVICE if node_id else TRUNCATED_ADVICE_SETTINGS + + async def report_cut(purpose: str, limit: int | None) -> VerifyOutcome: + """被截断、没交成结果:记一条 verify_truncated(purpose 是哪一步,limit 是那次的额度),原因照截断说。查过的照记。""" + text = TRUNCATED_ERROR.format(cut=cut_words("核查模型", limit), advice=advice) + await _announce(emit, EventType.LOG, level="warn", code="verify_truncated", purpose=purpose, max_tokens=limit, + message=text) + return outcome(limit=stopped, answered=answered, error=text) + # ---- 查库的循环 stopped: str | None = None answered = False @@ -788,7 +803,6 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou said = "" attempts = 0 done: dict[str, str] = {} # 成功过的查询(工具名 + 参数)→ 编号:一模一样的不再执行 - cut: tuple[str, int | None] | None = None # 最近一次因为输出到了上限被截断的调用:(哪一步, 那次的额度) for _ in range(steps): remaining = left() if remaining is not None and remaining <= 0: @@ -804,18 +818,17 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou if isinstance(failure, asyncio.TimeoutError): stopped = "timeout_s" break - if truncated_error(failure): - cut = ("verify", base_model.max_tokens) - break + if truncated_error(failure) or cut_off(response): + # 到了上限被截断(推理模型想完了额度也用完了):话没说完,不是模型自己停下来的;带着的工具调用是写到一半的 + # (Anthropic 的 tool_use 截在半路时 input 是空的,有的网关给半截的 SQL),不执行。也不收尾:一条都没查到的, + # 收尾只能交「查不到」(双盲复算会记成「盲算没有查到这个数」);查到过的,截断前查到的不一定是要的那个数 + # (模型可能正想改查询),同时长到点一样算半路被掐断。收尾那次多半同样被推理占满,也不为它多花一次额度 + return await report_cut("verify", base_model.max_tokens) if failure is not None or not isinstance(response, BaseMessage): return outcome(error=explain_model_error(failure, model_id) if failure else "核查模型没有给出回复") messages.append(response) calls = getattr(response, "tool_calls", None) or [] if not calls: - if cut_off(response): - # 话没说完、也没发查询就到了上限(推理模型想完了额度也用完了):不是模型自己停下来的,不能当成说完了去收尾 - cut, said = ("verify", base_model.max_tokens), message_text(response) or said - break answered, said = True, message_text(response) break said = message_text(response) or said @@ -880,26 +893,6 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou await _announce(emit, EventType.LOG, level="warn", code="verify_limit", limit=stopped, message=f"核查已达上限({words}),已停止查询,按已有结果收尾") - async def report_cut(delivered: bool) -> str: - """被截断过:记一条 verify_truncated,返回给人看的话(没交成结果的原因,或者交了结果时记进 notes 的说明)。""" - purpose, limit = cut or ("verify", None) - text = (TRUNCATED_SETTLED if delivered else TRUNCATED_ERROR).format(cut=cut_words("核查模型", limit), - advice=TRUNCATED_ADVICE) - await _announce(emit, EventType.LOG, level="warn", code="verify_truncated", purpose=purpose, max_tokens=limit, - delivered=delivered, message=text) - return text - - async def give_up(error: str, limit: str | None = None) -> VerifyOutcome: - """没交成结果,原因照 error 说。工具循环里被截断过的,时间线上照记一条:收尾失败可能就是它带出来的。""" - if cut is not None: - await report_cut(False) - return outcome(limit=limit or stopped, answered=answered, error=error) - - if cut is not None and not queries: - # 一条查询都没成功就被截断:收尾只能交「查不到」,双盲复算会记成「盲算没有查到这个数」,原因却是输出额度。 - # 收尾那次多半同样被推理占满,不再为它多花一次额度,直接交回截断的原因 - return outcome(limit=stopped, answered=answered, error=await report_cut(False)) - # ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容(经 OpenAI 兼容接入的推理模型 # 关不掉,所以额度和工具循环一样,退回小额度也和工具循环共用一个开关) settle_spec = spec.model_copy(update={"thinking": "off"}) @@ -907,8 +900,8 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou async with SessionLocal() as session: settle_model, _ = await get_chat_model(session, settle_spec) except Exception as e: # noqa: BLE001 - return await give_up(f"核查模型不可用({describe_exception(e)})") - settler = _with_fallback(settle_model, settle_spec, small, shared_with=base_model) + return outcome(limit=stopped, answered=answered, error=f"核查模型不可用({describe_exception(e)})") + settler = _with_fallback(settle_model, settle_spec, small, model_id, shared_with=base_model) final = _final_messages(head, task, queries, said, words) try: structured = settler.with_structured_output(schema, include_raw=True) @@ -928,15 +921,18 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou estimate = budget.estimate(_text_of(final) + schema_text, _OUT_FINAL) day, refused = await budget.reserve(estimate, kept=kept) if refused: - return await give_up(f"已达{words_of(refused)},没能交结果", refused) + return outcome(limit=stopped or refused, answered=answered, error=f"已达{words_of(refused)},没能交结果") got, raw, failure = await ask(final, structured.ainvoke, estimate, day or "", "verify_result", until_finish()) - parsed = None if failure is not None else _parsed(_split_structured(got)[1]) - if parsed is None and raw is not None: - # 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用 - for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: - if (parsed := _parsed(candidate)) is not None: - break - settle_cut = parsed is None and (truncated_error(failure) or bool(cut_off(raw))) + # 强制工具调用写到一半到了上限:Anthropic 截在 tool_use 里时 input 是空的,有的网关给半截对象(反方审稿的结论 + # 写完了、理由写到一半)。解析出来的、能从回复里抠出来的都说不清写完了没有,一律不采信 + settle_cut = truncated_error(failure) or bool(cut_off(raw)) + if failure is None and not settle_cut: + parsed = _parsed(_split_structured(got)[1]) + if parsed is None and raw is not None: + # 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用 + for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: + if (parsed := _parsed(candidate)) is not None: + break if parsed is None and not isinstance(failure, asyncio.TimeoutError) and not settle_cut: # 不支持结构化输出的模型:普通调用一次,结构写进消息,从回复里把 JSON 抠出来。结构化那次已经做过的, # 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱。结构化那次是被截断的不补救: @@ -947,54 +943,53 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou estimate = budget.estimate(_text_of(plain), _OUT_FINAL) day, refused = await budget.reserve(estimate, kept=kept and structured is None) if refused and structured is None: - return await give_up(f"已达{words_of(refused)},没能交结果", refused) + return outcome(limit=stopped or refused, answered=answered, + error=f"已达{words_of(refused)},没能交结果") if not refused: got, _, failure = await ask(plain, settler.ainvoke, estimate, day or "", "verify_result", remaining) - parsed = None if failure is not None else _parsed(message_text(got) if got is not None else None) - settle_cut = parsed is None and (truncated_error(failure) or bool(cut_off(got))) + # 同上:截断的回复里就算抠得出一段 JSON,后面还在写,说不清写完了没有 + settle_cut = truncated_error(failure) or bool(cut_off(got)) + if failure is None and not settle_cut: + parsed = _parsed(message_text(got) if got is not None else None) if parsed is None: if settle_cut: - cut = ("verify_result", settler.max_tokens) - return outcome(limit=stopped, answered=answered, error=await report_cut(False)) + return await report_cut("verify_result", settler.max_tokens) why = "收尾时超过时长上限,没能交结果" if isinstance(failure, asyncio.TimeoutError) else \ f"核查模型没有按结构交结果({explain_model_error(failure, model_id)})" if failure is not None else \ "核查模型没有按结构交结果" - return await give_up(why) + return outcome(limit=stopped, answered=answered, error=why) declared = list((resolved.get("properties") or {}) if _is_object(resolved) else {}) if declared and not any(k in parsed for k in declared): # 一个要求的字段都没交:不是「查不到」,是没按结构交。不这么分,字段名对不上的回复会被当成什么都没查到 - return await give_up("核查模型没有按结构交结果(交回的内容里没有要求的字段:" + "、".join(declared[:8]) - + ("等" if len(declared) > 8 else "") + ")") + return outcome(limit=stopped, answered=answered, + error="核查模型没有按结构交结果(交回的内容里没有要求的字段:" + + "、".join(declared[:8]) + ("等" if len(declared) > 8 else "") + ")") # ---- 取值、组装理由 cells = _Cells(queries) result = cells.walk(resolved, parsed) if _is_object(resolved) else {} - if cut is not None and any(c["status"] != "resolved" for c in cells.cells): - # 工具循环被截断,截断前查到的里面没有要的数(或者引用指不到):话没说完就停了,说「没查到」会让人以为数据里 - # 没有。不交这份残缺的结果,原因照截断说;查过的照记 - return outcome(cells=cells.cells, limit=stopped, answered=answered, error=await report_cut(False)) rationale = parsed.get(RATIONALE) doc = None if isinstance(rationale, str) and rationale.strip(): doc = compose_doc(rationale, build_catalog(nodes={}, ledger=ledger), node_id=owner, run_id=run_id or None) result[RATIONALE] = doc["markdown"] if doc else "" - if cut is not None: - # 截断前查到的够用:照常交结果,说明记进 notes,时间线上照记一条(结果可能不完整) - notes.append(await report_cut(True)) return outcome(result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered) -def _with_fallback(model: Any, spec: ModelSpec, small: int | None, *, shared_with: BudgetFallback | None = None - ) -> BudgetFallback: - """按 spec 建好的模型包一层:服务方嫌额度大时按 small 重建一个再调,要退回时才建,平常一个都不多建。small 为 None - 是额度本来就不比它大,没有可退的。经本模块的 get_chat_model 建,测试换成剧本时小额度的那个也是剧本。""" +def _with_fallback(model: Any, spec: ModelSpec, small: int | None, model_id: str, *, + shared_with: BudgetFallback | None = None) -> BudgetFallback: + """按 spec 建好的模型包一层:服务方嫌额度大(或者 SDK 在本地拒了)时按 small 重建一个再调,要退回时才建,平常一个 + 都不多建。small 为 None 是额度本来就不比它大,没有可退的。被嫌大过的按 (接入, 模型, 额度) 记在进程里,证据裁判 + (judge._with_fallback)用的是同一种记法:同一个模型在那边被嫌大过,这边也直接用小的。经本模块的 get_chat_model + 建,测试换成剧本时小额度的那个也是剧本。""" async def build() -> Any: async with SessionLocal() as session: return (await get_chat_model(session, spec.model_copy(update={"max_tokens": small})))[0] return BudgetFallback(model, build=build if small else None, limits=(spec.max_tokens, small or spec.max_tokens), - shared_with=shared_with) + shared_with=shared_with, rejected=budget_refused, + key=(spec.provider, model_id, spec.max_tokens)) async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, remaining: float | None, diff --git a/backend/tests/test_verify_truncation.py b/backend/tests/test_verify_truncation.py index 2d237caf..733a0066 100644 --- a/backend/tests/test_verify_truncation.py +++ b/backend/tests/test_verify_truncation.py @@ -6,36 +6,42 @@ thinking=off,思考就能把 4096 用光:这次调用 finish_reason=length ——都没说出真正的原因。这里守几件事: - 额度:默认 16384(judge.MAX_TOKENS,核查和裁判共用);服务方嫌大时退回 4096 再试一次,之后工具循环、收尾都用小的, - 只在还没有任何输出时重来;别的 400 不重试 -- 工具循环里被截断(没有工具调用、话没说完)不算模型自己停下:一条查询都没成功时不再收尾,直接交回截断的原因; - 查到过东西的照常按查到的收尾,取不到要的数时同样交回截断的原因 -- 收尾被截断(Anthropic 的 stop_reason=max_tokens、openai 结构化输出抛的 LengthFinishReasonError、普通调用的 - finish_reason=length):交回截断的原因,不再退回普通调用多花一次额度;截断那次的用量照记 + 只在还没有任何输出时重来;别的 400 不重试。被嫌大过的接入、模型记在进程里,之后的核查直接用小的;自建服务说的 + max_new_tokens、anthropic SDK 在本地就拒的额度也认 +- 被截断的回复一律不采信:工具循环里截断的回复带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input 是 + 空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾(同时长到点,算半路被掐断);收尾的强制工具调用被 + 截断时,解析出来的空对象、半截对象都不算交了结果,也不再退回普通调用多花一次额度。截断那次的用量照记 - 时间线上记一条 verify_truncated(节点里调用时),llm.end 标出这次被截断 -- 原因经现有的渠道交出去:双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错;没有截断时原有说法不变 +- 原因经现有的渠道交出去:双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错。开头只说「未能给出结果」, + 建议按模型从哪来的说:节点里节点上指定的模型优先,两处都说;质询只看设置。没有截断时原有说法不变,触顶原因的先后 + 也不变 -模型是剧本,数据源是现造的 SQLite 文件(借 test_verify_agent 的夹具),表名、数字都是虚构的。 +模型是剧本,另有真实的 ChatOpenAI、ChatAnthropic(只有 HTTP 层是假的);数据源是现造的 SQLite 文件(借 +test_verify_agent 的夹具),表名、数字都是虚构的。 """ from __future__ import annotations import json import uuid +import pytest from langchain_core.messages import AIMessage, BaseMessage from openai import LengthFinishReasonError from openai.types import CompletionUsage from openai.types.chat import ChatCompletion from app.core.events import EventType -from app.db.models import Provider +from app.db.base import SessionLocal +from app.db.models import Provider, Setting from app.engine import judge as judging from app.engine import review as rv -from app.engine import verify_agent -from app.engine.inquiry import _error_words -from app.engine.nodes.challenge import _claim_of +from app.engine import spend, verify_agent +from app.engine.inquiry import INQUIRY_SCHEMA, _error_words +from app.engine.nodes.challenge import CHALLENGE_SCHEMA, _claim_of from app.engine.nodes.dual_check import _slim +from app.providers import budget from app.providers.factory import build_chat_model -from tests.test_verify_agent import VALID_SUM, clean_spend, q, run, shop # noqa: F401 - 夹具按名字引入 +from tests.test_verify_agent import VALID_SUM, Script, clean_spend, priced, q, run, shop # noqa: F401 - 夹具按名字引入 try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上 from langchain_openai._compat import httpx @@ -43,9 +49,24 @@ except ImportError: # pragma: no cover - 老版本的 langchain_openai import httpx CUT_16K = "核查模型的输出达到上限(16384 token)被截断" -ADVICE = "通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型" +#: 节点里(双盲复算、反方审稿):节点上指定的模型优先于设置,两处都说 +ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型" +#: 不在节点里(审批质询):只看设置 +ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型" +FAILED = f"{CUT_16K},未能给出结果。{ADVICE}" +FAILED_SETTINGS = f"{CUT_16K},未能给出结果。{ADVICE_SETTINGS}" FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", "rationale": "未作废订单的金额合计是 [[v:Q1.r0.total]]。"} +#: 反方审稿那种结构写到一半:结论写完了,理由写到一半 +HALF_CHALLENGE = {"verdict": "stands", "rationale": "合计是 [[v:Q1.r0.total]],与报告"} + + +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() def cut_reply(output: int = 16384) -> AIMessage: @@ -55,11 +76,25 @@ def cut_reply(output: int = 16384) -> AIMessage: usage_metadata={"input_tokens": 900, "output_tokens": output, "total_tokens": 900 + output}) -def anthropic_cut() -> dict: - """结构化输出走强制工具调用(Anthropic)时被截断:原始回复 stop_reason=max_tokens,解析不出结果。""" - raw = AIMessage(content=[], response_metadata={"stop_reason": "max_tokens"}, +def cut_call(name: str, args: dict) -> AIMessage: + """工具调用写到一半到了上限(ChatAnthropic 的样子):tool_use 截在半路时 input 是空的,有的网关给半截对象; + stop_reason=max_tokens。""" + block = {"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": args} + return AIMessage(content=[{"type": "thinking", "thinking": "想了很久……", "signature": "s"}, block], + tool_calls=[{"name": block["name"], "args": args, "id": block["id"]}], + response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 900, "output_tokens": 16384, "total_tokens": 17284}) + + +def anthropic_cut(args: dict | None = None) -> dict: + """收尾的强制工具调用写到一半到了上限(结构化输出走 function_calling,ChatAnthropic 的缺省):原始回复 + stop_reason=max_tokens。args 为 None 是截在思考里、一个工具调用都没写出来;给了就是截在 tool_use 里——input 是 + 空的(Anthropic 非流式的样子)或半截对象(有的网关),解析出来的就是它。""" + content = [] if args is None else [{"type": "tool_use", "id": "toolu_1", "name": "verify_result", "input": args}] + calls = [] if args is None else [{"name": "verify_result", "args": args, "id": "toolu_1"}] + raw = AIMessage(content=content, tool_calls=calls, response_metadata={"stop_reason": "max_tokens"}, usage_metadata={"input_tokens": 700, "output_tokens": 16384, "total_tokens": 17084}) - return {"raw": raw, "parsed": None, "parsing_error": None} + return {"raw": raw, "parsed": args, "parsing_error": None} def length_error(prompt: int = 800, completion: int = 16384) -> LengthFinishReasonError: @@ -76,7 +111,9 @@ def query_step(name: str, sql: str = VALID_SUM, call_id: str = "c1") -> AIMessag class BudgetRejected(Exception): """服务方嫌输出额度太大时的 400(DeepSeek chat 系只收 8K)。""" - status_code = 400 + def __init__(self, message: str, status_code: int = 400) -> None: + super().__init__(message) + self.status_code = status_code class Reasoner: @@ -123,8 +160,9 @@ class _View: class Rejecting: """一开口就被拒的模型:工具循环、收尾都一样。""" - def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens"): - self.message = message + def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens", + status_code: int = 400): + self.message, self.status_code = message, status_code self.calls = 0 def bind_tools(self, tools, **kwargs): @@ -135,7 +173,7 @@ class Rejecting: async def ainvoke(self, messages, *args, **kwargs): self.calls += 1 - raise BudgetRejected(self.message) + raise BudgetRejected(self.message, self.status_code) class Models: @@ -175,43 +213,48 @@ async def test_thinking_that_eats_the_budget_in_the_loop_is_reported_as_truncati events, emit = collector() out = await run(name, emit=emit, node_id="dc") assert out.result is None and out.queries == [] - assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.error == FAILED assert model.calls == ["step"], "一条查询都没有:收尾只能交「查不到」,不再为它多花一次额度" assert out.answered is False, "被截断不是模型自己停下来的" assert out.usage["output_tokens"] == 16384, "截断那次的额度是真花出去了" [log] = truncated_logs(events) - assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["delivered"] is False + assert log["level"] == "warn" and log["max_tokens"] == 16384 and "delivered" not in log assert log["purpose"] == "verify" and log["message"] == out.error [end] = [d for k, d in events if k == EventType.LLM_END] assert end["truncated"] is True and end["max_tokens"] == 16384 and CUT_16K in end["error"] -async def test_a_cut_after_a_query_still_settles_on_what_was_found(monkeypatch, shop): +@pytest.mark.parametrize("args", [{}, {"sql": "SELECT SUM(amount) AS total FROM orders WHERE is_del"}], + ids=["empty-input", "half-sql"]) +async def test_a_tool_call_cut_off_halfway_is_not_run(monkeypatch, shop, args): + """Anthropic 系:工具调用写到一半到了上限。写到一半的调用不能执行——空参数只会报「缺少必填参数」,半截的 SQL + 查出来的数是错的——也不按它收尾,原因照截断说。""" + name, _ = shop + model = Reasoner([cut_call(name, args)], structured=FOUND) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step"], "不执行、不收尾,也不再问一轮" + assert out.result is None and out.queries == [] and out.error == FAILED + assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_END, EventType.TOOL_ERROR)] + [log] = truncated_logs(events) + assert log["purpose"] == "verify" and log["max_tokens"] == 16384 + + +async def test_a_cut_after_a_query_is_reported_without_settling(monkeypatch, shop): + """查过一次、下一轮被截断:截断前查到的不一定是要的那个数(模型可能正想改查询),按它收尾交出去的结果说不清完整 + 不完整,会照常参与双盲复算的比对、反方审稿的处置。同时长到点一样算半路被掐断:不收尾,原因照截断说,查过的照记。""" name, _ = shop model = Reasoner([query_step(name), cut_reply()], structured=FOUND) Models(monkeypatch, model) events, emit = collector() out = await run(name, emit=emit, node_id="dc") - assert model.calls == ["step", "step", "final"] - assert out.error is None and out.result["value"] == 230, "查到的数照常交:截断前的查询是真查过的" - assert out.answered is False - assert any(CUT_16K in n and "已按截断前查到的结果收尾" in n for n in out.notes) + assert model.calls == ["step", "step"], "不再为收尾多花一次额度" + assert out.result is None and out.error == FAILED and out.answered is False + assert [q["alias"] for q in out.queries] == ["Q1"], "查过的照记" [log] = truncated_logs(events) - assert log["delivered"] is True and log["purpose"] == "verify" and log["max_tokens"] == 16384 - - -async def test_a_cut_that_leaves_the_number_unfound_is_reported_as_truncation(monkeypatch, shop): - """查过、可要的数没查到就被截断了:说「没查到」会让人以为数据里没有,原因要写截断。""" - name, _ = shop - model = Reasoner([query_step(name, "SELECT COUNT(*) AS n FROM orders"), cut_reply()], - structured={"value": None, "verdict": "无法判断", "rationale": "还没查到合计"}) - Models(monkeypatch, model) - events, emit = collector() - out = await run(name, emit=emit, node_id="dc") - assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" - assert [c["status"] for c in out.cells] == ["missing"] and len(out.queries) == 1, "查过的照记" - [log] = truncated_logs(events) - assert log["delivered"] is False and log["purpose"] == "verify" + assert log["purpose"] == "verify" and log["max_tokens"] == 16384 + assert not [n for n in out.notes if "截断" in n], "没有「按截断前查到的结果收尾」这一说" # -------------------------------------------------------------------------- @@ -220,20 +263,41 @@ async def test_a_cut_that_leaves_the_number_unfound_is_reported_as_truncation(mo async def test_a_cut_off_settle_is_reported_and_not_retried_as_plain(monkeypatch, shop): - """Anthropic 系:强制工具调用写到一半到了上限。退回普通调用也多半被推理占满,不再白花一次额度。""" + """Anthropic 系:强制工具调用写出来之前就到了上限。退回普通调用也多半被推理占满,不再白花一次额度。""" name, _ = shop model = Reasoner([query_step(name)], structured=anthropic_cut(), plain=AIMessage(content=json.dumps(FOUND))) Models(monkeypatch, model) events, emit = collector() out = await run(name, emit=emit, node_id="dc") assert model.calls == ["step", "step", "final"], "没有再退回普通调用" - assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.result is None and out.error == FAILED [log] = truncated_logs(events) - assert log["purpose"] == "verify_result" and log["delivered"] is False + assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384 end = [d for k, d in events if k == EventType.LLM_END][-1] assert end["purpose"] == "verify_result" and end["truncated"] is True and CUT_16K in end["error"] +@pytest.mark.parametrize(("schema", "args"), [ + (None, {}), + (None, {"value": {"alias": "Q1"}}), + (CHALLENGE_SCHEMA, HALF_CHALLENGE), + (INQUIRY_SCHEMA, {"rationale": "合计是 [[v:Q1.r0.total]],比报告里写的"}), +], ids=["empty-input", "half-cell", "half-challenge", "half-inquiry"]) +async def test_a_settle_cut_inside_the_forced_tool_call_is_not_taken(monkeypatch, shop, schema, args): + """强制工具调用写到一半到了上限:解析出来的是空对象(Anthropic)或半截对象(有的网关)。空对象以前记成「没有按结构 + 交结果」,半截的格子引用记成「引用不可用」,反方审稿的半截对象(结论写完了、理由写到一半)照单全收成「经受住」, + 质询把半句话当成回答——截断的回复一律不采信,原因照截断说,也不退回普通调用。""" + name, _ = shop + model = Reasoner([query_step(name)], structured=anthropic_cut(args), plain=AIMessage(content=json.dumps(FOUND))) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {})) + assert model.calls == ["step", "step", "final"] + assert out.result is None and out.error == FAILED and out.cells == [] + [log] = truncated_logs(events) + assert log["purpose"] == "verify_result" + + async def test_an_openai_length_error_in_the_settle_is_truncation_not_a_failed_call(monkeypatch, shop): """openai 的结构化输出解析时抛 LengthFinishReasonError:不是「调用失败,请稍后重试」,是额度被推理占满。""" name, _ = shop @@ -242,7 +306,7 @@ async def test_an_openai_length_error_in_the_settle_is_truncation_not_a_failed_c events, emit = collector() out = await run(name, emit=emit, node_id="dc") assert model.calls == ["step", "step", "final"] - assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.error == FAILED assert out.usage["output_tokens"] == 16384 and out.usage["input_tokens"] >= 800, "异常里带着的用量照记" end = [d for k, d in events if k == EventType.LLM_END][-1] assert "请稍后重试" not in end["error"] and CUT_16K in end["error"] and end["output_tokens"] == 16384 @@ -255,7 +319,18 @@ async def test_a_plain_settle_cut_off_is_reported(monkeypatch, shop): Models(monkeypatch, model) out = await run(name) assert model.calls == ["step", "step", "plain"] - assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.result is None and out.error == FAILED_SETTINGS + + +async def test_a_plain_settle_cut_after_the_json_is_not_taken_either(monkeypatch, shop): + """普通调用的正文里有一段 JSON、回复却是截断的:JSON 后面还在写(多半是理由的补充),说不清写完了没有,同样不采信。""" + name, _ = shop + cut = AIMessage(content=json.dumps(FOUND, ensure_ascii=False) + "\n补充说明:合计里还应该", + response_metadata={"finish_reason": "length"}) + model = Reasoner([query_step(name)], plain=cut, unstructured=True) + Models(monkeypatch, model) + out = await run(name) + assert out.result is None and out.error == FAILED_SETTINGS async def test_without_truncation_the_old_wording_stays(monkeypatch, shop): @@ -270,6 +345,28 @@ async def test_without_truncation_the_old_wording_stays(monkeypatch, shop): assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END) +async def test_an_empty_object_without_truncation_keeps_the_shape_wording(monkeypatch, shop): + """没被截断、交回的是空对象:照旧是「没有按结构交结果」,不说成截断。""" + name, _ = shop + whole = {"raw": AIMessage(content="", response_metadata={"stop_reason": "tool_use"}), "parsed": {}} + Models(monkeypatch, Reasoner([query_step(name)], structured=whole)) + out = await run(name) + assert out.error == "核查模型没有按结构交结果(交回的内容里没有要求的字段:value、verdict)" + + +async def test_the_first_limit_hit_is_still_the_one_reported(monkeypatch, shop, priced): + """循环先因时长到点停下(一分钱还没留),收尾又被每日金额上限拦下:触顶原因照旧报先撞上的时长。双盲复算按它写 + 无法复算的原因,反方审稿按「每日上限」决定还派不派后面的句子——截断的改动不能把这个先后换掉。""" + name, _ = shop + async with SessionLocal() as session: + session.add(Setting(key=spend.SPEND_KEY, value={"date": spend._today(), "usd": 1.995, "calls": 9})) + await session.commit() + Script(monkeypatch, [q(name, VALID_SUM, "c1")], FOUND, model_id="priced-model") + out = await run(name, daily_max_usd=2.0, timeout_s=0) + assert out.limit == "timeout_s" + assert out.error == "已达全局每日核查金额上限 $2.00,没能交结果" + + # -------------------------------------------------------------------------- # 额度与退回 # -------------------------------------------------------------------------- @@ -296,6 +393,18 @@ async def test_a_rejected_budget_falls_back_to_4096_for_the_loop_and_the_settle( assert model.calls == ["step", "step", "final"] +async def test_a_rejected_budget_is_remembered_for_the_next_runs(monkeypatch, shop): + """双盲复算的几路盲算、反方审稿的每一句、每次质询各是一次核查:同一接入、模型被嫌大过一次,之后的核查直接用小的, + 不再每次先撞一次。""" + name, _ = shop + model = Reasoner([query_step(name), AIMessage(content="查完了。"), query_step(name, call_id="c2")], + structured=FOUND) + models = Models(monkeypatch, model, rejects_above=4096) + first, second = await run(name), await run(name) + assert first.error is None and second.error is None and second.result["value"] == 230 + assert models.rejecting.calls == 1, "第二次核查没有再按 16384 撞一次" + + async def test_a_cut_after_the_fallback_names_the_small_budget(monkeypatch, shop): name, _ = shop Models(monkeypatch, Reasoner([cut_reply(output=4096)]), rejects_above=4096) @@ -312,6 +421,18 @@ async def test_other_rejections_are_not_retried_with_the_small_budget(monkeypatc assert [s.max_tokens for s in models.specs] == [16384], "不是额度的问题:不拿小额度再撞一次" +async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, shop): + """text-generation-inference 一类自建服务:输入加额度超过 max_total_tokens 时回 422,说的是 max_new_tokens。""" + name, _ = shop + tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 1200 `inputs` tokens and 16384 `max_new_tokens`", status_code=422) + model = Reasoner([query_step(name)], structured=FOUND) + models = Models(monkeypatch, model, rejects_above=4096, rejecting=tgi) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096] and tgi.calls == 1 + + async def test_the_verify_spec_asks_for_the_larger_budget(): """核查模型的配置(api/settings.py 的 verify_model_spec 沿用证据裁判的额度):16384,思考照旧关掉。""" assert judging.MAX_TOKENS == 16384 and judging.FALLBACK_MAX_TOKENS == 4096 @@ -322,21 +443,28 @@ async def test_the_verify_spec_asks_for_the_larger_budget(): # -------------------------------------------------------------------------- -async def test_the_reason_reaches_dual_check_challenge_and_inquiry(monkeypatch, shop): +async def test_the_reason_reaches_dual_check_and_challenge(monkeypatch, shop): name, _ = shop Models(monkeypatch, Reasoner([cut_reply()])) - out = await run(name) - expected = f"{CUT_16K},核查未能完成。{ADVICE}" + out = await run(name, node_id="dc") # 双盲复算:无法复算的原因 slim = _slim(out) - assert slim["value"] is None and slim["reason"] == expected + assert slim["value"] is None and slim["reason"] == FAILED # 反方审稿:这句记为无法判断,理由写「反查未完成」,没能反查的原因就是截断 cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="", aliases=("Q1",)) claim, why = _claim_of(cand, out) - assert claim["verdict"] == "inconclusive" and claim["rationale"] == f"反查未完成:{expected}" and why == expected - # 审批质询:提问人看到的报错照原话 - assert _error_words(out, timeout_s=60, daily_max_usd=1.0) == expected + assert claim["verdict"] == "inconclusive" and claim["rationale"] == f"反查未完成:{FAILED}" and why == FAILED + + +async def test_an_inquiry_names_the_settings_and_not_a_check(monkeypatch, shop): + """审批质询不在节点里(核查模型只看设置):建议只说设置;开头只说「未能给出结果」——提问人问的是问题,不是在核查。""" + name, _ = shop + Models(monkeypatch, Reasoner([cut_reply()])) + out = await run(name, output_schema=INQUIRY_SCHEMA) + words = _error_words(out, timeout_s=60, daily_max_usd=1.0) + assert words == FAILED_SETTINGS + assert "核查未能完成" not in words and "节点" not in words def test_the_review_layer_knows_the_truncation_log(): @@ -344,13 +472,14 @@ def test_the_review_layer_knows_the_truncation_log(): 不落成笼统的 warn;不触发「重跑一次大概率能好」——同样的模型再跑一遍还是撞同一堵墙。""" assert rv.WARN_CODES["verify_truncated"] == ("verify_truncated", rv.DEGRADED) event = {"type": str(EventType.LOG), "node_id": "dc", - "data": {"level": "warn", "code": "verify_truncated", "message": f"{CUT_16K},核查未能完成。{ADVICE}"}} + "data": {"level": "warn", "code": "verify_truncated", "message": FAILED}} [signal] = rv.scan([event], {"结论": "本周订单 120 单。"}) assert (signal.kind, signal.severity) == ("verify_truncated", rv.DEGRADED) and not rv.should_retry([signal]) def test_the_reply_shapes_used_here_are_messages(): assert isinstance(cut_reply(), BaseMessage) and isinstance(anthropic_cut()["raw"], BaseMessage) + assert isinstance(cut_call("x", {}), BaseMessage) # -------------------------------------------------------------------------- @@ -409,18 +538,30 @@ async def test_a_real_client_cut_off_in_the_loop(monkeypatch, shop): name, _ = shop fake = gateway(monkeypatch, [THINKING_ONLY]) out = await run(name) - assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.error == FAILED_SETTINGS assert len(fake.bodies) == 1 and fake.bodies[0]["max_completion_tokens"] == 16384 assert "tools" in fake.bodies[0], "工具循环那次带着只读的查询工具" +async def test_a_real_client_cut_off_with_a_half_written_tool_call(monkeypatch, shop): + """OpenAI 系:工具调用的参数写到一半到了上限(finish_reason=length,arguments 是半截 JSON)。不执行、不收尾。""" + name, _ = shop + half = ({"role": "assistant", "content": None, "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": f"db_query__{name}", "arguments": '{"sql": "SELECT SUM(amount) AS total FROM ord'}}]}, + "length", 16384) + fake = gateway(monkeypatch, [half]) + out = await run(name) + assert out.error == FAILED_SETTINGS and out.queries == [] and len(fake.bodies) == 1 + + async def test_a_real_client_cut_off_in_the_json_schema_settle(monkeypatch, shop): """收尾走 openai 的结构化输出(json_schema):finish_reason=length 时 SDK 抛 LengthFinishReasonError。""" name, _ = shop half = ({"role": "assistant", "content": '{"value": {"alias": "Q1", "row'}, "length", 16384) fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), half]) out = await run(name) - assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" and len(out.queries) == 1 + assert out.error == FAILED_SETTINGS and len(out.queries) == 1 assert len(fake.bodies) == 3 and "response_format" in fake.bodies[2], "收尾被截断:没有再退回普通调用" assert out.usage["output_tokens"] == 120 + 20 + 16384, "异常里带着的那次用量照记" @@ -434,3 +575,106 @@ async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch assert out.error is None and out.result["value"] == 230 budgets = [b.get("max_completion_tokens") for b in fake.bodies] assert budgets == [16384, 4096, 4096, 4096], "第一次被拒后退回 4096,之后工具循环和收尾都用小的,不再先撞一次" + + +# -------------------------------------------------------------------------- +# 真实的 ChatAnthropic 和 anthropic SDK:只有 HTTP 层是假的 +# -------------------------------------------------------------------------- + +CLAUDE = "claude-x" + + +class Messages: + """Anthropic 的 Messages 接口:按顺序回 message(最后一个重复用)。每个回复是 (content, stop_reason, 输出 token 数), + content 可以是按请求体算的函数(收尾那次强制调用的工具名照请求里的写)。bodies 记下每个请求体。""" + + def __init__(self, replies: list) -> None: + self.replies = list(replies) + self.bodies: list[dict] = [] + + async def handle(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content or b"{}") + self.bodies.append(body) + content, stop, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0] + return httpx.Response(200, request=request, json={ + "id": "msg_1", "type": "message", "role": "assistant", "model": body.get("model"), + "content": content(body) if callable(content) else content, "stop_reason": stop, "stop_sequence": None, + "usage": {"input_tokens": 700, "output_tokens": spent}}) + + +def messages_api(monkeypatch, replies: list, *, model: str = CLAUDE, bearer: bool = False, + module=verify_agent) -> Messages: + """接上一个 Anthropic 接入(bearer 为真时按 Authorization: Bearer 鉴权,factory 会换上没设超时的 SDK 客户端)。""" + fake = Messages(replies) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle) + provider = Provider(name=f"an-{uuid.uuid4().hex[:8]}", kind="anthropic", base_url="http://an.example.test", + api_key="test-only-key", models=[{"id": model}], default_model=model, enabled=True, + extra={"auth_style": "bearer"} if bearer else {}) + + async def build(session, spec): + # 按这个接入的模型建:调用方的 spec 写的是别的模型名(mock-fast),SDK 在本地拦不拦要看真正发出去的 id + return build_chat_model(provider, spec.model_copy(update={"model": model})), model + + monkeypatch.setattr(module, "get_chat_model", build) + return fake + + +def forced(args: dict): + """收尾那次强制调用的工具:名字照请求里的写,input 是 args。""" + return lambda body: [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": args}] + + +def query_use(name: str) -> list: + return [{"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {"sql": VALID_SUM}}] + + +DONE = ([{"type": "text", "text": "查完了。"}], "end_turn", 10) + + +async def test_a_real_anthropic_cut_in_the_middle_of_a_tool_call(monkeypatch, shop): + """思考占满了额度、tool_use 截在半路(input 是空的):以前每一轮都执行一遍空参数的查询(报「缺少必填参数 sql」), + 撞到步数上限,最后记成「没有按结构交结果」。""" + name, _ = shop + cut = ([{"type": "thinking", "thinking": "想了很久……", "signature": "s"}, + {"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {}}], "max_tokens", 16384) + fake = messages_api(monkeypatch, [cut]) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", max_queries=2) + assert out.error == FAILED and out.queries == [] and out.limit is None + assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384 + assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_ERROR)] + [log] = truncated_logs(events) + assert log["purpose"] == "verify" + signals = rv.scan([{"type": str(k), "node_id": "dc", "data": d} for k, d in events], {"结论": "本周订单 120 单。"}) + assert [s.kind for s in signals] == ["verify_truncated"], "不是工具出错、也不是触顶" + + +@pytest.mark.parametrize(("schema", "args"), [(None, {}), (CHALLENGE_SCHEMA, HALF_CHALLENGE)], + ids=["empty-input", "half-challenge"]) +async def test_a_real_anthropic_settle_cut_inside_the_forced_tool_call(monkeypatch, shop, schema, args): + name, _ = shop + fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE, (forced(args), "max_tokens", 16384)]) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {})) + assert out.error == FAILED and out.result is None and len(out.queries) == 1 + assert len(fake.bodies) == 3 and fake.bodies[2]["tool_choice"]["type"] == "tool", "没有再退回普通调用" + [log] = truncated_logs(events) + assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384 + if schema is CHALLENGE_SCHEMA: + cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="", + aliases=("Q1",)) + claim, why = _claim_of(cand, out) + assert claim["verdict"] == "inconclusive" and why == FAILED, "半截的「经受住」不能当真" + + +async def test_an_anthropic_client_refusing_the_budget_locally_falls_back(monkeypatch, shop): + """Bearer 鉴权的 Anthropic 接入(factory 换上的 SDK 客户端没设超时):Opus 4、4.1 这一代的 id(经网关用的 Vertex、 + Bedrock 写法也是)非流式只能给 8192,16384 的请求 anthropic SDK 在本地就拒了(ValueError,请求没发出去)。照服务方 + 嫌额度大处理,退回 4096。这条守的是 SDK 的行为:anthropic 以后不再在本地拦这个 id,第一个请求就会按 16384 发出去, + 到时改期望。""" + name, _ = shop + fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE, + (forced(FOUND), "tool_use", 60)], model="claude-opus-4-1@20250805", bearer=True) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [b["max_tokens"] for b in fake.bodies] == [4096, 4096, 4096], "16384 的请求一个都没发出去" diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index cdd1af77..7e0cfb9a 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1039,16 +1039,17 @@ export const AGENT_GUARD_TEXT = { unlimitedShort: '不限', } as const +/** 「(16,384 token)」:输出额度,没给就不写 */ +const tokenCap = (maxTokens?: number) => (maxTokens ? `(${formatNumber(maxTokens)} token)` : '') + /** * 时间线上「核查模型的输出到了上限被截断」那一行(engine/verify_agent.py 的 verify_truncated 日志,双盲复算、反方审稿的 * 核查 Agent 发)。推理模型的思考也算在输出额度里,额度用完时回复是空的或只写了一半:照实说是输出额度不够,不说成 - * 「没查到」「格式不对」,下一步指到换模型。delivered:截断发生在查询过程中、已按截断前查到的结果收尾(结果可能不完整); - * undelivered:这次核查没有给出结果,节点按无法复算、未能反查处理 + * 「没查到」「格式不对」,下一步指到换模型。截断的回复一律不采信,这次核查没有结果,节点按无法复算、未能反查记缺口 */ export const VERIFY_TRUNCATED_TEXT = { - title: (maxTokens?: number) => `核查模型的输出达到上限${maxTokens ? `(${formatNumber(maxTokens)} token)` : ''}被截断`, - delivered: '已按截断前查到的结果收尾,结果可能不完整', - undelivered: '这次核查未能完成', + title: (maxTokens?: number) => `核查模型的输出达到上限${tokenCap(maxTokens)}被截断`, + sub: '这次核查未能给出结果', next: '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型', } as const diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index 8a3a92ce..8a80a716 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -1217,12 +1217,12 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.max_tokens : undefined return { title: VT.title(limit), - sub: d.delivered === true ? VT.delivered : d.delivered === false ? VT.undelivered : undefined, + sub: VT.sub, next: VT.next, fix: 'canvas', fixField: 'model', diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index 3804887b..fa9b0093 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -2319,29 +2319,29 @@ await section('核查类日志与出具缺口(期 4):无法复算、未能 rows.map((r) => `${r?.title}|${r?.sub ?? ''}`).join(' ; ')) }) -await section('核查模型的输出到了上限被截断(verify_truncated):说清额度、交没交出结果,下一步指到换模型', async () => { +await section('核查模型的输出到了上限被截断(verify_truncated):说清额度、这次核查没有结果,下一步指到换模型', async () => { // 推理模型的思考也算在输出额度里:核查 Agent 的一次调用 finish_reason=length、没有工具调用也没有 JSON 时,以前双盲复算 - // 记成「盲算没有查到这个数」、反方审稿记成「核查模型没有按结构交结果」。现在核查 Agent 另发 verify_truncated(max_tokens、 - // delivered、purpose),时间线按结构化字段说人话,原话留在展开区。先核对后端源码还是这么发的 + // 记成「盲算没有查到这个数」、反方审稿记成「核查模型没有按结构交结果」。现在截断的回复一律不采信,核查 Agent 另发 + // verify_truncated(max_tokens、purpose),时间线按结构化字段说人话,原话留在展开区。先核对后端源码还是这么发的 const verifyPy = readFileSync(`${root}backend/app/engine/verify_agent.py`, 'utf8') check('后端发截断日志的格式还是夹具里这一种', verifyPy.includes('code="verify_truncated"') && verifyPy.includes('max_tokens=limit') - && verifyPy.includes('delivered=delivered') && verifyPy.includes('purpose=purpose') - && verifyPy.includes('TRUNCATED_ERROR = "{cut},核查未能完成。{advice}"'), + && verifyPy.includes('purpose=purpose') && !verifyPy.includes('delivered=') + && verifyPy.includes('TRUNCATED_ERROR = "{cut},未能给出结果。{advice}"'), '改了 verify_agent.py 的字段或说法就同步改 decode.ts 的 explainLog 和这里的夹具') const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data }) - const ADVICE = '通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型' - const FAILED = `核查模型的输出达到上限(16384 token)被截断,核查未能完成。${ADVICE}` - const SETTLED = `核查模型的输出达到上限(16384 token)被截断,已按截断前查到的结果收尾,结果可能不完整。${ADVICE}` + const ADVICE = '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型' + const FAILED = `核查模型的输出达到上限(16384 token)被截断,未能给出结果。${ADVICE}` + const SMALL = `核查模型的输出达到上限(4096 token)被截断,未能给出结果。${ADVICE}` const all = flatten(mod.decodeRun([ E(1, 'run.started', null, { total: 2 }), E(2, 'node.started', 'dc', { node_type: 'dual_check', label: '复核销售额' }), - E(3, 'log', 'dc', { level: 'warn', code: 'verify_truncated', purpose: 'verify', max_tokens: 16384, delivered: false, message: FAILED }), + E(3, 'log', 'dc', { level: 'warn', code: 'verify_truncated', purpose: 'verify', max_tokens: 16384, message: FAILED }), E(4, 'node.finished', 'dc', { duration_ms: 20 }), E(5, 'node.started', 'ch', { node_type: 'challenge', label: '审月报' }), - E(6, 'log', 'ch', { level: 'warn', code: 'verify_truncated', purpose: 'verify_result', max_tokens: 4096, delivered: true, - message: SETTLED }), + // 退回小额度之后在收尾时被截断 + E(6, 'log', 'ch', { level: 'warn', code: 'verify_truncated', purpose: 'verify_result', max_tokens: 4096, message: SMALL }), // 缺字段、字段类型不对(别的服务端版本):只给不带数的标题,不瞎填 - E(7, 'log', 'ch', { level: 'warn', code: 'verify_truncated', max_tokens: '16384', delivered: 'no', message: FAILED }), + E(7, 'log', 'ch', { level: 'warn', code: 'verify_truncated', max_tokens: '16384', message: FAILED }), E(8, 'node.finished', 'ch', { duration_ms: 30 }), E(9, 'run.finished', null, { status: 'succeeded' }), ])) @@ -2349,19 +2349,20 @@ await section('核查模型的输出到了上限被截断(verify_truncated) const nodes = all.filter((s) => s.kind === 'node') check('每条一行,挂在自己的节点下', rows.length === 3 && nodes.find((s) => s.nodeId === 'dc')?.children?.filter((x) => x.code === 'verify_truncated').length === 1 && nodes.find((s) => s.nodeId === 'ch')?.children?.filter((x) => x.code === 'verify_truncated').length === 2, `${rows.length} 行`) - const [failed, settled, bare] = rows - check('没交出结果:标题说输出到了上限被截断、带额度,副标题说核查未能完成', failed?.title === '核查模型的输出达到上限(16,384 token)被截断' - && failed.sub === '这次核查未能完成', `${failed?.title} / ${failed?.sub}`) - check('交了结果:额度照实写(退回小额度之后是 4,096),副标题说按截断前查到的结果收尾、可能不完整', - settled?.title === '核查模型的输出达到上限(4,096 token)被截断' && settled.sub === '已按截断前查到的结果收尾,结果可能不完整', - `${settled?.title} / ${settled?.sub}`) - check('……下一步指到换不带推理的模型,入口落到节点的模型,提醒色', rows.every((s) => !!s.next?.includes('不带推理的模型') - && s.next.includes('设置 → 偏好设置 → 核查') && s.fix === 'canvas' && s.fixField === 'model' && s.level === 'warn'), + const [failed, small, bare] = rows + check('标题说输出到了上限被截断、带额度,副标题说这次核查没有结果', failed?.title === '核查模型的输出达到上限(16,384 token)被截断' + && failed.sub === '这次核查未能给出结果', `${failed?.title} / ${failed?.sub}`) + check('额度照实写(退回小额度之后是 4,096)', small?.title === '核查模型的输出达到上限(4,096 token)被截断' && small.sub === '这次核查未能给出结果', + `${small?.title} / ${small?.sub}`) + check('……下一步指到换不带推理的模型(节点上指定的优先,设置里的也说),入口落到节点的模型,提醒色', + rows.every((s) => !!s.next?.includes('在节点上指定不带推理的模型') && s.next.includes('设置 → 偏好设置 → 核查') + && s.fix === 'canvas' && s.fixField === 'model' && s.level === 'warn'), `${failed?.next} / ${failed?.fix} / ${failed?.fixField} / ${failed?.level}`) - check('……原话留在展开区', failed?.detail === FAILED && settled?.detail === SETTLED) - check('缺字段、类型不对:标题不带数,不写副标题,不出 undefined、NaN', bare?.title === '核查模型的输出达到上限被截断' && bare.sub === undefined - && !/undefined|NaN/.test(`${bare?.title}${bare?.next ?? ''}`), `${bare?.title} / ${bare?.sub}`) - check('标题、副标题里没有内部代码', !rows.some((s) => /verify_truncated|max_tokens|delivered/.test(`${s.title}${s.sub ?? ''}${s.next ?? ''}`))) + check('……没有「已按截断前查到的结果收尾」这一说', !rows.some((s) => /收尾|可能不完整/.test(`${s.title}${s.sub ?? ''}`))) + check('……原话留在展开区', failed?.detail === FAILED && small?.detail === SMALL) + check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '核查模型的输出达到上限被截断' + && !/undefined|NaN/.test(`${bare?.title}${bare?.sub ?? ''}${bare?.next ?? ''}`), `${bare?.title} / ${bare?.sub}`) + check('标题、副标题里没有内部代码', !rows.some((s) => /verify_truncated|max_tokens|purpose/.test(`${s.title}${s.sub ?? ''}${s.next ?? ''}`))) check('节点照常完成', nodes.every((s) => s.status === 'done')) }) From 48f656f3628cac4ea7a55f14660e11336fbd0ae9 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 14:30:15 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=E8=AF=81=E6=8D=AE=E8=A3=81=E5=88=A4?= =?UTF-8?q?=E6=88=AA=E6=96=AD=E7=9A=84=E5=9B=9E=E5=A4=8D=E6=95=B4=E6=89=B9?= =?UTF-8?q?=E4=B8=8D=E9=87=87=E4=BF=A1=EF=BC=8C=E6=88=AA=E6=96=AD=E5=8F=A6?= =?UTF-8?q?=E8=AE=B0=20judge=5Ftruncated=EF=BC=8C=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E7=BA=BF=E7=9A=84=E4=B8=8B=E4=B8=80=E6=AD=A5=E6=8C=87=E5=88=B0?= =?UTF-8?q?=E6=8D=A2=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复核第 1 轮的发现: - 有的网关把截断的强制工具调用解析成半截清单:第一条判定写完、第二条只写了编号。_ask 拿到清单就交出去,不再看截断,于是记成一句支持、一句格式不对、两句未给出判定,llm.end 也不标截断。现在回复被截断就整批记 truncated(最后一条可能只写了一半,也可能判定写完、理由写到一半,分不清哪条完整),纯文本调用同理 - 截断的那一批原来只发 llm.end(时间线标题「结论句裁判未能执行」)和 judge_failed(下一步「检查裁判模型的接入后重新运行」):截断时接入是好的,同一个模型重新运行多半还是截断。现在另发 judge_truncated(max_tokens、unjudged),judge_failed 不再带截断;时间线上调用那一行说「裁判模型的输出达到上限(N token)被截断」,judge_truncated 那一行的下一步指到在报告撰写节点的「结论句裁判」或设置里换不带推理的裁判模型,入口落到节点的结论句裁判;check-decode 加一节 - 复核层(review.py)认得 judge_truncated:是缺口,记成它自己的 kind - 退回小额度按 budget_refused 认,和核查同一种记法记在进程里:之后每份报告、每次点击直接用小的 测试补上半截清单(剧本和真实 ChatAnthropic)、纯文本清单后面还在写、截断和格式不对各记各的日志、跨报告记住被嫌大、自建服务的 max_new_tokens --- backend/app/engine/judge.py | 46 +++++--- backend/app/engine/review.py | 3 + backend/tests/test_judge_truncation.py | 145 +++++++++++++++++++++++-- frontend/src/lib/terms.ts | 14 +++ frontend/src/run/decode.ts | 18 ++- scripts/check-decode.mjs | 50 +++++++++ 6 files changed, 248 insertions(+), 28 deletions(-) diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index d4c3903d..25b03568 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -13,7 +13,8 @@ 查询在 SQL 之后另有来历行(来源、口径、已接受、区域外文字,_provenance_lines) 3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析, 再失败就记未裁判。thinking 关掉,输出额度 MAX_TOKENS(推理模型的思考也算在里面),服务方嫌大时退回 - FALLBACK_MAX_TOKENS;输出到了上限被截断的记未裁判(truncated),照实说是额度不够、不再退回纯文本多花一次 + FALLBACK_MAX_TOKENS,被嫌大过的接入、模型之后直接用小的;输出到了上限被截断的整批记未裁判(truncated), + 照实说是额度不够、不再退回纯文本多花一次,时间线上另记 judge_truncated 4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算, 超了按优先级截断;触顶就停,已判的保留,没判的记未裁判并说明是哪个上限——不报错 5. 每日计数落在设置表(SPEND_KEY),按本地日期滚动。目录里没有价格的模型估不出金额,金额上限对它 @@ -70,7 +71,7 @@ from app.engine.merge_query import MERGE_SOURCE from app.engine.nodes.llm import _split_structured, _usage_of from app.engine.state import message_text from app.providers import catalog as pricing -from app.providers.budget import BudgetFallback, cut_off, cut_words, error_usage, truncated_error +from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model logger = logging.getLogger(__name__) @@ -1081,8 +1082,9 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non 先走结构化输出;不成(模型不支持、给的不是合法结构)先从原始回复里找 JSON,再退回纯文本调用、 从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对,truncated 是输出到了上限被截断 (Anthropic 的 stop_reason=max_tokens、OpenAI 系的 finish_reason=length,以及 openai 结构化输出抛的 - LengthFinishReasonError)。结构化调用被截断的不再退回纯文本:推理占满额度时纯文本调用多半一样被截断,白花一次 - 额度。超时由调用方掐。 + LengthFinishReasonError)。截断的回复整批不采信:强制工具调用截在半路时 Anthropic 给的是空对象,有的网关给半截的 + 清单——最后一条可能只写了编号,也可能判定写完了、理由写到一半,分不清哪条是完整的。结构化调用被截断的不再退回 + 纯文本:推理占满额度时纯文本调用多半一样被截断,白花一次额度。超时由调用方掐。 """ raws: list[BaseMessage] = [] try: @@ -1090,6 +1092,8 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non raw, parsed = _split_structured(got) if raw is not None: raws.append(raw) + if cut_off(raw): + return None, raws, _TRUNCATED, 1 if (items := _items_of(parsed)) is not None: return items, raws, None, 1 if raw is not None: @@ -1097,8 +1101,6 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non for found in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: if (items := _items_of(found)) is not None: return items, raws, None, 1 - if cut_off(raw): - return None, raws, _TRUNCATED, 1 except Exception as e: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析 if truncated_error(e): return None, [*raws, *_spent(e)], _TRUNCATED, 1 @@ -1109,9 +1111,10 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non return None, [*raws, *_spent(e)], _TRUNCATED, 2 return None, raws, f"error:{describe_exception(e)}", 2 raws.append(reply) - items = _items_of(message_text(reply)) - if items is None and cut_off(reply): + if cut_off(reply): + # 正文里就算抠得出一段清单,后面还在写,说不清写完了没有:同样整批不采信 return None, raws, _TRUNCATED, 2 + items = _items_of(message_text(reply)) return items, raws, None if items is not None else "format", 2 @@ -1123,15 +1126,18 @@ def _spent(error: BaseException) -> list[BaseMessage]: return [AIMessage(content="", usage_metadata={"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out})] -def _with_fallback(model: Any, spec: ModelSpec) -> BudgetFallback: - """裁判模型包一层:服务方嫌 MAX_TOKENS 太大时按 FALLBACK_MAX_TOKENS 重建一个再问(要退回时才建,平常不多建),之后 - 各批都用小的。经本模块的 get_chat_model 建:测试换成剧本时,小额度的那个也是剧本。""" +def _with_fallback(model: Any, spec: ModelSpec, model_id: str) -> BudgetFallback: + """裁判模型包一层:服务方嫌 MAX_TOKENS 太大(或者 anthropic SDK 在本地就拒了)时按 FALLBACK_MAX_TOKENS 重建一个再问 + (要退回时才建,平常不多建),之后各批都用小的。被嫌大过的按 (接入, 模型, 额度) 记在进程里:之后每份报告、每次 + 点击直接用小的,核查(verify_agent._with_fallback)同一种记法、互相认。经本模块的 get_chat_model 建:测试换成剧本 + 时,小额度的那个也是剧本。""" async def build() -> Any: async with SessionLocal() as session: return (await get_chat_model(session, spec.model_copy(update={"max_tokens": FALLBACK_MAX_TOKENS})))[0] - return BudgetFallback(model, build=build, limits=(spec.max_tokens, FALLBACK_MAX_TOKENS)) + return BudgetFallback(model, build=build, limits=(spec.max_tokens, FALLBACK_MAX_TOKENS), rejected=budget_refused, + key=(spec.provider, model_id, spec.max_tokens)) def _unjudged(reason: str, rationale: str, *, judge: str | None, post_seal: bool) -> dict[str, Any]: @@ -1196,7 +1202,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, try: async with SessionLocal() as session: model, model_id = await get_chat_model(session, spec) - model = _with_fallback(model, spec) + model = _with_fallback(model, spec, model_id) except Exception as e: # noqa: BLE001 - 裁判模型没配好:这回判不了,报告照常 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) details["error"] = f"裁判模型不可用({why})" @@ -1233,6 +1239,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, spent = 0.0 width = 1 + cut_limit: int | None = None # 被截断的那次调用的额度:judge_truncated 日志照它写 while pending: remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started) if remaining is not None and remaining <= 0: @@ -1312,9 +1319,9 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, if problem == _TRUNCATED: # 输出到了上限被截断:这一批记未裁判,原因照实说是额度,不说「格式有误」「调用失败」。别的批照常问 # (同格式不对):截断取决于这一批要想多久,不像鉴权、网关出错那样后面几批多半一样 - limit = model.max_tokens - details["truncated"] = cut_words("裁判模型", limit) - emit(EventType.LLM_END, **end, error=details["truncated"], truncated=True, max_tokens=limit) + cut_limit = model.max_tokens + details["truncated"] = cut_words("裁判模型", cut_limit) + emit(EventType.LLM_END, **end, error=details["truncated"], truncated=True, max_tokens=cut_limit) stop(batch, "truncated", details["truncated"]) continue if problem is not None and problem != "format": @@ -1353,7 +1360,12 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, n = sum(outcome["unjudged"].get(r, 0) for r in hit) emit(EventType.LOG, level="warn", code="judge_limit", limits=hit, unjudged=n, message=f"结论句裁判已达上限({labels}):{n} 句未裁判,已裁判的结果保留") - if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS]: + # 输出被截断单独记一条(带那次的额度、几句未裁判):judge_failed 在时间线上的下一步是检查接入、重新运行,截断时接入 + # 是好的,同一个模型重新运行多半还是截断,该换的是模型 + if truncated := [g for r, g in outcome["_gaps"] if r == "truncated"]: + emit(EventType.LOG, level="warn", code="judge_truncated", max_tokens=cut_limit, + unjudged=outcome["unjudged"]["truncated"], message="结论句裁判未完成:" + ";".join(truncated)) + if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS and r != "truncated"]: emit(EventType.LOG, level="warn", code="judge_failed", message="结论句裁判未完成:" + ";".join(failed)) outcome.pop("_gaps") return outcome diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index bae1b65b..e56e4713 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -104,6 +104,9 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 结论句裁判(claims: judge):触顶、没跑成的句子记为未裁判,出具不能判完整——是缺口,要说明 "judge_limit": ("judge_limit", DEGRADED), "judge_failed": ("judge_failed", DEGRADED), + # 裁判模型的输出到了上限被截断(推理模型的思考也算在额度里):同 judge_failed 是缺口,记成它自己的 kind——说得清 + # 是输出额度的问题,该换的是模型 + "judge_truncated": ("judge_truncated", DEGRADED), # 证据不支持的句子交回写作者改过一次(和 report_repair 一样只记缺口);改写稿冒出新问题没采用的, # 原稿里那几句仍是证据不支持 "report_rewrite": ("report_rewrite", DEGRADED), diff --git a/backend/tests/test_judge_truncation.py b/backend/tests/test_judge_truncation.py index 9686bd2a..1b8a5f75 100644 --- a/backend/tests/test_judge_truncation.py +++ b/backend/tests/test_judge_truncation.py @@ -6,8 +6,12 @@ - 额度:16384(MAX_TOKENS);服务方嫌大时退回 4096(FALLBACK_MAX_TOKENS)再问一次,之后各批都用小的,只撞一次 - 输出到了上限被截断(stop_reason=max_tokens、finish_reason=length、LengthFinishReasonError):这一批记未裁判,原因 - 是 truncated,理由、缺口、llm.end、judge_failed 日志都写明「裁判模型的输出达到上限(N token)被截断」;结构化调用 - 被截断的不再退回纯文本多花一次额度;截断那次的用量照记 + 是 truncated,理由、缺口、llm.end 都写明「裁判模型的输出达到上限(N token)被截断」;截断的回复整批不采信(有的网关 + 把截断的强制工具调用解析成半截清单:最后一条只写了一半);结构化调用被截断的不再退回纯文本多花一次额度;截断那次 + 的用量照记 +- 时间线上另记一条 judge_truncated(带额度、几句未裁判),不混进 judge_failed:judge_failed 的下一步是「检查接入、 + 重新运行」,截断时接入是好的,同一个模型重新运行多半还是截断 +- 被嫌大过的接入、模型记在进程里,之后的裁判直接用小的;自建服务说的 max_new_tokens 也认 - 没有截断时原有说法不变(格式不对仍是 format) 模型都是剧本;证据借 test_evidence_judge 的虚构夹具(口径卡、一次查询、一次检索)。 @@ -18,6 +22,7 @@ import json import re import uuid +import pytest from langchain_core.messages import AIMessage from openai import LengthFinishReasonError from openai.types import CompletionUsage @@ -26,10 +31,13 @@ from openai.types.chat import ChatCompletion from app.core.events import EventType from app.db.models import Provider from app.engine import judge as judge_mod +from app.engine import review as rv from app.engine.evidence import compose_doc from app.engine.judge import BATCH_SIZE, FALLBACK_MAX_TOKENS, MAX_TOKENS, REASONS, judge_doc +from app.providers import budget from app.providers.factory import build_chat_model from tests.test_evidence_judge import PRICED, RAW, SPEC, UNLIMITED, clean_settings, make_catalog, stored # noqa: F401 +from tests.test_verify_truncation import messages_api try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上 from langchain_openai._compat import httpx @@ -40,6 +48,14 @@ CUT_16K = "裁判模型的输出达到上限(16384 token)被截断" ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型" +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() + + def length_error(prompt: int = 3000, completion: int = 16384) -> LengthFinishReasonError: usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) return LengthFinishReasonError(completion=ChatCompletion.model_construct( @@ -60,6 +76,17 @@ def thinking_only(units) -> AIMessage: usage_metadata={"input_tokens": 3000, "output_tokens": 4096, "total_tokens": 7096}) +def half_list(units) -> dict: + """有的网关把截断的强制工具调用解析成半截对象:第一条判定写完了,第二条只写了编号,后面的没写到。""" + items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []}, + {"unit": units[1]}] + raw = AIMessage(content=[{"type": "tool_use", "id": "toolu_1", "name": "judge", "input": {"items": items}}], + tool_calls=[{"name": "judge", "args": {"items": items}, "id": "toolu_1"}], + response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384}) + return {"raw": raw, "parsed": {"items": items}, "parsing_error": None} + + def supported(units) -> dict: items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] return {"raw": AIMessage(content="", response_metadata={"finish_reason": "stop"}), "parsed": {"items": items}} @@ -106,7 +133,9 @@ class _View: class Rejecting: - def __init__(self) -> None: + def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens", + status_code: int = 400) -> None: + self.message, self.status_code = message, status_code self.calls = 0 def with_structured_output(self, schema, **kwargs): @@ -114,13 +143,16 @@ class Rejecting: async def ainvoke(self, messages, *args, **kwargs): self.calls += 1 - raise BudgetRejected("max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens") + error = BudgetRejected(self.message) + error.status_code = self.status_code + raise error class Models: - def __init__(self, monkeypatch, model, *, rejects_above: int | None = None) -> None: + def __init__(self, monkeypatch, model, *, rejects_above: int | None = None, + rejecting: Rejecting | None = None) -> None: self.specs = [] - self.rejecting = Rejecting() + self.rejecting = rejecting or Rejecting() async def fake(session, spec): self.specs.append(spec) @@ -157,8 +189,40 @@ async def test_a_cut_off_structured_reply_is_unjudged_as_truncation_without_a_pl [end] = [d for k, d in events if k == EventType.LLM_END] assert end["purpose"] == "judge" and end["error"] == CUT_16K and end["truncated"] is True assert end["max_tokens"] == 16384 and end["output_tokens"] == 16384 - [failed] = [d for k, d in events if k == EventType.LOG and d.get("code") == "judge_failed"] - assert failed["message"] == f"结论句裁判未完成:有 4 句结论未裁判:{CUT_16K},{ADVICE}" + [log] = logs(events, "judge_truncated") + assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["unjudged"] == 4 + assert log["message"] == f"结论句裁判未完成:有 4 句结论未裁判:{CUT_16K},{ADVICE}" + assert logs(events, "judge_failed") == [], "不混进 judge_failed:那一行的下一步是检查接入、重新运行" + + +def logs(events, code: str) -> list[dict]: + return [d for k, d in events if k == EventType.LOG and d.get("code") == code] + + +async def test_a_half_written_list_in_a_cut_off_reply_is_not_taken(monkeypatch, stored): + """截断的回复整批不采信:写完的那条也可能只是理由写完了一半;以前第一条记成「证据支持」、第二条「判定无法识别」、 + 后面的「未给出判定」,llm.end 也没标截断。""" + model = Reasoner(structured=half_list, plain=lambda units: AIMessage(content=json.dumps({"items": []}))) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured"] + assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0 + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["truncated"] is True and end["error"] == CUT_16K + [log] = logs(events, "judge_truncated") + assert log["unjudged"] == 4 + + +async def test_a_plain_reply_cut_after_the_list_is_not_taken_either(monkeypatch, stored): + """纯文本调用:清单后面还在写、回复是截断的,说不清写完了没有,同样整批记截断。""" + def cut(units): + items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] + return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False) + "\n补充:第二句的依据还要", + response_metadata={"finish_reason": "length"}) + + Models(monkeypatch, Reasoner(plain=cut, unstructured=True)) + outcome, _ = await judged(stored) + assert outcome["unjudged"] == {"truncated": 4} async def test_an_openai_length_error_is_truncation_and_its_tokens_are_counted(monkeypatch, stored): @@ -199,6 +263,35 @@ async def test_a_truncated_batch_does_not_stop_the_others(monkeypatch, stored): assert outcome["counts"]["supported"] == 3 +async def test_truncation_and_other_failures_are_logged_apart(monkeypatch, stored): + """一批被截断、另一批格式不对:截断记 judge_truncated,格式不对照旧记 judge_failed,各说各的。""" + first: list[bool] = [] + + def once(units): + if not first: + first.append(True) + return anthropic_cut(units) + return {"raw": AIMessage(content="不是 JSON", response_metadata={"finish_reason": "stop"}), "parsed": None} + + Models(monkeypatch, Reasoner(structured=once, plain=lambda units: AIMessage(content="还是不是"))) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3)) + outcome, events = await judged(stored, raw) + assert outcome["unjudged"] == {"format": 3, "truncated": BATCH_SIZE} + [cut] = logs(events, "judge_truncated") + [failed] = logs(events, "judge_failed") + assert cut["unjudged"] == BATCH_SIZE and CUT_16K in cut["message"] + assert failed["message"] == "结论句裁判未完成:有 3 句结论未裁判:裁判返回的结果格式有误" + + +def test_the_review_layer_knows_the_truncation_log(): + """跑完之后的复核层认得 judge_truncated:同 judge_failed,是缺口(要说明),kind 是它自己。""" + assert rv.WARN_CODES["judge_truncated"] == ("judge_truncated", rv.DEGRADED) + event = {"type": str(EventType.LOG), "node_id": "write", + "data": {"level": "warn", "code": "judge_truncated", "message": f"有 4 句结论未裁判:{CUT_16K}"}} + [signal] = rv.scan([event], {"answer": "本周销售额 45,678.5元。"}) + assert (signal.kind, signal.severity) == ("judge_truncated", rv.DEGRADED) and not rv.should_retry([signal]) + + async def test_without_truncation_format_problems_keep_their_wording(monkeypatch, stored): model = Reasoner(structured=lambda units: {"raw": AIMessage(content="不是 JSON", response_metadata={"finish_reason": "stop"}), @@ -210,6 +303,7 @@ async def test_without_truncation_format_problems_keep_their_wording(monkeypatch assert outcome["unjudged"] == {"format": 4} assert outcome["gaps"] == ["有 4 句结论未裁判:裁判返回的结果格式有误"] assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END) + assert logs(events, "judge_truncated") == [] and len(logs(events, "judge_failed")) == 1 # -------------------------------------------------------------------------- @@ -236,6 +330,25 @@ async def test_a_rejected_budget_falls_back_to_4096_once_for_all_batches(monkeyp assert [kind for kind, _ in model.calls] == ["structured"] * 3 +async def test_a_rejected_budget_is_remembered_for_the_next_reports(monkeypatch, stored): + """每份报告、每次点击各问一次裁判:同一接入、模型被嫌大过一次,之后直接用小的,不再每次先撞一次。""" + model = Reasoner() + models = Models(monkeypatch, model, rejects_above=4096) + first, _ = await judged(stored) + second, _ = await judged(stored) + assert first["counts"]["supported"] == 4 and second["counts"]["supported"] == 4 + assert models.rejecting.calls == 1, "第二份报告没有再按 16384 撞一次" + + +async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, stored): + tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 3000 `inputs` tokens and 16384 `max_new_tokens`", status_code=422) + models = Models(monkeypatch, Reasoner(), rejects_above=4096, rejecting=tgi) + outcome, _ = await judged(stored) + assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == [] + assert [s.max_tokens for s in models.specs] == [16384, 4096] and tgi.calls == 1 + + async def test_a_truncation_after_the_fallback_names_the_small_budget(monkeypatch, stored): Models(monkeypatch, Reasoner(plain=thinking_only, unstructured=True), rejects_above=4096) outcome, _ = await judged(stored) @@ -306,3 +419,19 @@ async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch outcome, _ = await judged(stored) assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == [] assert [b.get("max_completion_tokens") for b in fake.bodies] == [16384, 4096] + + +async def test_a_real_anthropic_cut_with_a_half_written_list(monkeypatch, stored): + """真实的 ChatAnthropic:强制工具调用写到第二条判定时到了上限,网关给的是半截清单。整批记截断,llm.end 标出来。""" + def half(body): + units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M) + items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []}, + {"unit": units[1]}] + return [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": {"items": items}}] + + fake = messages_api(monkeypatch, [(half, "max_tokens", 16384)], model=PRICED, module=judge_mod) + outcome, events = await judged(stored) + assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0 + assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384, "没有再退回纯文本" + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["truncated"] is True and end["error"] == CUT_16K diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 7e0cfb9a..00f352b6 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1053,6 +1053,20 @@ export const VERIFY_TRUNCATED_TEXT = { next: '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型', } as const +/** + * 时间线上结论句裁判的输出到了上限被截断(engine/judge.py):被截断的那次调用(llm.end 带 truncated),和 + * judge_truncated 日志那一行。不用「裁判未能执行」「检查接入后重新运行」的说法:截断时接入是好的,同一个模型重新运行 + * 多半还是截断,该换的是模型 + */ +export const JUDGE_TRUNCATED_TEXT = { + /** 被截断的那次调用:原来叫「请裁判模型判断 N 句结论」的那一行 */ + call: (maxTokens?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断`, + title: (maxTokens?: number, unjudged?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断` + + (unjudged ? `,${formatNumber(unjudged)} 句结论未裁判` : ''), + next: '未裁判的结论句记为缺口,不能完整出具。通常是推理占满了输出额度:请在报告撰写节点的「结论句裁判」中指定不带推理的裁判模型,' + + '或在「设置 → 偏好设置 → 证据裁判」中更换,再重新运行', +} as const + /** * 上传表格:数据页「表格」标签的上传弹窗、导入回执和卡片。种类的说法和后端 data/tabular.py 的 * CONVERSION_KINDS、SHAPE_KINDS 对得上;后端以后多出来的种类,界面上只显示它给的那句话 diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index 8a80a716..fd01b6c7 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -3,7 +3,8 @@ import type { NodeState, RunPhase } from './trace' import { formatDuration, formatNumber } from '../lib/format' import { ABSENCE_CLAIM_TEXT, CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, SCHEMA_PARTIAL_TEXT, SQL_CHECK_TEXT, TYPE_LABEL, - CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, NODE_TYPE_LABEL, VERIFY_TRUNCATED_TEXT, + CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, JUDGE_TRUNCATED_TEXT, NODE_TYPE_LABEL, + VERIFY_TRUNCATED_TEXT, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel, } from '../lib/terms' @@ -1012,6 +1013,15 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.max_tokens : undefined + const n = typeof d.unjudged === 'number' && d.unjudged > 0 ? Math.round(d.unjudged) : undefined + return { + title: JUDGE_TRUNCATED_TEXT.title(limit, n), next: JUDGE_TRUNCATED_TEXT.next, fix: 'canvas', fixField: 'judge', + } + } case 'judge_failed': { // 「结论句裁判没跑完:有 2 句结论没裁判:裁判调用失败(…)」 return { @@ -1927,9 +1937,11 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt step.detail = [String(d.error), step.detail].filter(Boolean).join('\n') } if (d.purpose === 'judge' && d.error) { - // 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败 + // 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败。输出到了上限被截断的 + // 照实说是截断(不是「未能执行」:接入是好的,该换的是模型,下一步见 judge_truncated 那一行) + const cap = num(d.max_tokens) step.level = 'warn' - step.title = JUDGE_FAILED_TITLE + step.title = d.truncated === true ? JUDGE_TRUNCATED_TEXT.call(cap && cap > 0 ? cap : undefined) : JUDGE_FAILED_TITLE step.detail = [String(d.error), step.detail].filter(Boolean).join('\n') } pendingLlm.delete(nodeId ?? '_') diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index fa9b0093..8e361973 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -2366,6 +2366,56 @@ await section('核查模型的输出到了上限被截断(verify_truncated) check('节点照常完成', nodes.every((s) => s.status === 'done')) }) +await section('结论句裁判的输出到了上限被截断(judge_truncated):调用那一行说截断,日志另起一行,下一步指到换模型', async () => { + // 以前截断的那一批只发 llm.end(标题沿用「结论句裁判未能执行」)和 judge_failed(下一步是「检查接入后重新运行」): + // 截断时接入是好的,同一个模型重新运行多半还是截断。现在调用那一行照 truncated、max_tokens 说截断,另发 judge_truncated + // (max_tokens、unjudged),judge_failed 只说别的失败。先核对后端源码还是这么发的 + const judgePy = readFileSync(`${root}backend/app/engine/judge.py`, 'utf8') + check('后端发裁判截断的格式还是夹具里这一种', judgePy.includes('code="judge_truncated", max_tokens=cut_limit') + && judgePy.includes('unjudged=outcome["unjudged"]["truncated"]') && judgePy.includes('truncated=True, max_tokens=cut_limit') + && judgePy.includes('r not in LIMITS and r != "truncated"'), + '改了 judge.py 的字段或说法就同步改 decode.ts 的 explainLog、llm.end 和这里的夹具') + const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data }) + const CUT = '裁判模型的输出达到上限(16384 token)被截断' + const TRUNCATED = `结论句裁判未完成:有 4 句结论未裁判:${CUT},通常是推理占满了输出额度,可换用不带推理的裁判模型` + const all = flatten(mod.decodeRun([ + E(1, 'run.started', null, { total: 1 }), + E(2, 'node.started', 'write', { node_type: 'report', label: '写周报' }), + E(3, 'llm.start', 'write', { model: 'jm-r', structured: true, purpose: 'judge', message_count: 2, units: 4 }), + E(4, 'llm.end', 'write', { model: 'jm-r', purpose: 'judge', duration_ms: 41000, units: 4, input_tokens: 3000, + output_tokens: 16384, error: CUT, truncated: true, max_tokens: 16384 }), + E(5, 'log', 'write', { level: 'warn', code: 'judge_truncated', max_tokens: 16384, unjudged: 4, message: TRUNCATED }), + E(6, 'node.finished', 'write', { duration_ms: 43000 }), + E(7, 'run.finished', null, { status: 'succeeded' }), + ])) + const call = all.find((s) => s.kind === 'llm' && s.code === 'judge') + check('调用那一行:说输出到了上限被截断、带额度,不说「未能执行」;提醒色、照常完成,原话在展开区', + call?.title === '裁判模型的输出达到上限(16,384 token)被截断' && call.level === 'warn' && call.status === 'done' + && !!call.detail?.includes(CUT), `${call?.title} / ${call?.level} / ${call?.status}`) + const row = all.find((s) => s.code === 'judge_truncated') + check('judge_truncated:标题说截断、带额度和几句未裁判', row?.title === '裁判模型的输出达到上限(16,384 token)被截断,4 句结论未裁判', + row?.title) + check('……下一步指到换不带推理的裁判模型(节点的「结论句裁判」、设置都说),不说检查接入', !!row?.next?.includes('不带推理的裁判模型') + && row.next.includes('结论句裁判') && row.next.includes('设置 → 偏好设置 → 证据裁判') && !row.next.includes('接入') + && row.fix === 'canvas' && row.fixField === 'judge' && row.level === 'warn', `${row?.next} / ${row?.fix} / ${row?.fixField}`) + check('……原话留在展开区,标题里没有内部代码', row?.detail === TRUNCATED && !/judge_truncated|max_tokens|unjudged/.test(`${row?.title}${row?.next}`)) + check('没有混出一条 judge_failed', !all.some((s) => s.code === 'judge_failed')) + // 没截断的失败照旧:「未能执行」那一行和 judge_failed 的说法不变 + const other = flatten(mod.decodeRun([ + E(1, 'node.started', 'write', { node_type: 'report' }), + E(2, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 2 }), + E(3, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 300, units: 2, error: 'ConnectError: 连不上' }), + E(4, 'node.finished', 'write', { duration_ms: 400 }), + ])) + check('没截断的调用失败照旧叫「结论句裁判未能执行」', other.find((s) => s.kind === 'llm')?.title === '结论句裁判未能执行', + other.find((s) => s.kind === 'llm')?.title) + const bare = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report' }), + E(2, 'log', 'w', { level: 'warn', code: 'judge_truncated', max_tokens: 'x', message: '裁判被截断了' })])) + .find((s) => s.code === 'judge_truncated') + check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '裁判模型的输出达到上限被截断' + && !/undefined|NaN/.test(`${bare?.title}${bare?.next ?? ''}`), bare?.title) +}) + await section('审批质询 approval.inquiry(期 4):一行「审批人质询」,标题带问题前 40 字,副标题说回答是否完成、查了几次', async () => { // 前后端共用的约定:运行停在审批上时服务端追加 approval.inquiry,node_id 为空;data 里有问题、划选的原文、回答正文、 // 回答文档、成没成、触顶原因、查询(queries 和 inquiry_artifacts 两份)。以前这类事件落进「未识别的事件」