From 5afa7a9a70f47a071d7057f2b3bf1f36591cfd8c Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 14:29:48 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A0=B8=E6=9F=A5=E3=80=81=E8=AF=81=E6=8D=AE?= =?UTF-8?q?=E8=A3=81=E5=88=A4=E9=80=80=E5=9B=9E=E5=B0=8F=E9=A2=9D=E5=BA=A6?= =?UTF-8?q?=E6=97=B6=E5=A4=9A=E8=AE=A4=E4=B8=A4=E7=A7=8D=E6=8B=92=E7=BB=9D?= =?UTF-8?q?=EF=BC=8C=E8=A2=AB=E5=AB=8C=E5=A4=A7=E8=BF=87=E7=9A=84=E6=8E=A5?= =?UTF-8?q?=E5=85=A5=E8=AE=B0=E5=9C=A8=E8=BF=9B=E7=A8=8B=E9=87=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复核第 1 轮的发现： - 自建服务（text-generation-inference 一类）嫌额度大时回 422，原话说的是 max_new_tokens，不含 max_tokens，认不出就不退回；核查、裁判的额度从 4096 提到 16384 之后才容易撞上。新增 WIDER_BUDGET_WORDS 和 budget_refused，只给核查、裁判用；助手改图仍按 budget_rejected 认，BUDGET_WORDS 不动（data/recipe_ai.py 的测试拿它比对） - Bearer 鉴权的 Anthropic 接入（factory 换上的 SDK 客户端没设超时）调 Opus 4、4.1 这一代的 id 时，16384 的非流式请求 anthropic SDK 在本地就抛 ValueError（Streaming is required…），请求根本没发出去。budget_refused 把它也认成额度被挡下，退回小额度；重来不会重复输出，也没花钱 - 退回的开关原来只在一组调用里共用：双盲复算的几路盲算、反方审稿的每一句、每次质询、每次点击裁判各自先按 16384 撞一次。BudgetFallback 加 key：被嫌大一次就记进进程里的 _REJECTED，之后同一个 key 新建的一开始就用小的（同 factory 的 _NO_STREAM_USAGE，重启就忘）；不给 key 的（助手改图）不记也不查，行为不变 --- backend/app/providers/budget.py | 72 +++++++++++++++------ backend/tests/test_model_budget.py | 100 +++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 19 deletions(-) diff --git a/backend/app/providers/budget.py b/backend/app/providers/budget.py index a6da84e0..3d52402a 100644 --- a/backend/app/providers/budget.py +++ b/backend/app/providers/budget.py @@ -7,21 +7,28 @@ thinking=off，Claude 开了 adaptive thinking 也一样。额度小了，想完 - cut_off：这一块（流式）或这条回复是不是因为额度用完停下的 - truncated_error：这个异常是不是在说同一件事（openai 结构化输出解析时抛的 LengthFinishReasonError） -- budget_rejected：服务方一开口就嫌额度太大 +- budget_rejected：服务方一开口就嫌额度太大（助手改图按它认） +- budget_refused：核查、证据裁判按它认，多认自建服务说的 max_new_tokens 和 anthropic SDK 在本地的拒绝 - error_usage：截断的那次调用花掉的用量 - cut_words：给人看的「某某模型的输出达到上限（N token）被截断」 -- BudgetFallback：先按大额度调，被嫌大时按小额度重来，之后都用小的 +- BudgetFallback：先按大额度调，被嫌大时按小额度重来，之后都用小的；给了 key 的，被嫌大过就记在进程里 engine/nodes/llm.py（cite_fields 的抽取）、data/recipe_ai.py 各有一份同样的判断，是在这里之前写的，行为相同。 """ from __future__ import annotations -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Hashable from typing import Any #: 服务方嫌额度太大时报错里会出现的字样：超过模型的输出上限（DeepSeek chat 系只到 8K）， -#: 或者输入加额度超过上下文长度（本地 vLLM） +#: 或者输入加额度超过上下文长度（本地 vLLM）。助手改图按这一份认，data/recipe_ai.py 的测试拿它比对 BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") +#: 核查、证据裁判另外认的：text-generation-inference 一类自建服务说的是 max_new_tokens（「`inputs` tokens + +#: `max_new_tokens` must be <= 8192」）。这两处的额度从 4096 提到 16384 之后才容易撞上；助手改图的认法不动 +WIDER_BUDGET_WORDS = (*BUDGET_WORDS, "max_new_tokens") +#: anthropic SDK 在本地拒绝时的原话：客户端没设超时（Bearer 鉴权的接入，factory 换上的就是这种）、非流式调用、额度 +#: 超过这个模型不流式能给的上限（Opus 4、4.1 这一代只到 8192）或估计要写十分钟以上，请求没发出去就抛 ValueError +LOCAL_REFUSAL = "streaming is required for operations that may take longer than 10 minutes" #: 截断类报错的原话：openai 的 LengthFinishReasonError（结构化输出解析时 finish_reason=length） LENGTH_WORDS = ("length limit was reached", "finish_reason=length", "finish_reason='length'") @@ -46,9 +53,18 @@ def truncated_error(error: BaseException | None) -> bool: return any(word in text for word in LENGTH_WORDS) -def budget_rejected(e: BaseException) -> bool: +def budget_rejected(e: BaseException, words: tuple[str, ...] = BUDGET_WORDS) -> bool: status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None) - return status in (400, 422) and any(w in str(e).lower() for w in BUDGET_WORDS) + return status in (400, 422) and any(w in str(e).lower() for w in words) + + +def budget_refused(e: BaseException) -> bool: + """核查、证据裁判按这个认「额度被挡下了，换小的重来」：服务方嫌额度太大（多认 max_new_tokens），或者 anthropic SDK + 在本地就拒了。两种都是一个字还没写、请求要么被拒要么根本没发出去，重来不会重复输出，也没花钱。助手改图走流式， + SDK 不在本地拦，仍按 budget_rejected 认。""" + if isinstance(e, ValueError) and LOCAL_REFUSAL in str(e).lower(): + return True + return budget_rejected(e, WIDER_BUDGET_WORDS) def error_usage(error: BaseException | None) -> tuple[int, int]: @@ -63,13 +79,27 @@ def cut_words(who: str, limit: int | None) -> str: return f"{who}的输出达到上限{f'（{limit} token）' if limit else ''}被截断" +#: 嫌过大额度的接入：BudgetFallback 的 key（核查、证据裁判给的是 (接入, 模型, 大额度)）。双盲复算的几路盲算、反方 +#: 审稿的每一句、每次质询、每次点击裁判各是一组新建的调用，开关只在一组里共用；记在这里，之后新建的直接用小的， +#: 不再每组先撞一次。只记被拒过的；进程内有效，重启后再按大的试（服务方可能调高了上限），同 factory 的 _NO_STREAM_USAGE +_REJECTED: set[Hashable] = set() + + class _Switch: - """几次调用共用的「退回小额度了没有」：核查的工具循环和收尾、裁判的各批。""" + """几次调用共用的「退回小额度了没有」：核查的工具循环和收尾、裁判的各批。rejected 是怎么认「额度被挡下了」； + key 给了，开关一开始就照 _REJECTED 定，退回时记进去。""" - __slots__ = ("small",) + __slots__ = ("key", "rejected", "small") - def __init__(self) -> None: - self.small = False + def __init__(self, key: Hashable | None, rejected: Callable[[BaseException], bool]) -> None: + self.key = key + self.rejected = rejected + self.small = key is not None and key in _REJECTED + + def fall_back(self) -> None: + self.small = True + if self.key is not None: + _REJECTED.add(self.key) class BudgetFallback: @@ -81,17 +111,21 @@ class BudgetFallback: small 是现成的小额度模型；也可以不给，给 build（无参的协程函数，返回小额度的模型），要退回时才建，平常一个都不多建。 limits 是 (大额度, 小额度)：给了，max_tokens 就按它报（包过一层的模型身上不一定读得到额度）；不给就读眼下那个模型的 - max_tokens。 + max_tokens。rejected 是怎么认「额度被挡下了」，缺省 budget_rejected（助手改图），核查、证据裁判给 budget_refused。 + key 给了（核查、证据裁判给 (接入, 模型, 大额度)），被嫌大一次就记在进程里，之后同一个 key 新建的一开始就用小的； + 不给（助手改图）不记也不查。shared_with 给了，rejected、key 跟着它。 """ def __init__(self, big: Any, small: Any = None, *, build: Callable[[], Awaitable[Any]] | None = None, limits: tuple[int | None, int | None] | None = None, - shared_with: BudgetFallback | None = None) -> None: + shared_with: BudgetFallback | None = None, + rejected: Callable[[BaseException], bool] = budget_rejected, + key: Hashable | None = None) -> None: self._big = big self._small = small self._build = build self._limits = limits - self._switch = shared_with._switch if shared_with is not None else _Switch() + self._switch = shared_with._switch if shared_with is not None else _Switch(key, rejected) @property def model(self) -> Any: @@ -127,9 +161,9 @@ class BudgetFallback: try: return await used.ainvoke(messages, *args, **kwargs) except Exception as e: - if not self._can_fall_back(used) or not budget_rejected(e): + if not self._can_fall_back(used) or not self._switch.rejected(e): raise - self._switch.small = True + self._switch.fall_back() return await (await self._current()).ainvoke(messages, *args, **kwargs) async def astream(self, messages: Any, *args: Any, **kwargs: Any): @@ -141,9 +175,9 @@ class BudgetFallback: yield chunk return except Exception as e: - if started or not self._can_fall_back(used) or not budget_rejected(e): + if started or not self._can_fall_back(used) or not self._switch.rejected(e): raise - self._switch.small = True + self._switch.fall_back() async for chunk in (await self._current()).astream(messages, *args, **kwargs): yield chunk @@ -164,5 +198,5 @@ class BudgetFallback: return self._derived(lambda m: m.with_structured_output(schema, **kwargs)) -__all__ = ["BUDGET_WORDS", "LENGTH_WORDS", "BudgetFallback", "budget_rejected", "cut_off", "cut_words", "error_usage", - "truncated_error"] +__all__ = ["BUDGET_WORDS", "LENGTH_WORDS", "LOCAL_REFUSAL", "WIDER_BUDGET_WORDS", "BudgetFallback", "budget_refused", + "budget_rejected", "cut_off", "cut_words", "error_usage", "truncated_error"] diff --git a/backend/tests/test_model_budget.py b/backend/tests/test_model_budget.py index 21ee3ecc..4ede095f 100644 --- a/backend/tests/test_model_budget.py +++ b/backend/tests/test_model_budget.py @@ -7,6 +7,10 @@ - 截断的那次调用花掉的用量从异常里取得到（额度是真花出去了） - BudgetFallback：先按大额度调，被嫌大时按小额度重来一次，之后的调用都用小的；流式、非流式都认；bind_tools、 with_structured_output 包出来的和原来的共用同一个开关；只在还没吐出任何东西时重来；小额度的模型要退回时才建 +- 核查、证据裁判另认两种「额度被挡下」（budget_refused）：自建服务（text-generation-inference 一类）说的 + max_new_tokens，和 anthropic SDK 在本地就拒的额度（请求没发出去）；助手改图的认法（budget_rejected）不变 +- 给了 key（接入、模型、大额度）的，被嫌大一次就记在进程里：之后新建的调用直接用小的，不再每次先撞一次；没给 key 的 + （助手改图）不记也不查 模型都是剧本，不碰真接口。 """ @@ -18,9 +22,11 @@ from openai import LengthFinishReasonError from openai.types import CompletionUsage from openai.types.chat import ChatCompletion +from app.providers import budget from app.providers.budget import ( BUDGET_WORDS, BudgetFallback, + budget_refused, budget_rejected, cut_off, cut_words, @@ -29,6 +35,14 @@ from app.providers.budget import ( ) +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里：每个测试从没记过开始，也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() + + def length_error(prompt: int = 9000, completion: int = 16384) -> LengthFinishReasonError: """openai 的结构化输出在 finish_reason=length 时抛的那个错（原话和真实运行里一样）。""" usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) @@ -45,6 +59,12 @@ class Rejected(Exception): TOO_LARGE = "max_tokens: 16384 > 8192, which is the maximum allowed number of output tokens" +#: text-generation-inference 的 OpenAI 兼容接口：输入加额度超过 max_total_tokens 时的 422（原话的样子） +TGI_TOO_LARGE = ("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 1200 `inputs` tokens and 16384 `max_new_tokens`") +#: anthropic SDK 在本地拒绝时的原话：没设超时的客户端非流式调用、额度超过这个模型不流式能给的上限 +SDK_REFUSED = ("Streaming is required for operations that may take longer than 10 minutes. See " + "https://platform.claude.com/docs/en/cli-sdks-libraries/sdks/python#long-requests for more details") class Model: @@ -124,6 +144,25 @@ def test_other_rejections_are_not_a_budget_problem(): assert "max_tokens" in BUDGET_WORDS +def test_verify_and_judge_also_recognise_tgi_and_a_local_sdk_refusal(): + """核查、证据裁判的额度从 4096 提到了 16384，才撞得上这两种：自建服务说的是 max_new_tokens；Bearer 鉴权的 Anthropic + 接入（客户端没设超时）调 Opus 4、4.1 这一代时，SDK 嫌 16384 太大、请求没发出去就抛 ValueError。""" + assert budget_refused(Rejected(TGI_TOO_LARGE, status_code=422)) + assert budget_refused(ValueError(SDK_REFUSED)) + assert budget_refused(Rejected(TOO_LARGE)), "原来认的照样认" + assert not budget_refused(ValueError("别的本地错误")) + assert not budget_refused(RuntimeError(SDK_REFUSED)), "只认 SDK 抛的 ValueError" + assert not budget_refused(Rejected("model 'no-such-model' does not exist")) + assert not budget_refused(Rejected(TGI_TOO_LARGE, status_code=500)) + + +def test_the_copilot_wording_list_stays_as_it_was(): + """助手改图按 budget_rejected 认（流式调用，SDK 不在本地拦）：认法不变，data/recipe_ai.py 的测试拿 BUDGET_WORDS 比对。""" + assert BUDGET_WORDS == ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") + assert not budget_rejected(Rejected(TGI_TOO_LARGE, status_code=422)) + assert not budget_rejected(ValueError(SDK_REFUSED)) + + def test_the_tokens_spent_by_a_cut_off_call_come_from_the_error(): assert error_usage(length_error(prompt=9000, completion=16384)) == (9000, 16384) assert error_usage(RuntimeError("网关超时")) == (0, 0) and error_usage(None) == (0, 0) @@ -242,3 +281,64 @@ async def test_without_limits_max_tokens_reads_the_current_model(): assert model.max_tokens == 32768 await model.ainvoke(["问"]) assert model.max_tokens == 8192 + + +async def test_a_wider_check_can_be_given(): + """核查、证据裁判按 budget_refused 认：服务方说的是 max_new_tokens 也退回小额度。缺省（助手改图）不认。""" + tgi = Rejected(TGI_TOO_LARGE, status_code=422) + model = BudgetFallback(Model("big", fail=tgi), Model("small"), rejected=budget_refused) + assert (await model.ainvoke(["问"])).content == "small" + with pytest.raises(Rejected): + await BudgetFallback(Model("big", fail=tgi), Model("small")).ainvoke(["问"]) + local = BudgetFallback(Model("big", fail=ValueError(SDK_REFUSED)), Model("small"), rejected=budget_refused) + structured = local.with_structured_output({"type": "object"}) + assert (await structured.ainvoke(["交结果"])).content == "small", "包出来的视图照样按这个认" + + +async def test_a_rejection_is_remembered_for_later_calls_to_the_same_model(): + """双盲复算的几路盲算、反方审稿的每一句、每次质询、每次点击裁判，各是一组新建的调用：同一接入、模型、额度被嫌大 + 过一次，之后新建的直接用小的，不再先撞一次。别的模型、别的额度照旧先按大的试。""" + built: list[str] = [] + + def builder(name: str): + async def build(): + built.append(name) + return Model(name) + return build + + key = ("gw", "reasoner-x", 16384) + first = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=builder("small-1"), limits=(16384, 4096), + key=key) + assert first.max_tokens == 16384 + assert (await first.ainvoke(["问"])).content == "small-1" + later_big = Model("big", fail=Rejected(TOO_LARGE)) + later = BudgetFallback(later_big, build=builder("small-2"), limits=(16384, 4096), key=key) + assert later.max_tokens == 4096, "还没调用就知道要用小的：截断时报的额度对得上" + assert (await later.bind_tools(["db_query"]).ainvoke(["再问"])).content == "small-2" + assert later_big.calls == [], "不再先按大额度撞一次" + other = Model("other-big") + assert (await BudgetFallback(other, build=builder("x"), key=("gw", "other", 16384)).ainvoke(["问"])).content \ + == "other-big" + bigger = Model("bigger") + assert (await BudgetFallback(bigger, build=builder("y"), key=("gw", "reasoner-x", 32768)).ainvoke(["问"])).content \ + == "bigger" + assert built == ["small-1", "small-2"] + + +async def test_without_a_key_nothing_is_remembered(): + """助手改图不给 key：每次改图照旧先按大额度试（行为和挪过来之前一样）。""" + await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), Model("small")).ainvoke(["问"]) + assert budget._REJECTED == set() + fresh = Model("big") + assert (await BudgetFallback(fresh, Model("small")).ainvoke(["问"])).content == "big" + + +async def test_a_streaming_rejection_is_remembered_too(): + key = ("gw", "streamer", 16384) + big = Model("big", fail=Rejected(TOO_LARGE), stream=[]) + model = BudgetFallback(big, Model("small", stream=[AIMessageChunk(content="a")]), key=key) + assert [c.content async for c in model.astream(["问"])] == ["a"] + again = Model("big", stream=[AIMessageChunk(content="b")]) + assert [c.content async for c in BudgetFallback(again, Model("small", stream=[AIMessageChunk(content="c")]), + key=key).astream(["问"])] == ["c"] + assert again.calls == []