diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 33ea5d50..14e19738 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -27,6 +27,10 @@ from app.db.models import Workflow from app.engine.layout import CORRIDOR_MIN, MIN_ROW_GAP, NODE_W, _height, auto_layout from app.engine.schema import GraphSpec, NodeType, ValidationIssue, text_parse_issues, validate_graph from app.engine.state import message_text +# 额度用完的认法、退回小额度的做法挪到了 providers/budget.py(核查、证据裁判也用),这里留着老名字; +# _BUDGET_WORDS 另有 data/recipe_ai.py 的测试拿来比对 +from app.providers.budget import BUDGET_WORDS as _BUDGET_WORDS # noqa: F401 +from app.providers.budget import BudgetFallback as _BudgetFallback, cut_off as _cut_off from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model from app.tools.registry import all_specs @@ -1053,54 +1057,12 @@ class CopilotIncomplete(Exception): self.detail = detail -def _cut_off(chunk: Any) -> str: - """这一块是不是在说「额度用完了」:OpenAI 系是 finish_reason=length,Anthropic 是 stop_reason=max_tokens。""" - meta = getattr(chunk, "response_metadata", None) or {} - reason = meta.get("finish_reason") or meta.get("stop_reason") - return str(reason) if reason in ("length", "max_tokens") else "" - - #: 流式改图的输出额度。会思考的模型(DeepSeek、Claude 的 adaptive thinking)思考也算在里面: #: 8192 时复杂的改图常常想完就没额度了,正文一个字没写(2026-09 部署实例上同类请求 5 轮坏了 4 轮) COPILOT_MAX_TOKENS = 32768 -#: 服务方不收大额度时退回这个(原来的默认值) +#: 服务方不收大额度时退回这个(原来的默认值)。退回的做法(_BudgetFallback:只在还没吐出任何东西时重来, +#: 之后的调用即自查修正轮都用小的)和截断的认法(_cut_off)在 providers/budget.py,核查、证据裁判也用它 _FALLBACK_MAX_TOKENS = 8192 -#: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K), -#: 或者输入加额度超过上下文长度(本地 vLLM) -_BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") - - -def _budget_rejected(e: BaseException) -> bool: - status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None) - return status in (400, 422) and any(w in str(e).lower() for w in _BUDGET_WORDS) - - -class _BudgetFallback: - """先按大额度调;服务方一开口就嫌额度太大时按原来的额度重来,之后的调用(自查修正轮)都用小的。 - - 只在还没吐出任何东西时重来:这类拒绝都发生在请求刚发出去的时候,重来不会重复输出。 - """ - - def __init__(self, big: Any, small: Any) -> None: - self.model, self._small = big, small - - @property - def max_tokens(self) -> Any: - return getattr(self.model, "max_tokens", None) - - async def astream(self, messages: list[Any]): - started = False - try: - async for chunk in self.model.astream(messages): - started = True - yield chunk - return - except Exception as e: - if started or self.model is self._small or not _budget_rejected(e): - raise - self.model = self._small - async for chunk in self.model.astream(messages): - yield chunk async def _op_model(session: AsyncSession, payload: GenerateIn) -> tuple[Any, str]: diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 566fae74..5c98ba86 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -549,7 +549,9 @@ async def verify_model_spec(session: AsyncSession, node_config: dict[str, Any] | """核查用哪个模型:节点配置里的 provider / model > 设置里的核查模型 > 证据裁判模型的取值链(裁判设置 > 助手模型 > 默认接入,见 judge.judge_model_spec)。哪一级写了就整组用哪一级,不跨级拼接入和模型。 - 和裁判一样关掉 thinking、max_tokens 4096:核查最后要按结构交结果(强制工具调用),和思考模式不兼容。 + 和裁判一样关掉 thinking:核查最后要按结构交结果(强制工具调用),和思考模式不兼容。额度也和裁判一样(judge.MAX_TOKENS, + 16384):经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考算在额度里,4096 常常想完就没额度查库、交结果了; + 服务方嫌大时核查 agent 退回 judge.FALLBACK_MAX_TOKENS(engine/verify_agent.py)。 建议和被核查的模型不同,否则等于自己核自己;比较、提醒由调用方做(它知道被核查的是哪个模型)。 """ node = node_config if isinstance(node_config, dict) else {} diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index 1dc0e67d..25b03568 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -12,7 +12,9 @@ (按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过。带期 4 文档标记的报告,上传表格的 查询在 SQL 之后另有来历行(来源、口径、已接受、区域外文字,_provenance_lines) 3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析, - 再失败就记未裁判。thinking 关掉,max_tokens 4096 + 再失败就记未裁判。thinking 关掉,输出额度 MAX_TOKENS(推理模型的思考也算在里面),服务方嫌大时退回 + FALLBACK_MAX_TOKENS,被嫌大过的接入、模型之后直接用小的;输出到了上限被截断的整批记未裁判(truncated), + 照实说是额度不够、不再退回纯文本多花一次,时间线上另记 judge_truncated 4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算, 超了按优先级截断;触顶就停,已判的保留,没判的记未裁判并说明是哪个上限——不报错 5. 每日计数落在设置表(SPEND_KEY),按本地日期滚动。目录里没有价格的模型估不出金额,金额上限对它 @@ -38,7 +40,7 @@ from dataclasses import asdict, dataclass from datetime import datetime from typing import Any -from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage from sqlalchemy import select from app.core.artifact_store import canonical_json, content_hash @@ -69,6 +71,7 @@ from app.engine.merge_query import MERGE_SOURCE from app.engine.nodes.llm import _split_structured, _usage_of from app.engine.state import message_text from app.providers import catalog as pricing +from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model logger = logging.getLogger(__name__) @@ -104,7 +107,13 @@ BATCH_SIZE = 8 #: 一波最多几批同时问:几批共用剩下的时长,句数上限内的句子一两波就判完。第一批单独问:接入调不通(鉴权、 #: 网关挂了)时只撞这一次,不让一整波一起失败 PARALLEL_BATCHES = 4 -MAX_TOKENS = 4096 +#: 裁判(和核查:api/settings.py 的 verify_model_spec、engine/verify_agent.py 沿用这一档)的输出额度。thinking 关掉只对 +#: Anthropic 有用:经 OpenAI 兼容接入的推理模型(DeepSeek、GLM、Kimi 这一类)收不到 thinking=off,思考算在额度里,原来的 +#: 4096 常常想完就没额度写判定了(cite_fields 的抽取在 2026-10 撞过同一堵墙,那边也改成了 16384)。16384 是常见网关、 +#: 模型接受的输出上限里偏大的一档(GPT-4o 的上限就是它);更小的服务方会拒收,那时退回 FALLBACK_MAX_TOKENS +MAX_TOKENS = 16384 +#: 服务方嫌 MAX_TOKENS 太大(400 / 422,报错里有 max_tokens 一类字样)时退回这个:原来的额度,各家都收。之后的调用都用它 +FALLBACK_MAX_TOKENS = 4096 #: 证据摘录:SQL 前几个字、每次查询最多几行几列、每段知识库片段最多几个字、整份引用的检索给几段。 #: 列数上限只管没被引用的列:句子引用的格子所在的列一律给(遮罩的除外),再用别的列补满到这么多列 SQL_CHARS = 300 @@ -124,9 +133,12 @@ SHORT_CHARS = 6 _OUT_PER_UNIT = 120 _OUT_BASE = 20 -#: 未裁判的原因。前四个是上限,报告里写「已到上限」 +#: 未裁判的原因。前四个是上限,报告里写「已到上限」;truncated 是裁判模型的输出到了上限被截断(推理模型的思考也算在 +#: 额度里),和格式不对分开记:说成「格式有误」,人会去查提示词,真正该换的是模型 LIMITS = ("max_claims", "max_cost_usd", "daily_max_usd", "timeout_s") -REASONS = (*LIMITS, "error", "format", "missing", "on_demand") +REASONS = (*LIMITS, "error", "format", "truncated", "missing", "on_demand") +#: 输出被截断时给人看的建议,接在 budget.cut_words 的「裁判模型的输出达到上限(N token)被截断」后面 +TRUNCATED_ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型" JUDGE_ROLE = "你是报告核查员:逐句判断报告里的话,能不能被它挂的证据支持。你不写报告,也不改报告。" @@ -897,7 +909,8 @@ def _text(value: Any) -> str | None: async def judge_model_spec(session: Any, node_judge: dict[str, Any] | None = None, *, settings: dict[str, Any] | None = None) -> ModelSpec: """裁判用哪个模型:节点上的 judge.model(连同 provider)> 设置里的证据裁判模型 > Copilot 模型 > - 默认 provider。哪一级写了就整组用哪一级,不跨级拼 provider 和模型。thinking 关、max_tokens 4096。 + 默认 provider。哪一级写了就整组用哪一级,不跨级拼 provider 和模型。thinking 关、额度 MAX_TOKENS(推理模型关不掉 + 思考,见 MAX_TOKENS 的说明;服务方嫌大时 run_request 退回 FALLBACK_MAX_TOKENS)。 建议和写作模型不同,否则等于自己审自己(报告节点发现相同会发 judge_same_model 警告)。 """ @@ -1058,12 +1071,20 @@ def _items_of(value: Any) -> list[Any] | None: return value if isinstance(value, list) else None +#: _ask 交回的没成的原因:输出到了上限被截断 +_TRUNCATED = "truncated" + + async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | None, list[BaseMessage], str | None, int]: """问一次裁判:(判定清单, 模型的原始回复, 没成的原因, 调了几次模型)。 先走结构化输出;不成(模型不支持、给的不是合法结构)先从原始回复里找 JSON,再退回纯文本调用、 - 从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对。超时由调用方掐。 + 从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对,truncated 是输出到了上限被截断 + (Anthropic 的 stop_reason=max_tokens、OpenAI 系的 finish_reason=length,以及 openai 结构化输出抛的 + LengthFinishReasonError)。截断的回复整批不采信:强制工具调用截在半路时 Anthropic 给的是空对象,有的网关给半截的 + 清单——最后一条可能只写了编号,也可能判定写完了、理由写到一半,分不清哪条是完整的。结构化调用被截断的不再退回 + 纯文本:推理占满额度时纯文本调用多半一样被截断,白花一次额度。超时由调用方掐。 """ raws: list[BaseMessage] = [] try: @@ -1071,6 +1092,8 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non raw, parsed = _split_structured(got) if raw is not None: raws.append(raw) + if cut_off(raw): + return None, raws, _TRUNCATED, 1 if (items := _items_of(parsed)) is not None: return items, raws, None, 1 if raw is not None: @@ -1078,17 +1101,45 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non for found in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: if (items := _items_of(found)) is not None: return items, raws, None, 1 - except Exception: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析 - pass + except Exception as e: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析 + if truncated_error(e): + return None, [*raws, *_spent(e)], _TRUNCATED, 1 try: reply = await model.ainvoke(messages) except Exception as e: # noqa: BLE001 + if truncated_error(e): + return None, [*raws, *_spent(e)], _TRUNCATED, 2 return None, raws, f"error:{describe_exception(e)}", 2 raws.append(reply) + if cut_off(reply): + # 正文里就算抠得出一段清单,后面还在写,说不清写完了没有:同样整批不采信 + return None, raws, _TRUNCATED, 2 items = _items_of(message_text(reply)) return items, raws, None if items is not None else "format", 2 +def _spent(error: BaseException) -> list[BaseMessage]: + """截断时 openai 的异常带着这次的用量:折成一条空回复交给记账(额度是真花出去了,每日上限不能漏掉它)。""" + inp, out = error_usage(error) + if not (inp or out): + return [] + return [AIMessage(content="", usage_metadata={"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out})] + + +def _with_fallback(model: Any, spec: ModelSpec, model_id: str) -> BudgetFallback: + """裁判模型包一层:服务方嫌 MAX_TOKENS 太大(或者 anthropic SDK 在本地就拒了)时按 FALLBACK_MAX_TOKENS 重建一个再问 + (要退回时才建,平常不多建),之后各批都用小的。被嫌大过的按 (接入, 模型, 额度) 记在进程里:之后每份报告、每次 + 点击直接用小的,核查(verify_agent._with_fallback)同一种记法、互相认。经本模块的 get_chat_model 建:测试换成剧本 + 时,小额度的那个也是剧本。""" + + async def build() -> Any: + async with SessionLocal() as session: + return (await get_chat_model(session, spec.model_copy(update={"max_tokens": FALLBACK_MAX_TOKENS})))[0] + + return BudgetFallback(model, build=build, limits=(spec.max_tokens, FALLBACK_MAX_TOKENS), rejected=budget_refused, + key=(spec.provider, model_id, spec.max_tokens)) + + def _unjudged(reason: str, rationale: str, *, judge: str | None, post_seal: bool) -> dict[str, Any]: return {"status": "unjudged", "rationale": rationale, "judge": judge, "post_seal": post_seal, "reason": reason} @@ -1116,6 +1167,8 @@ def _why_unjudged(reason: str, budget: Budget, detail: str = "", *, click: bool return f"裁判失败:{detail}"[:120] if reason == "format": return detail or "裁判返回的结果格式有误,这句未裁判" + if reason == "truncated": + return f"{detail or cut_words('裁判模型', None)},这句未裁判。{TRUNCATED_ADVICE}" if reason == "missing": return "裁判未对这句给出判定" return "探索运行按需裁判:点开这句并请模型判断后才会裁判" @@ -1149,6 +1202,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, try: async with SessionLocal() as session: model, model_id = await get_chat_model(session, spec) + model = _with_fallback(model, spec, model_id) except Exception as e: # noqa: BLE001 - 裁判模型没配好:这回判不了,报告照常 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) details["error"] = f"裁判模型不可用({why})" @@ -1185,6 +1239,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, spent = 0.0 width = 1 + cut_limit: int | None = None # 被截断的那次调用的额度:judge_truncated 日志照它写 while pending: remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started) if remaining is not None and remaining <= 0: @@ -1261,6 +1316,14 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, end = {"model": model_id, "purpose": "judge", "duration_ms": int((time.perf_counter() - began) * 1000), "input_tokens": inp, "output_tokens": out, "total_tokens": inp + out, "cost_usd": round(actual, 6), "calls": calls, "units": len(batch)} + if problem == _TRUNCATED: + # 输出到了上限被截断:这一批记未裁判,原因照实说是额度,不说「格式有误」「调用失败」。别的批照常问 + # (同格式不对):截断取决于这一批要想多久,不像鉴权、网关出错那样后面几批多半一样 + cut_limit = model.max_tokens + details["truncated"] = cut_words("裁判模型", cut_limit) + emit(EventType.LLM_END, **end, error=details["truncated"], truncated=True, max_tokens=cut_limit) + stop(batch, "truncated", details["truncated"]) + continue if problem is not None and problem != "format": reason = "timeout_s" if problem == "timeout_s" else "error" detail = problem.partition(":")[2] @@ -1297,7 +1360,12 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, n = sum(outcome["unjudged"].get(r, 0) for r in hit) emit(EventType.LOG, level="warn", code="judge_limit", limits=hit, unjudged=n, message=f"结论句裁判已达上限({labels}):{n} 句未裁判,已裁判的结果保留") - if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS]: + # 输出被截断单独记一条(带那次的额度、几句未裁判):judge_failed 在时间线上的下一步是检查接入、重新运行,截断时接入 + # 是好的,同一个模型重新运行多半还是截断,该换的是模型 + if truncated := [g for r, g in outcome["_gaps"] if r == "truncated"]: + emit(EventType.LOG, level="warn", code="judge_truncated", max_tokens=cut_limit, + unjudged=outcome["unjudged"]["truncated"], message="结论句裁判未完成:" + ";".join(truncated)) + if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS and r != "truncated"]: emit(EventType.LOG, level="warn", code="judge_failed", message="结论句裁判未完成:" + ";".join(failed)) outcome.pop("_gaps") return outcome @@ -1394,6 +1462,8 @@ def _outcome(request: JudgeRequest, verdicts: dict[str, dict[str, Any]], budget: text = f"有 {n} 句结论未裁判:裁判调用失败({details.get('error') or '原因不明'})" elif reason == "format": text = f"有 {n} 句结论未裁判:裁判返回的结果格式有误" + elif reason == "truncated": + text = f"有 {n} 句结论未裁判:{details.get('truncated') or cut_words('裁判模型', None)},{TRUNCATED_ADVICE}" else: text = f"有 {n} 句结论未获得裁判结果" gaps.append((reason, text)) diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 7b4cc70b..e56e4713 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -104,6 +104,9 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 结论句裁判(claims: judge):触顶、没跑成的句子记为未裁判,出具不能判完整——是缺口,要说明 "judge_limit": ("judge_limit", DEGRADED), "judge_failed": ("judge_failed", DEGRADED), + # 裁判模型的输出到了上限被截断(推理模型的思考也算在额度里):同 judge_failed 是缺口,记成它自己的 kind——说得清 + # 是输出额度的问题,该换的是模型 + "judge_truncated": ("judge_truncated", DEGRADED), # 证据不支持的句子交回写作者改过一次(和 report_repair 一样只记缺口);改写稿冒出新问题没采用的, # 原稿里那几句仍是证据不支持 "report_rewrite": ("report_rewrite", DEGRADED), @@ -128,6 +131,9 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 核查 agent 自己触顶、按已有结果收尾:多半和节点自己的 *_limit 说的是同一次触顶,照样是缺口,记成它自己的 kind, # 不落成笼统的 warn——步数上限(max_steps)只有它说 "verify_limit": ("verify_limit", DEGRADED), + # 核查 agent 的输出到了上限被截断(推理模型的思考也算在额度里):没交成结果,由节点按无法复算、未能反查记缺口。 + # 同 verify_limit,是缺口不是断裂,记成它自己的 kind:说得清是输出额度的问题,换个模型就好 + "verify_truncated": ("verify_truncated", DEGRADED), } # 这些 kind 重跑一次大概率能好:原因说得清,且不是「问题本身没问明白」。 diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index ad0a39d2..b127eaef 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -25,6 +25,15 @@ query_artifact 和 role: verify——随运行封存、证据面板能下钻。 每日上限不传就按设置里的「核查每日金额上限」,明确传 None 才是不限。模型不在价格目录里时估不出金额,金额上限 对它无效,照裁判的说法记一条说明。每次调用都记进核查的每日账(engine/spend.py)。 +**输出额度。** 经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考也算在输出额度里:额度按 spec(缺省 judge.MAX_TOKENS), +服务方嫌大时(或者 anthropic SDK 在本地就拒了)退回 judge.FALLBACK_MAX_TOKENS 再试一次,之后工具循环、收尾都用小的; +同一接入、模型被嫌大过,之后的核查一开始就用小的(providers/budget.py)。每次调用都看有没有到了上限被截断,截断的回复 +一律不采信:工具循环里截断的回复不算模型自己停下,带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input +是空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾——同时长到点一样算半路被掐断,按截断前查到的收尾 +交出去的结果说不清完整不完整,还会照常参与比对和处置。收尾的强制工具调用被截断时,解析出来的空对象、半截对象都不算 +交了结果,也不再退回普通调用多花一次额度。因为截断没交成结果的,error 照实说是输出额度(TRUNCATED_ERROR),不说成 +「没查到」「没按结构交」;节点里调用时另记一条 verify_truncated。 + 永远不抛:模型没配好、调用失败、格式不对、触顶、库写不进去,都落在返回值里(error / limit)。模型调用都经 get_chat_model,测试换成剧本。 """ @@ -49,11 +58,12 @@ from app.db.base import SessionLocal from app.engine import spend from app.engine.evidence import build_catalog, cell_eid, compose_doc, query_entry_fields, schema_ledger_entry from app.engine.expressions import CellError, column_kind, locate_cell, numeric_text -from app.engine.judge import _json_in, _tokens, _usd +from app.engine.judge import FALLBACK_MAX_TOKENS, MAX_TOKENS, _json_in, _tokens, _usd from app.engine.nodes.llm import _DIGEST_BUDGET, _digest, _split_structured, _typed, explain_model_error from app.engine.state import message_text from app.engine.toolcalls import ToolTimeout, limit_fields, limit_of, run_bounded from app.providers import catalog as pricing +from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error from app.providers.factory import ModelSpec, ProviderNotConfigured, bind_tools_safely, get_chat_model from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX, build_datasource_tools from app.tools.registry import ToolArgsError, ToolContext, Versions, args_model_of, prepare_args @@ -79,6 +89,14 @@ _REF_DEPTH = 8 RATIONALE = "rationale" #: 触顶原因 LIMITS = ("max_queries", "timeout_s", "max_cost_usd", "daily_max_usd", "max_steps") +#: 核查模型的输出到了上限被截断、没交成结果时给人看的话:双盲复算的无法复算、反方审稿的「反查未完成」、质询的报错都照 +#: 它说,也是 verify_truncated 日志的原话。cut 是 budget.cut_words 的「核查模型的输出达到上限(N token)被截断」。开头只说 +#: 「未能给出结果」:质询里提问人看到的就是这句,它问的是问题,不是在核查 +TRUNCATED_ERROR = "{cut},未能给出结果。{advice}" +#: 截断的建议:该换的是模型。节点里(双盲复算、反方审稿)节点上指定的模型优先于设置(api/settings.py 的 +#: verify_model_spec),两处都说,和时间线上的下一步一致;不在节点里(审批质询)核查模型只看设置 +TRUNCATED_ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型" +TRUNCATED_ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型" VERIFY_SYSTEM = ( "你是数据核查员:只读地查库,用查询结果完成交给你的任务。\n" @@ -129,7 +147,8 @@ class VerifyOutcome: source, tool, call_id}。 rationale:理由组装成的报告文档(evidence.compose_doc 的形状,violations 里有裸数字)。usage:{input_tokens, output_tokens, total_tokens, cost_usd, calls, unpriced_calls}。limit:触顶原因(LIMITS 之一),没触顶为 None。 - answered:模型是不是自己停下来的(没被上限掐断)。 + answered:模型是不是自己停下来的(没被上限掐断,也没因为输出到了上限被截断)。被截断的一律没有结果,error 是 + TRUNCATED_ERROR。 """ result: dict[str, Any] | None @@ -686,13 +705,16 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou def left() -> float | None: return None if deadline is None else deadline - time.monotonic() - # ---- 模型 + # ---- 模型。额度没写的按 MAX_TOKENS:推理模型的思考算在输出额度里,4096 常常想完就没额度发查询、交结果了 + spec = spec.model_copy(update={"max_tokens": spec.max_tokens or MAX_TOKENS}) + small = FALLBACK_MAX_TOKENS if spec.max_tokens > FALLBACK_MAX_TOKENS else None try: async with SessionLocal() as session: - base_model, model_id = await get_chat_model(session, spec) + first, model_id = await get_chat_model(session, spec) except Exception as e: # noqa: BLE001 - 核查模型没配好:这回核不了,照实交回 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) return outcome(error=f"核查模型不可用({why})") + base_model = _with_fallback(first, spec, small, model_id) state.model = model_id if daily_max_usd is Default.SETTINGS: from app.api.settings import verify_settings # 在这里导入:api.settings 导入了 engine 里的模块 @@ -743,7 +765,8 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou raw = got if isinstance(got, BaseMessage) else _split_structured(got)[0] except (asyncio.TimeoutError, Exception) as e: # noqa: BLE001 - 调用失败落在返回值里 failure = e - inp, out = _usage(raw) + # openai 的结构化输出被截断时抛异常,用量在异常里:额度是真花出去了 + inp, out = _usage(raw) if raw is not None else error_usage(failure) cost = await budget.settle(day, estimate, inp, out) usage["input_tokens"] += inp usage["output_tokens"] += out @@ -756,9 +779,23 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou if failure is not None: end["error"] = "超过核查的时长上限,已放弃这次调用" if isinstance(failure, asyncio.TimeoutError) \ else explain_model_error(failure, model_id) + if truncated_error(failure) or cut_off(raw): + # 到了上限被截断:不是「调用失败,请稍后重试」,重试解决不了额度问题 + limit = base_model.max_tokens + end.update(truncated=True, max_tokens=limit, error=cut_words("核查模型", limit)) await _announce(emit, EventType.LLM_END, **end) return got, raw, failure + # 截断的建议按模型从哪来的说:节点里节点上指定的优先,不在节点里(质询)只看设置 + advice = TRUNCATED_ADVICE if node_id else TRUNCATED_ADVICE_SETTINGS + + async def report_cut(purpose: str, limit: int | None) -> VerifyOutcome: + """被截断、没交成结果:记一条 verify_truncated(purpose 是哪一步,limit 是那次的额度),原因照截断说。查过的照记。""" + text = TRUNCATED_ERROR.format(cut=cut_words("核查模型", limit), advice=advice) + await _announce(emit, EventType.LOG, level="warn", code="verify_truncated", purpose=purpose, max_tokens=limit, + message=text) + return outcome(limit=stopped, answered=answered, error=text) + # ---- 查库的循环 stopped: str | None = None answered = False @@ -781,6 +818,12 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou if isinstance(failure, asyncio.TimeoutError): stopped = "timeout_s" break + if truncated_error(failure) or cut_off(response): + # 到了上限被截断(推理模型想完了额度也用完了):话没说完,不是模型自己停下来的;带着的工具调用是写到一半的 + # (Anthropic 的 tool_use 截在半路时 input 是空的,有的网关给半截的 SQL),不执行。也不收尾:一条都没查到的, + # 收尾只能交「查不到」(双盲复算会记成「盲算没有查到这个数」);查到过的,截断前查到的不一定是要的那个数 + # (模型可能正想改查询),同时长到点一样算半路被掐断。收尾那次多半同样被推理占满,也不为它多花一次额度 + return await report_cut("verify", base_model.max_tokens) if failure is not None or not isinstance(response, BaseMessage): return outcome(error=explain_model_error(failure, model_id) if failure else "核查模型没有给出回复") messages.append(response) @@ -850,12 +893,15 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou await _announce(emit, EventType.LOG, level="warn", code="verify_limit", limit=stopped, message=f"核查已达上限({words}),已停止查询,按已有结果收尾") - # ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容 + # ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容(经 OpenAI 兼容接入的推理模型 + # 关不掉,所以额度和工具循环一样,退回小额度也和工具循环共用一个开关) + settle_spec = spec.model_copy(update={"thinking": "off"}) try: async with SessionLocal() as session: - settler, _ = await get_chat_model(session, spec.model_copy(update={"thinking": "off"})) + settle_model, _ = await get_chat_model(session, settle_spec) except Exception as e: # noqa: BLE001 return outcome(limit=stopped, answered=answered, error=f"核查模型不可用({describe_exception(e)})") + settler = _with_fallback(settle_model, settle_spec, small, model_id, shared_with=base_model) final = _final_messages(head, task, queries, said, words) try: structured = settler.with_structured_output(schema, include_raw=True) @@ -870,21 +916,27 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou parsed: dict[str, Any] | None = None failure: BaseException | None = None + settle_cut = False # 收尾那次到了上限被截断 if structured is not None: estimate = budget.estimate(_text_of(final) + schema_text, _OUT_FINAL) day, refused = await budget.reserve(estimate, kept=kept) if refused: return outcome(limit=stopped or refused, answered=answered, error=f"已达{words_of(refused)},没能交结果") got, raw, failure = await ask(final, structured.ainvoke, estimate, day or "", "verify_result", until_finish()) - parsed = None if failure is not None else _parsed(_split_structured(got)[1]) - if parsed is None and raw is not None: - # 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用 - for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: - if (parsed := _parsed(candidate)) is not None: - break - if parsed is None and not isinstance(failure, asyncio.TimeoutError): + # 强制工具调用写到一半到了上限:Anthropic 截在 tool_use 里时 input 是空的,有的网关给半截对象(反方审稿的结论 + # 写完了、理由写到一半)。解析出来的、能从回复里抠出来的都说不清写完了没有,一律不采信 + settle_cut = truncated_error(failure) or bool(cut_off(raw)) + if failure is None and not settle_cut: + parsed = _parsed(_split_structured(got)[1]) + if parsed is None and raw is not None: + # 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用 + for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: + if (parsed := _parsed(candidate)) is not None: + break + if parsed is None and not isinstance(failure, asyncio.TimeoutError) and not settle_cut: # 不支持结构化输出的模型:普通调用一次,结构写进消息,从回复里把 JSON 抠出来。结构化那次已经做过的, - # 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱 + # 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱。结构化那次是被截断的不补救: + # 推理占满额度时普通调用多半一样被截断,白花一次额度 plain = _plain_messages(final, schema) remaining = until_finish() if remaining is None or remaining > 0: @@ -895,8 +947,13 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou error=f"已达{words_of(refused)},没能交结果") if not refused: got, _, failure = await ask(plain, settler.ainvoke, estimate, day or "", "verify_result", remaining) - parsed = None if failure is not None else _parsed(message_text(got) if got is not None else None) + # 同上:截断的回复里就算抠得出一段 JSON,后面还在写,说不清写完了没有 + settle_cut = truncated_error(failure) or bool(cut_off(got)) + if failure is None and not settle_cut: + parsed = _parsed(message_text(got) if got is not None else None) if parsed is None: + if settle_cut: + return await report_cut("verify_result", settler.max_tokens) why = "收尾时超过时长上限,没能交结果" if isinstance(failure, asyncio.TimeoutError) else \ f"核查模型没有按结构交结果({explain_model_error(failure, model_id)})" if failure is not None else \ "核查模型没有按结构交结果" @@ -919,6 +976,22 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou return outcome(result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered) +def _with_fallback(model: Any, spec: ModelSpec, small: int | None, model_id: str, *, + shared_with: BudgetFallback | None = None) -> BudgetFallback: + """按 spec 建好的模型包一层:服务方嫌额度大(或者 SDK 在本地拒了)时按 small 重建一个再调,要退回时才建,平常一个 + 都不多建。small 为 None 是额度本来就不比它大,没有可退的。被嫌大过的按 (接入, 模型, 额度) 记在进程里,证据裁判 + (judge._with_fallback)用的是同一种记法:同一个模型在那边被嫌大过,这边也直接用小的。经本模块的 get_chat_model + 建,测试换成剧本时小额度的那个也是剧本。""" + + async def build() -> Any: + async with SessionLocal() as session: + return (await get_chat_model(session, spec.model_copy(update={"max_tokens": small})))[0] + + return BudgetFallback(model, build=build if small else None, limits=(spec.max_tokens, small or spec.max_tokens), + shared_with=shared_with, rejected=budget_refused, + key=(spec.provider, model_id, spec.max_tokens)) + + async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, remaining: float | None, emit: Emit | None, queries: list[dict[str, Any]], ledger: list[dict[str, Any]], owner: str) -> str | None: diff --git a/backend/app/providers/budget.py b/backend/app/providers/budget.py new file mode 100644 index 00000000..3d52402a --- /dev/null +++ b/backend/app/providers/budget.py @@ -0,0 +1,202 @@ +"""模型调用的输出额度:认出「输出到了上限被截断」,服务方嫌额度太大时退回小额度重来。 + +助手改图(api/copilot.py)、核查(engine/verify_agent.py:双盲复算、反方审稿、审批质询共用)和证据裁判(engine/judge.py) +共用这一份。会思考的模型思考也算在输出额度里:经 OpenAI 兼容接入的推理模型(DeepSeek、GLM、Kimi 这一类)收不到 +thinking=off,Claude 开了 adaptive thinking 也一样。额度小了,想完就没额度写正文——回复是空的,或者 JSON 只写了一半。 +这时要照实说是输出额度不够,不能当成「模型没按格式交」「没查到」。 + +- cut_off:这一块(流式)或这条回复是不是因为额度用完停下的 +- truncated_error:这个异常是不是在说同一件事(openai 结构化输出解析时抛的 LengthFinishReasonError) +- budget_rejected:服务方一开口就嫌额度太大(助手改图按它认) +- budget_refused:核查、证据裁判按它认,多认自建服务说的 max_new_tokens 和 anthropic SDK 在本地的拒绝 +- error_usage:截断的那次调用花掉的用量 +- cut_words:给人看的「某某模型的输出达到上限(N token)被截断」 +- BudgetFallback:先按大额度调,被嫌大时按小额度重来,之后都用小的;给了 key 的,被嫌大过就记在进程里 + +engine/nodes/llm.py(cite_fields 的抽取)、data/recipe_ai.py 各有一份同样的判断,是在这里之前写的,行为相同。 +""" +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Hashable +from typing import Any + +#: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K), +#: 或者输入加额度超过上下文长度(本地 vLLM)。助手改图按这一份认,data/recipe_ai.py 的测试拿它比对 +BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") +#: 核查、证据裁判另外认的:text-generation-inference 一类自建服务说的是 max_new_tokens(「`inputs` tokens + +#: `max_new_tokens` must be <= 8192」)。这两处的额度从 4096 提到 16384 之后才容易撞上;助手改图的认法不动 +WIDER_BUDGET_WORDS = (*BUDGET_WORDS, "max_new_tokens") +#: anthropic SDK 在本地拒绝时的原话:客户端没设超时(Bearer 鉴权的接入,factory 换上的就是这种)、非流式调用、额度 +#: 超过这个模型不流式能给的上限(Opus 4、4.1 这一代只到 8192)或估计要写十分钟以上,请求没发出去就抛 ValueError +LOCAL_REFUSAL = "streaming is required for operations that may take longer than 10 minutes" +#: 截断类报错的原话:openai 的 LengthFinishReasonError(结构化输出解析时 finish_reason=length) +LENGTH_WORDS = ("length limit was reached", "finish_reason=length", "finish_reason='length'") + + +def cut_off(chunk: Any) -> str: + """这一块是不是在说「额度用完了」:OpenAI 系是 finish_reason=length,Anthropic 是 stop_reason=max_tokens。""" + meta = getattr(chunk, "response_metadata", None) or {} + reason = meta.get("finish_reason") or meta.get("stop_reason") + return str(reason) if reason in ("length", "max_tokens") else "" + + +def truncated_error(error: BaseException | None) -> bool: + """这个异常是不是在说「输出被截断」:openai 的 LengthFinishReasonError,或者 langchain、网关转述的同一件事。 + + 服务方嫌额度太大的 400(报错里也有 max_tokens)不算:那是请求被拒,模型一个字都没写。 + """ + if error is None: + return False + if type(error).__name__ == "LengthFinishReasonError": + return True + text = str(error).lower() + return any(word in text for word in LENGTH_WORDS) + + +def budget_rejected(e: BaseException, words: tuple[str, ...] = BUDGET_WORDS) -> bool: + status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None) + return status in (400, 422) and any(w in str(e).lower() for w in words) + + +def budget_refused(e: BaseException) -> bool: + """核查、证据裁判按这个认「额度被挡下了,换小的重来」:服务方嫌额度太大(多认 max_new_tokens),或者 anthropic SDK + 在本地就拒了。两种都是一个字还没写、请求要么被拒要么根本没发出去,重来不会重复输出,也没花钱。助手改图走流式, + SDK 不在本地拦,仍按 budget_rejected 认。""" + if isinstance(e, ValueError) and LOCAL_REFUSAL in str(e).lower(): + return True + return budget_rejected(e, WIDER_BUDGET_WORDS) + + +def error_usage(error: BaseException | None) -> tuple[int, int]: + """失败的那次调用花掉的 (输入, 输出) token。截断时 openai 的异常带着 completion.usage:额度是真花出去了,要记账, + 不然每日上限会漏掉最贵的那几次。别的异常没有用量,是 (0, 0)。""" + usage = getattr(getattr(error, "completion", None), "usage", None) + return int(getattr(usage, "prompt_tokens", 0) or 0), int(getattr(usage, "completion_tokens", 0) or 0) + + +def cut_words(who: str, limit: int | None) -> str: + """「核查模型的输出达到上限(16384 token)被截断」。额度不知道(没给 max_tokens、按接入的默认值)就不写数。""" + return f"{who}的输出达到上限{f'({limit} token)' if limit else ''}被截断" + + +#: 嫌过大额度的接入:BudgetFallback 的 key(核查、证据裁判给的是 (接入, 模型, 大额度))。双盲复算的几路盲算、反方 +#: 审稿的每一句、每次质询、每次点击裁判各是一组新建的调用,开关只在一组里共用;记在这里,之后新建的直接用小的, +#: 不再每组先撞一次。只记被拒过的;进程内有效,重启后再按大的试(服务方可能调高了上限),同 factory 的 _NO_STREAM_USAGE +_REJECTED: set[Hashable] = set() + + +class _Switch: + """几次调用共用的「退回小额度了没有」:核查的工具循环和收尾、裁判的各批。rejected 是怎么认「额度被挡下了」; + key 给了,开关一开始就照 _REJECTED 定,退回时记进去。""" + + __slots__ = ("key", "rejected", "small") + + def __init__(self, key: Hashable | None, rejected: Callable[[BaseException], bool]) -> None: + self.key = key + self.rejected = rejected + self.small = key is not None and key in _REJECTED + + def fall_back(self) -> None: + self.small = True + if self.key is not None: + _REJECTED.add(self.key) + + +class BudgetFallback: + """先按大额度调;服务方一开口就嫌额度太大时按小额度重来,之后的调用都用小的。 + + 只在还没吐出任何东西时重来:这类拒绝都发生在请求刚发出去的时候,重来不会重复输出。流式(astream)、非流式 + (ainvoke)都认。bind_tools、with_structured_output 照样包一层,包出来的和原来的共用同一个开关;shared_with 给了, + 另一份配置建的模型(核查收尾时关掉思考的那个)也和它共用:工具循环里退回过,收尾那次直接用小的,不再先撞一次。 + + small 是现成的小额度模型;也可以不给,给 build(无参的协程函数,返回小额度的模型),要退回时才建,平常一个都不多建。 + limits 是 (大额度, 小额度):给了,max_tokens 就按它报(包过一层的模型身上不一定读得到额度);不给就读眼下那个模型的 + max_tokens。rejected 是怎么认「额度被挡下了」,缺省 budget_rejected(助手改图),核查、证据裁判给 budget_refused。 + key 给了(核查、证据裁判给 (接入, 模型, 大额度)),被嫌大一次就记在进程里,之后同一个 key 新建的一开始就用小的; + 不给(助手改图)不记也不查。shared_with 给了,rejected、key 跟着它。 + """ + + def __init__(self, big: Any, small: Any = None, *, build: Callable[[], Awaitable[Any]] | None = None, + limits: tuple[int | None, int | None] | None = None, + shared_with: BudgetFallback | None = None, + rejected: Callable[[BaseException], bool] = budget_rejected, + key: Hashable | None = None) -> None: + self._big = big + self._small = small + self._build = build + self._limits = limits + self._switch = shared_with._switch if shared_with is not None else _Switch(key, rejected) + + @property + def model(self) -> Any: + """眼下用的那个:退回过、小额度的也建好了就是小的,否则是大的。""" + return self._small if self._switch.small and self._small is not None else self._big + + @property + def max_tokens(self) -> Any: + if self._limits is not None: + return self._limits[1] if self._switch.small else self._limits[0] + return getattr(self.model, "max_tokens", None) + + def _can_fall_back(self, used: Any) -> bool: + """刚才那次用的是大额度、又有小额度可退。大小是同一个模型时重来也是同样的额度,不算可退。""" + if used is not self._big: + return False + return self._build is not None if self._small is None else self._small is not self._big + + async def _small_model(self) -> Any: + if self._small is None and self._build is not None: + built = await self._build() + if self._small is None: # 并发的几次调用同时要建时,留先建好的那个 + self._small = built + return self._small + + async def _current(self) -> Any: + if self._switch.small and (small := await self._small_model()) is not None: + return small + return self._big + + async def ainvoke(self, messages: Any, *args: Any, **kwargs: Any) -> Any: + used = await self._current() + try: + return await used.ainvoke(messages, *args, **kwargs) + except Exception as e: + if not self._can_fall_back(used) or not self._switch.rejected(e): + raise + self._switch.fall_back() + return await (await self._current()).ainvoke(messages, *args, **kwargs) + + async def astream(self, messages: Any, *args: Any, **kwargs: Any): + used = await self._current() + started = False + try: + async for chunk in used.astream(messages, *args, **kwargs): + started = True + yield chunk + return + except Exception as e: + if started or not self._can_fall_back(used) or not self._switch.rejected(e): + raise + self._switch.fall_back() + async for chunk in (await self._current()).astream(messages, *args, **kwargs): + yield chunk + + def _derived(self, make: Callable[[Any], Any]) -> BudgetFallback: + """对大小两个模型做同一件事(绑工具、要结构化输出),包成共用开关的一层。小的还没建时,建好再做。""" + small, build = None, None + if self._small is not None: + small = make(self._small) + elif self._build is not None: + async def build() -> Any: + return make(await self._small_model()) + return BudgetFallback(make(self._big), small, build=build, limits=self._limits, shared_with=self) + + def bind_tools(self, tools: Any, **kwargs: Any) -> BudgetFallback: + return self._derived(lambda m: m.bind_tools(tools, **kwargs)) + + def with_structured_output(self, schema: Any, **kwargs: Any) -> BudgetFallback: + return self._derived(lambda m: m.with_structured_output(schema, **kwargs)) + + +__all__ = ["BUDGET_WORDS", "LENGTH_WORDS", "LOCAL_REFUSAL", "WIDER_BUDGET_WORDS", "BudgetFallback", "budget_refused", + "budget_rejected", "cut_off", "cut_words", "error_usage", "truncated_error"] diff --git a/backend/tests/test_evidence_judge.py b/backend/tests/test_evidence_judge.py index 75a3b995..f46d6a62 100644 --- a/backend/tests/test_evidence_judge.py +++ b/backend/tests/test_evidence_judge.py @@ -6,7 +6,7 @@ - 一份报告合并成一次调用(超过每批的句数就分批),每句给 supported / partial / contradicted / insufficient / not_a_claim - 模型不支持结构化输出、或者给的不是合法结构时退回 JSON 解析,再失败就记「未裁判」,不编判定 -- 裁判模型关掉 thinking,max_tokens 4096 +- 裁判模型关掉 thinking,输出额度 16384(推理模型的思考也算在里面);服务方嫌大时退回 4096,截断照实说(见 test_judge_truncation) - 预算(每份报告的句数、金额、时长,全局每日金额)每一项都能写 null 表示不限;触顶就停、已判的保留、 没判的记未裁判并说明是哪个上限——不报错、不让报告失败。金额超了按优先级截断:先判挂了依据的, 再判有方向词、因果词的 @@ -207,7 +207,7 @@ async def test_the_judge_model_runs_without_thinking(monkeypatch, stored): max_tokens=64000), budget=Budget(max_claims=None, max_cost_usd=None, timeout_s=30, daily_max_usd=None)) [spec] = judge.specs - assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 4096 + assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 16384 assert spec.model == PRICED and spec.timeout and spec.timeout <= 30 diff --git a/backend/tests/test_evidence_judge_api.py b/backend/tests/test_evidence_judge_api.py index 598d5cad..923aec55 100644 --- a/backend/tests/test_evidence_judge_api.py +++ b/backend/tests/test_evidence_judge_api.py @@ -350,7 +350,7 @@ async def test_the_judge_model_follows_node_then_settings(client, monkeypatch): row = await finish(weekly(claims="judge")) await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]]) [spec] = models.specs - assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 4096 + assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 16384 row = await finish(weekly(claims="judge", judge={"model": "judge-from-node"})) body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json() diff --git a/backend/tests/test_judge_settings.py b/backend/tests/test_judge_settings.py index 88fd6de3..80934b2d 100644 --- a/backend/tests/test_judge_settings.py +++ b/backend/tests/test_judge_settings.py @@ -17,6 +17,7 @@ from app.db.base import SessionLocal from app.db.models import Setting from app.engine.judge import ( JUDGE_DEFAULTS, + MAX_TOKENS, SETTING_GROUP, SPEND_KEY, Budget, @@ -166,7 +167,7 @@ async def test_the_judge_model_priority(): async with SessionLocal() as session: spec = await judge_model_spec(session, None) assert (spec.provider, spec.model) == (None, None), "什么都没配:默认 provider" - assert spec.thinking == "off" and spec.max_tokens == 4096 + assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 16384 session.add(Setting(key="copilot", value={"provider": "", "model": "copilot-model"})) await session.commit() diff --git a/backend/tests/test_judge_truncation.py b/backend/tests/test_judge_truncation.py new file mode 100644 index 00000000..1b8a5f75 --- /dev/null +++ b/backend/tests/test_judge_truncation.py @@ -0,0 +1,437 @@ +"""结论句裁判(engine/judge.py)的输出额度:截断照实说,服务方嫌额度大时退回小额度。 + +裁判原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考就能把 4096 用光: +结构化调用写不出判定(openai 抛 LengthFinishReasonError,Anthropic 的强制工具调用写到一半),退回纯文本调用也一样, +最后记成「裁判返回的结果格式有误」——没说出真正的原因。这里守几件事: + +- 额度:16384(MAX_TOKENS);服务方嫌大时退回 4096(FALLBACK_MAX_TOKENS)再问一次,之后各批都用小的,只撞一次 +- 输出到了上限被截断(stop_reason=max_tokens、finish_reason=length、LengthFinishReasonError):这一批记未裁判,原因 + 是 truncated,理由、缺口、llm.end 都写明「裁判模型的输出达到上限(N token)被截断」;截断的回复整批不采信(有的网关 + 把截断的强制工具调用解析成半截清单:最后一条只写了一半);结构化调用被截断的不再退回纯文本多花一次额度;截断那次 + 的用量照记 +- 时间线上另记一条 judge_truncated(带额度、几句未裁判),不混进 judge_failed:judge_failed 的下一步是「检查接入、 + 重新运行」,截断时接入是好的,同一个模型重新运行多半还是截断 +- 被嫌大过的接入、模型记在进程里,之后的裁判直接用小的;自建服务说的 max_new_tokens 也认 +- 没有截断时原有说法不变(格式不对仍是 format) + +模型都是剧本;证据借 test_evidence_judge 的虚构夹具(口径卡、一次查询、一次检索)。 +""" +from __future__ import annotations + +import json +import re +import uuid + +import pytest +from langchain_core.messages import AIMessage +from openai import LengthFinishReasonError +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion + +from app.core.events import EventType +from app.db.models import Provider +from app.engine import judge as judge_mod +from app.engine import review as rv +from app.engine.evidence import compose_doc +from app.engine.judge import BATCH_SIZE, FALLBACK_MAX_TOKENS, MAX_TOKENS, REASONS, judge_doc +from app.providers import budget +from app.providers.factory import build_chat_model +from tests.test_evidence_judge import PRICED, RAW, SPEC, UNLIMITED, clean_settings, make_catalog, stored # noqa: F401 +from tests.test_verify_truncation import messages_api + +try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上 + from langchain_openai._compat import httpx +except ImportError: # pragma: no cover - 老版本的 langchain_openai + import httpx + +CUT_16K = "裁判模型的输出达到上限(16384 token)被截断" +ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型" + + +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() + + +def length_error(prompt: int = 3000, completion: int = 16384) -> LengthFinishReasonError: + usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) + return LengthFinishReasonError(completion=ChatCompletion.model_construct( + id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage)) + + +def anthropic_cut(units) -> dict: + """强制工具调用写到一半到了上限:原始回复 stop_reason=max_tokens,解析不出判定。""" + raw = AIMessage(content=[], response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384}) + return {"raw": raw, "parsed": None, "parsing_error": None} + + +def thinking_only(units) -> AIMessage: + """纯文本调用:额度全用在思考上,正文为空,finish_reason=length(推理模型经 OpenAI 兼容接入的样子)。""" + return AIMessage(content="", additional_kwargs={"reasoning_content": "逐句看证据……"}, + response_metadata={"finish_reason": "length"}, + usage_metadata={"input_tokens": 3000, "output_tokens": 4096, "total_tokens": 7096}) + + +def half_list(units) -> dict: + """有的网关把截断的强制工具调用解析成半截对象:第一条判定写完了,第二条只写了编号,后面的没写到。""" + items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []}, + {"unit": units[1]}] + raw = AIMessage(content=[{"type": "tool_use", "id": "toolu_1", "name": "judge", "input": {"items": items}}], + tool_calls=[{"name": "judge", "args": {"items": items}, "id": "toolu_1"}], + response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384}) + return {"raw": raw, "parsed": {"items": items}, "parsing_error": None} + + +def supported(units) -> dict: + items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] + return {"raw": AIMessage(content="", response_metadata={"finish_reason": "stop"}), "parsed": {"items": items}} + + +class BudgetRejected(Exception): + """服务方嫌输出额度太大时的 400。""" + + status_code = 400 + + +class Reasoner: + """裁判模型的剧本。structured(units) / plain(units) 给出结构化调用、退回纯文本调用的回复(返回异常就抛它); + unstructured 为真时不支持结构化输出。calls 记下每次调用的种类和判的句子。""" + + def __init__(self, *, structured=supported, plain=None, unstructured: bool = False) -> None: + self.structured, self.plain, self.unstructured = structured, plain, unstructured + self.calls: list[tuple[str, list[str]]] = [] + + def with_structured_output(self, schema, **kwargs): + if self.unstructured: + raise NotImplementedError("这个模型不支持结构化输出") + return _View(self) + + async def ainvoke(self, messages, *args, **kwargs): + return self.answer("plain", messages) + + def answer(self, kind: str, messages): + units = re.findall(r"^\[(u\d+)\]", str(messages[-1].content), re.M) + self.calls.append((kind, units)) + make = self.structured if kind == "structured" else self.plain + reply = make(units) + if isinstance(reply, BaseException): + raise reply + return reply + + +class _View: + def __init__(self, model: Reasoner) -> None: + self.model = model + + async def ainvoke(self, messages, *args, **kwargs): + return self.model.answer("structured", messages) + + +class Rejecting: + def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens", + status_code: int = 400) -> None: + self.message, self.status_code = message, status_code + self.calls = 0 + + def with_structured_output(self, schema, **kwargs): + return self + + async def ainvoke(self, messages, *args, **kwargs): + self.calls += 1 + error = BudgetRejected(self.message) + error.status_code = self.status_code + raise error + + +class Models: + def __init__(self, monkeypatch, model, *, rejects_above: int | None = None, + rejecting: Rejecting | None = None) -> None: + self.specs = [] + self.rejecting = rejecting or Rejecting() + + async def fake(session, spec): + self.specs.append(spec) + if rejects_above is not None and (spec.max_tokens or 0) > rejects_above: + return self.rejecting, PRICED + return model, PRICED + + monkeypatch.setattr(judge_mod, "get_chat_model", fake) + + +async def judged(stored, raw: str = RAW): + catalog = make_catalog(stored) + events: list[tuple] = [] + outcome = await judge_doc(compose_doc(raw, catalog), catalog, spec=SPEC, budget=UNLIMITED, + emit=lambda kind, **data: events.append((kind, data))) + return outcome, events + + +# -------------------------------------------------------------------------- +# 截断 +# -------------------------------------------------------------------------- + + +async def test_a_cut_off_structured_reply_is_unjudged_as_truncation_without_a_plain_retry(monkeypatch, stored): + model = Reasoner(structured=anthropic_cut, plain=lambda units: AIMessage(content=json.dumps({"items": []}))) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured"], "判定写到一半到了上限:不再退回纯文本多花一次额度" + verdicts = list(outcome["verdicts"].values()) + assert {(v["status"], v["reason"]) for v in verdicts} == {("unjudged", "truncated")} + assert verdicts[0]["rationale"] == f"{CUT_16K},这句未裁判。{ADVICE}" + assert outcome["unjudged"] == {"truncated": 4} and outcome["limits_hit"] == [] + assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"] + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["purpose"] == "judge" and end["error"] == CUT_16K and end["truncated"] is True + assert end["max_tokens"] == 16384 and end["output_tokens"] == 16384 + [log] = logs(events, "judge_truncated") + assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["unjudged"] == 4 + assert log["message"] == f"结论句裁判未完成:有 4 句结论未裁判:{CUT_16K},{ADVICE}" + assert logs(events, "judge_failed") == [], "不混进 judge_failed:那一行的下一步是检查接入、重新运行" + + +def logs(events, code: str) -> list[dict]: + return [d for k, d in events if k == EventType.LOG and d.get("code") == code] + + +async def test_a_half_written_list_in_a_cut_off_reply_is_not_taken(monkeypatch, stored): + """截断的回复整批不采信:写完的那条也可能只是理由写完了一半;以前第一条记成「证据支持」、第二条「判定无法识别」、 + 后面的「未给出判定」,llm.end 也没标截断。""" + model = Reasoner(structured=half_list, plain=lambda units: AIMessage(content=json.dumps({"items": []}))) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured"] + assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0 + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["truncated"] is True and end["error"] == CUT_16K + [log] = logs(events, "judge_truncated") + assert log["unjudged"] == 4 + + +async def test_a_plain_reply_cut_after_the_list_is_not_taken_either(monkeypatch, stored): + """纯文本调用:清单后面还在写、回复是截断的,说不清写完了没有,同样整批记截断。""" + def cut(units): + items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] + return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False) + "\n补充:第二句的依据还要", + response_metadata={"finish_reason": "length"}) + + Models(monkeypatch, Reasoner(plain=cut, unstructured=True)) + outcome, _ = await judged(stored) + assert outcome["unjudged"] == {"truncated": 4} + + +async def test_an_openai_length_error_is_truncation_and_its_tokens_are_counted(monkeypatch, stored): + model = Reasoner(structured=lambda units: length_error(completion=16384)) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured"] + assert outcome["unjudged"] == {"truncated": 4} + assert outcome["usage"]["output_tokens"] == 16384 and outcome["cost_usd"] > 0, "截断那次的额度是真花出去了" + end = [d for k, d in events if k == EventType.LLM_END][-1] + assert "调用" not in end["error"] and end["error"] == CUT_16K + + +async def test_a_plain_reply_with_only_thinking_is_truncation(monkeypatch, stored): + """不支持结构化输出的模型直接退回纯文本调用:回复只有思考、finish_reason=length。""" + model = Reasoner(plain=thinking_only, unstructured=True) + Models(monkeypatch, model) + outcome, _ = await judged(stored) + assert [kind for kind, _ in model.calls] == ["plain"] and outcome["calls"] == 2 + assert outcome["unjudged"] == {"truncated": 4} + assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"] + + +async def test_a_truncated_batch_does_not_stop_the_others(monkeypatch, stored): + """只有一批被截断:别的批照常判(和格式不对一样,不像调用失败那样后面的都不再问)。""" + first: list[bool] = [] + + def once(units): + if not first: + first.append(True) + return anthropic_cut(units) + return supported(units) + + Models(monkeypatch, Reasoner(structured=once)) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3)) + outcome, _ = await judged(stored, raw) + assert outcome["unjudged"] == {"truncated": BATCH_SIZE} + assert outcome["counts"]["supported"] == 3 + + +async def test_truncation_and_other_failures_are_logged_apart(monkeypatch, stored): + """一批被截断、另一批格式不对:截断记 judge_truncated,格式不对照旧记 judge_failed,各说各的。""" + first: list[bool] = [] + + def once(units): + if not first: + first.append(True) + return anthropic_cut(units) + return {"raw": AIMessage(content="不是 JSON", response_metadata={"finish_reason": "stop"}), "parsed": None} + + Models(monkeypatch, Reasoner(structured=once, plain=lambda units: AIMessage(content="还是不是"))) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3)) + outcome, events = await judged(stored, raw) + assert outcome["unjudged"] == {"format": 3, "truncated": BATCH_SIZE} + [cut] = logs(events, "judge_truncated") + [failed] = logs(events, "judge_failed") + assert cut["unjudged"] == BATCH_SIZE and CUT_16K in cut["message"] + assert failed["message"] == "结论句裁判未完成:有 3 句结论未裁判:裁判返回的结果格式有误" + + +def test_the_review_layer_knows_the_truncation_log(): + """跑完之后的复核层认得 judge_truncated:同 judge_failed,是缺口(要说明),kind 是它自己。""" + assert rv.WARN_CODES["judge_truncated"] == ("judge_truncated", rv.DEGRADED) + event = {"type": str(EventType.LOG), "node_id": "write", + "data": {"level": "warn", "code": "judge_truncated", "message": f"有 4 句结论未裁判:{CUT_16K}"}} + [signal] = rv.scan([event], {"answer": "本周销售额 45,678.5元。"}) + assert (signal.kind, signal.severity) == ("judge_truncated", rv.DEGRADED) and not rv.should_retry([signal]) + + +async def test_without_truncation_format_problems_keep_their_wording(monkeypatch, stored): + model = Reasoner(structured=lambda units: {"raw": AIMessage(content="不是 JSON", + response_metadata={"finish_reason": "stop"}), + "parsed": None}, + plain=lambda units: AIMessage(content="还是不是", response_metadata={"finish_reason": "stop"})) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured", "plain"] + assert outcome["unjudged"] == {"format": 4} + assert outcome["gaps"] == ["有 4 句结论未裁判:裁判返回的结果格式有误"] + assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END) + assert logs(events, "judge_truncated") == [] and len(logs(events, "judge_failed")) == 1 + + +# -------------------------------------------------------------------------- +# 额度与退回 +# -------------------------------------------------------------------------- + + +async def test_the_judge_asks_for_the_larger_budget(monkeypatch, stored): + models = Models(monkeypatch, Reasoner()) + outcome, _ = await judged(stored) + assert outcome["counts"]["supported"] == 4 + assert [s.max_tokens for s in models.specs] == [MAX_TOKENS] == [16384], "平常只建一个模型" + assert "truncated" in REASONS and FALLBACK_MAX_TOKENS == 4096 + + +async def test_a_rejected_budget_falls_back_to_4096_once_for_all_batches(monkeypatch, stored): + model = Reasoner() + models = Models(monkeypatch, model, rejects_above=4096) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE * 2 + 3)) + outcome, _ = await judged(stored, raw) + assert outcome["counts"]["supported"] == BATCH_SIZE * 2 + 3 and outcome["gaps"] == [] + assert [s.max_tokens for s in models.specs] == [16384, 4096] + assert models.rejecting.calls == 1, "第一批退回过,后面几批直接用小的" + assert [kind for kind, _ in model.calls] == ["structured"] * 3 + + +async def test_a_rejected_budget_is_remembered_for_the_next_reports(monkeypatch, stored): + """每份报告、每次点击各问一次裁判:同一接入、模型被嫌大过一次,之后直接用小的,不再每次先撞一次。""" + model = Reasoner() + models = Models(monkeypatch, model, rejects_above=4096) + first, _ = await judged(stored) + second, _ = await judged(stored) + assert first["counts"]["supported"] == 4 and second["counts"]["supported"] == 4 + assert models.rejecting.calls == 1, "第二份报告没有再按 16384 撞一次" + + +async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, stored): + tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 3000 `inputs` tokens and 16384 `max_new_tokens`", status_code=422) + models = Models(monkeypatch, Reasoner(), rejects_above=4096, rejecting=tgi) + outcome, _ = await judged(stored) + assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == [] + assert [s.max_tokens for s in models.specs] == [16384, 4096] and tgi.calls == 1 + + +async def test_a_truncation_after_the_fallback_names_the_small_budget(monkeypatch, stored): + Models(monkeypatch, Reasoner(plain=thinking_only, unstructured=True), rejects_above=4096) + outcome, _ = await judged(stored) + [verdict, *_] = outcome["verdicts"].values() + assert verdict["reason"] == "truncated" + assert verdict["rationale"].startswith("裁判模型的输出达到上限(4096 token)被截断") + + +# -------------------------------------------------------------------------- +# 真实的 ChatOpenAI 和 openai SDK:只有 HTTP 层是假的(经 OpenAI 兼容网关接入的推理模型就是这一路) +# -------------------------------------------------------------------------- + + +class Gateway: + """OpenAI 兼容网关:reply(这一批的句子编号) 给出 (message, finish_reason, 输出 token 数)。cap 给了,请求的额度超过它 + 就回 400。bodies 记下每个请求体。""" + + def __init__(self, reply, *, cap: int | None = None) -> None: + self.reply, self.cap = reply, cap + self.bodies: list[dict] = [] + + async def handle(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content or b"{}") + self.bodies.append(body) + budget = body.get("max_completion_tokens") or body.get("max_tokens") + if self.cap and budget and budget > self.cap: + message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens" + return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}}, + request=request) + units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M) + message, finish, spent = self.reply(units) + return httpx.Response(200, request=request, json={ + "id": "c-1", "object": "chat.completion", "created": 1, "model": PRICED, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": {"prompt_tokens": 3000, "completion_tokens": spent, "total_tokens": 3000 + spent}}) + + +def gateway(monkeypatch, reply, *, cap: int | None = None) -> Gateway: + fake = Gateway(reply, cap=cap) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle) + provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1", + api_key=None, models=[{"id": PRICED}], default_model=PRICED, enabled=True, extra={}) + + async def build(session, spec): + return build_chat_model(provider, spec), PRICED + + monkeypatch.setattr(judge_mod, "get_chat_model", build) + return fake + + +async def test_a_real_client_cut_off_in_the_json_schema_call(monkeypatch, stored): + """json_schema 的结构化输出:finish_reason=length 时 openai SDK 抛 LengthFinishReasonError,不再退回纯文本。""" + fake = gateway(monkeypatch, lambda units: ({"role": "assistant", "content": '{"items": [{"unit": "u'}, "length", 16384)) + outcome, _ = await judged(stored) + assert outcome["unjudged"] == {"truncated": 4} + assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"] + assert len(fake.bodies) == 1 and "response_format" in fake.bodies[0] + assert fake.bodies[0]["max_completion_tokens"] == 16384 + assert outcome["usage"]["output_tokens"] == 16384, "异常里带着的那次用量照记" + + +async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, stored): + def answer(units): + items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] + return {"role": "assistant", "content": json.dumps({"items": items}, ensure_ascii=False)}, "stop", 80 + + fake = gateway(monkeypatch, answer, cap=8192) + outcome, _ = await judged(stored) + assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == [] + assert [b.get("max_completion_tokens") for b in fake.bodies] == [16384, 4096] + + +async def test_a_real_anthropic_cut_with_a_half_written_list(monkeypatch, stored): + """真实的 ChatAnthropic:强制工具调用写到第二条判定时到了上限,网关给的是半截清单。整批记截断,llm.end 标出来。""" + def half(body): + units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M) + items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []}, + {"unit": units[1]}] + return [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": {"items": items}}] + + fake = messages_api(monkeypatch, [(half, "max_tokens", 16384)], model=PRICED, module=judge_mod) + outcome, events = await judged(stored) + assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0 + assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384, "没有再退回纯文本" + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["truncated"] is True and end["error"] == CUT_16K diff --git a/backend/tests/test_model_budget.py b/backend/tests/test_model_budget.py new file mode 100644 index 00000000..4ede095f --- /dev/null +++ b/backend/tests/test_model_budget.py @@ -0,0 +1,344 @@ +"""模型调用的输出额度(providers/budget.py):助手改图、核查、证据裁判共用的那几样。 + +推理模型的思考也算在输出额度里,额度小了,想完就没额度写正文:回复是空的,或者 JSON 只写了一半。这里守几件事: +- 认得出「输出到了上限被截断」:OpenAI 系 finish_reason=length、Anthropic stop_reason=max_tokens,以及 openai 结构化 + 输出解析时抛的 LengthFinishReasonError;服务方嫌额度太大的 400 不算截断(那是请求被拒,一个字都没写) +- 认得出「服务方嫌额度太大」:400 / 422,报错里有 max_tokens 一类字样;别的 400 不算 +- 截断的那次调用花掉的用量从异常里取得到(额度是真花出去了) +- BudgetFallback:先按大额度调,被嫌大时按小额度重来一次,之后的调用都用小的;流式、非流式都认;bind_tools、 + with_structured_output 包出来的和原来的共用同一个开关;只在还没吐出任何东西时重来;小额度的模型要退回时才建 +- 核查、证据裁判另认两种「额度被挡下」(budget_refused):自建服务(text-generation-inference 一类)说的 + max_new_tokens,和 anthropic SDK 在本地就拒的额度(请求没发出去);助手改图的认法(budget_rejected)不变 +- 给了 key(接入、模型、大额度)的,被嫌大一次就记在进程里:之后新建的调用直接用小的,不再每次先撞一次;没给 key 的 + (助手改图)不记也不查 + +模型都是剧本,不碰真接口。 +""" +from __future__ import annotations + +import pytest +from langchain_core.messages import AIMessage, AIMessageChunk +from openai import LengthFinishReasonError +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion + +from app.providers import budget +from app.providers.budget import ( + BUDGET_WORDS, + BudgetFallback, + budget_refused, + budget_rejected, + cut_off, + cut_words, + error_usage, + truncated_error, +) + + +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里:每个测试从没记过开始,也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() + + +def length_error(prompt: int = 9000, completion: int = 16384) -> LengthFinishReasonError: + """openai 的结构化输出在 finish_reason=length 时抛的那个错(原话和真实运行里一样)。""" + usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) + return LengthFinishReasonError(completion=ChatCompletion.model_construct( + id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage)) + + +class Rejected(Exception): + """仿 openai / anthropic SDK 的 400:一发请求就被拒,还没吐出任何东西。""" + + def __init__(self, message: str, status_code: int = 400) -> None: + super().__init__(message) + self.status_code = status_code + + +TOO_LARGE = "max_tokens: 16384 > 8192, which is the maximum allowed number of output tokens" +#: text-generation-inference 的 OpenAI 兼容接口:输入加额度超过 max_total_tokens 时的 422(原话的样子) +TGI_TOO_LARGE = ("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 1200 `inputs` tokens and 16384 `max_new_tokens`") +#: anthropic SDK 在本地拒绝时的原话:没设超时的客户端非流式调用、额度超过这个模型不流式能给的上限 +SDK_REFUSED = ("Streaming is required for operations that may take longer than 10 minutes. See " + "https://platform.claude.com/docs/en/cli-sdks-libraries/sdks/python#long-requests for more details") + + +class Model: + """记下每次调用的剧本模型。fail:每次调用都抛它;stream:流式时吐的块,吐到 boom 处抛 fail。""" + + def __init__(self, name: str, *, fail: Exception | None = None, stream: list | None = None, + boom: int | None = None, max_tokens: int | None = None) -> None: + self.name = name + self.fail = fail + self.stream = stream if stream is not None else [AIMessageChunk(content=name)] + self.boom = boom + self.max_tokens = max_tokens + self.calls: list[tuple[str, object]] = [] + self.bound: list[tuple[str, object, dict]] = [] + + async def ainvoke(self, messages, *args, **kwargs): + self.calls.append(("ainvoke", messages)) + if self.fail is not None: + raise self.fail + return AIMessage(content=self.name) + + async def astream(self, messages, *args, **kwargs): + self.calls.append(("astream", messages)) + for i, chunk in enumerate(self.stream): + if self.boom is not None and i == self.boom: + raise self.fail + yield chunk + if self.fail is not None and self.boom is None: + raise self.fail + + def bind_tools(self, tools, **kwargs): + self.bound.append(("tools", tools, kwargs)) + return self + + def with_structured_output(self, schema, **kwargs): + self.bound.append(("structured", schema, kwargs)) + return self + + +# -------------------------------------------------------------------------- +# 认出截断、认出嫌额度太大 +# -------------------------------------------------------------------------- + + +def test_cut_off_replies_are_recognised_for_both_families(): + assert cut_off(AIMessage(content="", response_metadata={"finish_reason": "length"})) == "length" + assert cut_off(AIMessageChunk(content=[], response_metadata={"stop_reason": "max_tokens"})) == "max_tokens" + assert cut_off(AIMessage(content="好了", response_metadata={"finish_reason": "stop"})) == "" + assert cut_off(AIMessage(content="", response_metadata={"stop_reason": "end_turn"})) == "" + assert cut_off(None) == "" and cut_off({"finish_reason": "length"}) == "" + + +def test_length_errors_are_recognised_but_a_rejected_budget_is_not(): + assert truncated_error(length_error()) + assert truncated_error(RuntimeError("Could not parse response content as the length limit was reached")) + assert not truncated_error(None) + assert not truncated_error(RuntimeError("网关超时")) + assert not truncated_error(Rejected(TOO_LARGE)), "嫌额度太大是请求被拒,模型一个字都没写,不是截断" + + +@pytest.mark.parametrize("message", [ + "Invalid max_tokens value, the valid range of max_tokens is [1, 8192]", + TOO_LARGE, + "max_completion_tokens is too large: 16384. This model supports at most 8192 completion tokens", + "This model's maximum context length is 32768 tokens. However, you requested 40369 tokens " + "(7601 in the messages, 32768 in the completion).", +]) +def test_a_rejected_budget_is_recognised(message): + assert budget_rejected(Rejected(message)) + assert budget_rejected(Rejected(message, status_code=422)) + + +def test_other_rejections_are_not_a_budget_problem(): + assert not budget_rejected(Rejected("model 'no-such-model' does not exist")) + assert not budget_rejected(Rejected(TOO_LARGE, status_code=500)), "服务方自己出错不是嫌额度大" + assert not budget_rejected(RuntimeError(TOO_LARGE)), "没有状态码的不认" + assert "max_tokens" in BUDGET_WORDS + + +def test_verify_and_judge_also_recognise_tgi_and_a_local_sdk_refusal(): + """核查、证据裁判的额度从 4096 提到了 16384,才撞得上这两种:自建服务说的是 max_new_tokens;Bearer 鉴权的 Anthropic + 接入(客户端没设超时)调 Opus 4、4.1 这一代时,SDK 嫌 16384 太大、请求没发出去就抛 ValueError。""" + assert budget_refused(Rejected(TGI_TOO_LARGE, status_code=422)) + assert budget_refused(ValueError(SDK_REFUSED)) + assert budget_refused(Rejected(TOO_LARGE)), "原来认的照样认" + assert not budget_refused(ValueError("别的本地错误")) + assert not budget_refused(RuntimeError(SDK_REFUSED)), "只认 SDK 抛的 ValueError" + assert not budget_refused(Rejected("model 'no-such-model' does not exist")) + assert not budget_refused(Rejected(TGI_TOO_LARGE, status_code=500)) + + +def test_the_copilot_wording_list_stays_as_it_was(): + """助手改图按 budget_rejected 认(流式调用,SDK 不在本地拦):认法不变,data/recipe_ai.py 的测试拿 BUDGET_WORDS 比对。""" + assert BUDGET_WORDS == ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") + assert not budget_rejected(Rejected(TGI_TOO_LARGE, status_code=422)) + assert not budget_rejected(ValueError(SDK_REFUSED)) + + +def test_the_tokens_spent_by_a_cut_off_call_come_from_the_error(): + assert error_usage(length_error(prompt=9000, completion=16384)) == (9000, 16384) + assert error_usage(RuntimeError("网关超时")) == (0, 0) and error_usage(None) == (0, 0) + + +def test_the_words_name_who_was_cut_off_and_the_limit(): + assert cut_words("核查模型", 16384) == "核查模型的输出达到上限(16384 token)被截断" + assert cut_words("裁判模型", None) == "裁判模型的输出达到上限被截断", "额度不知道就不写数" + + +# -------------------------------------------------------------------------- +# 退回小额度 +# -------------------------------------------------------------------------- + + +async def test_a_rejected_budget_is_retried_once_with_the_small_one_and_stays_small(): + big, small = Model("big", fail=Rejected(TOO_LARGE)), Model("small") + model = BudgetFallback(big, small) + assert (await model.ainvoke(["问"])).content == "small" + assert (await model.ainvoke(["再问"])).content == "small", "之后的调用都用小的,不再先撞一次" + assert len(big.calls) == 1 and len(small.calls) == 2 + assert model.model is small + + +async def test_other_errors_are_not_retried(): + big, small = Model("big", fail=Rejected("model 'no-such-model' does not exist")), Model("small") + with pytest.raises(Rejected): + await BudgetFallback(big, small).ainvoke(["问"]) + assert small.calls == [] + + +async def test_the_small_model_is_only_built_when_needed(): + built: list[str] = [] + small = Model("small") + + async def build(): + built.append("small") + return small + + fine = BudgetFallback(Model("big"), build=build, limits=(16384, 4096)) + assert (await fine.ainvoke(["问"])).content == "big" + assert built == [] and fine.max_tokens == 16384, "平常一个小额度的模型都不多建" + + rejecting = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=build, limits=(16384, 4096)) + assert (await rejecting.ainvoke(["问"])).content == "small" + assert (await rejecting.ainvoke(["再问"])).content == "small" + assert built == ["small"], "退回之后建一次,之后沿用" + assert rejecting.max_tokens == 4096 + + +async def test_without_a_small_model_the_rejection_is_raised(): + with pytest.raises(Rejected): + await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE))).ainvoke(["问"]) + same = Model("same", fail=Rejected(TOO_LARGE)) + with pytest.raises(Rejected): + await BudgetFallback(same, same).ainvoke(["问"]) + assert len(same.calls) == 1, "大小是同一个模型:重来也是同样的额度,不白撞第二次" + + +async def test_bound_and_structured_views_share_the_switch(): + """核查的工具循环里退回过小额度,收尾那次直接用小的;包出来的视图各自转给对应的小模型。""" + big, small = Model("big", fail=Rejected(TOO_LARGE)), Model("small") + model = BudgetFallback(big, small, limits=(16384, 4096)) + tools = model.bind_tools(["db_query"], parallel_tool_calls=False) + assert big.bound == [("tools", ["db_query"], {"parallel_tool_calls": False})] + assert small.bound == [("tools", ["db_query"], {"parallel_tool_calls": False})] + assert (await tools.ainvoke(["查"])).content == "small" + structured = model.with_structured_output({"type": "object"}, include_raw=True) + assert (await structured.ainvoke(["交结果"])).content == "small" + assert len(big.calls) == 1, "收尾那次不再先按大额度撞一次" + assert structured.max_tokens == 4096 and model.max_tokens == 4096 + + +async def test_a_sibling_shares_the_switch_and_builds_its_own_small_model(): + """收尾用的是另一份配置(关掉思考)建的模型:和工具循环的模型共用开关,要退回时按自己的配置建小的。""" + built: list[str] = [] + loop_small, settle_small = Model("loop-small"), Model("settle-small") + + async def build_loop(): + built.append("loop") + return loop_small + + async def build_settle(): + built.append("settle") + return settle_small + + loop = BudgetFallback(Model("loop-big", fail=Rejected(TOO_LARGE)), build=build_loop, limits=(16384, 4096)) + assert (await loop.ainvoke(["查"])).content == "loop-small" + settle_big = Model("settle-big") + settle = BudgetFallback(settle_big, build=build_settle, limits=(16384, 4096), shared_with=loop) + assert (await settle.with_structured_output({"type": "object"}).ainvoke(["交结果"])).content == "settle-small" + assert settle_big.calls == [] and built == ["loop", "settle"] + + +async def test_streaming_falls_back_only_before_anything_came_out(): + big = Model("big", fail=Rejected(TOO_LARGE), stream=[]) + small = Model("small", stream=[AIMessageChunk(content="a"), AIMessageChunk(content="b")]) + model = BudgetFallback(big, small) + assert [c.content async for c in model.astream(["问"])] == ["a", "b"] + assert len(big.calls) == 1 and len(small.calls) == 1 + + started = Model("big", fail=Rejected(TOO_LARGE), stream=[AIMessageChunk(content="x"), AIMessageChunk(content="y")], + boom=1) + unused = Model("small") + got: list[str] = [] + with pytest.raises(Rejected): + async for chunk in BudgetFallback(started, unused).astream(["问"]): + got.append(chunk.content) + assert got == ["x"] and unused.calls == [], "已经吐出东西了:重来会重复输出,原样抛出" + + +async def test_without_limits_max_tokens_reads_the_current_model(): + big = Model("big", fail=Rejected(TOO_LARGE), max_tokens=32768) + small = Model("small", max_tokens=8192) + model = BudgetFallback(big, small) + assert model.max_tokens == 32768 + await model.ainvoke(["问"]) + assert model.max_tokens == 8192 + + +async def test_a_wider_check_can_be_given(): + """核查、证据裁判按 budget_refused 认:服务方说的是 max_new_tokens 也退回小额度。缺省(助手改图)不认。""" + tgi = Rejected(TGI_TOO_LARGE, status_code=422) + model = BudgetFallback(Model("big", fail=tgi), Model("small"), rejected=budget_refused) + assert (await model.ainvoke(["问"])).content == "small" + with pytest.raises(Rejected): + await BudgetFallback(Model("big", fail=tgi), Model("small")).ainvoke(["问"]) + local = BudgetFallback(Model("big", fail=ValueError(SDK_REFUSED)), Model("small"), rejected=budget_refused) + structured = local.with_structured_output({"type": "object"}) + assert (await structured.ainvoke(["交结果"])).content == "small", "包出来的视图照样按这个认" + + +async def test_a_rejection_is_remembered_for_later_calls_to_the_same_model(): + """双盲复算的几路盲算、反方审稿的每一句、每次质询、每次点击裁判,各是一组新建的调用:同一接入、模型、额度被嫌大 + 过一次,之后新建的直接用小的,不再先撞一次。别的模型、别的额度照旧先按大的试。""" + built: list[str] = [] + + def builder(name: str): + async def build(): + built.append(name) + return Model(name) + return build + + key = ("gw", "reasoner-x", 16384) + first = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=builder("small-1"), limits=(16384, 4096), + key=key) + assert first.max_tokens == 16384 + assert (await first.ainvoke(["问"])).content == "small-1" + later_big = Model("big", fail=Rejected(TOO_LARGE)) + later = BudgetFallback(later_big, build=builder("small-2"), limits=(16384, 4096), key=key) + assert later.max_tokens == 4096, "还没调用就知道要用小的:截断时报的额度对得上" + assert (await later.bind_tools(["db_query"]).ainvoke(["再问"])).content == "small-2" + assert later_big.calls == [], "不再先按大额度撞一次" + other = Model("other-big") + assert (await BudgetFallback(other, build=builder("x"), key=("gw", "other", 16384)).ainvoke(["问"])).content \ + == "other-big" + bigger = Model("bigger") + assert (await BudgetFallback(bigger, build=builder("y"), key=("gw", "reasoner-x", 32768)).ainvoke(["问"])).content \ + == "bigger" + assert built == ["small-1", "small-2"] + + +async def test_without_a_key_nothing_is_remembered(): + """助手改图不给 key:每次改图照旧先按大额度试(行为和挪过来之前一样)。""" + await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), Model("small")).ainvoke(["问"]) + assert budget._REJECTED == set() + fresh = Model("big") + assert (await BudgetFallback(fresh, Model("small")).ainvoke(["问"])).content == "big" + + +async def test_a_streaming_rejection_is_remembered_too(): + key = ("gw", "streamer", 16384) + big = Model("big", fail=Rejected(TOO_LARGE), stream=[]) + model = BudgetFallback(big, Model("small", stream=[AIMessageChunk(content="a")]), key=key) + assert [c.content async for c in model.astream(["问"])] == ["a"] + again = Model("big", stream=[AIMessageChunk(content="b")]) + assert [c.content async for c in BudgetFallback(again, Model("small", stream=[AIMessageChunk(content="c")]), + key=key).astream(["问"])] == ["c"] + assert again.calls == [] diff --git a/backend/tests/test_verify_truncation.py b/backend/tests/test_verify_truncation.py new file mode 100644 index 00000000..733a0066 --- /dev/null +++ b/backend/tests/test_verify_truncation.py @@ -0,0 +1,680 @@ +"""核查 agent(engine/verify_agent.py)的输出额度:截断照实说,服务方嫌额度大时退回小额度。 + +核查(双盲复算、反方审稿、审批质询共用)原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到 +thinking=off,思考就能把 4096 用光:这次调用 finish_reason=length,没有工具调用也没有 JSON。于是工具循环把它当成 +「模型说完了」去收尾,双盲复算记成「盲算没有查到这个数」;收尾被截断时记成「核查模型没有按结构交结果」;质询回答失败 +——都没说出真正的原因。这里守几件事: + +- 额度:默认 16384(judge.MAX_TOKENS,核查和裁判共用);服务方嫌大时退回 4096 再试一次,之后工具循环、收尾都用小的, + 只在还没有任何输出时重来;别的 400 不重试。被嫌大过的接入、模型记在进程里,之后的核查直接用小的;自建服务说的 + max_new_tokens、anthropic SDK 在本地就拒的额度也认 +- 被截断的回复一律不采信:工具循环里截断的回复带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input 是 + 空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾(同时长到点,算半路被掐断);收尾的强制工具调用被 + 截断时,解析出来的空对象、半截对象都不算交了结果,也不再退回普通调用多花一次额度。截断那次的用量照记 +- 时间线上记一条 verify_truncated(节点里调用时),llm.end 标出这次被截断 +- 原因经现有的渠道交出去:双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错。开头只说「未能给出结果」, + 建议按模型从哪来的说:节点里节点上指定的模型优先,两处都说;质询只看设置。没有截断时原有说法不变,触顶原因的先后 + 也不变 + +模型是剧本,另有真实的 ChatOpenAI、ChatAnthropic(只有 HTTP 层是假的);数据源是现造的 SQLite 文件(借 +test_verify_agent 的夹具),表名、数字都是虚构的。 +""" +from __future__ import annotations + +import json +import uuid + +import pytest +from langchain_core.messages import AIMessage, BaseMessage +from openai import LengthFinishReasonError +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion + +from app.core.events import EventType +from app.db.base import SessionLocal +from app.db.models import Provider, Setting +from app.engine import judge as judging +from app.engine import review as rv +from app.engine import spend, verify_agent +from app.engine.inquiry import INQUIRY_SCHEMA, _error_words +from app.engine.nodes.challenge import CHALLENGE_SCHEMA, _claim_of +from app.engine.nodes.dual_check import _slim +from app.providers import budget +from app.providers.factory import build_chat_model +from tests.test_verify_agent import VALID_SUM, Script, clean_spend, priced, q, run, shop # noqa: F401 - 夹具按名字引入 + +try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上 + from langchain_openai._compat import httpx +except ImportError: # pragma: no cover - 老版本的 langchain_openai + import httpx + +CUT_16K = "核查模型的输出达到上限(16384 token)被截断" +#: 节点里(双盲复算、反方审稿):节点上指定的模型优先于设置,两处都说 +ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型" +#: 不在节点里(审批质询):只看设置 +ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型" +FAILED = f"{CUT_16K},未能给出结果。{ADVICE}" +FAILED_SETTINGS = f"{CUT_16K},未能给出结果。{ADVICE_SETTINGS}" +FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", + "rationale": "未作废订单的金额合计是 [[v:Q1.r0.total]]。"} +#: 反方审稿那种结构写到一半:结论写完了,理由写到一半 +HALF_CHALLENGE = {"verdict": "stands", "rationale": "合计是 [[v:Q1.r0.total]],与报告"} + + +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() + + +def cut_reply(output: int = 16384) -> AIMessage: + """推理模型把额度全用在思考上:正文为空、没有工具调用,finish_reason=length(DeepSeek 经 LiteLLM 的样子)。""" + return AIMessage(content="", additional_kwargs={"reasoning_content": "先想想该查哪张表……"}, + response_metadata={"finish_reason": "length"}, + usage_metadata={"input_tokens": 900, "output_tokens": output, "total_tokens": 900 + output}) + + +def cut_call(name: str, args: dict) -> AIMessage: + """工具调用写到一半到了上限(ChatAnthropic 的样子):tool_use 截在半路时 input 是空的,有的网关给半截对象; + stop_reason=max_tokens。""" + block = {"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": args} + return AIMessage(content=[{"type": "thinking", "thinking": "想了很久……", "signature": "s"}, block], + tool_calls=[{"name": block["name"], "args": args, "id": block["id"]}], + response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 900, "output_tokens": 16384, "total_tokens": 17284}) + + +def anthropic_cut(args: dict | None = None) -> dict: + """收尾的强制工具调用写到一半到了上限(结构化输出走 function_calling,ChatAnthropic 的缺省):原始回复 + stop_reason=max_tokens。args 为 None 是截在思考里、一个工具调用都没写出来;给了就是截在 tool_use 里——input 是 + 空的(Anthropic 非流式的样子)或半截对象(有的网关),解析出来的就是它。""" + content = [] if args is None else [{"type": "tool_use", "id": "toolu_1", "name": "verify_result", "input": args}] + calls = [] if args is None else [{"name": "verify_result", "args": args, "id": "toolu_1"}] + raw = AIMessage(content=content, tool_calls=calls, response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 700, "output_tokens": 16384, "total_tokens": 17084}) + return {"raw": raw, "parsed": args, "parsing_error": None} + + +def length_error(prompt: int = 800, completion: int = 16384) -> LengthFinishReasonError: + """openai 的结构化输出(json_schema)在 finish_reason=length 时抛的那个错。""" + usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) + return LengthFinishReasonError(completion=ChatCompletion.model_construct( + id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage)) + + +def query_step(name: str, sql: str = VALID_SUM, call_id: str = "c1") -> AIMessage: + return AIMessage(content="查一下。", tool_calls=[{"name": n, "args": a, "id": i} for n, a, i in q(name, sql, call_id)]) + + +class BudgetRejected(Exception): + """服务方嫌输出额度太大时的 400(DeepSeek chat 系只收 8K)。""" + + def __init__(self, message: str, status_code: int = 400) -> None: + super().__init__(message) + self.status_code = status_code + + +class Reasoner: + """核查模型的剧本。steps:工具循环里每一轮的回复,用完了回「查完了。」;structured / plain:收尾时结构化调用、 + 普通调用的回复(AIMessage、结构化结果,或者一个异常就抛它)。calls 按顺序记下每次调用是哪一种。""" + + def __init__(self, steps=(), *, structured=None, plain=None, unstructured: bool = False) -> None: + self.steps = list(steps) + self.structured = structured + self.plain = plain + self.unstructured = unstructured + self.calls: list[str] = [] + + def bind_tools(self, tools, **kwargs): + return _View(self, "step") + + def with_structured_output(self, schema, **kwargs): + if self.unstructured: + raise NotImplementedError("这个模型不支持结构化输出") + return _View(self, "final") + + async def ainvoke(self, messages, *args, **kwargs): + return self.answer("plain") + + def answer(self, kind: str): + self.calls.append(kind) + if kind == "step": + reply = self.steps.pop(0) if self.steps else AIMessage(content="查完了。") + else: + reply = self.structured if kind == "final" else self.plain + if isinstance(reply, BaseException): + raise reply + return reply + + +class _View: + def __init__(self, model: Reasoner, kind: str) -> None: + self.model, self.kind = model, kind + + async def ainvoke(self, messages, *args, **kwargs): + return self.model.answer(self.kind) + + +class Rejecting: + """一开口就被拒的模型:工具循环、收尾都一样。""" + + def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens", + status_code: int = 400): + self.message, self.status_code = message, status_code + self.calls = 0 + + def bind_tools(self, tools, **kwargs): + return self + + def with_structured_output(self, schema, **kwargs): + return self + + async def ainvoke(self, messages, *args, **kwargs): + self.calls += 1 + raise BudgetRejected(self.message, self.status_code) + + +class Models: + """按额度给模型:rejects_above 给了,额度比它大的一律交 Rejecting。specs 记下每次建模型的配置。""" + + def __init__(self, monkeypatch, model, *, rejects_above: int | None = None, rejecting: Rejecting | None = None): + self.specs = [] + self.rejecting = rejecting or Rejecting() + + async def fake(session, spec): + self.specs.append(spec) + if rejects_above is not None and (spec.max_tokens or 0) > rejects_above: + return self.rejecting, "reasoner-x" + return model, "reasoner-x" + + monkeypatch.setattr(verify_agent, "get_chat_model", fake) + + +def collector(): + events: list[tuple] = [] + return events, (lambda kind, **data: events.append((kind, data))) + + +def truncated_logs(events) -> list[dict]: + return [d for k, d in events if k == EventType.LOG and d.get("code") == "verify_truncated"] + + +# -------------------------------------------------------------------------- +# 工具循环里被截断 +# -------------------------------------------------------------------------- + + +async def test_thinking_that_eats_the_budget_in_the_loop_is_reported_as_truncation(monkeypatch, shop): + name, _ = shop + model = Reasoner([cut_reply()], structured={"value": None, "verdict": "无法判断", "rationale": "没有查询结果"}) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert out.result is None and out.queries == [] + assert out.error == FAILED + assert model.calls == ["step"], "一条查询都没有:收尾只能交「查不到」,不再为它多花一次额度" + assert out.answered is False, "被截断不是模型自己停下来的" + assert out.usage["output_tokens"] == 16384, "截断那次的额度是真花出去了" + [log] = truncated_logs(events) + assert log["level"] == "warn" and log["max_tokens"] == 16384 and "delivered" not in log + assert log["purpose"] == "verify" and log["message"] == out.error + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["truncated"] is True and end["max_tokens"] == 16384 and CUT_16K in end["error"] + + +@pytest.mark.parametrize("args", [{}, {"sql": "SELECT SUM(amount) AS total FROM orders WHERE is_del"}], + ids=["empty-input", "half-sql"]) +async def test_a_tool_call_cut_off_halfway_is_not_run(monkeypatch, shop, args): + """Anthropic 系:工具调用写到一半到了上限。写到一半的调用不能执行——空参数只会报「缺少必填参数」,半截的 SQL + 查出来的数是错的——也不按它收尾,原因照截断说。""" + name, _ = shop + model = Reasoner([cut_call(name, args)], structured=FOUND) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step"], "不执行、不收尾,也不再问一轮" + assert out.result is None and out.queries == [] and out.error == FAILED + assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_END, EventType.TOOL_ERROR)] + [log] = truncated_logs(events) + assert log["purpose"] == "verify" and log["max_tokens"] == 16384 + + +async def test_a_cut_after_a_query_is_reported_without_settling(monkeypatch, shop): + """查过一次、下一轮被截断:截断前查到的不一定是要的那个数(模型可能正想改查询),按它收尾交出去的结果说不清完整 + 不完整,会照常参与双盲复算的比对、反方审稿的处置。同时长到点一样算半路被掐断:不收尾,原因照截断说,查过的照记。""" + name, _ = shop + model = Reasoner([query_step(name), cut_reply()], structured=FOUND) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step", "step"], "不再为收尾多花一次额度" + assert out.result is None and out.error == FAILED and out.answered is False + assert [q["alias"] for q in out.queries] == ["Q1"], "查过的照记" + [log] = truncated_logs(events) + assert log["purpose"] == "verify" and log["max_tokens"] == 16384 + assert not [n for n in out.notes if "截断" in n], "没有「按截断前查到的结果收尾」这一说" + + +# -------------------------------------------------------------------------- +# 收尾被截断 +# -------------------------------------------------------------------------- + + +async def test_a_cut_off_settle_is_reported_and_not_retried_as_plain(monkeypatch, shop): + """Anthropic 系:强制工具调用写出来之前就到了上限。退回普通调用也多半被推理占满,不再白花一次额度。""" + name, _ = shop + model = Reasoner([query_step(name)], structured=anthropic_cut(), plain=AIMessage(content=json.dumps(FOUND))) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step", "step", "final"], "没有再退回普通调用" + assert out.result is None and out.error == FAILED + [log] = truncated_logs(events) + assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384 + end = [d for k, d in events if k == EventType.LLM_END][-1] + assert end["purpose"] == "verify_result" and end["truncated"] is True and CUT_16K in end["error"] + + +@pytest.mark.parametrize(("schema", "args"), [ + (None, {}), + (None, {"value": {"alias": "Q1"}}), + (CHALLENGE_SCHEMA, HALF_CHALLENGE), + (INQUIRY_SCHEMA, {"rationale": "合计是 [[v:Q1.r0.total]],比报告里写的"}), +], ids=["empty-input", "half-cell", "half-challenge", "half-inquiry"]) +async def test_a_settle_cut_inside_the_forced_tool_call_is_not_taken(monkeypatch, shop, schema, args): + """强制工具调用写到一半到了上限:解析出来的是空对象(Anthropic)或半截对象(有的网关)。空对象以前记成「没有按结构 + 交结果」,半截的格子引用记成「引用不可用」,反方审稿的半截对象(结论写完了、理由写到一半)照单全收成「经受住」, + 质询把半句话当成回答——截断的回复一律不采信,原因照截断说,也不退回普通调用。""" + name, _ = shop + model = Reasoner([query_step(name)], structured=anthropic_cut(args), plain=AIMessage(content=json.dumps(FOUND))) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {})) + assert model.calls == ["step", "step", "final"] + assert out.result is None and out.error == FAILED and out.cells == [] + [log] = truncated_logs(events) + assert log["purpose"] == "verify_result" + + +async def test_an_openai_length_error_in_the_settle_is_truncation_not_a_failed_call(monkeypatch, shop): + """openai 的结构化输出解析时抛 LengthFinishReasonError:不是「调用失败,请稍后重试」,是额度被推理占满。""" + name, _ = shop + model = Reasoner([query_step(name)], structured=length_error(completion=16384)) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step", "step", "final"] + assert out.error == FAILED + assert out.usage["output_tokens"] == 16384 and out.usage["input_tokens"] >= 800, "异常里带着的用量照记" + end = [d for k, d in events if k == EventType.LLM_END][-1] + assert "请稍后重试" not in end["error"] and CUT_16K in end["error"] and end["output_tokens"] == 16384 + + +async def test_a_plain_settle_cut_off_is_reported(monkeypatch, shop): + """不支持结构化输出的模型:收尾直接走普通调用,回复只有思考、finish_reason=length。""" + name, _ = shop + model = Reasoner([query_step(name)], plain=cut_reply(), unstructured=True) + Models(monkeypatch, model) + out = await run(name) + assert model.calls == ["step", "step", "plain"] + assert out.result is None and out.error == FAILED_SETTINGS + + +async def test_a_plain_settle_cut_after_the_json_is_not_taken_either(monkeypatch, shop): + """普通调用的正文里有一段 JSON、回复却是截断的:JSON 后面还在写(多半是理由的补充),说不清写完了没有,同样不采信。""" + name, _ = shop + cut = AIMessage(content=json.dumps(FOUND, ensure_ascii=False) + "\n补充说明:合计里还应该", + response_metadata={"finish_reason": "length"}) + model = Reasoner([query_step(name)], plain=cut, unstructured=True) + Models(monkeypatch, model) + out = await run(name) + assert out.result is None and out.error == FAILED_SETTINGS + + +async def test_without_truncation_the_old_wording_stays(monkeypatch, shop): + name, _ = shop + model = Reasoner([query_step(name)], structured="不是 JSON", plain=AIMessage(content="还是不是 JSON")) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step", "step", "final", "plain"], "格式不对照旧退回普通调用补救一次" + assert out.error == "核查模型没有按结构交结果" + assert truncated_logs(events) == [] + assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END) + + +async def test_an_empty_object_without_truncation_keeps_the_shape_wording(monkeypatch, shop): + """没被截断、交回的是空对象:照旧是「没有按结构交结果」,不说成截断。""" + name, _ = shop + whole = {"raw": AIMessage(content="", response_metadata={"stop_reason": "tool_use"}), "parsed": {}} + Models(monkeypatch, Reasoner([query_step(name)], structured=whole)) + out = await run(name) + assert out.error == "核查模型没有按结构交结果(交回的内容里没有要求的字段:value、verdict)" + + +async def test_the_first_limit_hit_is_still_the_one_reported(monkeypatch, shop, priced): + """循环先因时长到点停下(一分钱还没留),收尾又被每日金额上限拦下:触顶原因照旧报先撞上的时长。双盲复算按它写 + 无法复算的原因,反方审稿按「每日上限」决定还派不派后面的句子——截断的改动不能把这个先后换掉。""" + name, _ = shop + async with SessionLocal() as session: + session.add(Setting(key=spend.SPEND_KEY, value={"date": spend._today(), "usd": 1.995, "calls": 9})) + await session.commit() + Script(monkeypatch, [q(name, VALID_SUM, "c1")], FOUND, model_id="priced-model") + out = await run(name, daily_max_usd=2.0, timeout_s=0) + assert out.limit == "timeout_s" + assert out.error == "已达全局每日核查金额上限 $2.00,没能交结果" + + +# -------------------------------------------------------------------------- +# 额度与退回 +# -------------------------------------------------------------------------- + + +async def test_the_default_budget_leaves_room_for_reasoning(monkeypatch, shop): + name, _ = shop + models = Models(monkeypatch, Reasoner([query_step(name)], structured=FOUND)) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [s.max_tokens for s in models.specs] == [16384, 16384], "工具循环、收尾都按 16384,不多建模型" + assert models.specs[1].thinking == "off" + + +async def test_a_rejected_budget_falls_back_to_4096_for_the_loop_and_the_settle(monkeypatch, shop): + name, _ = shop + model = Reasoner([query_step(name)], structured=FOUND) + models = Models(monkeypatch, model, rejects_above=4096) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096] + assert [s.thinking for s in models.specs][2:] == ["off", "off"], "收尾的小额度模型同样关掉思考" + assert models.rejecting.calls == 1, "退回过一次之后收尾直接用小的,不再先撞一次" + assert model.calls == ["step", "step", "final"] + + +async def test_a_rejected_budget_is_remembered_for_the_next_runs(monkeypatch, shop): + """双盲复算的几路盲算、反方审稿的每一句、每次质询各是一次核查:同一接入、模型被嫌大过一次,之后的核查直接用小的, + 不再每次先撞一次。""" + name, _ = shop + model = Reasoner([query_step(name), AIMessage(content="查完了。"), query_step(name, call_id="c2")], + structured=FOUND) + models = Models(monkeypatch, model, rejects_above=4096) + first, second = await run(name), await run(name) + assert first.error is None and second.error is None and second.result["value"] == 230 + assert models.rejecting.calls == 1, "第二次核查没有再按 16384 撞一次" + + +async def test_a_cut_after_the_fallback_names_the_small_budget(monkeypatch, shop): + name, _ = shop + Models(monkeypatch, Reasoner([cut_reply(output=4096)]), rejects_above=4096) + out = await run(name) + assert out.error.startswith("核查模型的输出达到上限(4096 token)被截断") + + +async def test_other_rejections_are_not_retried_with_the_small_budget(monkeypatch, shop): + name, _ = shop + models = Models(monkeypatch, Reasoner(), rejects_above=4096, + rejecting=Rejecting("model 'reasoner-x' does not exist")) + out = await run(name) + assert out.result is None and "reasoner-x" in out.error + assert [s.max_tokens for s in models.specs] == [16384], "不是额度的问题:不拿小额度再撞一次" + + +async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, shop): + """text-generation-inference 一类自建服务:输入加额度超过 max_total_tokens 时回 422,说的是 max_new_tokens。""" + name, _ = shop + tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 1200 `inputs` tokens and 16384 `max_new_tokens`", status_code=422) + model = Reasoner([query_step(name)], structured=FOUND) + models = Models(monkeypatch, model, rejects_above=4096, rejecting=tgi) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096] and tgi.calls == 1 + + +async def test_the_verify_spec_asks_for_the_larger_budget(): + """核查模型的配置(api/settings.py 的 verify_model_spec 沿用证据裁判的额度):16384,思考照旧关掉。""" + assert judging.MAX_TOKENS == 16384 and judging.FALLBACK_MAX_TOKENS == 4096 + + +# -------------------------------------------------------------------------- +# 原因经现有的渠道交出去 +# -------------------------------------------------------------------------- + + +async def test_the_reason_reaches_dual_check_and_challenge(monkeypatch, shop): + name, _ = shop + Models(monkeypatch, Reasoner([cut_reply()])) + out = await run(name, node_id="dc") + # 双盲复算:无法复算的原因 + slim = _slim(out) + assert slim["value"] is None and slim["reason"] == FAILED + # 反方审稿:这句记为无法判断,理由写「反查未完成」,没能反查的原因就是截断 + cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="", + aliases=("Q1",)) + claim, why = _claim_of(cand, out) + assert claim["verdict"] == "inconclusive" and claim["rationale"] == f"反查未完成:{FAILED}" and why == FAILED + + +async def test_an_inquiry_names_the_settings_and_not_a_check(monkeypatch, shop): + """审批质询不在节点里(核查模型只看设置):建议只说设置;开头只说「未能给出结果」——提问人问的是问题,不是在核查。""" + name, _ = shop + Models(monkeypatch, Reasoner([cut_reply()])) + out = await run(name, output_schema=INQUIRY_SCHEMA) + words = _error_words(out, timeout_s=60, daily_max_usd=1.0) + assert words == FAILED_SETTINGS + assert "核查未能完成" not in words and "节点" not in words + + +def test_the_review_layer_knows_the_truncation_log(): + """跑完之后的复核层(engine/review.py)认得 verify_truncated:同 verify_limit,是缺口不是断裂,kind 是它自己, + 不落成笼统的 warn;不触发「重跑一次大概率能好」——同样的模型再跑一遍还是撞同一堵墙。""" + assert rv.WARN_CODES["verify_truncated"] == ("verify_truncated", rv.DEGRADED) + event = {"type": str(EventType.LOG), "node_id": "dc", + "data": {"level": "warn", "code": "verify_truncated", "message": FAILED}} + [signal] = rv.scan([event], {"结论": "本周订单 120 单。"}) + assert (signal.kind, signal.severity) == ("verify_truncated", rv.DEGRADED) and not rv.should_retry([signal]) + + +def test_the_reply_shapes_used_here_are_messages(): + assert isinstance(cut_reply(), BaseMessage) and isinstance(anthropic_cut()["raw"], BaseMessage) + assert isinstance(cut_call("x", {}), BaseMessage) + + +# -------------------------------------------------------------------------- +# 真实的 ChatOpenAI 和 openai SDK:只有 HTTP 层是假的(经 OpenAI 兼容网关接入的推理模型就是这一路) +# -------------------------------------------------------------------------- + +GATEWAY_MODEL = "reasoner-v4" + + +class Gateway: + """OpenAI 兼容网关:按顺序回 chat.completion(最后一个重复用)。cap 给了,请求的额度超过它就回 400(DeepSeek chat 系 + 只收 8K 那样)。bodies 记下每个请求体。""" + + def __init__(self, replies: list, *, cap: int | None = None) -> None: + self.replies, self.cap = list(replies), cap + self.bodies: list[dict] = [] + + async def handle(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content or b"{}") + self.bodies.append(body) + budget = body.get("max_completion_tokens") or body.get("max_tokens") + if self.cap and budget and budget > self.cap: + message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens" + return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}}, + request=request) + message, finish, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0] + return httpx.Response(200, request=request, json={ + "id": "c-1", "object": "chat.completion", "created": 1, "model": GATEWAY_MODEL, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": {"prompt_tokens": 900, "completion_tokens": spent, "total_tokens": 900 + spent}}) + + +def gateway(monkeypatch, replies: list, *, cap: int | None = None) -> Gateway: + fake = Gateway(replies, cap=cap) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle) + provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1", + api_key=None, models=[{"id": GATEWAY_MODEL}], default_model=GATEWAY_MODEL, enabled=True, extra={}) + + async def build(session, spec): + return build_chat_model(provider, spec), GATEWAY_MODEL + + monkeypatch.setattr(verify_agent, "get_chat_model", build) + return fake + + +THINKING_ONLY = ({"role": "assistant", "content": "", "reasoning_content": "先看看有哪些表……"}, "length", 16384) + + +def tool_call(name: str) -> tuple: + return ({"role": "assistant", "content": None, "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": f"db_query__{name}", "arguments": json.dumps({"sql": VALID_SUM})}}]}, "tool_calls", 120) + + +async def test_a_real_client_cut_off_in_the_loop(monkeypatch, shop): + name, _ = shop + fake = gateway(monkeypatch, [THINKING_ONLY]) + out = await run(name) + assert out.error == FAILED_SETTINGS + assert len(fake.bodies) == 1 and fake.bodies[0]["max_completion_tokens"] == 16384 + assert "tools" in fake.bodies[0], "工具循环那次带着只读的查询工具" + + +async def test_a_real_client_cut_off_with_a_half_written_tool_call(monkeypatch, shop): + """OpenAI 系:工具调用的参数写到一半到了上限(finish_reason=length,arguments 是半截 JSON)。不执行、不收尾。""" + name, _ = shop + half = ({"role": "assistant", "content": None, "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": f"db_query__{name}", "arguments": '{"sql": "SELECT SUM(amount) AS total FROM ord'}}]}, + "length", 16384) + fake = gateway(monkeypatch, [half]) + out = await run(name) + assert out.error == FAILED_SETTINGS and out.queries == [] and len(fake.bodies) == 1 + + +async def test_a_real_client_cut_off_in_the_json_schema_settle(monkeypatch, shop): + """收尾走 openai 的结构化输出(json_schema):finish_reason=length 时 SDK 抛 LengthFinishReasonError。""" + name, _ = shop + half = ({"role": "assistant", "content": '{"value": {"alias": "Q1", "row'}, "length", 16384) + fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), half]) + out = await run(name) + assert out.error == FAILED_SETTINGS and len(out.queries) == 1 + assert len(fake.bodies) == 3 and "response_format" in fake.bodies[2], "收尾被截断:没有再退回普通调用" + assert out.usage["output_tokens"] == 120 + 20 + 16384, "异常里带着的那次用量照记" + + +async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, shop): + name, _ = shop + done = ({"role": "assistant", "content": json.dumps(FOUND, ensure_ascii=False)}, "stop", 60) + fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), done], + cap=8192) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + budgets = [b.get("max_completion_tokens") for b in fake.bodies] + assert budgets == [16384, 4096, 4096, 4096], "第一次被拒后退回 4096,之后工具循环和收尾都用小的,不再先撞一次" + + +# -------------------------------------------------------------------------- +# 真实的 ChatAnthropic 和 anthropic SDK:只有 HTTP 层是假的 +# -------------------------------------------------------------------------- + +CLAUDE = "claude-x" + + +class Messages: + """Anthropic 的 Messages 接口:按顺序回 message(最后一个重复用)。每个回复是 (content, stop_reason, 输出 token 数), + content 可以是按请求体算的函数(收尾那次强制调用的工具名照请求里的写)。bodies 记下每个请求体。""" + + def __init__(self, replies: list) -> None: + self.replies = list(replies) + self.bodies: list[dict] = [] + + async def handle(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content or b"{}") + self.bodies.append(body) + content, stop, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0] + return httpx.Response(200, request=request, json={ + "id": "msg_1", "type": "message", "role": "assistant", "model": body.get("model"), + "content": content(body) if callable(content) else content, "stop_reason": stop, "stop_sequence": None, + "usage": {"input_tokens": 700, "output_tokens": spent}}) + + +def messages_api(monkeypatch, replies: list, *, model: str = CLAUDE, bearer: bool = False, + module=verify_agent) -> Messages: + """接上一个 Anthropic 接入(bearer 为真时按 Authorization: Bearer 鉴权,factory 会换上没设超时的 SDK 客户端)。""" + fake = Messages(replies) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle) + provider = Provider(name=f"an-{uuid.uuid4().hex[:8]}", kind="anthropic", base_url="http://an.example.test", + api_key="test-only-key", models=[{"id": model}], default_model=model, enabled=True, + extra={"auth_style": "bearer"} if bearer else {}) + + async def build(session, spec): + # 按这个接入的模型建:调用方的 spec 写的是别的模型名(mock-fast),SDK 在本地拦不拦要看真正发出去的 id + return build_chat_model(provider, spec.model_copy(update={"model": model})), model + + monkeypatch.setattr(module, "get_chat_model", build) + return fake + + +def forced(args: dict): + """收尾那次强制调用的工具:名字照请求里的写,input 是 args。""" + return lambda body: [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": args}] + + +def query_use(name: str) -> list: + return [{"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {"sql": VALID_SUM}}] + + +DONE = ([{"type": "text", "text": "查完了。"}], "end_turn", 10) + + +async def test_a_real_anthropic_cut_in_the_middle_of_a_tool_call(monkeypatch, shop): + """思考占满了额度、tool_use 截在半路(input 是空的):以前每一轮都执行一遍空参数的查询(报「缺少必填参数 sql」), + 撞到步数上限,最后记成「没有按结构交结果」。""" + name, _ = shop + cut = ([{"type": "thinking", "thinking": "想了很久……", "signature": "s"}, + {"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {}}], "max_tokens", 16384) + fake = messages_api(monkeypatch, [cut]) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", max_queries=2) + assert out.error == FAILED and out.queries == [] and out.limit is None + assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384 + assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_ERROR)] + [log] = truncated_logs(events) + assert log["purpose"] == "verify" + signals = rv.scan([{"type": str(k), "node_id": "dc", "data": d} for k, d in events], {"结论": "本周订单 120 单。"}) + assert [s.kind for s in signals] == ["verify_truncated"], "不是工具出错、也不是触顶" + + +@pytest.mark.parametrize(("schema", "args"), [(None, {}), (CHALLENGE_SCHEMA, HALF_CHALLENGE)], + ids=["empty-input", "half-challenge"]) +async def test_a_real_anthropic_settle_cut_inside_the_forced_tool_call(monkeypatch, shop, schema, args): + name, _ = shop + fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE, (forced(args), "max_tokens", 16384)]) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {})) + assert out.error == FAILED and out.result is None and len(out.queries) == 1 + assert len(fake.bodies) == 3 and fake.bodies[2]["tool_choice"]["type"] == "tool", "没有再退回普通调用" + [log] = truncated_logs(events) + assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384 + if schema is CHALLENGE_SCHEMA: + cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="", + aliases=("Q1",)) + claim, why = _claim_of(cand, out) + assert claim["verdict"] == "inconclusive" and why == FAILED, "半截的「经受住」不能当真" + + +async def test_an_anthropic_client_refusing_the_budget_locally_falls_back(monkeypatch, shop): + """Bearer 鉴权的 Anthropic 接入(factory 换上的 SDK 客户端没设超时):Opus 4、4.1 这一代的 id(经网关用的 Vertex、 + Bedrock 写法也是)非流式只能给 8192,16384 的请求 anthropic SDK 在本地就拒了(ValueError,请求没发出去)。照服务方 + 嫌额度大处理,退回 4096。这条守的是 SDK 的行为:anthropic 以后不再在本地拦这个 id,第一个请求就会按 16384 发出去, + 到时改期望。""" + name, _ = shop + fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE, + (forced(FOUND), "tool_use", 60)], model="claude-opus-4-1@20250805", bearer=True) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [b["max_tokens"] for b in fake.bodies] == [4096, 4096, 4096], "16384 的请求一个都没发出去" diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 25def9a1..9ef3d6f2 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1039,6 +1039,34 @@ export const AGENT_GUARD_TEXT = { unlimitedShort: '不限', } as const +/** 「(16,384 token)」:输出额度,没给就不写 */ +const tokenCap = (maxTokens?: number) => (maxTokens ? `(${formatNumber(maxTokens)} token)` : '') + +/** + * 时间线上「核查模型的输出到了上限被截断」那一行(engine/verify_agent.py 的 verify_truncated 日志,双盲复算、反方审稿的 + * 核查 Agent 发)。推理模型的思考也算在输出额度里,额度用完时回复是空的或只写了一半:照实说是输出额度不够,不说成 + * 「没查到」「格式不对」,下一步指到换模型。截断的回复一律不采信,这次核查没有结果,节点按无法复算、未能反查记缺口 + */ +export const VERIFY_TRUNCATED_TEXT = { + title: (maxTokens?: number) => `核查模型的输出达到上限${tokenCap(maxTokens)}被截断`, + sub: '这次核查未能给出结果', + next: '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型', +} as const + +/** + * 时间线上结论句裁判的输出到了上限被截断(engine/judge.py):被截断的那次调用(llm.end 带 truncated),和 + * judge_truncated 日志那一行。不用「裁判未能执行」「检查接入后重新运行」的说法:截断时接入是好的,同一个模型重新运行 + * 多半还是截断,该换的是模型 + */ +export const JUDGE_TRUNCATED_TEXT = { + /** 被截断的那次调用:原来叫「请裁判模型判断 N 句结论」的那一行 */ + call: (maxTokens?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断`, + title: (maxTokens?: number, unjudged?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断` + + (unjudged ? `,${formatNumber(unjudged)} 句结论未裁判` : ''), + next: '未裁判的结论句记为缺口,不能完整出具。通常是推理占满了输出额度:请在报告撰写节点的「结论句裁判」中指定不带推理的裁判模型,' + + '或在「设置 → 偏好设置 → 证据裁判」中更换,再重新运行', +} as const + /** * 上传表格:数据页「表格」标签的上传弹窗、导入回执和卡片。种类的说法和后端 data/tabular.py 的 * CONVERSION_KINDS、SHAPE_KINDS 对得上;后端以后多出来的种类,界面上只显示它给的那句话 diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index f8799259..fd01b6c7 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -3,7 +3,8 @@ import type { NodeState, RunPhase } from './trace' import { formatDuration, formatNumber } from '../lib/format' import { ABSENCE_CLAIM_TEXT, CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, SCHEMA_PARTIAL_TEXT, SQL_CHECK_TEXT, TYPE_LABEL, - CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, NODE_TYPE_LABEL, + CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, JUDGE_TRUNCATED_TEXT, NODE_TYPE_LABEL, + VERIFY_TRUNCATED_TEXT, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel, } from '../lib/terms' @@ -765,17 +766,19 @@ function gateTitle(tool: string, verdict: 'allow' | 'escalate' | 'team', reason: * 双盲复算、反方审稿的日志代码(期 3 前后端共用的约定)。这几条按 code 和结构化字段解读,不看原话;后端发成 info 的 * (一致、裁决、记入数据目录、审稿完成)也上时间线,不然这两种节点就是一个跑了几十秒、什么都没说的黑盒。 * 无法复算(dual_check_unchecked)、未能反查(challenge_unreviewed)、只查了部分数据源(challenge_sources_lost)是 - * 期 4 补上的:以前认不得,只能照原话显示 + * 期 4 补上的:以前认不得,只能照原话显示。核查模型的输出到了上限被截断(verify_truncated,两种节点共用的核查 Agent 发) + * 是期 5 验收前补上的 */ const VERIFY_LOG_CODES = new Set([ 'dual_check_skipped', 'dual_check_same_model', 'dual_check_agreed', 'dual_check_disagreed', 'dual_check_resolved', 'dual_check_catalog', 'dual_check_limit', 'dual_check_unchecked', 'challenge_skipped', 'challenge_same_model', 'challenge_done', 'challenge_refuted', 'challenge_limit', - 'challenge_unreviewed', 'challenge_sources_lost', + 'challenge_unreviewed', 'challenge_sources_lost', 'verify_truncated', ]) const DC = DUAL_CHECK_TEXT const CL = CHALLENGE_TEXT.log +const VT = VERIFY_TRUNCATED_TEXT /** * 出具缺口里哪几条是双盲复算、反方审稿报上来的:出具契约给上游节点报的缺口加了「节点类型「节点名」:」的前缀 @@ -1010,6 +1013,15 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.max_tokens : undefined + const n = typeof d.unjudged === 'number' && d.unjudged > 0 ? Math.round(d.unjudged) : undefined + return { + title: JUDGE_TRUNCATED_TEXT.title(limit, n), next: JUDGE_TRUNCATED_TEXT.next, fix: 'canvas', fixField: 'judge', + } + } case 'judge_failed': { // 「结论句裁判没跑完:有 2 句结论没裁判:裁判调用失败(…)」 return { @@ -1214,6 +1226,19 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.max_tokens : undefined + return { + title: VT.title(limit), + sub: VT.sub, + next: VT.next, + fix: 'canvas', + fixField: 'model', + level: 'warn', + } + } default: return null } @@ -1912,9 +1937,11 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt step.detail = [String(d.error), step.detail].filter(Boolean).join('\n') } if (d.purpose === 'judge' && d.error) { - // 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败 + // 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败。输出到了上限被截断的 + // 照实说是截断(不是「未能执行」:接入是好的,该换的是模型,下一步见 judge_truncated 那一行) + const cap = num(d.max_tokens) step.level = 'warn' - step.title = JUDGE_FAILED_TITLE + step.title = d.truncated === true ? JUDGE_TRUNCATED_TEXT.call(cap && cap > 0 ? cap : undefined) : JUDGE_FAILED_TITLE step.detail = [String(d.error), step.detail].filter(Boolean).join('\n') } pendingLlm.delete(nodeId ?? '_') diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index 68f283cd..8e361973 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -2319,6 +2319,103 @@ await section('核查类日志与出具缺口(期 4):无法复算、未能 rows.map((r) => `${r?.title}|${r?.sub ?? ''}`).join(' ; ')) }) +await section('核查模型的输出到了上限被截断(verify_truncated):说清额度、这次核查没有结果,下一步指到换模型', async () => { + // 推理模型的思考也算在输出额度里:核查 Agent 的一次调用 finish_reason=length、没有工具调用也没有 JSON 时,以前双盲复算 + // 记成「盲算没有查到这个数」、反方审稿记成「核查模型没有按结构交结果」。现在截断的回复一律不采信,核查 Agent 另发 + // verify_truncated(max_tokens、purpose),时间线按结构化字段说人话,原话留在展开区。先核对后端源码还是这么发的 + const verifyPy = readFileSync(`${root}backend/app/engine/verify_agent.py`, 'utf8') + check('后端发截断日志的格式还是夹具里这一种', verifyPy.includes('code="verify_truncated"') && verifyPy.includes('max_tokens=limit') + && verifyPy.includes('purpose=purpose') && !verifyPy.includes('delivered=') + && verifyPy.includes('TRUNCATED_ERROR = "{cut},未能给出结果。{advice}"'), + '改了 verify_agent.py 的字段或说法就同步改 decode.ts 的 explainLog 和这里的夹具') + const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data }) + const ADVICE = '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型' + const FAILED = `核查模型的输出达到上限(16384 token)被截断,未能给出结果。${ADVICE}` + const SMALL = `核查模型的输出达到上限(4096 token)被截断,未能给出结果。${ADVICE}` + const all = flatten(mod.decodeRun([ + E(1, 'run.started', null, { total: 2 }), + E(2, 'node.started', 'dc', { node_type: 'dual_check', label: '复核销售额' }), + E(3, 'log', 'dc', { level: 'warn', code: 'verify_truncated', purpose: 'verify', max_tokens: 16384, message: FAILED }), + E(4, 'node.finished', 'dc', { duration_ms: 20 }), + E(5, 'node.started', 'ch', { node_type: 'challenge', label: '审月报' }), + // 退回小额度之后在收尾时被截断 + E(6, 'log', 'ch', { level: 'warn', code: 'verify_truncated', purpose: 'verify_result', max_tokens: 4096, message: SMALL }), + // 缺字段、字段类型不对(别的服务端版本):只给不带数的标题,不瞎填 + E(7, 'log', 'ch', { level: 'warn', code: 'verify_truncated', max_tokens: '16384', message: FAILED }), + E(8, 'node.finished', 'ch', { duration_ms: 30 }), + E(9, 'run.finished', null, { status: 'succeeded' }), + ])) + const rows = all.filter((s) => s.code === 'verify_truncated') + const nodes = all.filter((s) => s.kind === 'node') + check('每条一行,挂在自己的节点下', rows.length === 3 && nodes.find((s) => s.nodeId === 'dc')?.children?.filter((x) => x.code === 'verify_truncated').length === 1 + && nodes.find((s) => s.nodeId === 'ch')?.children?.filter((x) => x.code === 'verify_truncated').length === 2, `${rows.length} 行`) + const [failed, small, bare] = rows + check('标题说输出到了上限被截断、带额度,副标题说这次核查没有结果', failed?.title === '核查模型的输出达到上限(16,384 token)被截断' + && failed.sub === '这次核查未能给出结果', `${failed?.title} / ${failed?.sub}`) + check('额度照实写(退回小额度之后是 4,096)', small?.title === '核查模型的输出达到上限(4,096 token)被截断' && small.sub === '这次核查未能给出结果', + `${small?.title} / ${small?.sub}`) + check('……下一步指到换不带推理的模型(节点上指定的优先,设置里的也说),入口落到节点的模型,提醒色', + rows.every((s) => !!s.next?.includes('在节点上指定不带推理的模型') && s.next.includes('设置 → 偏好设置 → 核查') + && s.fix === 'canvas' && s.fixField === 'model' && s.level === 'warn'), + `${failed?.next} / ${failed?.fix} / ${failed?.fixField} / ${failed?.level}`) + check('……没有「已按截断前查到的结果收尾」这一说', !rows.some((s) => /收尾|可能不完整/.test(`${s.title}${s.sub ?? ''}`))) + check('……原话留在展开区', failed?.detail === FAILED && small?.detail === SMALL) + check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '核查模型的输出达到上限被截断' + && !/undefined|NaN/.test(`${bare?.title}${bare?.sub ?? ''}${bare?.next ?? ''}`), `${bare?.title} / ${bare?.sub}`) + check('标题、副标题里没有内部代码', !rows.some((s) => /verify_truncated|max_tokens|purpose/.test(`${s.title}${s.sub ?? ''}${s.next ?? ''}`))) + check('节点照常完成', nodes.every((s) => s.status === 'done')) +}) + +await section('结论句裁判的输出到了上限被截断(judge_truncated):调用那一行说截断,日志另起一行,下一步指到换模型', async () => { + // 以前截断的那一批只发 llm.end(标题沿用「结论句裁判未能执行」)和 judge_failed(下一步是「检查接入后重新运行」): + // 截断时接入是好的,同一个模型重新运行多半还是截断。现在调用那一行照 truncated、max_tokens 说截断,另发 judge_truncated + // (max_tokens、unjudged),judge_failed 只说别的失败。先核对后端源码还是这么发的 + const judgePy = readFileSync(`${root}backend/app/engine/judge.py`, 'utf8') + check('后端发裁判截断的格式还是夹具里这一种', judgePy.includes('code="judge_truncated", max_tokens=cut_limit') + && judgePy.includes('unjudged=outcome["unjudged"]["truncated"]') && judgePy.includes('truncated=True, max_tokens=cut_limit') + && judgePy.includes('r not in LIMITS and r != "truncated"'), + '改了 judge.py 的字段或说法就同步改 decode.ts 的 explainLog、llm.end 和这里的夹具') + const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data }) + const CUT = '裁判模型的输出达到上限(16384 token)被截断' + const TRUNCATED = `结论句裁判未完成:有 4 句结论未裁判:${CUT},通常是推理占满了输出额度,可换用不带推理的裁判模型` + const all = flatten(mod.decodeRun([ + E(1, 'run.started', null, { total: 1 }), + E(2, 'node.started', 'write', { node_type: 'report', label: '写周报' }), + E(3, 'llm.start', 'write', { model: 'jm-r', structured: true, purpose: 'judge', message_count: 2, units: 4 }), + E(4, 'llm.end', 'write', { model: 'jm-r', purpose: 'judge', duration_ms: 41000, units: 4, input_tokens: 3000, + output_tokens: 16384, error: CUT, truncated: true, max_tokens: 16384 }), + E(5, 'log', 'write', { level: 'warn', code: 'judge_truncated', max_tokens: 16384, unjudged: 4, message: TRUNCATED }), + E(6, 'node.finished', 'write', { duration_ms: 43000 }), + E(7, 'run.finished', null, { status: 'succeeded' }), + ])) + const call = all.find((s) => s.kind === 'llm' && s.code === 'judge') + check('调用那一行:说输出到了上限被截断、带额度,不说「未能执行」;提醒色、照常完成,原话在展开区', + call?.title === '裁判模型的输出达到上限(16,384 token)被截断' && call.level === 'warn' && call.status === 'done' + && !!call.detail?.includes(CUT), `${call?.title} / ${call?.level} / ${call?.status}`) + const row = all.find((s) => s.code === 'judge_truncated') + check('judge_truncated:标题说截断、带额度和几句未裁判', row?.title === '裁判模型的输出达到上限(16,384 token)被截断,4 句结论未裁判', + row?.title) + check('……下一步指到换不带推理的裁判模型(节点的「结论句裁判」、设置都说),不说检查接入', !!row?.next?.includes('不带推理的裁判模型') + && row.next.includes('结论句裁判') && row.next.includes('设置 → 偏好设置 → 证据裁判') && !row.next.includes('接入') + && row.fix === 'canvas' && row.fixField === 'judge' && row.level === 'warn', `${row?.next} / ${row?.fix} / ${row?.fixField}`) + check('……原话留在展开区,标题里没有内部代码', row?.detail === TRUNCATED && !/judge_truncated|max_tokens|unjudged/.test(`${row?.title}${row?.next}`)) + check('没有混出一条 judge_failed', !all.some((s) => s.code === 'judge_failed')) + // 没截断的失败照旧:「未能执行」那一行和 judge_failed 的说法不变 + const other = flatten(mod.decodeRun([ + E(1, 'node.started', 'write', { node_type: 'report' }), + E(2, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 2 }), + E(3, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 300, units: 2, error: 'ConnectError: 连不上' }), + E(4, 'node.finished', 'write', { duration_ms: 400 }), + ])) + check('没截断的调用失败照旧叫「结论句裁判未能执行」', other.find((s) => s.kind === 'llm')?.title === '结论句裁判未能执行', + other.find((s) => s.kind === 'llm')?.title) + const bare = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report' }), + E(2, 'log', 'w', { level: 'warn', code: 'judge_truncated', max_tokens: 'x', message: '裁判被截断了' })])) + .find((s) => s.code === 'judge_truncated') + check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '裁判模型的输出达到上限被截断' + && !/undefined|NaN/.test(`${bare?.title}${bare?.next ?? ''}`), bare?.title) +}) + await section('审批质询 approval.inquiry(期 4):一行「审批人质询」,标题带问题前 40 字,副标题说回答是否完成、查了几次', async () => { // 前后端共用的约定:运行停在审批上时服务端追加 approval.inquiry,node_id 为空;data 里有问题、划选的原文、回答正文、 // 回答文档、成没成、触顶原因、查询(queries 和 inquiry_artifacts 两份)。以前这类事件落进「未识别的事件」