合入期 5 前置修复 K2:核查与裁判模型的输出额度提到 16384,服务方嫌大时退回 4096(和助手共用 providers/budget.py);每次调用认截断,因推理占满额度没交出结果时照实说(verify_truncated、judge_truncated),不再说成盲算没查到、没按结构交结果

This commit is contained in:
ThearchyHelios
2026-10-06 16:26:37 +08:00
15 changed files with 2009 additions and 80 deletions
+6 -44
View File
@@ -27,6 +27,10 @@ from app.db.models import Workflow
from app.engine.layout import CORRIDOR_MIN, MIN_ROW_GAP, NODE_W, _height, auto_layout
from app.engine.schema import GraphSpec, NodeType, ValidationIssue, text_parse_issues, validate_graph
from app.engine.state import message_text
# 额度用完的认法、退回小额度的做法挪到了 providers/budget.py(核查、证据裁判也用),这里留着老名字;
# _BUDGET_WORDS 另有 data/recipe_ai.py 的测试拿来比对
from app.providers.budget import BUDGET_WORDS as _BUDGET_WORDS # noqa: F401
from app.providers.budget import BudgetFallback as _BudgetFallback, cut_off as _cut_off
from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model
from app.tools.registry import all_specs
@@ -1053,54 +1057,12 @@ class CopilotIncomplete(Exception):
self.detail = detail
def _cut_off(chunk: Any) -> str:
"""这一块是不是在说「额度用完了」:OpenAI 系是 finish_reason=length,Anthropic 是 stop_reason=max_tokens。"""
meta = getattr(chunk, "response_metadata", None) or {}
reason = meta.get("finish_reason") or meta.get("stop_reason")
return str(reason) if reason in ("length", "max_tokens") else ""
#: 流式改图的输出额度。会思考的模型(DeepSeek、Claude 的 adaptive thinking)思考也算在里面:
#: 8192 时复杂的改图常常想完就没额度了,正文一个字没写(2026-09 部署实例上同类请求 5 轮坏了 4 轮)
COPILOT_MAX_TOKENS = 32768
#: 服务方不收大额度时退回这个(原来的默认值)
#: 服务方不收大额度时退回这个(原来的默认值)。退回的做法(_BudgetFallback:只在还没吐出任何东西时重来,
#: 之后的调用即自查修正轮都用小的)和截断的认法(_cut_off)在 providers/budget.py,核查、证据裁判也用它
_FALLBACK_MAX_TOKENS = 8192
#: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K),
#: 或者输入加额度超过上下文长度(本地 vLLM)
_BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length")
def _budget_rejected(e: BaseException) -> bool:
status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None)
return status in (400, 422) and any(w in str(e).lower() for w in _BUDGET_WORDS)
class _BudgetFallback:
"""先按大额度调;服务方一开口就嫌额度太大时按原来的额度重来,之后的调用(自查修正轮)都用小的。
只在还没吐出任何东西时重来:这类拒绝都发生在请求刚发出去的时候,重来不会重复输出。
"""
def __init__(self, big: Any, small: Any) -> None:
self.model, self._small = big, small
@property
def max_tokens(self) -> Any:
return getattr(self.model, "max_tokens", None)
async def astream(self, messages: list[Any]):
started = False
try:
async for chunk in self.model.astream(messages):
started = True
yield chunk
return
except Exception as e:
if started or self.model is self._small or not _budget_rejected(e):
raise
self.model = self._small
async for chunk in self.model.astream(messages):
yield chunk
async def _op_model(session: AsyncSession, payload: GenerateIn) -> tuple[Any, str]:
+3 -1
View File
@@ -549,7 +549,9 @@ async def verify_model_spec(session: AsyncSession, node_config: dict[str, Any] |
"""核查用哪个模型:节点配置里的 provider / model > 设置里的核查模型 > 证据裁判模型的取值链(裁判设置 >
助手模型 > 默认接入,见 judge.judge_model_spec)。哪一级写了就整组用哪一级,不跨级拼接入和模型。
和裁判一样关掉 thinking、max_tokens 4096:核查最后要按结构交结果(强制工具调用),和思考模式不兼容。
和裁判一样关掉 thinking:核查最后要按结构交结果(强制工具调用),和思考模式不兼容。额度也和裁判一样(judge.MAX_TOKENS,
16384):经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考算在额度里,4096 常常想完就没额度查库、交结果了;
服务方嫌大时核查 agent 退回 judge.FALLBACK_MAX_TOKENS(engine/verify_agent.py)。
建议和被核查的模型不同,否则等于自己核自己;比较、提醒由调用方做(它知道被核查的是哪个模型)。
"""
node = node_config if isinstance(node_config, dict) else {}
+80 -10
View File
@@ -12,7 +12,9 @@
(按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过。带期 4 文档标记的报告,上传表格的
查询在 SQL 之后另有来历行(来源、口径、已接受、区域外文字,_provenance_lines)
3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析,
再失败就记未裁判。thinking 关掉,max_tokens 4096
再失败就记未裁判。thinking 关掉,输出额度 MAX_TOKENS(推理模型的思考也算在里面),服务方嫌大时退回
FALLBACK_MAX_TOKENS,被嫌大过的接入、模型之后直接用小的;输出到了上限被截断的整批记未裁判(truncated),
照实说是额度不够、不再退回纯文本多花一次,时间线上另记 judge_truncated
4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算,
超了按优先级截断;触顶就停,已判的保留,没判的记未裁判并说明是哪个上限——不报错
5. 每日计数落在设置表(SPEND_KEY),按本地日期滚动。目录里没有价格的模型估不出金额,金额上限对它
@@ -38,7 +40,7 @@ from dataclasses import asdict, dataclass
from datetime import datetime
from typing import Any
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
from sqlalchemy import select
from app.core.artifact_store import canonical_json, content_hash
@@ -69,6 +71,7 @@ from app.engine.merge_query import MERGE_SOURCE
from app.engine.nodes.llm import _split_structured, _usage_of
from app.engine.state import message_text
from app.providers import catalog as pricing
from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error
from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model
logger = logging.getLogger(__name__)
@@ -104,7 +107,13 @@ BATCH_SIZE = 8
#: 一波最多几批同时问:几批共用剩下的时长,句数上限内的句子一两波就判完。第一批单独问:接入调不通(鉴权、
#: 网关挂了)时只撞这一次,不让一整波一起失败
PARALLEL_BATCHES = 4
MAX_TOKENS = 4096
#: 裁判(和核查:api/settings.py 的 verify_model_spec、engine/verify_agent.py 沿用这一档)的输出额度。thinking 关掉只对
#: Anthropic 有用:经 OpenAI 兼容接入的推理模型(DeepSeek、GLM、Kimi 这一类)收不到 thinking=off,思考算在额度里,原来的
#: 4096 常常想完就没额度写判定了(cite_fields 的抽取在 2026-10 撞过同一堵墙,那边也改成了 16384)。16384 是常见网关、
#: 模型接受的输出上限里偏大的一档(GPT-4o 的上限就是它);更小的服务方会拒收,那时退回 FALLBACK_MAX_TOKENS
MAX_TOKENS = 16384
#: 服务方嫌 MAX_TOKENS 太大(400 / 422,报错里有 max_tokens 一类字样)时退回这个:原来的额度,各家都收。之后的调用都用它
FALLBACK_MAX_TOKENS = 4096
#: 证据摘录:SQL 前几个字、每次查询最多几行几列、每段知识库片段最多几个字、整份引用的检索给几段。
#: 列数上限只管没被引用的列:句子引用的格子所在的列一律给(遮罩的除外),再用别的列补满到这么多列
SQL_CHARS = 300
@@ -124,9 +133,12 @@ SHORT_CHARS = 6
_OUT_PER_UNIT = 120
_OUT_BASE = 20
#: 未裁判的原因。前四个是上限,报告里写「已到上限」
#: 未裁判的原因。前四个是上限,报告里写「已到上限」;truncated 是裁判模型的输出到了上限被截断(推理模型的思考也算在
#: 额度里),和格式不对分开记:说成「格式有误」,人会去查提示词,真正该换的是模型
LIMITS = ("max_claims", "max_cost_usd", "daily_max_usd", "timeout_s")
REASONS = (*LIMITS, "error", "format", "missing", "on_demand")
REASONS = (*LIMITS, "error", "format", "truncated", "missing", "on_demand")
#: 输出被截断时给人看的建议,接在 budget.cut_words 的「裁判模型的输出达到上限(N token)被截断」后面
TRUNCATED_ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型"
JUDGE_ROLE = "你是报告核查员:逐句判断报告里的话,能不能被它挂的证据支持。你不写报告,也不改报告。"
@@ -897,7 +909,8 @@ def _text(value: Any) -> str | None:
async def judge_model_spec(session: Any, node_judge: dict[str, Any] | None = None, *,
settings: dict[str, Any] | None = None) -> ModelSpec:
"""裁判用哪个模型:节点上的 judge.model(连同 provider)> 设置里的证据裁判模型 > Copilot 模型 >
默认 provider。哪一级写了就整组用哪一级,不跨级拼 provider 和模型。thinking 关、max_tokens 4096。
默认 provider。哪一级写了就整组用哪一级,不跨级拼 provider 和模型。thinking 关、额度 MAX_TOKENS(推理模型关不掉
思考,见 MAX_TOKENS 的说明;服务方嫌大时 run_request 退回 FALLBACK_MAX_TOKENS)。
建议和写作模型不同,否则等于自己审自己(报告节点发现相同会发 judge_same_model 警告)。
"""
@@ -1058,12 +1071,20 @@ def _items_of(value: Any) -> list[Any] | None:
return value if isinstance(value, list) else None
#: _ask 交回的没成的原因:输出到了上限被截断
_TRUNCATED = "truncated"
async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | None, list[BaseMessage], str | None,
int]:
"""问一次裁判:(判定清单, 模型的原始回复, 没成的原因, 调了几次模型)。
先走结构化输出;不成(模型不支持、给的不是合法结构)先从原始回复里找 JSON,再退回纯文本调用、
从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对。超时由调用方掐。
从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对,truncated 是输出到了上限被截断
(Anthropic 的 stop_reason=max_tokens、OpenAI 系的 finish_reason=length,以及 openai 结构化输出抛的
LengthFinishReasonError)。截断的回复整批不采信:强制工具调用截在半路时 Anthropic 给的是空对象,有的网关给半截的
清单——最后一条可能只写了编号,也可能判定写完了、理由写到一半,分不清哪条是完整的。结构化调用被截断的不再退回
纯文本:推理占满额度时纯文本调用多半一样被截断,白花一次额度。超时由调用方掐。
"""
raws: list[BaseMessage] = []
try:
@@ -1071,6 +1092,8 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non
raw, parsed = _split_structured(got)
if raw is not None:
raws.append(raw)
if cut_off(raw):
return None, raws, _TRUNCATED, 1
if (items := _items_of(parsed)) is not None:
return items, raws, None, 1
if raw is not None:
@@ -1078,17 +1101,45 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non
for found in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]:
if (items := _items_of(found)) is not None:
return items, raws, None, 1
except Exception: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析
pass
except Exception as e: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析
if truncated_error(e):
return None, [*raws, *_spent(e)], _TRUNCATED, 1
try:
reply = await model.ainvoke(messages)
except Exception as e: # noqa: BLE001
if truncated_error(e):
return None, [*raws, *_spent(e)], _TRUNCATED, 2
return None, raws, f"error:{describe_exception(e)}", 2
raws.append(reply)
if cut_off(reply):
# 正文里就算抠得出一段清单,后面还在写,说不清写完了没有:同样整批不采信
return None, raws, _TRUNCATED, 2
items = _items_of(message_text(reply))
return items, raws, None if items is not None else "format", 2
def _spent(error: BaseException) -> list[BaseMessage]:
"""截断时 openai 的异常带着这次的用量:折成一条空回复交给记账(额度是真花出去了,每日上限不能漏掉它)。"""
inp, out = error_usage(error)
if not (inp or out):
return []
return [AIMessage(content="", usage_metadata={"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out})]
def _with_fallback(model: Any, spec: ModelSpec, model_id: str) -> BudgetFallback:
"""裁判模型包一层:服务方嫌 MAX_TOKENS 太大(或者 anthropic SDK 在本地就拒了)时按 FALLBACK_MAX_TOKENS 重建一个再问
(要退回时才建,平常不多建),之后各批都用小的。被嫌大过的按 (接入, 模型, 额度) 记在进程里:之后每份报告、每次
点击直接用小的,核查(verify_agent._with_fallback)同一种记法、互相认。经本模块的 get_chat_model 建:测试换成剧本
时,小额度的那个也是剧本。"""
async def build() -> Any:
async with SessionLocal() as session:
return (await get_chat_model(session, spec.model_copy(update={"max_tokens": FALLBACK_MAX_TOKENS})))[0]
return BudgetFallback(model, build=build, limits=(spec.max_tokens, FALLBACK_MAX_TOKENS), rejected=budget_refused,
key=(spec.provider, model_id, spec.max_tokens))
def _unjudged(reason: str, rationale: str, *, judge: str | None, post_seal: bool) -> dict[str, Any]:
return {"status": "unjudged", "rationale": rationale, "judge": judge, "post_seal": post_seal, "reason": reason}
@@ -1116,6 +1167,8 @@ def _why_unjudged(reason: str, budget: Budget, detail: str = "", *, click: bool
return f"裁判失败:{detail}"[:120]
if reason == "format":
return detail or "裁判返回的结果格式有误,这句未裁判"
if reason == "truncated":
return f"{detail or cut_words('裁判模型', None)},这句未裁判。{TRUNCATED_ADVICE}"
if reason == "missing":
return "裁判未对这句给出判定"
return "探索运行按需裁判:点开这句并请模型判断后才会裁判"
@@ -1149,6 +1202,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
try:
async with SessionLocal() as session:
model, model_id = await get_chat_model(session, spec)
model = _with_fallback(model, spec, model_id)
except Exception as e: # noqa: BLE001 - 裁判模型没配好:这回判不了,报告照常
why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e)
details["error"] = f"裁判模型不可用({why})"
@@ -1185,6 +1239,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
spent = 0.0
width = 1
cut_limit: int | None = None # 被截断的那次调用的额度:judge_truncated 日志照它写
while pending:
remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started)
if remaining is not None and remaining <= 0:
@@ -1261,6 +1316,14 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
end = {"model": model_id, "purpose": "judge", "duration_ms": int((time.perf_counter() - began) * 1000),
"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out,
"cost_usd": round(actual, 6), "calls": calls, "units": len(batch)}
if problem == _TRUNCATED:
# 输出到了上限被截断:这一批记未裁判,原因照实说是额度,不说「格式有误」「调用失败」。别的批照常问
# (同格式不对):截断取决于这一批要想多久,不像鉴权、网关出错那样后面几批多半一样
cut_limit = model.max_tokens
details["truncated"] = cut_words("裁判模型", cut_limit)
emit(EventType.LLM_END, **end, error=details["truncated"], truncated=True, max_tokens=cut_limit)
stop(batch, "truncated", details["truncated"])
continue
if problem is not None and problem != "format":
reason = "timeout_s" if problem == "timeout_s" else "error"
detail = problem.partition(":")[2]
@@ -1297,7 +1360,12 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
n = sum(outcome["unjudged"].get(r, 0) for r in hit)
emit(EventType.LOG, level="warn", code="judge_limit", limits=hit, unjudged=n,
message=f"结论句裁判已达上限({labels}):{n} 句未裁判,已裁判的结果保留")
if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS]:
# 输出被截断单独记一条(带那次的额度、几句未裁判):judge_failed 在时间线上的下一步是检查接入、重新运行,截断时接入
# 是好的,同一个模型重新运行多半还是截断,该换的是模型
if truncated := [g for r, g in outcome["_gaps"] if r == "truncated"]:
emit(EventType.LOG, level="warn", code="judge_truncated", max_tokens=cut_limit,
unjudged=outcome["unjudged"]["truncated"], message="结论句裁判未完成:" + ";".join(truncated))
if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS and r != "truncated"]:
emit(EventType.LOG, level="warn", code="judge_failed", message="结论句裁判未完成:" + ";".join(failed))
outcome.pop("_gaps")
return outcome
@@ -1394,6 +1462,8 @@ def _outcome(request: JudgeRequest, verdicts: dict[str, dict[str, Any]], budget:
text = f"有 {n} 句结论未裁判:裁判调用失败({details.get('error') or '原因不明'})"
elif reason == "format":
text = f"有 {n} 句结论未裁判:裁判返回的结果格式有误"
elif reason == "truncated":
text = f"有 {n} 句结论未裁判:{details.get('truncated') or cut_words('裁判模型', None)},{TRUNCATED_ADVICE}"
else:
text = f"有 {n} 句结论未获得裁判结果"
gaps.append((reason, text))
+6
View File
@@ -104,6 +104,9 @@ WARN_CODES: dict[str, tuple[str, str]] = {
# 结论句裁判(claims: judge):触顶、没跑成的句子记为未裁判,出具不能判完整——是缺口,要说明
"judge_limit": ("judge_limit", DEGRADED),
"judge_failed": ("judge_failed", DEGRADED),
# 裁判模型的输出到了上限被截断(推理模型的思考也算在额度里):同 judge_failed 是缺口,记成它自己的 kind——说得清
# 是输出额度的问题,该换的是模型
"judge_truncated": ("judge_truncated", DEGRADED),
# 证据不支持的句子交回写作者改过一次(和 report_repair 一样只记缺口);改写稿冒出新问题没采用的,
# 原稿里那几句仍是证据不支持
"report_rewrite": ("report_rewrite", DEGRADED),
@@ -128,6 +131,9 @@ WARN_CODES: dict[str, tuple[str, str]] = {
# 核查 agent 自己触顶、按已有结果收尾:多半和节点自己的 *_limit 说的是同一次触顶,照样是缺口,记成它自己的 kind,
# 不落成笼统的 warn——步数上限(max_steps)只有它说
"verify_limit": ("verify_limit", DEGRADED),
# 核查 agent 的输出到了上限被截断(推理模型的思考也算在额度里):没交成结果,由节点按无法复算、未能反查记缺口。
# 同 verify_limit,是缺口不是断裂,记成它自己的 kind:说得清是输出额度的问题,换个模型就好
"verify_truncated": ("verify_truncated", DEGRADED),
}
# 这些 kind 重跑一次大概率能好:原因说得清,且不是「问题本身没问明白」。
+89 -16
View File
@@ -25,6 +25,15 @@ query_artifact 和 role: verify——随运行封存、证据面板能下钻。
每日上限不传就按设置里的「核查每日金额上限」,明确传 None 才是不限。模型不在价格目录里时估不出金额,金额上限
对它无效,照裁判的说法记一条说明。每次调用都记进核查的每日账(engine/spend.py)。
**输出额度。** 经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考也算在输出额度里:额度按 spec(缺省 judge.MAX_TOKENS),
服务方嫌大时(或者 anthropic SDK 在本地就拒了)退回 judge.FALLBACK_MAX_TOKENS 再试一次,之后工具循环、收尾都用小的;
同一接入、模型被嫌大过,之后的核查一开始就用小的(providers/budget.py)。每次调用都看有没有到了上限被截断,截断的回复
一律不采信:工具循环里截断的回复不算模型自己停下,带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input
是空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾——同时长到点一样算半路被掐断,按截断前查到的收尾
交出去的结果说不清完整不完整,还会照常参与比对和处置。收尾的强制工具调用被截断时,解析出来的空对象、半截对象都不算
交了结果,也不再退回普通调用多花一次额度。因为截断没交成结果的,error 照实说是输出额度(TRUNCATED_ERROR),不说成
「没查到」「没按结构交」;节点里调用时另记一条 verify_truncated。
永远不抛:模型没配好、调用失败、格式不对、触顶、库写不进去,都落在返回值里(error / limit)。模型调用都经
get_chat_model,测试换成剧本。
"""
@@ -49,11 +58,12 @@ from app.db.base import SessionLocal
from app.engine import spend
from app.engine.evidence import build_catalog, cell_eid, compose_doc, query_entry_fields, schema_ledger_entry
from app.engine.expressions import CellError, column_kind, locate_cell, numeric_text
from app.engine.judge import _json_in, _tokens, _usd
from app.engine.judge import FALLBACK_MAX_TOKENS, MAX_TOKENS, _json_in, _tokens, _usd
from app.engine.nodes.llm import _DIGEST_BUDGET, _digest, _split_structured, _typed, explain_model_error
from app.engine.state import message_text
from app.engine.toolcalls import ToolTimeout, limit_fields, limit_of, run_bounded
from app.providers import catalog as pricing
from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error
from app.providers.factory import ModelSpec, ProviderNotConfigured, bind_tools_safely, get_chat_model
from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX, build_datasource_tools
from app.tools.registry import ToolArgsError, ToolContext, Versions, args_model_of, prepare_args
@@ -79,6 +89,14 @@ _REF_DEPTH = 8
RATIONALE = "rationale"
#: 触顶原因
LIMITS = ("max_queries", "timeout_s", "max_cost_usd", "daily_max_usd", "max_steps")
#: 核查模型的输出到了上限被截断、没交成结果时给人看的话:双盲复算的无法复算、反方审稿的「反查未完成」、质询的报错都照
#: 它说,也是 verify_truncated 日志的原话。cut 是 budget.cut_words 的「核查模型的输出达到上限(N token)被截断」。开头只说
#: 「未能给出结果」:质询里提问人看到的就是这句,它问的是问题,不是在核查
TRUNCATED_ERROR = "{cut},未能给出结果。{advice}"
#: 截断的建议:该换的是模型。节点里(双盲复算、反方审稿)节点上指定的模型优先于设置(api/settings.py 的
#: verify_model_spec),两处都说,和时间线上的下一步一致;不在节点里(审批质询)核查模型只看设置
TRUNCATED_ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型"
TRUNCATED_ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型"
VERIFY_SYSTEM = (
"你是数据核查员:只读地查库,用查询结果完成交给你的任务。\n"
@@ -129,7 +147,8 @@ class VerifyOutcome:
source, tool, call_id}。
rationale:理由组装成的报告文档(evidence.compose_doc 的形状,violations 里有裸数字)。usage:{input_tokens,
output_tokens, total_tokens, cost_usd, calls, unpriced_calls}。limit:触顶原因(LIMITS 之一),没触顶为 None。
answered:模型是不是自己停下来的(没被上限掐断)。
answered:模型是不是自己停下来的(没被上限掐断,也没因为输出到了上限被截断)。被截断的一律没有结果,error 是
TRUNCATED_ERROR。
"""
result: dict[str, Any] | None
@@ -686,13 +705,16 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
def left() -> float | None:
return None if deadline is None else deadline - time.monotonic()
# ---- 模型
# ---- 模型。额度没写的按 MAX_TOKENS:推理模型的思考算在输出额度里,4096 常常想完就没额度发查询、交结果了
spec = spec.model_copy(update={"max_tokens": spec.max_tokens or MAX_TOKENS})
small = FALLBACK_MAX_TOKENS if spec.max_tokens > FALLBACK_MAX_TOKENS else None
try:
async with SessionLocal() as session:
base_model, model_id = await get_chat_model(session, spec)
first, model_id = await get_chat_model(session, spec)
except Exception as e: # noqa: BLE001 - 核查模型没配好:这回核不了,照实交回
why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e)
return outcome(error=f"核查模型不可用({why})")
base_model = _with_fallback(first, spec, small, model_id)
state.model = model_id
if daily_max_usd is Default.SETTINGS:
from app.api.settings import verify_settings # 在这里导入:api.settings 导入了 engine 里的模块
@@ -743,7 +765,8 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
raw = got if isinstance(got, BaseMessage) else _split_structured(got)[0]
except (asyncio.TimeoutError, Exception) as e: # noqa: BLE001 - 调用失败落在返回值里
failure = e
inp, out = _usage(raw)
# openai 的结构化输出被截断时抛异常,用量在异常里:额度是真花出去了
inp, out = _usage(raw) if raw is not None else error_usage(failure)
cost = await budget.settle(day, estimate, inp, out)
usage["input_tokens"] += inp
usage["output_tokens"] += out
@@ -756,9 +779,23 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
if failure is not None:
end["error"] = "超过核查的时长上限,已放弃这次调用" if isinstance(failure, asyncio.TimeoutError) \
else explain_model_error(failure, model_id)
if truncated_error(failure) or cut_off(raw):
# 到了上限被截断:不是「调用失败,请稍后重试」,重试解决不了额度问题
limit = base_model.max_tokens
end.update(truncated=True, max_tokens=limit, error=cut_words("核查模型", limit))
await _announce(emit, EventType.LLM_END, **end)
return got, raw, failure
# 截断的建议按模型从哪来的说:节点里节点上指定的优先,不在节点里(质询)只看设置
advice = TRUNCATED_ADVICE if node_id else TRUNCATED_ADVICE_SETTINGS
async def report_cut(purpose: str, limit: int | None) -> VerifyOutcome:
"""被截断、没交成结果:记一条 verify_truncated(purpose 是哪一步,limit 是那次的额度),原因照截断说。查过的照记。"""
text = TRUNCATED_ERROR.format(cut=cut_words("核查模型", limit), advice=advice)
await _announce(emit, EventType.LOG, level="warn", code="verify_truncated", purpose=purpose, max_tokens=limit,
message=text)
return outcome(limit=stopped, answered=answered, error=text)
# ---- 查库的循环
stopped: str | None = None
answered = False
@@ -781,6 +818,12 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
if isinstance(failure, asyncio.TimeoutError):
stopped = "timeout_s"
break
if truncated_error(failure) or cut_off(response):
# 到了上限被截断(推理模型想完了额度也用完了):话没说完,不是模型自己停下来的;带着的工具调用是写到一半的
# (Anthropic 的 tool_use 截在半路时 input 是空的,有的网关给半截的 SQL),不执行。也不收尾:一条都没查到的,
# 收尾只能交「查不到」(双盲复算会记成「盲算没有查到这个数」);查到过的,截断前查到的不一定是要的那个数
# (模型可能正想改查询),同时长到点一样算半路被掐断。收尾那次多半同样被推理占满,也不为它多花一次额度
return await report_cut("verify", base_model.max_tokens)
if failure is not None or not isinstance(response, BaseMessage):
return outcome(error=explain_model_error(failure, model_id) if failure else "核查模型没有给出回复")
messages.append(response)
@@ -850,12 +893,15 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
await _announce(emit, EventType.LOG, level="warn", code="verify_limit", limit=stopped,
message=f"核查已达上限({words}),已停止查询,按已有结果收尾")
# ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容
# ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容(经 OpenAI 兼容接入的推理模型
# 关不掉,所以额度和工具循环一样,退回小额度也和工具循环共用一个开关)
settle_spec = spec.model_copy(update={"thinking": "off"})
try:
async with SessionLocal() as session:
settler, _ = await get_chat_model(session, spec.model_copy(update={"thinking": "off"}))
settle_model, _ = await get_chat_model(session, settle_spec)
except Exception as e: # noqa: BLE001
return outcome(limit=stopped, answered=answered, error=f"核查模型不可用({describe_exception(e)})")
settler = _with_fallback(settle_model, settle_spec, small, model_id, shared_with=base_model)
final = _final_messages(head, task, queries, said, words)
try:
structured = settler.with_structured_output(schema, include_raw=True)
@@ -870,21 +916,27 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
parsed: dict[str, Any] | None = None
failure: BaseException | None = None
settle_cut = False # 收尾那次到了上限被截断
if structured is not None:
estimate = budget.estimate(_text_of(final) + schema_text, _OUT_FINAL)
day, refused = await budget.reserve(estimate, kept=kept)
if refused:
return outcome(limit=stopped or refused, answered=answered, error=f"已达{words_of(refused)},没能交结果")
got, raw, failure = await ask(final, structured.ainvoke, estimate, day or "", "verify_result", until_finish())
parsed = None if failure is not None else _parsed(_split_structured(got)[1])
if parsed is None and raw is not None:
# 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用
for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]:
if (parsed := _parsed(candidate)) is not None:
break
if parsed is None and not isinstance(failure, asyncio.TimeoutError):
# 强制工具调用写到一半到了上限:Anthropic 截在 tool_use 里时 input 是空的,有的网关给半截对象(反方审稿的结论
# 写完了、理由写到一半)。解析出来的、能从回复里抠出来的都说不清写完了没有,一律不采信
settle_cut = truncated_error(failure) or bool(cut_off(raw))
if failure is None and not settle_cut:
parsed = _parsed(_split_structured(got)[1])
if parsed is None and raw is not None:
# 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用
for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]:
if (parsed := _parsed(candidate)) is not None:
break
if parsed is None and not isinstance(failure, asyncio.TimeoutError) and not settle_cut:
# 不支持结构化输出的模型:普通调用一次,结构写进消息,从回复里把 JSON 抠出来。结构化那次已经做过的,
# 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱
# 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱。结构化那次是被截断的不补救:
# 推理占满额度时普通调用多半一样被截断,白花一次额度
plain = _plain_messages(final, schema)
remaining = until_finish()
if remaining is None or remaining > 0:
@@ -895,8 +947,13 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
error=f"已达{words_of(refused)},没能交结果")
if not refused:
got, _, failure = await ask(plain, settler.ainvoke, estimate, day or "", "verify_result", remaining)
parsed = None if failure is not None else _parsed(message_text(got) if got is not None else None)
# 同上:截断的回复里就算抠得出一段 JSON,后面还在写,说不清写完了没有
settle_cut = truncated_error(failure) or bool(cut_off(got))
if failure is None and not settle_cut:
parsed = _parsed(message_text(got) if got is not None else None)
if parsed is None:
if settle_cut:
return await report_cut("verify_result", settler.max_tokens)
why = "收尾时超过时长上限,没能交结果" if isinstance(failure, asyncio.TimeoutError) else \
f"核查模型没有按结构交结果({explain_model_error(failure, model_id)})" if failure is not None else \
"核查模型没有按结构交结果"
@@ -919,6 +976,22 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
return outcome(result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered)
def _with_fallback(model: Any, spec: ModelSpec, small: int | None, model_id: str, *,
shared_with: BudgetFallback | None = None) -> BudgetFallback:
"""按 spec 建好的模型包一层:服务方嫌额度大(或者 SDK 在本地拒了)时按 small 重建一个再调,要退回时才建,平常一个
都不多建。small 为 None 是额度本来就不比它大,没有可退的。被嫌大过的按 (接入, 模型, 额度) 记在进程里,证据裁判
(judge._with_fallback)用的是同一种记法:同一个模型在那边被嫌大过,这边也直接用小的。经本模块的 get_chat_model
建,测试换成剧本时小额度的那个也是剧本。"""
async def build() -> Any:
async with SessionLocal() as session:
return (await get_chat_model(session, spec.model_copy(update={"max_tokens": small})))[0]
return BudgetFallback(model, build=build if small else None, limits=(spec.max_tokens, small or spec.max_tokens),
shared_with=shared_with, rejected=budget_refused,
key=(spec.provider, model_id, spec.max_tokens))
async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, remaining: float | None,
emit: Emit | None, queries: list[dict[str, Any]], ledger: list[dict[str, Any]],
owner: str) -> str | None:
+202
View File
@@ -0,0 +1,202 @@
"""模型调用的输出额度:认出「输出到了上限被截断」,服务方嫌额度太大时退回小额度重来。
助手改图(api/copilot.py)、核查(engine/verify_agent.py:双盲复算、反方审稿、审批质询共用)和证据裁判(engine/judge.py)
共用这一份。会思考的模型思考也算在输出额度里:经 OpenAI 兼容接入的推理模型(DeepSeek、GLM、Kimi 这一类)收不到
thinking=off,Claude 开了 adaptive thinking 也一样。额度小了,想完就没额度写正文——回复是空的,或者 JSON 只写了一半。
这时要照实说是输出额度不够,不能当成「模型没按格式交」「没查到」。
- cut_off:这一块(流式)或这条回复是不是因为额度用完停下的
- truncated_error:这个异常是不是在说同一件事(openai 结构化输出解析时抛的 LengthFinishReasonError)
- budget_rejected:服务方一开口就嫌额度太大(助手改图按它认)
- budget_refused:核查、证据裁判按它认,多认自建服务说的 max_new_tokens 和 anthropic SDK 在本地的拒绝
- error_usage:截断的那次调用花掉的用量
- cut_words:给人看的「某某模型的输出达到上限(N token)被截断」
- BudgetFallback:先按大额度调,被嫌大时按小额度重来,之后都用小的;给了 key 的,被嫌大过就记在进程里
engine/nodes/llm.py(cite_fields 的抽取)、data/recipe_ai.py 各有一份同样的判断,是在这里之前写的,行为相同。
"""
from __future__ import annotations
from collections.abc import Awaitable, Callable, Hashable
from typing import Any
#: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K),
#: 或者输入加额度超过上下文长度(本地 vLLM)。助手改图按这一份认,data/recipe_ai.py 的测试拿它比对
BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length")
#: 核查、证据裁判另外认的:text-generation-inference 一类自建服务说的是 max_new_tokens(「`inputs` tokens +
#: `max_new_tokens` must be <= 8192」)。这两处的额度从 4096 提到 16384 之后才容易撞上;助手改图的认法不动
WIDER_BUDGET_WORDS = (*BUDGET_WORDS, "max_new_tokens")
#: anthropic SDK 在本地拒绝时的原话:客户端没设超时(Bearer 鉴权的接入,factory 换上的就是这种)、非流式调用、额度
#: 超过这个模型不流式能给的上限(Opus 4、4.1 这一代只到 8192)或估计要写十分钟以上,请求没发出去就抛 ValueError
LOCAL_REFUSAL = "streaming is required for operations that may take longer than 10 minutes"
#: 截断类报错的原话:openai 的 LengthFinishReasonError(结构化输出解析时 finish_reason=length)
LENGTH_WORDS = ("length limit was reached", "finish_reason=length", "finish_reason='length'")
def cut_off(chunk: Any) -> str:
"""这一块是不是在说「额度用完了」:OpenAI 系是 finish_reason=length,Anthropic 是 stop_reason=max_tokens。"""
meta = getattr(chunk, "response_metadata", None) or {}
reason = meta.get("finish_reason") or meta.get("stop_reason")
return str(reason) if reason in ("length", "max_tokens") else ""
def truncated_error(error: BaseException | None) -> bool:
"""这个异常是不是在说「输出被截断」:openai 的 LengthFinishReasonError,或者 langchain、网关转述的同一件事。
服务方嫌额度太大的 400(报错里也有 max_tokens)不算:那是请求被拒,模型一个字都没写。
"""
if error is None:
return False
if type(error).__name__ == "LengthFinishReasonError":
return True
text = str(error).lower()
return any(word in text for word in LENGTH_WORDS)
def budget_rejected(e: BaseException, words: tuple[str, ...] = BUDGET_WORDS) -> bool:
status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None)
return status in (400, 422) and any(w in str(e).lower() for w in words)
def budget_refused(e: BaseException) -> bool:
"""核查、证据裁判按这个认「额度被挡下了,换小的重来」:服务方嫌额度太大(多认 max_new_tokens),或者 anthropic SDK
在本地就拒了。两种都是一个字还没写、请求要么被拒要么根本没发出去,重来不会重复输出,也没花钱。助手改图走流式,
SDK 不在本地拦,仍按 budget_rejected 认。"""
if isinstance(e, ValueError) and LOCAL_REFUSAL in str(e).lower():
return True
return budget_rejected(e, WIDER_BUDGET_WORDS)
def error_usage(error: BaseException | None) -> tuple[int, int]:
"""失败的那次调用花掉的 (输入, 输出) token。截断时 openai 的异常带着 completion.usage:额度是真花出去了,要记账,
不然每日上限会漏掉最贵的那几次。别的异常没有用量,是 (0, 0)。"""
usage = getattr(getattr(error, "completion", None), "usage", None)
return int(getattr(usage, "prompt_tokens", 0) or 0), int(getattr(usage, "completion_tokens", 0) or 0)
def cut_words(who: str, limit: int | None) -> str:
"""「核查模型的输出达到上限(16384 token)被截断」。额度不知道(没给 max_tokens、按接入的默认值)就不写数。"""
return f"{who}的输出达到上限{f'({limit} token)' if limit else ''}被截断"
#: 嫌过大额度的接入:BudgetFallback 的 key(核查、证据裁判给的是 (接入, 模型, 大额度))。双盲复算的几路盲算、反方
#: 审稿的每一句、每次质询、每次点击裁判各是一组新建的调用,开关只在一组里共用;记在这里,之后新建的直接用小的,
#: 不再每组先撞一次。只记被拒过的;进程内有效,重启后再按大的试(服务方可能调高了上限),同 factory 的 _NO_STREAM_USAGE
_REJECTED: set[Hashable] = set()
class _Switch:
"""几次调用共用的「退回小额度了没有」:核查的工具循环和收尾、裁判的各批。rejected 是怎么认「额度被挡下了」;
key 给了,开关一开始就照 _REJECTED 定,退回时记进去。"""
__slots__ = ("key", "rejected", "small")
def __init__(self, key: Hashable | None, rejected: Callable[[BaseException], bool]) -> None:
self.key = key
self.rejected = rejected
self.small = key is not None and key in _REJECTED
def fall_back(self) -> None:
self.small = True
if self.key is not None:
_REJECTED.add(self.key)
class BudgetFallback:
"""先按大额度调;服务方一开口就嫌额度太大时按小额度重来,之后的调用都用小的。
只在还没吐出任何东西时重来:这类拒绝都发生在请求刚发出去的时候,重来不会重复输出。流式(astream)、非流式
(ainvoke)都认。bind_tools、with_structured_output 照样包一层,包出来的和原来的共用同一个开关;shared_with 给了,
另一份配置建的模型(核查收尾时关掉思考的那个)也和它共用:工具循环里退回过,收尾那次直接用小的,不再先撞一次。
small 是现成的小额度模型;也可以不给,给 build(无参的协程函数,返回小额度的模型),要退回时才建,平常一个都不多建。
limits 是 (大额度, 小额度):给了,max_tokens 就按它报(包过一层的模型身上不一定读得到额度);不给就读眼下那个模型的
max_tokens。rejected 是怎么认「额度被挡下了」,缺省 budget_rejected(助手改图),核查、证据裁判给 budget_refused。
key 给了(核查、证据裁判给 (接入, 模型, 大额度)),被嫌大一次就记在进程里,之后同一个 key 新建的一开始就用小的;
不给(助手改图)不记也不查。shared_with 给了,rejected、key 跟着它。
"""
def __init__(self, big: Any, small: Any = None, *, build: Callable[[], Awaitable[Any]] | None = None,
limits: tuple[int | None, int | None] | None = None,
shared_with: BudgetFallback | None = None,
rejected: Callable[[BaseException], bool] = budget_rejected,
key: Hashable | None = None) -> None:
self._big = big
self._small = small
self._build = build
self._limits = limits
self._switch = shared_with._switch if shared_with is not None else _Switch(key, rejected)
@property
def model(self) -> Any:
"""眼下用的那个:退回过、小额度的也建好了就是小的,否则是大的。"""
return self._small if self._switch.small and self._small is not None else self._big
@property
def max_tokens(self) -> Any:
if self._limits is not None:
return self._limits[1] if self._switch.small else self._limits[0]
return getattr(self.model, "max_tokens", None)
def _can_fall_back(self, used: Any) -> bool:
"""刚才那次用的是大额度、又有小额度可退。大小是同一个模型时重来也是同样的额度,不算可退。"""
if used is not self._big:
return False
return self._build is not None if self._small is None else self._small is not self._big
async def _small_model(self) -> Any:
if self._small is None and self._build is not None:
built = await self._build()
if self._small is None: # 并发的几次调用同时要建时,留先建好的那个
self._small = built
return self._small
async def _current(self) -> Any:
if self._switch.small and (small := await self._small_model()) is not None:
return small
return self._big
async def ainvoke(self, messages: Any, *args: Any, **kwargs: Any) -> Any:
used = await self._current()
try:
return await used.ainvoke(messages, *args, **kwargs)
except Exception as e:
if not self._can_fall_back(used) or not self._switch.rejected(e):
raise
self._switch.fall_back()
return await (await self._current()).ainvoke(messages, *args, **kwargs)
async def astream(self, messages: Any, *args: Any, **kwargs: Any):
used = await self._current()
started = False
try:
async for chunk in used.astream(messages, *args, **kwargs):
started = True
yield chunk
return
except Exception as e:
if started or not self._can_fall_back(used) or not self._switch.rejected(e):
raise
self._switch.fall_back()
async for chunk in (await self._current()).astream(messages, *args, **kwargs):
yield chunk
def _derived(self, make: Callable[[Any], Any]) -> BudgetFallback:
"""对大小两个模型做同一件事(绑工具、要结构化输出),包成共用开关的一层。小的还没建时,建好再做。"""
small, build = None, None
if self._small is not None:
small = make(self._small)
elif self._build is not None:
async def build() -> Any:
return make(await self._small_model())
return BudgetFallback(make(self._big), small, build=build, limits=self._limits, shared_with=self)
def bind_tools(self, tools: Any, **kwargs: Any) -> BudgetFallback:
return self._derived(lambda m: m.bind_tools(tools, **kwargs))
def with_structured_output(self, schema: Any, **kwargs: Any) -> BudgetFallback:
return self._derived(lambda m: m.with_structured_output(schema, **kwargs))
__all__ = ["BUDGET_WORDS", "LENGTH_WORDS", "LOCAL_REFUSAL", "WIDER_BUDGET_WORDS", "BudgetFallback", "budget_refused",
"budget_rejected", "cut_off", "cut_words", "error_usage", "truncated_error"]
+2 -2
View File
@@ -6,7 +6,7 @@
- 一份报告合并成一次调用(超过每批的句数就分批),每句给 supported / partial / contradicted / insufficient /
not_a_claim
- 模型不支持结构化输出、或者给的不是合法结构时退回 JSON 解析,再失败就记「未裁判」,不编判定
- 裁判模型关掉 thinking,max_tokens 4096
- 裁判模型关掉 thinking,输出额度 16384(推理模型的思考也算在里面);服务方嫌大时退回 4096,截断照实说(见 test_judge_truncation)
- 预算(每份报告的句数、金额、时长,全局每日金额)每一项都能写 null 表示不限;触顶就停、已判的保留、
没判的记未裁判并说明是哪个上限——不报错、不让报告失败。金额超了按优先级截断:先判挂了依据的,
再判有方向词、因果词的
@@ -207,7 +207,7 @@ async def test_the_judge_model_runs_without_thinking(monkeypatch, stored):
max_tokens=64000),
budget=Budget(max_claims=None, max_cost_usd=None, timeout_s=30, daily_max_usd=None))
[spec] = judge.specs
assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 4096
assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 16384
assert spec.model == PRICED and spec.timeout and spec.timeout <= 30
+1 -1
View File
@@ -350,7 +350,7 @@ async def test_the_judge_model_follows_node_then_settings(client, monkeypatch):
row = await finish(weekly(claims="judge"))
await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])
[spec] = models.specs
assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 4096
assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 16384
row = await finish(weekly(claims="judge", judge={"model": "judge-from-node"}))
body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json()
+2 -1
View File
@@ -17,6 +17,7 @@ from app.db.base import SessionLocal
from app.db.models import Setting
from app.engine.judge import (
JUDGE_DEFAULTS,
MAX_TOKENS,
SETTING_GROUP,
SPEND_KEY,
Budget,
@@ -166,7 +167,7 @@ async def test_the_judge_model_priority():
async with SessionLocal() as session:
spec = await judge_model_spec(session, None)
assert (spec.provider, spec.model) == (None, None), "什么都没配:默认 provider"
assert spec.thinking == "off" and spec.max_tokens == 4096
assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 16384
session.add(Setting(key="copilot", value={"provider": "", "model": "copilot-model"}))
await session.commit()
+437
View File
@@ -0,0 +1,437 @@
"""结论句裁判(engine/judge.py)的输出额度:截断照实说,服务方嫌额度大时退回小额度。
裁判原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考就能把 4096 用光:
结构化调用写不出判定(openai 抛 LengthFinishReasonError,Anthropic 的强制工具调用写到一半),退回纯文本调用也一样,
最后记成「裁判返回的结果格式有误」——没说出真正的原因。这里守几件事:
- 额度:16384(MAX_TOKENS);服务方嫌大时退回 4096(FALLBACK_MAX_TOKENS)再问一次,之后各批都用小的,只撞一次
- 输出到了上限被截断(stop_reason=max_tokens、finish_reason=length、LengthFinishReasonError):这一批记未裁判,原因
是 truncated,理由、缺口、llm.end 都写明「裁判模型的输出达到上限(N token)被截断」;截断的回复整批不采信(有的网关
把截断的强制工具调用解析成半截清单:最后一条只写了一半);结构化调用被截断的不再退回纯文本多花一次额度;截断那次
的用量照记
- 时间线上另记一条 judge_truncated(带额度、几句未裁判),不混进 judge_failed:judge_failed 的下一步是「检查接入、
重新运行」,截断时接入是好的,同一个模型重新运行多半还是截断
- 被嫌大过的接入、模型记在进程里,之后的裁判直接用小的;自建服务说的 max_new_tokens 也认
- 没有截断时原有说法不变(格式不对仍是 format)
模型都是剧本;证据借 test_evidence_judge 的虚构夹具(口径卡、一次查询、一次检索)。
"""
from __future__ import annotations
import json
import re
import uuid
import pytest
from langchain_core.messages import AIMessage
from openai import LengthFinishReasonError
from openai.types import CompletionUsage
from openai.types.chat import ChatCompletion
from app.core.events import EventType
from app.db.models import Provider
from app.engine import judge as judge_mod
from app.engine import review as rv
from app.engine.evidence import compose_doc
from app.engine.judge import BATCH_SIZE, FALLBACK_MAX_TOKENS, MAX_TOKENS, REASONS, judge_doc
from app.providers import budget
from app.providers.factory import build_chat_model
from tests.test_evidence_judge import PRICED, RAW, SPEC, UNLIMITED, clean_settings, make_catalog, stored # noqa: F401
from tests.test_verify_truncation import messages_api
try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上
from langchain_openai._compat import httpx
except ImportError: # pragma: no cover - 老版本的 langchain_openai
import httpx
CUT_16K = "裁判模型的输出达到上限(16384 token)被截断"
ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型"
@pytest.fixture(autouse=True)
def forget_rejections():
"""被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。"""
budget._REJECTED.clear()
yield
budget._REJECTED.clear()
def length_error(prompt: int = 3000, completion: int = 16384) -> LengthFinishReasonError:
usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion)
return LengthFinishReasonError(completion=ChatCompletion.model_construct(
id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage))
def anthropic_cut(units) -> dict:
"""强制工具调用写到一半到了上限:原始回复 stop_reason=max_tokens,解析不出判定。"""
raw = AIMessage(content=[], response_metadata={"stop_reason": "max_tokens"},
usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384})
return {"raw": raw, "parsed": None, "parsing_error": None}
def thinking_only(units) -> AIMessage:
"""纯文本调用:额度全用在思考上,正文为空,finish_reason=length(推理模型经 OpenAI 兼容接入的样子)。"""
return AIMessage(content="", additional_kwargs={"reasoning_content": "逐句看证据……"},
response_metadata={"finish_reason": "length"},
usage_metadata={"input_tokens": 3000, "output_tokens": 4096, "total_tokens": 7096})
def half_list(units) -> dict:
"""有的网关把截断的强制工具调用解析成半截对象:第一条判定写完了,第二条只写了编号,后面的没写到。"""
items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []},
{"unit": units[1]}]
raw = AIMessage(content=[{"type": "tool_use", "id": "toolu_1", "name": "judge", "input": {"items": items}}],
tool_calls=[{"name": "judge", "args": {"items": items}, "id": "toolu_1"}],
response_metadata={"stop_reason": "max_tokens"},
usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384})
return {"raw": raw, "parsed": {"items": items}, "parsing_error": None}
def supported(units) -> dict:
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
return {"raw": AIMessage(content="", response_metadata={"finish_reason": "stop"}), "parsed": {"items": items}}
class BudgetRejected(Exception):
"""服务方嫌输出额度太大时的 400。"""
status_code = 400
class Reasoner:
"""裁判模型的剧本。structured(units) / plain(units) 给出结构化调用、退回纯文本调用的回复(返回异常就抛它);
unstructured 为真时不支持结构化输出。calls 记下每次调用的种类和判的句子。"""
def __init__(self, *, structured=supported, plain=None, unstructured: bool = False) -> None:
self.structured, self.plain, self.unstructured = structured, plain, unstructured
self.calls: list[tuple[str, list[str]]] = []
def with_structured_output(self, schema, **kwargs):
if self.unstructured:
raise NotImplementedError("这个模型不支持结构化输出")
return _View(self)
async def ainvoke(self, messages, *args, **kwargs):
return self.answer("plain", messages)
def answer(self, kind: str, messages):
units = re.findall(r"^\[(u\d+)\]", str(messages[-1].content), re.M)
self.calls.append((kind, units))
make = self.structured if kind == "structured" else self.plain
reply = make(units)
if isinstance(reply, BaseException):
raise reply
return reply
class _View:
def __init__(self, model: Reasoner) -> None:
self.model = model
async def ainvoke(self, messages, *args, **kwargs):
return self.model.answer("structured", messages)
class Rejecting:
def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens",
status_code: int = 400) -> None:
self.message, self.status_code = message, status_code
self.calls = 0
def with_structured_output(self, schema, **kwargs):
return self
async def ainvoke(self, messages, *args, **kwargs):
self.calls += 1
error = BudgetRejected(self.message)
error.status_code = self.status_code
raise error
class Models:
def __init__(self, monkeypatch, model, *, rejects_above: int | None = None,
rejecting: Rejecting | None = None) -> None:
self.specs = []
self.rejecting = rejecting or Rejecting()
async def fake(session, spec):
self.specs.append(spec)
if rejects_above is not None and (spec.max_tokens or 0) > rejects_above:
return self.rejecting, PRICED
return model, PRICED
monkeypatch.setattr(judge_mod, "get_chat_model", fake)
async def judged(stored, raw: str = RAW):
catalog = make_catalog(stored)
events: list[tuple] = []
outcome = await judge_doc(compose_doc(raw, catalog), catalog, spec=SPEC, budget=UNLIMITED,
emit=lambda kind, **data: events.append((kind, data)))
return outcome, events
# --------------------------------------------------------------------------
# 截断
# --------------------------------------------------------------------------
async def test_a_cut_off_structured_reply_is_unjudged_as_truncation_without_a_plain_retry(monkeypatch, stored):
model = Reasoner(structured=anthropic_cut, plain=lambda units: AIMessage(content=json.dumps({"items": []})))
Models(monkeypatch, model)
outcome, events = await judged(stored)
assert [kind for kind, _ in model.calls] == ["structured"], "判定写到一半到了上限:不再退回纯文本多花一次额度"
verdicts = list(outcome["verdicts"].values())
assert {(v["status"], v["reason"]) for v in verdicts} == {("unjudged", "truncated")}
assert verdicts[0]["rationale"] == f"{CUT_16K},这句未裁判。{ADVICE}"
assert outcome["unjudged"] == {"truncated": 4} and outcome["limits_hit"] == []
assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"]
[end] = [d for k, d in events if k == EventType.LLM_END]
assert end["purpose"] == "judge" and end["error"] == CUT_16K and end["truncated"] is True
assert end["max_tokens"] == 16384 and end["output_tokens"] == 16384
[log] = logs(events, "judge_truncated")
assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["unjudged"] == 4
assert log["message"] == f"结论句裁判未完成:有 4 句结论未裁判:{CUT_16K},{ADVICE}"
assert logs(events, "judge_failed") == [], "不混进 judge_failed:那一行的下一步是检查接入、重新运行"
def logs(events, code: str) -> list[dict]:
return [d for k, d in events if k == EventType.LOG and d.get("code") == code]
async def test_a_half_written_list_in_a_cut_off_reply_is_not_taken(monkeypatch, stored):
"""截断的回复整批不采信:写完的那条也可能只是理由写完了一半;以前第一条记成「证据支持」、第二条「判定无法识别」、
后面的「未给出判定」,llm.end 也没标截断。"""
model = Reasoner(structured=half_list, plain=lambda units: AIMessage(content=json.dumps({"items": []})))
Models(monkeypatch, model)
outcome, events = await judged(stored)
assert [kind for kind, _ in model.calls] == ["structured"]
assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0
[end] = [d for k, d in events if k == EventType.LLM_END]
assert end["truncated"] is True and end["error"] == CUT_16K
[log] = logs(events, "judge_truncated")
assert log["unjudged"] == 4
async def test_a_plain_reply_cut_after_the_list_is_not_taken_either(monkeypatch, stored):
"""纯文本调用:清单后面还在写、回复是截断的,说不清写完了没有,同样整批记截断。"""
def cut(units):
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False) + "\n补充:第二句的依据还要",
response_metadata={"finish_reason": "length"})
Models(monkeypatch, Reasoner(plain=cut, unstructured=True))
outcome, _ = await judged(stored)
assert outcome["unjudged"] == {"truncated": 4}
async def test_an_openai_length_error_is_truncation_and_its_tokens_are_counted(monkeypatch, stored):
model = Reasoner(structured=lambda units: length_error(completion=16384))
Models(monkeypatch, model)
outcome, events = await judged(stored)
assert [kind for kind, _ in model.calls] == ["structured"]
assert outcome["unjudged"] == {"truncated": 4}
assert outcome["usage"]["output_tokens"] == 16384 and outcome["cost_usd"] > 0, "截断那次的额度是真花出去了"
end = [d for k, d in events if k == EventType.LLM_END][-1]
assert "调用" not in end["error"] and end["error"] == CUT_16K
async def test_a_plain_reply_with_only_thinking_is_truncation(monkeypatch, stored):
"""不支持结构化输出的模型直接退回纯文本调用:回复只有思考、finish_reason=length。"""
model = Reasoner(plain=thinking_only, unstructured=True)
Models(monkeypatch, model)
outcome, _ = await judged(stored)
assert [kind for kind, _ in model.calls] == ["plain"] and outcome["calls"] == 2
assert outcome["unjudged"] == {"truncated": 4}
assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"]
async def test_a_truncated_batch_does_not_stop_the_others(monkeypatch, stored):
"""只有一批被截断:别的批照常判(和格式不对一样,不像调用失败那样后面的都不再问)。"""
first: list[bool] = []
def once(units):
if not first:
first.append(True)
return anthropic_cut(units)
return supported(units)
Models(monkeypatch, Reasoner(structured=once))
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3))
outcome, _ = await judged(stored, raw)
assert outcome["unjudged"] == {"truncated": BATCH_SIZE}
assert outcome["counts"]["supported"] == 3
async def test_truncation_and_other_failures_are_logged_apart(monkeypatch, stored):
"""一批被截断、另一批格式不对:截断记 judge_truncated,格式不对照旧记 judge_failed,各说各的。"""
first: list[bool] = []
def once(units):
if not first:
first.append(True)
return anthropic_cut(units)
return {"raw": AIMessage(content="不是 JSON", response_metadata={"finish_reason": "stop"}), "parsed": None}
Models(monkeypatch, Reasoner(structured=once, plain=lambda units: AIMessage(content="还是不是")))
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3))
outcome, events = await judged(stored, raw)
assert outcome["unjudged"] == {"format": 3, "truncated": BATCH_SIZE}
[cut] = logs(events, "judge_truncated")
[failed] = logs(events, "judge_failed")
assert cut["unjudged"] == BATCH_SIZE and CUT_16K in cut["message"]
assert failed["message"] == "结论句裁判未完成:有 3 句结论未裁判:裁判返回的结果格式有误"
def test_the_review_layer_knows_the_truncation_log():
"""跑完之后的复核层认得 judge_truncated:同 judge_failed,是缺口(要说明),kind 是它自己。"""
assert rv.WARN_CODES["judge_truncated"] == ("judge_truncated", rv.DEGRADED)
event = {"type": str(EventType.LOG), "node_id": "write",
"data": {"level": "warn", "code": "judge_truncated", "message": f"有 4 句结论未裁判:{CUT_16K}"}}
[signal] = rv.scan([event], {"answer": "本周销售额 45,678.5元。"})
assert (signal.kind, signal.severity) == ("judge_truncated", rv.DEGRADED) and not rv.should_retry([signal])
async def test_without_truncation_format_problems_keep_their_wording(monkeypatch, stored):
model = Reasoner(structured=lambda units: {"raw": AIMessage(content="不是 JSON",
response_metadata={"finish_reason": "stop"}),
"parsed": None},
plain=lambda units: AIMessage(content="还是不是", response_metadata={"finish_reason": "stop"}))
Models(monkeypatch, model)
outcome, events = await judged(stored)
assert [kind for kind, _ in model.calls] == ["structured", "plain"]
assert outcome["unjudged"] == {"format": 4}
assert outcome["gaps"] == ["有 4 句结论未裁判:裁判返回的结果格式有误"]
assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END)
assert logs(events, "judge_truncated") == [] and len(logs(events, "judge_failed")) == 1
# --------------------------------------------------------------------------
# 额度与退回
# --------------------------------------------------------------------------
async def test_the_judge_asks_for_the_larger_budget(monkeypatch, stored):
models = Models(monkeypatch, Reasoner())
outcome, _ = await judged(stored)
assert outcome["counts"]["supported"] == 4
assert [s.max_tokens for s in models.specs] == [MAX_TOKENS] == [16384], "平常只建一个模型"
assert "truncated" in REASONS and FALLBACK_MAX_TOKENS == 4096
async def test_a_rejected_budget_falls_back_to_4096_once_for_all_batches(monkeypatch, stored):
model = Reasoner()
models = Models(monkeypatch, model, rejects_above=4096)
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE * 2 + 3))
outcome, _ = await judged(stored, raw)
assert outcome["counts"]["supported"] == BATCH_SIZE * 2 + 3 and outcome["gaps"] == []
assert [s.max_tokens for s in models.specs] == [16384, 4096]
assert models.rejecting.calls == 1, "第一批退回过,后面几批直接用小的"
assert [kind for kind, _ in model.calls] == ["structured"] * 3
async def test_a_rejected_budget_is_remembered_for_the_next_reports(monkeypatch, stored):
"""每份报告、每次点击各问一次裁判:同一接入、模型被嫌大过一次,之后直接用小的,不再每次先撞一次。"""
model = Reasoner()
models = Models(monkeypatch, model, rejects_above=4096)
first, _ = await judged(stored)
second, _ = await judged(stored)
assert first["counts"]["supported"] == 4 and second["counts"]["supported"] == 4
assert models.rejecting.calls == 1, "第二份报告没有再按 16384 撞一次"
async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, stored):
tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. "
"Given: 3000 `inputs` tokens and 16384 `max_new_tokens`", status_code=422)
models = Models(monkeypatch, Reasoner(), rejects_above=4096, rejecting=tgi)
outcome, _ = await judged(stored)
assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == []
assert [s.max_tokens for s in models.specs] == [16384, 4096] and tgi.calls == 1
async def test_a_truncation_after_the_fallback_names_the_small_budget(monkeypatch, stored):
Models(monkeypatch, Reasoner(plain=thinking_only, unstructured=True), rejects_above=4096)
outcome, _ = await judged(stored)
[verdict, *_] = outcome["verdicts"].values()
assert verdict["reason"] == "truncated"
assert verdict["rationale"].startswith("裁判模型的输出达到上限(4096 token)被截断")
# --------------------------------------------------------------------------
# 真实的 ChatOpenAI 和 openai SDK:只有 HTTP 层是假的(经 OpenAI 兼容网关接入的推理模型就是这一路)
# --------------------------------------------------------------------------
class Gateway:
"""OpenAI 兼容网关:reply(这一批的句子编号) 给出 (message, finish_reason, 输出 token 数)。cap 给了,请求的额度超过它
就回 400。bodies 记下每个请求体。"""
def __init__(self, reply, *, cap: int | None = None) -> None:
self.reply, self.cap = reply, cap
self.bodies: list[dict] = []
async def handle(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content or b"{}")
self.bodies.append(body)
budget = body.get("max_completion_tokens") or body.get("max_tokens")
if self.cap and budget and budget > self.cap:
message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens"
return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}},
request=request)
units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M)
message, finish, spent = self.reply(units)
return httpx.Response(200, request=request, json={
"id": "c-1", "object": "chat.completion", "created": 1, "model": PRICED,
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
"usage": {"prompt_tokens": 3000, "completion_tokens": spent, "total_tokens": 3000 + spent}})
def gateway(monkeypatch, reply, *, cap: int | None = None) -> Gateway:
fake = Gateway(reply, cap=cap)
monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle)
provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1",
api_key=None, models=[{"id": PRICED}], default_model=PRICED, enabled=True, extra={})
async def build(session, spec):
return build_chat_model(provider, spec), PRICED
monkeypatch.setattr(judge_mod, "get_chat_model", build)
return fake
async def test_a_real_client_cut_off_in_the_json_schema_call(monkeypatch, stored):
"""json_schema 的结构化输出:finish_reason=length 时 openai SDK 抛 LengthFinishReasonError,不再退回纯文本。"""
fake = gateway(monkeypatch, lambda units: ({"role": "assistant", "content": '{"items": [{"unit": "u'}, "length", 16384))
outcome, _ = await judged(stored)
assert outcome["unjudged"] == {"truncated": 4}
assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"]
assert len(fake.bodies) == 1 and "response_format" in fake.bodies[0]
assert fake.bodies[0]["max_completion_tokens"] == 16384
assert outcome["usage"]["output_tokens"] == 16384, "异常里带着的那次用量照记"
async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, stored):
def answer(units):
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
return {"role": "assistant", "content": json.dumps({"items": items}, ensure_ascii=False)}, "stop", 80
fake = gateway(monkeypatch, answer, cap=8192)
outcome, _ = await judged(stored)
assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == []
assert [b.get("max_completion_tokens") for b in fake.bodies] == [16384, 4096]
async def test_a_real_anthropic_cut_with_a_half_written_list(monkeypatch, stored):
"""真实的 ChatAnthropic:强制工具调用写到第二条判定时到了上限,网关给的是半截清单。整批记截断,llm.end 标出来。"""
def half(body):
units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M)
items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []},
{"unit": units[1]}]
return [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": {"items": items}}]
fake = messages_api(monkeypatch, [(half, "max_tokens", 16384)], model=PRICED, module=judge_mod)
outcome, events = await judged(stored)
assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0
assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384, "没有再退回纯文本"
[end] = [d for k, d in events if k == EventType.LLM_END]
assert end["truncated"] is True and end["error"] == CUT_16K
+344
View File
@@ -0,0 +1,344 @@
"""模型调用的输出额度(providers/budget.py):助手改图、核查、证据裁判共用的那几样。
推理模型的思考也算在输出额度里,额度小了,想完就没额度写正文:回复是空的,或者 JSON 只写了一半。这里守几件事:
- 认得出「输出到了上限被截断」:OpenAI 系 finish_reason=length、Anthropic stop_reason=max_tokens,以及 openai 结构化
输出解析时抛的 LengthFinishReasonError;服务方嫌额度太大的 400 不算截断(那是请求被拒,一个字都没写)
- 认得出「服务方嫌额度太大」:400 / 422,报错里有 max_tokens 一类字样;别的 400 不算
- 截断的那次调用花掉的用量从异常里取得到(额度是真花出去了)
- BudgetFallback:先按大额度调,被嫌大时按小额度重来一次,之后的调用都用小的;流式、非流式都认;bind_tools、
with_structured_output 包出来的和原来的共用同一个开关;只在还没吐出任何东西时重来;小额度的模型要退回时才建
- 核查、证据裁判另认两种「额度被挡下」(budget_refused):自建服务(text-generation-inference 一类)说的
max_new_tokens,和 anthropic SDK 在本地就拒的额度(请求没发出去);助手改图的认法(budget_rejected)不变
- 给了 key(接入、模型、大额度)的,被嫌大一次就记在进程里:之后新建的调用直接用小的,不再每次先撞一次;没给 key 的
(助手改图)不记也不查
模型都是剧本,不碰真接口。
"""
from __future__ import annotations
import pytest
from langchain_core.messages import AIMessage, AIMessageChunk
from openai import LengthFinishReasonError
from openai.types import CompletionUsage
from openai.types.chat import ChatCompletion
from app.providers import budget
from app.providers.budget import (
BUDGET_WORDS,
BudgetFallback,
budget_refused,
budget_rejected,
cut_off,
cut_words,
error_usage,
truncated_error,
)
@pytest.fixture(autouse=True)
def forget_rejections():
"""被嫌大过的接入记在进程里:每个测试从没记过开始,也不留给后面的测试。"""
budget._REJECTED.clear()
yield
budget._REJECTED.clear()
def length_error(prompt: int = 9000, completion: int = 16384) -> LengthFinishReasonError:
"""openai 的结构化输出在 finish_reason=length 时抛的那个错(原话和真实运行里一样)。"""
usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion)
return LengthFinishReasonError(completion=ChatCompletion.model_construct(
id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage))
class Rejected(Exception):
"""仿 openai / anthropic SDK 的 400:一发请求就被拒,还没吐出任何东西。"""
def __init__(self, message: str, status_code: int = 400) -> None:
super().__init__(message)
self.status_code = status_code
TOO_LARGE = "max_tokens: 16384 > 8192, which is the maximum allowed number of output tokens"
#: text-generation-inference 的 OpenAI 兼容接口:输入加额度超过 max_total_tokens 时的 422(原话的样子)
TGI_TOO_LARGE = ("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. "
"Given: 1200 `inputs` tokens and 16384 `max_new_tokens`")
#: anthropic SDK 在本地拒绝时的原话:没设超时的客户端非流式调用、额度超过这个模型不流式能给的上限
SDK_REFUSED = ("Streaming is required for operations that may take longer than 10 minutes. See "
"https://platform.claude.com/docs/en/cli-sdks-libraries/sdks/python#long-requests for more details")
class Model:
"""记下每次调用的剧本模型。fail:每次调用都抛它;stream:流式时吐的块,吐到 boom 处抛 fail。"""
def __init__(self, name: str, *, fail: Exception | None = None, stream: list | None = None,
boom: int | None = None, max_tokens: int | None = None) -> None:
self.name = name
self.fail = fail
self.stream = stream if stream is not None else [AIMessageChunk(content=name)]
self.boom = boom
self.max_tokens = max_tokens
self.calls: list[tuple[str, object]] = []
self.bound: list[tuple[str, object, dict]] = []
async def ainvoke(self, messages, *args, **kwargs):
self.calls.append(("ainvoke", messages))
if self.fail is not None:
raise self.fail
return AIMessage(content=self.name)
async def astream(self, messages, *args, **kwargs):
self.calls.append(("astream", messages))
for i, chunk in enumerate(self.stream):
if self.boom is not None and i == self.boom:
raise self.fail
yield chunk
if self.fail is not None and self.boom is None:
raise self.fail
def bind_tools(self, tools, **kwargs):
self.bound.append(("tools", tools, kwargs))
return self
def with_structured_output(self, schema, **kwargs):
self.bound.append(("structured", schema, kwargs))
return self
# --------------------------------------------------------------------------
# 认出截断、认出嫌额度太大
# --------------------------------------------------------------------------
def test_cut_off_replies_are_recognised_for_both_families():
assert cut_off(AIMessage(content="", response_metadata={"finish_reason": "length"})) == "length"
assert cut_off(AIMessageChunk(content=[], response_metadata={"stop_reason": "max_tokens"})) == "max_tokens"
assert cut_off(AIMessage(content="好了", response_metadata={"finish_reason": "stop"})) == ""
assert cut_off(AIMessage(content="", response_metadata={"stop_reason": "end_turn"})) == ""
assert cut_off(None) == "" and cut_off({"finish_reason": "length"}) == ""
def test_length_errors_are_recognised_but_a_rejected_budget_is_not():
assert truncated_error(length_error())
assert truncated_error(RuntimeError("Could not parse response content as the length limit was reached"))
assert not truncated_error(None)
assert not truncated_error(RuntimeError("网关超时"))
assert not truncated_error(Rejected(TOO_LARGE)), "嫌额度太大是请求被拒,模型一个字都没写,不是截断"
@pytest.mark.parametrize("message", [
"Invalid max_tokens value, the valid range of max_tokens is [1, 8192]",
TOO_LARGE,
"max_completion_tokens is too large: 16384. This model supports at most 8192 completion tokens",
"This model's maximum context length is 32768 tokens. However, you requested 40369 tokens "
"(7601 in the messages, 32768 in the completion).",
])
def test_a_rejected_budget_is_recognised(message):
assert budget_rejected(Rejected(message))
assert budget_rejected(Rejected(message, status_code=422))
def test_other_rejections_are_not_a_budget_problem():
assert not budget_rejected(Rejected("model 'no-such-model' does not exist"))
assert not budget_rejected(Rejected(TOO_LARGE, status_code=500)), "服务方自己出错不是嫌额度大"
assert not budget_rejected(RuntimeError(TOO_LARGE)), "没有状态码的不认"
assert "max_tokens" in BUDGET_WORDS
def test_verify_and_judge_also_recognise_tgi_and_a_local_sdk_refusal():
"""核查、证据裁判的额度从 4096 提到了 16384,才撞得上这两种:自建服务说的是 max_new_tokens;Bearer 鉴权的 Anthropic
接入(客户端没设超时)调 Opus 4、4.1 这一代时,SDK 嫌 16384 太大、请求没发出去就抛 ValueError。"""
assert budget_refused(Rejected(TGI_TOO_LARGE, status_code=422))
assert budget_refused(ValueError(SDK_REFUSED))
assert budget_refused(Rejected(TOO_LARGE)), "原来认的照样认"
assert not budget_refused(ValueError("别的本地错误"))
assert not budget_refused(RuntimeError(SDK_REFUSED)), "只认 SDK 抛的 ValueError"
assert not budget_refused(Rejected("model 'no-such-model' does not exist"))
assert not budget_refused(Rejected(TGI_TOO_LARGE, status_code=500))
def test_the_copilot_wording_list_stays_as_it_was():
"""助手改图按 budget_rejected 认(流式调用,SDK 不在本地拦):认法不变,data/recipe_ai.py 的测试拿 BUDGET_WORDS 比对。"""
assert BUDGET_WORDS == ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length")
assert not budget_rejected(Rejected(TGI_TOO_LARGE, status_code=422))
assert not budget_rejected(ValueError(SDK_REFUSED))
def test_the_tokens_spent_by_a_cut_off_call_come_from_the_error():
assert error_usage(length_error(prompt=9000, completion=16384)) == (9000, 16384)
assert error_usage(RuntimeError("网关超时")) == (0, 0) and error_usage(None) == (0, 0)
def test_the_words_name_who_was_cut_off_and_the_limit():
assert cut_words("核查模型", 16384) == "核查模型的输出达到上限(16384 token)被截断"
assert cut_words("裁判模型", None) == "裁判模型的输出达到上限被截断", "额度不知道就不写数"
# --------------------------------------------------------------------------
# 退回小额度
# --------------------------------------------------------------------------
async def test_a_rejected_budget_is_retried_once_with_the_small_one_and_stays_small():
big, small = Model("big", fail=Rejected(TOO_LARGE)), Model("small")
model = BudgetFallback(big, small)
assert (await model.ainvoke(["问"])).content == "small"
assert (await model.ainvoke(["再问"])).content == "small", "之后的调用都用小的,不再先撞一次"
assert len(big.calls) == 1 and len(small.calls) == 2
assert model.model is small
async def test_other_errors_are_not_retried():
big, small = Model("big", fail=Rejected("model 'no-such-model' does not exist")), Model("small")
with pytest.raises(Rejected):
await BudgetFallback(big, small).ainvoke(["问"])
assert small.calls == []
async def test_the_small_model_is_only_built_when_needed():
built: list[str] = []
small = Model("small")
async def build():
built.append("small")
return small
fine = BudgetFallback(Model("big"), build=build, limits=(16384, 4096))
assert (await fine.ainvoke(["问"])).content == "big"
assert built == [] and fine.max_tokens == 16384, "平常一个小额度的模型都不多建"
rejecting = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=build, limits=(16384, 4096))
assert (await rejecting.ainvoke(["问"])).content == "small"
assert (await rejecting.ainvoke(["再问"])).content == "small"
assert built == ["small"], "退回之后建一次,之后沿用"
assert rejecting.max_tokens == 4096
async def test_without_a_small_model_the_rejection_is_raised():
with pytest.raises(Rejected):
await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE))).ainvoke(["问"])
same = Model("same", fail=Rejected(TOO_LARGE))
with pytest.raises(Rejected):
await BudgetFallback(same, same).ainvoke(["问"])
assert len(same.calls) == 1, "大小是同一个模型:重来也是同样的额度,不白撞第二次"
async def test_bound_and_structured_views_share_the_switch():
"""核查的工具循环里退回过小额度,收尾那次直接用小的;包出来的视图各自转给对应的小模型。"""
big, small = Model("big", fail=Rejected(TOO_LARGE)), Model("small")
model = BudgetFallback(big, small, limits=(16384, 4096))
tools = model.bind_tools(["db_query"], parallel_tool_calls=False)
assert big.bound == [("tools", ["db_query"], {"parallel_tool_calls": False})]
assert small.bound == [("tools", ["db_query"], {"parallel_tool_calls": False})]
assert (await tools.ainvoke(["查"])).content == "small"
structured = model.with_structured_output({"type": "object"}, include_raw=True)
assert (await structured.ainvoke(["交结果"])).content == "small"
assert len(big.calls) == 1, "收尾那次不再先按大额度撞一次"
assert structured.max_tokens == 4096 and model.max_tokens == 4096
async def test_a_sibling_shares_the_switch_and_builds_its_own_small_model():
"""收尾用的是另一份配置(关掉思考)建的模型:和工具循环的模型共用开关,要退回时按自己的配置建小的。"""
built: list[str] = []
loop_small, settle_small = Model("loop-small"), Model("settle-small")
async def build_loop():
built.append("loop")
return loop_small
async def build_settle():
built.append("settle")
return settle_small
loop = BudgetFallback(Model("loop-big", fail=Rejected(TOO_LARGE)), build=build_loop, limits=(16384, 4096))
assert (await loop.ainvoke(["查"])).content == "loop-small"
settle_big = Model("settle-big")
settle = BudgetFallback(settle_big, build=build_settle, limits=(16384, 4096), shared_with=loop)
assert (await settle.with_structured_output({"type": "object"}).ainvoke(["交结果"])).content == "settle-small"
assert settle_big.calls == [] and built == ["loop", "settle"]
async def test_streaming_falls_back_only_before_anything_came_out():
big = Model("big", fail=Rejected(TOO_LARGE), stream=[])
small = Model("small", stream=[AIMessageChunk(content="a"), AIMessageChunk(content="b")])
model = BudgetFallback(big, small)
assert [c.content async for c in model.astream(["问"])] == ["a", "b"]
assert len(big.calls) == 1 and len(small.calls) == 1
started = Model("big", fail=Rejected(TOO_LARGE), stream=[AIMessageChunk(content="x"), AIMessageChunk(content="y")],
boom=1)
unused = Model("small")
got: list[str] = []
with pytest.raises(Rejected):
async for chunk in BudgetFallback(started, unused).astream(["问"]):
got.append(chunk.content)
assert got == ["x"] and unused.calls == [], "已经吐出东西了:重来会重复输出,原样抛出"
async def test_without_limits_max_tokens_reads_the_current_model():
big = Model("big", fail=Rejected(TOO_LARGE), max_tokens=32768)
small = Model("small", max_tokens=8192)
model = BudgetFallback(big, small)
assert model.max_tokens == 32768
await model.ainvoke(["问"])
assert model.max_tokens == 8192
async def test_a_wider_check_can_be_given():
"""核查、证据裁判按 budget_refused 认:服务方说的是 max_new_tokens 也退回小额度。缺省(助手改图)不认。"""
tgi = Rejected(TGI_TOO_LARGE, status_code=422)
model = BudgetFallback(Model("big", fail=tgi), Model("small"), rejected=budget_refused)
assert (await model.ainvoke(["问"])).content == "small"
with pytest.raises(Rejected):
await BudgetFallback(Model("big", fail=tgi), Model("small")).ainvoke(["问"])
local = BudgetFallback(Model("big", fail=ValueError(SDK_REFUSED)), Model("small"), rejected=budget_refused)
structured = local.with_structured_output({"type": "object"})
assert (await structured.ainvoke(["交结果"])).content == "small", "包出来的视图照样按这个认"
async def test_a_rejection_is_remembered_for_later_calls_to_the_same_model():
"""双盲复算的几路盲算、反方审稿的每一句、每次质询、每次点击裁判,各是一组新建的调用:同一接入、模型、额度被嫌大
过一次,之后新建的直接用小的,不再先撞一次。别的模型、别的额度照旧先按大的试。"""
built: list[str] = []
def builder(name: str):
async def build():
built.append(name)
return Model(name)
return build
key = ("gw", "reasoner-x", 16384)
first = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=builder("small-1"), limits=(16384, 4096),
key=key)
assert first.max_tokens == 16384
assert (await first.ainvoke(["问"])).content == "small-1"
later_big = Model("big", fail=Rejected(TOO_LARGE))
later = BudgetFallback(later_big, build=builder("small-2"), limits=(16384, 4096), key=key)
assert later.max_tokens == 4096, "还没调用就知道要用小的:截断时报的额度对得上"
assert (await later.bind_tools(["db_query"]).ainvoke(["再问"])).content == "small-2"
assert later_big.calls == [], "不再先按大额度撞一次"
other = Model("other-big")
assert (await BudgetFallback(other, build=builder("x"), key=("gw", "other", 16384)).ainvoke(["问"])).content \
== "other-big"
bigger = Model("bigger")
assert (await BudgetFallback(bigger, build=builder("y"), key=("gw", "reasoner-x", 32768)).ainvoke(["问"])).content \
== "bigger"
assert built == ["small-1", "small-2"]
async def test_without_a_key_nothing_is_remembered():
"""助手改图不给 key:每次改图照旧先按大额度试(行为和挪过来之前一样)。"""
await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), Model("small")).ainvoke(["问"])
assert budget._REJECTED == set()
fresh = Model("big")
assert (await BudgetFallback(fresh, Model("small")).ainvoke(["问"])).content == "big"
async def test_a_streaming_rejection_is_remembered_too():
key = ("gw", "streamer", 16384)
big = Model("big", fail=Rejected(TOO_LARGE), stream=[])
model = BudgetFallback(big, Model("small", stream=[AIMessageChunk(content="a")]), key=key)
assert [c.content async for c in model.astream(["问"])] == ["a"]
again = Model("big", stream=[AIMessageChunk(content="b")])
assert [c.content async for c in BudgetFallback(again, Model("small", stream=[AIMessageChunk(content="c")]),
key=key).astream(["问"])] == ["c"]
assert again.calls == []
+680
View File
@@ -0,0 +1,680 @@
"""核查 agent(engine/verify_agent.py)的输出额度:截断照实说,服务方嫌额度大时退回小额度。
核查(双盲复算、反方审稿、审批质询共用)原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到
thinking=off,思考就能把 4096 用光:这次调用 finish_reason=length,没有工具调用也没有 JSON。于是工具循环把它当成
「模型说完了」去收尾,双盲复算记成「盲算没有查到这个数」;收尾被截断时记成「核查模型没有按结构交结果」;质询回答失败
——都没说出真正的原因。这里守几件事:
- 额度:默认 16384(judge.MAX_TOKENS,核查和裁判共用);服务方嫌大时退回 4096 再试一次,之后工具循环、收尾都用小的,
只在还没有任何输出时重来;别的 400 不重试。被嫌大过的接入、模型记在进程里,之后的核查直接用小的;自建服务说的
max_new_tokens、anthropic SDK 在本地就拒的额度也认
- 被截断的回复一律不采信:工具循环里截断的回复带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input 是
空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾(同时长到点,算半路被掐断);收尾的强制工具调用被
截断时,解析出来的空对象、半截对象都不算交了结果,也不再退回普通调用多花一次额度。截断那次的用量照记
- 时间线上记一条 verify_truncated(节点里调用时),llm.end 标出这次被截断
- 原因经现有的渠道交出去:双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错。开头只说「未能给出结果」,
建议按模型从哪来的说:节点里节点上指定的模型优先,两处都说;质询只看设置。没有截断时原有说法不变,触顶原因的先后
也不变
模型是剧本,另有真实的 ChatOpenAI、ChatAnthropic(只有 HTTP 层是假的);数据源是现造的 SQLite 文件(借
test_verify_agent 的夹具),表名、数字都是虚构的。
"""
from __future__ import annotations
import json
import uuid
import pytest
from langchain_core.messages import AIMessage, BaseMessage
from openai import LengthFinishReasonError
from openai.types import CompletionUsage
from openai.types.chat import ChatCompletion
from app.core.events import EventType
from app.db.base import SessionLocal
from app.db.models import Provider, Setting
from app.engine import judge as judging
from app.engine import review as rv
from app.engine import spend, verify_agent
from app.engine.inquiry import INQUIRY_SCHEMA, _error_words
from app.engine.nodes.challenge import CHALLENGE_SCHEMA, _claim_of
from app.engine.nodes.dual_check import _slim
from app.providers import budget
from app.providers.factory import build_chat_model
from tests.test_verify_agent import VALID_SUM, Script, clean_spend, priced, q, run, shop # noqa: F401 - 夹具按名字引入
try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上
from langchain_openai._compat import httpx
except ImportError: # pragma: no cover - 老版本的 langchain_openai
import httpx
CUT_16K = "核查模型的输出达到上限(16384 token)被截断"
#: 节点里(双盲复算、反方审稿):节点上指定的模型优先于设置,两处都说
ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型"
#: 不在节点里(审批质询):只看设置
ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型"
FAILED = f"{CUT_16K},未能给出结果。{ADVICE}"
FAILED_SETTINGS = f"{CUT_16K},未能给出结果。{ADVICE_SETTINGS}"
FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致",
"rationale": "未作废订单的金额合计是 [[v:Q1.r0.total]]。"}
#: 反方审稿那种结构写到一半:结论写完了,理由写到一半
HALF_CHALLENGE = {"verdict": "stands", "rationale": "合计是 [[v:Q1.r0.total]],与报告"}
@pytest.fixture(autouse=True)
def forget_rejections():
"""被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。"""
budget._REJECTED.clear()
yield
budget._REJECTED.clear()
def cut_reply(output: int = 16384) -> AIMessage:
"""推理模型把额度全用在思考上:正文为空、没有工具调用,finish_reason=length(DeepSeek 经 LiteLLM 的样子)。"""
return AIMessage(content="", additional_kwargs={"reasoning_content": "先想想该查哪张表……"},
response_metadata={"finish_reason": "length"},
usage_metadata={"input_tokens": 900, "output_tokens": output, "total_tokens": 900 + output})
def cut_call(name: str, args: dict) -> AIMessage:
"""工具调用写到一半到了上限(ChatAnthropic 的样子):tool_use 截在半路时 input 是空的,有的网关给半截对象;
stop_reason=max_tokens。"""
block = {"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": args}
return AIMessage(content=[{"type": "thinking", "thinking": "想了很久……", "signature": "s"}, block],
tool_calls=[{"name": block["name"], "args": args, "id": block["id"]}],
response_metadata={"stop_reason": "max_tokens"},
usage_metadata={"input_tokens": 900, "output_tokens": 16384, "total_tokens": 17284})
def anthropic_cut(args: dict | None = None) -> dict:
"""收尾的强制工具调用写到一半到了上限(结构化输出走 function_calling,ChatAnthropic 的缺省):原始回复
stop_reason=max_tokens。args 为 None 是截在思考里、一个工具调用都没写出来;给了就是截在 tool_use 里——input 是
空的(Anthropic 非流式的样子)或半截对象(有的网关),解析出来的就是它。"""
content = [] if args is None else [{"type": "tool_use", "id": "toolu_1", "name": "verify_result", "input": args}]
calls = [] if args is None else [{"name": "verify_result", "args": args, "id": "toolu_1"}]
raw = AIMessage(content=content, tool_calls=calls, response_metadata={"stop_reason": "max_tokens"},
usage_metadata={"input_tokens": 700, "output_tokens": 16384, "total_tokens": 17084})
return {"raw": raw, "parsed": args, "parsing_error": None}
def length_error(prompt: int = 800, completion: int = 16384) -> LengthFinishReasonError:
"""openai 的结构化输出(json_schema)在 finish_reason=length 时抛的那个错。"""
usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion)
return LengthFinishReasonError(completion=ChatCompletion.model_construct(
id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage))
def query_step(name: str, sql: str = VALID_SUM, call_id: str = "c1") -> AIMessage:
return AIMessage(content="查一下。", tool_calls=[{"name": n, "args": a, "id": i} for n, a, i in q(name, sql, call_id)])
class BudgetRejected(Exception):
"""服务方嫌输出额度太大时的 400(DeepSeek chat 系只收 8K)。"""
def __init__(self, message: str, status_code: int = 400) -> None:
super().__init__(message)
self.status_code = status_code
class Reasoner:
"""核查模型的剧本。steps:工具循环里每一轮的回复,用完了回「查完了。」;structured / plain:收尾时结构化调用、
普通调用的回复(AIMessage、结构化结果,或者一个异常就抛它)。calls 按顺序记下每次调用是哪一种。"""
def __init__(self, steps=(), *, structured=None, plain=None, unstructured: bool = False) -> None:
self.steps = list(steps)
self.structured = structured
self.plain = plain
self.unstructured = unstructured
self.calls: list[str] = []
def bind_tools(self, tools, **kwargs):
return _View(self, "step")
def with_structured_output(self, schema, **kwargs):
if self.unstructured:
raise NotImplementedError("这个模型不支持结构化输出")
return _View(self, "final")
async def ainvoke(self, messages, *args, **kwargs):
return self.answer("plain")
def answer(self, kind: str):
self.calls.append(kind)
if kind == "step":
reply = self.steps.pop(0) if self.steps else AIMessage(content="查完了。")
else:
reply = self.structured if kind == "final" else self.plain
if isinstance(reply, BaseException):
raise reply
return reply
class _View:
def __init__(self, model: Reasoner, kind: str) -> None:
self.model, self.kind = model, kind
async def ainvoke(self, messages, *args, **kwargs):
return self.model.answer(self.kind)
class Rejecting:
"""一开口就被拒的模型:工具循环、收尾都一样。"""
def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens",
status_code: int = 400):
self.message, self.status_code = message, status_code
self.calls = 0
def bind_tools(self, tools, **kwargs):
return self
def with_structured_output(self, schema, **kwargs):
return self
async def ainvoke(self, messages, *args, **kwargs):
self.calls += 1
raise BudgetRejected(self.message, self.status_code)
class Models:
"""按额度给模型:rejects_above 给了,额度比它大的一律交 Rejecting。specs 记下每次建模型的配置。"""
def __init__(self, monkeypatch, model, *, rejects_above: int | None = None, rejecting: Rejecting | None = None):
self.specs = []
self.rejecting = rejecting or Rejecting()
async def fake(session, spec):
self.specs.append(spec)
if rejects_above is not None and (spec.max_tokens or 0) > rejects_above:
return self.rejecting, "reasoner-x"
return model, "reasoner-x"
monkeypatch.setattr(verify_agent, "get_chat_model", fake)
def collector():
events: list[tuple] = []
return events, (lambda kind, **data: events.append((kind, data)))
def truncated_logs(events) -> list[dict]:
return [d for k, d in events if k == EventType.LOG and d.get("code") == "verify_truncated"]
# --------------------------------------------------------------------------
# 工具循环里被截断
# --------------------------------------------------------------------------
async def test_thinking_that_eats_the_budget_in_the_loop_is_reported_as_truncation(monkeypatch, shop):
name, _ = shop
model = Reasoner([cut_reply()], structured={"value": None, "verdict": "无法判断", "rationale": "没有查询结果"})
Models(monkeypatch, model)
events, emit = collector()
out = await run(name, emit=emit, node_id="dc")
assert out.result is None and out.queries == []
assert out.error == FAILED
assert model.calls == ["step"], "一条查询都没有:收尾只能交「查不到」,不再为它多花一次额度"
assert out.answered is False, "被截断不是模型自己停下来的"
assert out.usage["output_tokens"] == 16384, "截断那次的额度是真花出去了"
[log] = truncated_logs(events)
assert log["level"] == "warn" and log["max_tokens"] == 16384 and "delivered" not in log
assert log["purpose"] == "verify" and log["message"] == out.error
[end] = [d for k, d in events if k == EventType.LLM_END]
assert end["truncated"] is True and end["max_tokens"] == 16384 and CUT_16K in end["error"]
@pytest.mark.parametrize("args", [{}, {"sql": "SELECT SUM(amount) AS total FROM orders WHERE is_del"}],
ids=["empty-input", "half-sql"])
async def test_a_tool_call_cut_off_halfway_is_not_run(monkeypatch, shop, args):
"""Anthropic 系:工具调用写到一半到了上限。写到一半的调用不能执行——空参数只会报「缺少必填参数」,半截的 SQL
查出来的数是错的——也不按它收尾,原因照截断说。"""
name, _ = shop
model = Reasoner([cut_call(name, args)], structured=FOUND)
Models(monkeypatch, model)
events, emit = collector()
out = await run(name, emit=emit, node_id="dc")
assert model.calls == ["step"], "不执行、不收尾,也不再问一轮"
assert out.result is None and out.queries == [] and out.error == FAILED
assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_END, EventType.TOOL_ERROR)]
[log] = truncated_logs(events)
assert log["purpose"] == "verify" and log["max_tokens"] == 16384
async def test_a_cut_after_a_query_is_reported_without_settling(monkeypatch, shop):
"""查过一次、下一轮被截断:截断前查到的不一定是要的那个数(模型可能正想改查询),按它收尾交出去的结果说不清完整
不完整,会照常参与双盲复算的比对、反方审稿的处置。同时长到点一样算半路被掐断:不收尾,原因照截断说,查过的照记。"""
name, _ = shop
model = Reasoner([query_step(name), cut_reply()], structured=FOUND)
Models(monkeypatch, model)
events, emit = collector()
out = await run(name, emit=emit, node_id="dc")
assert model.calls == ["step", "step"], "不再为收尾多花一次额度"
assert out.result is None and out.error == FAILED and out.answered is False
assert [q["alias"] for q in out.queries] == ["Q1"], "查过的照记"
[log] = truncated_logs(events)
assert log["purpose"] == "verify" and log["max_tokens"] == 16384
assert not [n for n in out.notes if "截断" in n], "没有「按截断前查到的结果收尾」这一说"
# --------------------------------------------------------------------------
# 收尾被截断
# --------------------------------------------------------------------------
async def test_a_cut_off_settle_is_reported_and_not_retried_as_plain(monkeypatch, shop):
"""Anthropic 系:强制工具调用写出来之前就到了上限。退回普通调用也多半被推理占满,不再白花一次额度。"""
name, _ = shop
model = Reasoner([query_step(name)], structured=anthropic_cut(), plain=AIMessage(content=json.dumps(FOUND)))
Models(monkeypatch, model)
events, emit = collector()
out = await run(name, emit=emit, node_id="dc")
assert model.calls == ["step", "step", "final"], "没有再退回普通调用"
assert out.result is None and out.error == FAILED
[log] = truncated_logs(events)
assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384
end = [d for k, d in events if k == EventType.LLM_END][-1]
assert end["purpose"] == "verify_result" and end["truncated"] is True and CUT_16K in end["error"]
@pytest.mark.parametrize(("schema", "args"), [
(None, {}),
(None, {"value": {"alias": "Q1"}}),
(CHALLENGE_SCHEMA, HALF_CHALLENGE),
(INQUIRY_SCHEMA, {"rationale": "合计是 [[v:Q1.r0.total]],比报告里写的"}),
], ids=["empty-input", "half-cell", "half-challenge", "half-inquiry"])
async def test_a_settle_cut_inside_the_forced_tool_call_is_not_taken(monkeypatch, shop, schema, args):
"""强制工具调用写到一半到了上限:解析出来的是空对象(Anthropic)或半截对象(有的网关)。空对象以前记成「没有按结构
交结果」,半截的格子引用记成「引用不可用」,反方审稿的半截对象(结论写完了、理由写到一半)照单全收成「经受住」,
质询把半句话当成回答——截断的回复一律不采信,原因照截断说,也不退回普通调用。"""
name, _ = shop
model = Reasoner([query_step(name)], structured=anthropic_cut(args), plain=AIMessage(content=json.dumps(FOUND)))
Models(monkeypatch, model)
events, emit = collector()
out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {}))
assert model.calls == ["step", "step", "final"]
assert out.result is None and out.error == FAILED and out.cells == []
[log] = truncated_logs(events)
assert log["purpose"] == "verify_result"
async def test_an_openai_length_error_in_the_settle_is_truncation_not_a_failed_call(monkeypatch, shop):
"""openai 的结构化输出解析时抛 LengthFinishReasonError:不是「调用失败,请稍后重试」,是额度被推理占满。"""
name, _ = shop
model = Reasoner([query_step(name)], structured=length_error(completion=16384))
Models(monkeypatch, model)
events, emit = collector()
out = await run(name, emit=emit, node_id="dc")
assert model.calls == ["step", "step", "final"]
assert out.error == FAILED
assert out.usage["output_tokens"] == 16384 and out.usage["input_tokens"] >= 800, "异常里带着的用量照记"
end = [d for k, d in events if k == EventType.LLM_END][-1]
assert "请稍后重试" not in end["error"] and CUT_16K in end["error"] and end["output_tokens"] == 16384
async def test_a_plain_settle_cut_off_is_reported(monkeypatch, shop):
"""不支持结构化输出的模型:收尾直接走普通调用,回复只有思考、finish_reason=length。"""
name, _ = shop
model = Reasoner([query_step(name)], plain=cut_reply(), unstructured=True)
Models(monkeypatch, model)
out = await run(name)
assert model.calls == ["step", "step", "plain"]
assert out.result is None and out.error == FAILED_SETTINGS
async def test_a_plain_settle_cut_after_the_json_is_not_taken_either(monkeypatch, shop):
"""普通调用的正文里有一段 JSON、回复却是截断的:JSON 后面还在写(多半是理由的补充),说不清写完了没有,同样不采信。"""
name, _ = shop
cut = AIMessage(content=json.dumps(FOUND, ensure_ascii=False) + "\n补充说明:合计里还应该",
response_metadata={"finish_reason": "length"})
model = Reasoner([query_step(name)], plain=cut, unstructured=True)
Models(monkeypatch, model)
out = await run(name)
assert out.result is None and out.error == FAILED_SETTINGS
async def test_without_truncation_the_old_wording_stays(monkeypatch, shop):
name, _ = shop
model = Reasoner([query_step(name)], structured="不是 JSON", plain=AIMessage(content="还是不是 JSON"))
Models(monkeypatch, model)
events, emit = collector()
out = await run(name, emit=emit, node_id="dc")
assert model.calls == ["step", "step", "final", "plain"], "格式不对照旧退回普通调用补救一次"
assert out.error == "核查模型没有按结构交结果"
assert truncated_logs(events) == []
assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END)
async def test_an_empty_object_without_truncation_keeps_the_shape_wording(monkeypatch, shop):
"""没被截断、交回的是空对象:照旧是「没有按结构交结果」,不说成截断。"""
name, _ = shop
whole = {"raw": AIMessage(content="", response_metadata={"stop_reason": "tool_use"}), "parsed": {}}
Models(monkeypatch, Reasoner([query_step(name)], structured=whole))
out = await run(name)
assert out.error == "核查模型没有按结构交结果(交回的内容里没有要求的字段:value、verdict)"
async def test_the_first_limit_hit_is_still_the_one_reported(monkeypatch, shop, priced):
"""循环先因时长到点停下(一分钱还没留),收尾又被每日金额上限拦下:触顶原因照旧报先撞上的时长。双盲复算按它写
无法复算的原因,反方审稿按「每日上限」决定还派不派后面的句子——截断的改动不能把这个先后换掉。"""
name, _ = shop
async with SessionLocal() as session:
session.add(Setting(key=spend.SPEND_KEY, value={"date": spend._today(), "usd": 1.995, "calls": 9}))
await session.commit()
Script(monkeypatch, [q(name, VALID_SUM, "c1")], FOUND, model_id="priced-model")
out = await run(name, daily_max_usd=2.0, timeout_s=0)
assert out.limit == "timeout_s"
assert out.error == "已达全局每日核查金额上限 $2.00,没能交结果"
# --------------------------------------------------------------------------
# 额度与退回
# --------------------------------------------------------------------------
async def test_the_default_budget_leaves_room_for_reasoning(monkeypatch, shop):
name, _ = shop
models = Models(monkeypatch, Reasoner([query_step(name)], structured=FOUND))
out = await run(name)
assert out.error is None and out.result["value"] == 230
assert [s.max_tokens for s in models.specs] == [16384, 16384], "工具循环、收尾都按 16384,不多建模型"
assert models.specs[1].thinking == "off"
async def test_a_rejected_budget_falls_back_to_4096_for_the_loop_and_the_settle(monkeypatch, shop):
name, _ = shop
model = Reasoner([query_step(name)], structured=FOUND)
models = Models(monkeypatch, model, rejects_above=4096)
out = await run(name)
assert out.error is None and out.result["value"] == 230
assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096]
assert [s.thinking for s in models.specs][2:] == ["off", "off"], "收尾的小额度模型同样关掉思考"
assert models.rejecting.calls == 1, "退回过一次之后收尾直接用小的,不再先撞一次"
assert model.calls == ["step", "step", "final"]
async def test_a_rejected_budget_is_remembered_for_the_next_runs(monkeypatch, shop):
"""双盲复算的几路盲算、反方审稿的每一句、每次质询各是一次核查:同一接入、模型被嫌大过一次,之后的核查直接用小的,
不再每次先撞一次。"""
name, _ = shop
model = Reasoner([query_step(name), AIMessage(content="查完了。"), query_step(name, call_id="c2")],
structured=FOUND)
models = Models(monkeypatch, model, rejects_above=4096)
first, second = await run(name), await run(name)
assert first.error is None and second.error is None and second.result["value"] == 230
assert models.rejecting.calls == 1, "第二次核查没有再按 16384 撞一次"
async def test_a_cut_after_the_fallback_names_the_small_budget(monkeypatch, shop):
name, _ = shop
Models(monkeypatch, Reasoner([cut_reply(output=4096)]), rejects_above=4096)
out = await run(name)
assert out.error.startswith("核查模型的输出达到上限(4096 token)被截断")
async def test_other_rejections_are_not_retried_with_the_small_budget(monkeypatch, shop):
name, _ = shop
models = Models(monkeypatch, Reasoner(), rejects_above=4096,
rejecting=Rejecting("model 'reasoner-x' does not exist"))
out = await run(name)
assert out.result is None and "reasoner-x" in out.error
assert [s.max_tokens for s in models.specs] == [16384], "不是额度的问题:不拿小额度再撞一次"
async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, shop):
"""text-generation-inference 一类自建服务:输入加额度超过 max_total_tokens 时回 422,说的是 max_new_tokens。"""
name, _ = shop
tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. "
"Given: 1200 `inputs` tokens and 16384 `max_new_tokens`", status_code=422)
model = Reasoner([query_step(name)], structured=FOUND)
models = Models(monkeypatch, model, rejects_above=4096, rejecting=tgi)
out = await run(name)
assert out.error is None and out.result["value"] == 230
assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096] and tgi.calls == 1
async def test_the_verify_spec_asks_for_the_larger_budget():
"""核查模型的配置(api/settings.py 的 verify_model_spec 沿用证据裁判的额度):16384,思考照旧关掉。"""
assert judging.MAX_TOKENS == 16384 and judging.FALLBACK_MAX_TOKENS == 4096
# --------------------------------------------------------------------------
# 原因经现有的渠道交出去
# --------------------------------------------------------------------------
async def test_the_reason_reaches_dual_check_and_challenge(monkeypatch, shop):
name, _ = shop
Models(monkeypatch, Reasoner([cut_reply()]))
out = await run(name, node_id="dc")
# 双盲复算:无法复算的原因
slim = _slim(out)
assert slim["value"] is None and slim["reason"] == FAILED
# 反方审稿:这句记为无法判断,理由写「反查未完成」,没能反查的原因就是截断
cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="",
aliases=("Q1",))
claim, why = _claim_of(cand, out)
assert claim["verdict"] == "inconclusive" and claim["rationale"] == f"反查未完成:{FAILED}" and why == FAILED
async def test_an_inquiry_names_the_settings_and_not_a_check(monkeypatch, shop):
"""审批质询不在节点里(核查模型只看设置):建议只说设置;开头只说「未能给出结果」——提问人问的是问题,不是在核查。"""
name, _ = shop
Models(monkeypatch, Reasoner([cut_reply()]))
out = await run(name, output_schema=INQUIRY_SCHEMA)
words = _error_words(out, timeout_s=60, daily_max_usd=1.0)
assert words == FAILED_SETTINGS
assert "核查未能完成" not in words and "节点" not in words
def test_the_review_layer_knows_the_truncation_log():
"""跑完之后的复核层(engine/review.py)认得 verify_truncated:同 verify_limit,是缺口不是断裂,kind 是它自己,
不落成笼统的 warn;不触发「重跑一次大概率能好」——同样的模型再跑一遍还是撞同一堵墙。"""
assert rv.WARN_CODES["verify_truncated"] == ("verify_truncated", rv.DEGRADED)
event = {"type": str(EventType.LOG), "node_id": "dc",
"data": {"level": "warn", "code": "verify_truncated", "message": FAILED}}
[signal] = rv.scan([event], {"结论": "本周订单 120 单。"})
assert (signal.kind, signal.severity) == ("verify_truncated", rv.DEGRADED) and not rv.should_retry([signal])
def test_the_reply_shapes_used_here_are_messages():
assert isinstance(cut_reply(), BaseMessage) and isinstance(anthropic_cut()["raw"], BaseMessage)
assert isinstance(cut_call("x", {}), BaseMessage)
# --------------------------------------------------------------------------
# 真实的 ChatOpenAI 和 openai SDK:只有 HTTP 层是假的(经 OpenAI 兼容网关接入的推理模型就是这一路)
# --------------------------------------------------------------------------
GATEWAY_MODEL = "reasoner-v4"
class Gateway:
"""OpenAI 兼容网关:按顺序回 chat.completion(最后一个重复用)。cap 给了,请求的额度超过它就回 400(DeepSeek chat 系
只收 8K 那样)。bodies 记下每个请求体。"""
def __init__(self, replies: list, *, cap: int | None = None) -> None:
self.replies, self.cap = list(replies), cap
self.bodies: list[dict] = []
async def handle(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content or b"{}")
self.bodies.append(body)
budget = body.get("max_completion_tokens") or body.get("max_tokens")
if self.cap and budget and budget > self.cap:
message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens"
return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}},
request=request)
message, finish, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0]
return httpx.Response(200, request=request, json={
"id": "c-1", "object": "chat.completion", "created": 1, "model": GATEWAY_MODEL,
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
"usage": {"prompt_tokens": 900, "completion_tokens": spent, "total_tokens": 900 + spent}})
def gateway(monkeypatch, replies: list, *, cap: int | None = None) -> Gateway:
fake = Gateway(replies, cap=cap)
monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle)
provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1",
api_key=None, models=[{"id": GATEWAY_MODEL}], default_model=GATEWAY_MODEL, enabled=True, extra={})
async def build(session, spec):
return build_chat_model(provider, spec), GATEWAY_MODEL
monkeypatch.setattr(verify_agent, "get_chat_model", build)
return fake
THINKING_ONLY = ({"role": "assistant", "content": "", "reasoning_content": "先看看有哪些表……"}, "length", 16384)
def tool_call(name: str) -> tuple:
return ({"role": "assistant", "content": None, "tool_calls": [{
"id": "c1", "type": "function",
"function": {"name": f"db_query__{name}", "arguments": json.dumps({"sql": VALID_SUM})}}]}, "tool_calls", 120)
async def test_a_real_client_cut_off_in_the_loop(monkeypatch, shop):
name, _ = shop
fake = gateway(monkeypatch, [THINKING_ONLY])
out = await run(name)
assert out.error == FAILED_SETTINGS
assert len(fake.bodies) == 1 and fake.bodies[0]["max_completion_tokens"] == 16384
assert "tools" in fake.bodies[0], "工具循环那次带着只读的查询工具"
async def test_a_real_client_cut_off_with_a_half_written_tool_call(monkeypatch, shop):
"""OpenAI 系:工具调用的参数写到一半到了上限(finish_reason=length,arguments 是半截 JSON)。不执行、不收尾。"""
name, _ = shop
half = ({"role": "assistant", "content": None, "tool_calls": [{
"id": "c1", "type": "function",
"function": {"name": f"db_query__{name}", "arguments": '{"sql": "SELECT SUM(amount) AS total FROM ord'}}]},
"length", 16384)
fake = gateway(monkeypatch, [half])
out = await run(name)
assert out.error == FAILED_SETTINGS and out.queries == [] and len(fake.bodies) == 1
async def test_a_real_client_cut_off_in_the_json_schema_settle(monkeypatch, shop):
"""收尾走 openai 的结构化输出(json_schema):finish_reason=length 时 SDK 抛 LengthFinishReasonError。"""
name, _ = shop
half = ({"role": "assistant", "content": '{"value": {"alias": "Q1", "row'}, "length", 16384)
fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), half])
out = await run(name)
assert out.error == FAILED_SETTINGS and len(out.queries) == 1
assert len(fake.bodies) == 3 and "response_format" in fake.bodies[2], "收尾被截断:没有再退回普通调用"
assert out.usage["output_tokens"] == 120 + 20 + 16384, "异常里带着的那次用量照记"
async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, shop):
name, _ = shop
done = ({"role": "assistant", "content": json.dumps(FOUND, ensure_ascii=False)}, "stop", 60)
fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), done],
cap=8192)
out = await run(name)
assert out.error is None and out.result["value"] == 230
budgets = [b.get("max_completion_tokens") for b in fake.bodies]
assert budgets == [16384, 4096, 4096, 4096], "第一次被拒后退回 4096,之后工具循环和收尾都用小的,不再先撞一次"
# --------------------------------------------------------------------------
# 真实的 ChatAnthropic 和 anthropic SDK:只有 HTTP 层是假的
# --------------------------------------------------------------------------
CLAUDE = "claude-x"
class Messages:
"""Anthropic 的 Messages 接口:按顺序回 message(最后一个重复用)。每个回复是 (content, stop_reason, 输出 token 数),
content 可以是按请求体算的函数(收尾那次强制调用的工具名照请求里的写)。bodies 记下每个请求体。"""
def __init__(self, replies: list) -> None:
self.replies = list(replies)
self.bodies: list[dict] = []
async def handle(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content or b"{}")
self.bodies.append(body)
content, stop, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0]
return httpx.Response(200, request=request, json={
"id": "msg_1", "type": "message", "role": "assistant", "model": body.get("model"),
"content": content(body) if callable(content) else content, "stop_reason": stop, "stop_sequence": None,
"usage": {"input_tokens": 700, "output_tokens": spent}})
def messages_api(monkeypatch, replies: list, *, model: str = CLAUDE, bearer: bool = False,
module=verify_agent) -> Messages:
"""接上一个 Anthropic 接入(bearer 为真时按 Authorization: Bearer 鉴权,factory 会换上没设超时的 SDK 客户端)。"""
fake = Messages(replies)
monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle)
provider = Provider(name=f"an-{uuid.uuid4().hex[:8]}", kind="anthropic", base_url="http://an.example.test",
api_key="test-only-key", models=[{"id": model}], default_model=model, enabled=True,
extra={"auth_style": "bearer"} if bearer else {})
async def build(session, spec):
# 按这个接入的模型建:调用方的 spec 写的是别的模型名(mock-fast),SDK 在本地拦不拦要看真正发出去的 id
return build_chat_model(provider, spec.model_copy(update={"model": model})), model
monkeypatch.setattr(module, "get_chat_model", build)
return fake
def forced(args: dict):
"""收尾那次强制调用的工具:名字照请求里的写,input 是 args。"""
return lambda body: [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": args}]
def query_use(name: str) -> list:
return [{"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {"sql": VALID_SUM}}]
DONE = ([{"type": "text", "text": "查完了。"}], "end_turn", 10)
async def test_a_real_anthropic_cut_in_the_middle_of_a_tool_call(monkeypatch, shop):
"""思考占满了额度、tool_use 截在半路(input 是空的):以前每一轮都执行一遍空参数的查询(报「缺少必填参数 sql」),
撞到步数上限,最后记成「没有按结构交结果」。"""
name, _ = shop
cut = ([{"type": "thinking", "thinking": "想了很久……", "signature": "s"},
{"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {}}], "max_tokens", 16384)
fake = messages_api(monkeypatch, [cut])
events, emit = collector()
out = await run(name, emit=emit, node_id="dc", max_queries=2)
assert out.error == FAILED and out.queries == [] and out.limit is None
assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384
assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_ERROR)]
[log] = truncated_logs(events)
assert log["purpose"] == "verify"
signals = rv.scan([{"type": str(k), "node_id": "dc", "data": d} for k, d in events], {"结论": "本周订单 120 单。"})
assert [s.kind for s in signals] == ["verify_truncated"], "不是工具出错、也不是触顶"
@pytest.mark.parametrize(("schema", "args"), [(None, {}), (CHALLENGE_SCHEMA, HALF_CHALLENGE)],
ids=["empty-input", "half-challenge"])
async def test_a_real_anthropic_settle_cut_inside_the_forced_tool_call(monkeypatch, shop, schema, args):
name, _ = shop
fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE, (forced(args), "max_tokens", 16384)])
events, emit = collector()
out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {}))
assert out.error == FAILED and out.result is None and len(out.queries) == 1
assert len(fake.bodies) == 3 and fake.bodies[2]["tool_choice"]["type"] == "tool", "没有再退回普通调用"
[log] = truncated_logs(events)
assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384
if schema is CHALLENGE_SCHEMA:
cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="",
aliases=("Q1",))
claim, why = _claim_of(cand, out)
assert claim["verdict"] == "inconclusive" and why == FAILED, "半截的「经受住」不能当真"
async def test_an_anthropic_client_refusing_the_budget_locally_falls_back(monkeypatch, shop):
"""Bearer 鉴权的 Anthropic 接入(factory 换上的 SDK 客户端没设超时):Opus 4、4.1 这一代的 id(经网关用的 Vertex、
Bedrock 写法也是)非流式只能给 8192,16384 的请求 anthropic SDK 在本地就拒了(ValueError,请求没发出去)。照服务方
嫌额度大处理,退回 4096。这条守的是 SDK 的行为:anthropic 以后不再在本地拦这个 id,第一个请求就会按 16384 发出去,
到时改期望。"""
name, _ = shop
fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE,
(forced(FOUND), "tool_use", 60)], model="claude-opus-4-1@20250805", bearer=True)
out = await run(name)
assert out.error is None and out.result["value"] == 230
assert [b["max_tokens"] for b in fake.bodies] == [4096, 4096, 4096], "16384 的请求一个都没发出去"
+28
View File
@@ -1039,6 +1039,34 @@ export const AGENT_GUARD_TEXT = {
unlimitedShort: '不限',
} as const
/** 「(16,384 token)」:输出额度,没给就不写 */
const tokenCap = (maxTokens?: number) => (maxTokens ? `(${formatNumber(maxTokens)} token)` : '')
/**
* 时间线上「核查模型的输出到了上限被截断」那一行(engine/verify_agent.py 的 verify_truncated 日志,双盲复算、反方审稿的
* 核查 Agent 发)。推理模型的思考也算在输出额度里,额度用完时回复是空的或只写了一半:照实说是输出额度不够,不说成
* 「没查到」「格式不对」,下一步指到换模型。截断的回复一律不采信,这次核查没有结果,节点按无法复算、未能反查记缺口
*/
export const VERIFY_TRUNCATED_TEXT = {
title: (maxTokens?: number) => `核查模型的输出达到上限${tokenCap(maxTokens)}被截断`,
sub: '这次核查未能给出结果',
next: '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型',
} as const
/**
* 时间线上结论句裁判的输出到了上限被截断(engine/judge.py):被截断的那次调用(llm.end 带 truncated),和
* judge_truncated 日志那一行。不用「裁判未能执行」「检查接入后重新运行」的说法:截断时接入是好的,同一个模型重新运行
* 多半还是截断,该换的是模型
*/
export const JUDGE_TRUNCATED_TEXT = {
/** 被截断的那次调用:原来叫「请裁判模型判断 N 句结论」的那一行 */
call: (maxTokens?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断`,
title: (maxTokens?: number, unjudged?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断`
+ (unjudged ? `,${formatNumber(unjudged)} 句结论未裁判` : ''),
next: '未裁判的结论句记为缺口,不能完整出具。通常是推理占满了输出额度:请在报告撰写节点的「结论句裁判」中指定不带推理的裁判模型,'
+ '或在「设置 → 偏好设置 → 证据裁判」中更换,再重新运行',
} as const
/**
* 上传表格:数据页「表格」标签的上传弹窗、导入回执和卡片。种类的说法和后端 data/tabular.py 的
* CONVERSION_KINDS、SHAPE_KINDS 对得上;后端以后多出来的种类,界面上只显示它给的那句话
+32 -5
View File
@@ -3,7 +3,8 @@ import type { NodeState, RunPhase } from './trace'
import { formatDuration, formatNumber } from '../lib/format'
import {
ABSENCE_CLAIM_TEXT, CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, SCHEMA_PARTIAL_TEXT, SQL_CHECK_TEXT, TYPE_LABEL,
CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, NODE_TYPE_LABEL,
CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, JUDGE_TRUNCATED_TEXT, NODE_TYPE_LABEL,
VERIFY_TRUNCATED_TEXT,
claimTally,
evidenceTally, issuanceLabel, nodeTypeLabel,
} from '../lib/terms'
@@ -765,17 +766,19 @@ function gateTitle(tool: string, verdict: 'allow' | 'escalate' | 'team', reason:
* 双盲复算、反方审稿的日志代码(期 3 前后端共用的约定)。这几条按 code 和结构化字段解读,不看原话;后端发成 info 的
* (一致、裁决、记入数据目录、审稿完成)也上时间线,不然这两种节点就是一个跑了几十秒、什么都没说的黑盒。
* 无法复算(dual_check_unchecked)、未能反查(challenge_unreviewed)、只查了部分数据源(challenge_sources_lost)是
* 期 4 补上的:以前认不得,只能照原话显示
* 期 4 补上的:以前认不得,只能照原话显示。核查模型的输出到了上限被截断(verify_truncated,两种节点共用的核查 Agent 发)
* 是期 5 验收前补上的
*/
const VERIFY_LOG_CODES = new Set([
'dual_check_skipped', 'dual_check_same_model', 'dual_check_agreed', 'dual_check_disagreed', 'dual_check_resolved',
'dual_check_catalog', 'dual_check_limit', 'dual_check_unchecked',
'challenge_skipped', 'challenge_same_model', 'challenge_done', 'challenge_refuted', 'challenge_limit',
'challenge_unreviewed', 'challenge_sources_lost',
'challenge_unreviewed', 'challenge_sources_lost', 'verify_truncated',
])
const DC = DUAL_CHECK_TEXT
const CL = CHALLENGE_TEXT.log
const VT = VERIFY_TRUNCATED_TEXT
/**
* 出具缺口里哪几条是双盲复算、反方审稿报上来的:出具契约给上游节点报的缺口加了「节点类型「节点名」:」的前缀
@@ -1010,6 +1013,15 @@ function explainLog(code: string | undefined, message: string, d: Record<string,
fix: 'canvas',
}
}
case 'judge_truncated': {
// 裁判模型的输出到了上限被截断(judge.py run_request):max_tokens 是那次调用的额度,unjudged 是因此未裁判的句数。
// 读结构化字段,不从原话里认;缺了字段只给不带数的标题。下一步指到换模型:同一个模型重新运行多半还是截断
const limit = typeof d.max_tokens === 'number' && d.max_tokens > 0 ? d.max_tokens : undefined
const n = typeof d.unjudged === 'number' && d.unjudged > 0 ? Math.round(d.unjudged) : undefined
return {
title: JUDGE_TRUNCATED_TEXT.title(limit, n), next: JUDGE_TRUNCATED_TEXT.next, fix: 'canvas', fixField: 'judge',
}
}
case 'judge_failed': {
// 「结论句裁判没跑完:有 2 句结论没裁判:裁判调用失败(…)」
return {
@@ -1214,6 +1226,19 @@ function explainLog(code: string | undefined, message: string, d: Record<string,
level: 'warn',
}
}
case 'verify_truncated': {
// 核查 Agent 的输出到了上限被截断(verify_agent.py):max_tokens 是那次调用的输出额度。截断的回复一律不采信,这次核查
// 没有结果。读结构化字段,不从原话里认;缺了字段(别的服务端版本)只给不带数的标题。入口落到节点的模型:该换的是模型
const limit = typeof d.max_tokens === 'number' && d.max_tokens > 0 ? d.max_tokens : undefined
return {
title: VT.title(limit),
sub: VT.sub,
next: VT.next,
fix: 'canvas',
fixField: 'model',
level: 'warn',
}
}
default:
return null
}
@@ -1912,9 +1937,11 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt
step.detail = [String(d.error), step.detail].filter(Boolean).join('\n')
}
if (d.purpose === 'judge' && d.error) {
// 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败
// 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败。输出到了上限被截断的
// 照实说是截断(不是「未能执行」:接入是好的,该换的是模型,下一步见 judge_truncated 那一行)
const cap = num(d.max_tokens)
step.level = 'warn'
step.title = JUDGE_FAILED_TITLE
step.title = d.truncated === true ? JUDGE_TRUNCATED_TEXT.call(cap && cap > 0 ? cap : undefined) : JUDGE_FAILED_TITLE
step.detail = [String(d.error), step.detail].filter(Boolean).join('\n')
}
pendingLlm.delete(nodeId ?? '_')
+97
View File
@@ -2319,6 +2319,103 @@ await section('核查类日志与出具缺口(期 4):无法复算、未能
rows.map((r) => `${r?.title}|${r?.sub ?? ''}`).join(' ; '))
})
await section('核查模型的输出到了上限被截断(verify_truncated):说清额度、这次核查没有结果,下一步指到换模型', async () => {
// 推理模型的思考也算在输出额度里:核查 Agent 的一次调用 finish_reason=length、没有工具调用也没有 JSON 时,以前双盲复算
// 记成「盲算没有查到这个数」、反方审稿记成「核查模型没有按结构交结果」。现在截断的回复一律不采信,核查 Agent 另发
// verify_truncated(max_tokens、purpose),时间线按结构化字段说人话,原话留在展开区。先核对后端源码还是这么发的
const verifyPy = readFileSync(`${root}backend/app/engine/verify_agent.py`, 'utf8')
check('后端发截断日志的格式还是夹具里这一种', verifyPy.includes('code="verify_truncated"') && verifyPy.includes('max_tokens=limit')
&& verifyPy.includes('purpose=purpose') && !verifyPy.includes('delivered=')
&& verifyPy.includes('TRUNCATED_ERROR = "{cut},未能给出结果。{advice}"'),
'改了 verify_agent.py 的字段或说法就同步改 decode.ts 的 explainLog 和这里的夹具')
const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data })
const ADVICE = '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型'
const FAILED = `核查模型的输出达到上限(16384 token)被截断,未能给出结果。${ADVICE}`
const SMALL = `核查模型的输出达到上限(4096 token)被截断,未能给出结果。${ADVICE}`
const all = flatten(mod.decodeRun([
E(1, 'run.started', null, { total: 2 }),
E(2, 'node.started', 'dc', { node_type: 'dual_check', label: '复核销售额' }),
E(3, 'log', 'dc', { level: 'warn', code: 'verify_truncated', purpose: 'verify', max_tokens: 16384, message: FAILED }),
E(4, 'node.finished', 'dc', { duration_ms: 20 }),
E(5, 'node.started', 'ch', { node_type: 'challenge', label: '审月报' }),
// 退回小额度之后在收尾时被截断
E(6, 'log', 'ch', { level: 'warn', code: 'verify_truncated', purpose: 'verify_result', max_tokens: 4096, message: SMALL }),
// 缺字段、字段类型不对(别的服务端版本):只给不带数的标题,不瞎填
E(7, 'log', 'ch', { level: 'warn', code: 'verify_truncated', max_tokens: '16384', message: FAILED }),
E(8, 'node.finished', 'ch', { duration_ms: 30 }),
E(9, 'run.finished', null, { status: 'succeeded' }),
]))
const rows = all.filter((s) => s.code === 'verify_truncated')
const nodes = all.filter((s) => s.kind === 'node')
check('每条一行,挂在自己的节点下', rows.length === 3 && nodes.find((s) => s.nodeId === 'dc')?.children?.filter((x) => x.code === 'verify_truncated').length === 1
&& nodes.find((s) => s.nodeId === 'ch')?.children?.filter((x) => x.code === 'verify_truncated').length === 2, `${rows.length} 行`)
const [failed, small, bare] = rows
check('标题说输出到了上限被截断、带额度,副标题说这次核查没有结果', failed?.title === '核查模型的输出达到上限(16,384 token)被截断'
&& failed.sub === '这次核查未能给出结果', `${failed?.title} / ${failed?.sub}`)
check('额度照实写(退回小额度之后是 4,096)', small?.title === '核查模型的输出达到上限(4,096 token)被截断' && small.sub === '这次核查未能给出结果',
`${small?.title} / ${small?.sub}`)
check('……下一步指到换不带推理的模型(节点上指定的优先,设置里的也说),入口落到节点的模型,提醒色',
rows.every((s) => !!s.next?.includes('在节点上指定不带推理的模型') && s.next.includes('设置 → 偏好设置 → 核查')
&& s.fix === 'canvas' && s.fixField === 'model' && s.level === 'warn'),
`${failed?.next} / ${failed?.fix} / ${failed?.fixField} / ${failed?.level}`)
check('……没有「已按截断前查到的结果收尾」这一说', !rows.some((s) => /收尾|可能不完整/.test(`${s.title}${s.sub ?? ''}`)))
check('……原话留在展开区', failed?.detail === FAILED && small?.detail === SMALL)
check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '核查模型的输出达到上限被截断'
&& !/undefined|NaN/.test(`${bare?.title}${bare?.sub ?? ''}${bare?.next ?? ''}`), `${bare?.title} / ${bare?.sub}`)
check('标题、副标题里没有内部代码', !rows.some((s) => /verify_truncated|max_tokens|purpose/.test(`${s.title}${s.sub ?? ''}${s.next ?? ''}`)))
check('节点照常完成', nodes.every((s) => s.status === 'done'))
})
await section('结论句裁判的输出到了上限被截断(judge_truncated):调用那一行说截断,日志另起一行,下一步指到换模型', async () => {
// 以前截断的那一批只发 llm.end(标题沿用「结论句裁判未能执行」)和 judge_failed(下一步是「检查接入后重新运行」):
// 截断时接入是好的,同一个模型重新运行多半还是截断。现在调用那一行照 truncated、max_tokens 说截断,另发 judge_truncated
// (max_tokens、unjudged),judge_failed 只说别的失败。先核对后端源码还是这么发的
const judgePy = readFileSync(`${root}backend/app/engine/judge.py`, 'utf8')
check('后端发裁判截断的格式还是夹具里这一种', judgePy.includes('code="judge_truncated", max_tokens=cut_limit')
&& judgePy.includes('unjudged=outcome["unjudged"]["truncated"]') && judgePy.includes('truncated=True, max_tokens=cut_limit')
&& judgePy.includes('r not in LIMITS and r != "truncated"'),
'改了 judge.py 的字段或说法就同步改 decode.ts 的 explainLog、llm.end 和这里的夹具')
const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data })
const CUT = '裁判模型的输出达到上限(16384 token)被截断'
const TRUNCATED = `结论句裁判未完成:有 4 句结论未裁判:${CUT},通常是推理占满了输出额度,可换用不带推理的裁判模型`
const all = flatten(mod.decodeRun([
E(1, 'run.started', null, { total: 1 }),
E(2, 'node.started', 'write', { node_type: 'report', label: '写周报' }),
E(3, 'llm.start', 'write', { model: 'jm-r', structured: true, purpose: 'judge', message_count: 2, units: 4 }),
E(4, 'llm.end', 'write', { model: 'jm-r', purpose: 'judge', duration_ms: 41000, units: 4, input_tokens: 3000,
output_tokens: 16384, error: CUT, truncated: true, max_tokens: 16384 }),
E(5, 'log', 'write', { level: 'warn', code: 'judge_truncated', max_tokens: 16384, unjudged: 4, message: TRUNCATED }),
E(6, 'node.finished', 'write', { duration_ms: 43000 }),
E(7, 'run.finished', null, { status: 'succeeded' }),
]))
const call = all.find((s) => s.kind === 'llm' && s.code === 'judge')
check('调用那一行:说输出到了上限被截断、带额度,不说「未能执行」;提醒色、照常完成,原话在展开区',
call?.title === '裁判模型的输出达到上限(16,384 token)被截断' && call.level === 'warn' && call.status === 'done'
&& !!call.detail?.includes(CUT), `${call?.title} / ${call?.level} / ${call?.status}`)
const row = all.find((s) => s.code === 'judge_truncated')
check('judge_truncated:标题说截断、带额度和几句未裁判', row?.title === '裁判模型的输出达到上限(16,384 token)被截断,4 句结论未裁判',
row?.title)
check('……下一步指到换不带推理的裁判模型(节点的「结论句裁判」、设置都说),不说检查接入', !!row?.next?.includes('不带推理的裁判模型')
&& row.next.includes('结论句裁判') && row.next.includes('设置 → 偏好设置 → 证据裁判') && !row.next.includes('接入')
&& row.fix === 'canvas' && row.fixField === 'judge' && row.level === 'warn', `${row?.next} / ${row?.fix} / ${row?.fixField}`)
check('……原话留在展开区,标题里没有内部代码', row?.detail === TRUNCATED && !/judge_truncated|max_tokens|unjudged/.test(`${row?.title}${row?.next}`))
check('没有混出一条 judge_failed', !all.some((s) => s.code === 'judge_failed'))
// 没截断的失败照旧:「未能执行」那一行和 judge_failed 的说法不变
const other = flatten(mod.decodeRun([
E(1, 'node.started', 'write', { node_type: 'report' }),
E(2, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 2 }),
E(3, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 300, units: 2, error: 'ConnectError: 连不上' }),
E(4, 'node.finished', 'write', { duration_ms: 400 }),
]))
check('没截断的调用失败照旧叫「结论句裁判未能执行」', other.find((s) => s.kind === 'llm')?.title === '结论句裁判未能执行',
other.find((s) => s.kind === 'llm')?.title)
const bare = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report' }),
E(2, 'log', 'w', { level: 'warn', code: 'judge_truncated', max_tokens: 'x', message: '裁判被截断了' })]))
.find((s) => s.code === 'judge_truncated')
check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '裁判模型的输出达到上限被截断'
&& !/undefined|NaN/.test(`${bare?.title}${bare?.next ?? ''}`), bare?.title)
})
await section('审批质询 approval.inquiry(期 4):一行「审批人质询」,标题带问题前 40 字,副标题说回答是否完成、查了几次', async () => {
// 前后端共用的约定:运行停在审批上时服务端追加 approval.inquiry,node_id 为空;data 里有问题、划选的原文、回答正文、
// 回答文档、成没成、触顶原因、查询(queries 和 inquiry_artifacts 两份)。以前这类事件落进「未识别的事件」