mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 00:22:05 +00:00
合入期 5 前置修复 K2:核查与裁判模型的输出额度提到 16384,服务方嫌大时退回 4096(和助手共用 providers/budget.py);每次调用认截断,因推理占满额度没交出结果时照实说(verify_truncated、judge_truncated),不再说成盲算没查到、没按结构交结果
This commit is contained in:
@@ -27,6 +27,10 @@ from app.db.models import Workflow
|
||||
from app.engine.layout import CORRIDOR_MIN, MIN_ROW_GAP, NODE_W, _height, auto_layout
|
||||
from app.engine.schema import GraphSpec, NodeType, ValidationIssue, text_parse_issues, validate_graph
|
||||
from app.engine.state import message_text
|
||||
# 额度用完的认法、退回小额度的做法挪到了 providers/budget.py(核查、证据裁判也用),这里留着老名字;
|
||||
# _BUDGET_WORDS 另有 data/recipe_ai.py 的测试拿来比对
|
||||
from app.providers.budget import BUDGET_WORDS as _BUDGET_WORDS # noqa: F401
|
||||
from app.providers.budget import BudgetFallback as _BudgetFallback, cut_off as _cut_off
|
||||
from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model
|
||||
from app.tools.registry import all_specs
|
||||
|
||||
@@ -1053,54 +1057,12 @@ class CopilotIncomplete(Exception):
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def _cut_off(chunk: Any) -> str:
|
||||
"""这一块是不是在说「额度用完了」:OpenAI 系是 finish_reason=length,Anthropic 是 stop_reason=max_tokens。"""
|
||||
meta = getattr(chunk, "response_metadata", None) or {}
|
||||
reason = meta.get("finish_reason") or meta.get("stop_reason")
|
||||
return str(reason) if reason in ("length", "max_tokens") else ""
|
||||
|
||||
|
||||
#: 流式改图的输出额度。会思考的模型(DeepSeek、Claude 的 adaptive thinking)思考也算在里面:
|
||||
#: 8192 时复杂的改图常常想完就没额度了,正文一个字没写(2026-09 部署实例上同类请求 5 轮坏了 4 轮)
|
||||
COPILOT_MAX_TOKENS = 32768
|
||||
#: 服务方不收大额度时退回这个(原来的默认值)
|
||||
#: 服务方不收大额度时退回这个(原来的默认值)。退回的做法(_BudgetFallback:只在还没吐出任何东西时重来,
|
||||
#: 之后的调用即自查修正轮都用小的)和截断的认法(_cut_off)在 providers/budget.py,核查、证据裁判也用它
|
||||
_FALLBACK_MAX_TOKENS = 8192
|
||||
#: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K),
|
||||
#: 或者输入加额度超过上下文长度(本地 vLLM)
|
||||
_BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length")
|
||||
|
||||
|
||||
def _budget_rejected(e: BaseException) -> bool:
|
||||
status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None)
|
||||
return status in (400, 422) and any(w in str(e).lower() for w in _BUDGET_WORDS)
|
||||
|
||||
|
||||
class _BudgetFallback:
|
||||
"""先按大额度调;服务方一开口就嫌额度太大时按原来的额度重来,之后的调用(自查修正轮)都用小的。
|
||||
|
||||
只在还没吐出任何东西时重来:这类拒绝都发生在请求刚发出去的时候,重来不会重复输出。
|
||||
"""
|
||||
|
||||
def __init__(self, big: Any, small: Any) -> None:
|
||||
self.model, self._small = big, small
|
||||
|
||||
@property
|
||||
def max_tokens(self) -> Any:
|
||||
return getattr(self.model, "max_tokens", None)
|
||||
|
||||
async def astream(self, messages: list[Any]):
|
||||
started = False
|
||||
try:
|
||||
async for chunk in self.model.astream(messages):
|
||||
started = True
|
||||
yield chunk
|
||||
return
|
||||
except Exception as e:
|
||||
if started or self.model is self._small or not _budget_rejected(e):
|
||||
raise
|
||||
self.model = self._small
|
||||
async for chunk in self.model.astream(messages):
|
||||
yield chunk
|
||||
|
||||
|
||||
async def _op_model(session: AsyncSession, payload: GenerateIn) -> tuple[Any, str]:
|
||||
|
||||
@@ -549,7 +549,9 @@ async def verify_model_spec(session: AsyncSession, node_config: dict[str, Any] |
|
||||
"""核查用哪个模型:节点配置里的 provider / model > 设置里的核查模型 > 证据裁判模型的取值链(裁判设置 >
|
||||
助手模型 > 默认接入,见 judge.judge_model_spec)。哪一级写了就整组用哪一级,不跨级拼接入和模型。
|
||||
|
||||
和裁判一样关掉 thinking、max_tokens 4096:核查最后要按结构交结果(强制工具调用),和思考模式不兼容。
|
||||
和裁判一样关掉 thinking:核查最后要按结构交结果(强制工具调用),和思考模式不兼容。额度也和裁判一样(judge.MAX_TOKENS,
|
||||
16384):经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考算在额度里,4096 常常想完就没额度查库、交结果了;
|
||||
服务方嫌大时核查 agent 退回 judge.FALLBACK_MAX_TOKENS(engine/verify_agent.py)。
|
||||
建议和被核查的模型不同,否则等于自己核自己;比较、提醒由调用方做(它知道被核查的是哪个模型)。
|
||||
"""
|
||||
node = node_config if isinstance(node_config, dict) else {}
|
||||
|
||||
+80
-10
@@ -12,7 +12,9 @@
|
||||
(按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过。带期 4 文档标记的报告,上传表格的
|
||||
查询在 SQL 之后另有来历行(来源、口径、已接受、区域外文字,_provenance_lines)
|
||||
3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析,
|
||||
再失败就记未裁判。thinking 关掉,max_tokens 4096
|
||||
再失败就记未裁判。thinking 关掉,输出额度 MAX_TOKENS(推理模型的思考也算在里面),服务方嫌大时退回
|
||||
FALLBACK_MAX_TOKENS,被嫌大过的接入、模型之后直接用小的;输出到了上限被截断的整批记未裁判(truncated),
|
||||
照实说是额度不够、不再退回纯文本多花一次,时间线上另记 judge_truncated
|
||||
4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算,
|
||||
超了按优先级截断;触顶就停,已判的保留,没判的记未裁判并说明是哪个上限——不报错
|
||||
5. 每日计数落在设置表(SPEND_KEY),按本地日期滚动。目录里没有价格的模型估不出金额,金额上限对它
|
||||
@@ -38,7 +40,7 @@ from dataclasses import asdict, dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
|
||||
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
@@ -69,6 +71,7 @@ from app.engine.merge_query import MERGE_SOURCE
|
||||
from app.engine.nodes.llm import _split_structured, _usage_of
|
||||
from app.engine.state import message_text
|
||||
from app.providers import catalog as pricing
|
||||
from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error
|
||||
from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -104,7 +107,13 @@ BATCH_SIZE = 8
|
||||
#: 一波最多几批同时问:几批共用剩下的时长,句数上限内的句子一两波就判完。第一批单独问:接入调不通(鉴权、
|
||||
#: 网关挂了)时只撞这一次,不让一整波一起失败
|
||||
PARALLEL_BATCHES = 4
|
||||
MAX_TOKENS = 4096
|
||||
#: 裁判(和核查:api/settings.py 的 verify_model_spec、engine/verify_agent.py 沿用这一档)的输出额度。thinking 关掉只对
|
||||
#: Anthropic 有用:经 OpenAI 兼容接入的推理模型(DeepSeek、GLM、Kimi 这一类)收不到 thinking=off,思考算在额度里,原来的
|
||||
#: 4096 常常想完就没额度写判定了(cite_fields 的抽取在 2026-10 撞过同一堵墙,那边也改成了 16384)。16384 是常见网关、
|
||||
#: 模型接受的输出上限里偏大的一档(GPT-4o 的上限就是它);更小的服务方会拒收,那时退回 FALLBACK_MAX_TOKENS
|
||||
MAX_TOKENS = 16384
|
||||
#: 服务方嫌 MAX_TOKENS 太大(400 / 422,报错里有 max_tokens 一类字样)时退回这个:原来的额度,各家都收。之后的调用都用它
|
||||
FALLBACK_MAX_TOKENS = 4096
|
||||
#: 证据摘录:SQL 前几个字、每次查询最多几行几列、每段知识库片段最多几个字、整份引用的检索给几段。
|
||||
#: 列数上限只管没被引用的列:句子引用的格子所在的列一律给(遮罩的除外),再用别的列补满到这么多列
|
||||
SQL_CHARS = 300
|
||||
@@ -124,9 +133,12 @@ SHORT_CHARS = 6
|
||||
_OUT_PER_UNIT = 120
|
||||
_OUT_BASE = 20
|
||||
|
||||
#: 未裁判的原因。前四个是上限,报告里写「已到上限」
|
||||
#: 未裁判的原因。前四个是上限,报告里写「已到上限」;truncated 是裁判模型的输出到了上限被截断(推理模型的思考也算在
|
||||
#: 额度里),和格式不对分开记:说成「格式有误」,人会去查提示词,真正该换的是模型
|
||||
LIMITS = ("max_claims", "max_cost_usd", "daily_max_usd", "timeout_s")
|
||||
REASONS = (*LIMITS, "error", "format", "missing", "on_demand")
|
||||
REASONS = (*LIMITS, "error", "format", "truncated", "missing", "on_demand")
|
||||
#: 输出被截断时给人看的建议,接在 budget.cut_words 的「裁判模型的输出达到上限(N token)被截断」后面
|
||||
TRUNCATED_ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型"
|
||||
|
||||
JUDGE_ROLE = "你是报告核查员:逐句判断报告里的话,能不能被它挂的证据支持。你不写报告,也不改报告。"
|
||||
|
||||
@@ -897,7 +909,8 @@ def _text(value: Any) -> str | None:
|
||||
async def judge_model_spec(session: Any, node_judge: dict[str, Any] | None = None, *,
|
||||
settings: dict[str, Any] | None = None) -> ModelSpec:
|
||||
"""裁判用哪个模型:节点上的 judge.model(连同 provider)> 设置里的证据裁判模型 > Copilot 模型 >
|
||||
默认 provider。哪一级写了就整组用哪一级,不跨级拼 provider 和模型。thinking 关、max_tokens 4096。
|
||||
默认 provider。哪一级写了就整组用哪一级,不跨级拼 provider 和模型。thinking 关、额度 MAX_TOKENS(推理模型关不掉
|
||||
思考,见 MAX_TOKENS 的说明;服务方嫌大时 run_request 退回 FALLBACK_MAX_TOKENS)。
|
||||
|
||||
建议和写作模型不同,否则等于自己审自己(报告节点发现相同会发 judge_same_model 警告)。
|
||||
"""
|
||||
@@ -1058,12 +1071,20 @@ def _items_of(value: Any) -> list[Any] | None:
|
||||
return value if isinstance(value, list) else None
|
||||
|
||||
|
||||
#: _ask 交回的没成的原因:输出到了上限被截断
|
||||
_TRUNCATED = "truncated"
|
||||
|
||||
|
||||
async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | None, list[BaseMessage], str | None,
|
||||
int]:
|
||||
"""问一次裁判:(判定清单, 模型的原始回复, 没成的原因, 调了几次模型)。
|
||||
|
||||
先走结构化输出;不成(模型不支持、给的不是合法结构)先从原始回复里找 JSON,再退回纯文本调用、
|
||||
从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对。超时由调用方掐。
|
||||
从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对,truncated 是输出到了上限被截断
|
||||
(Anthropic 的 stop_reason=max_tokens、OpenAI 系的 finish_reason=length,以及 openai 结构化输出抛的
|
||||
LengthFinishReasonError)。截断的回复整批不采信:强制工具调用截在半路时 Anthropic 给的是空对象,有的网关给半截的
|
||||
清单——最后一条可能只写了编号,也可能判定写完了、理由写到一半,分不清哪条是完整的。结构化调用被截断的不再退回
|
||||
纯文本:推理占满额度时纯文本调用多半一样被截断,白花一次额度。超时由调用方掐。
|
||||
"""
|
||||
raws: list[BaseMessage] = []
|
||||
try:
|
||||
@@ -1071,6 +1092,8 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non
|
||||
raw, parsed = _split_structured(got)
|
||||
if raw is not None:
|
||||
raws.append(raw)
|
||||
if cut_off(raw):
|
||||
return None, raws, _TRUNCATED, 1
|
||||
if (items := _items_of(parsed)) is not None:
|
||||
return items, raws, None, 1
|
||||
if raw is not None:
|
||||
@@ -1078,17 +1101,45 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non
|
||||
for found in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]:
|
||||
if (items := _items_of(found)) is not None:
|
||||
return items, raws, None, 1
|
||||
except Exception: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析
|
||||
pass
|
||||
except Exception as e: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析
|
||||
if truncated_error(e):
|
||||
return None, [*raws, *_spent(e)], _TRUNCATED, 1
|
||||
try:
|
||||
reply = await model.ainvoke(messages)
|
||||
except Exception as e: # noqa: BLE001
|
||||
if truncated_error(e):
|
||||
return None, [*raws, *_spent(e)], _TRUNCATED, 2
|
||||
return None, raws, f"error:{describe_exception(e)}", 2
|
||||
raws.append(reply)
|
||||
if cut_off(reply):
|
||||
# 正文里就算抠得出一段清单,后面还在写,说不清写完了没有:同样整批不采信
|
||||
return None, raws, _TRUNCATED, 2
|
||||
items = _items_of(message_text(reply))
|
||||
return items, raws, None if items is not None else "format", 2
|
||||
|
||||
|
||||
def _spent(error: BaseException) -> list[BaseMessage]:
|
||||
"""截断时 openai 的异常带着这次的用量:折成一条空回复交给记账(额度是真花出去了,每日上限不能漏掉它)。"""
|
||||
inp, out = error_usage(error)
|
||||
if not (inp or out):
|
||||
return []
|
||||
return [AIMessage(content="", usage_metadata={"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out})]
|
||||
|
||||
|
||||
def _with_fallback(model: Any, spec: ModelSpec, model_id: str) -> BudgetFallback:
|
||||
"""裁判模型包一层:服务方嫌 MAX_TOKENS 太大(或者 anthropic SDK 在本地就拒了)时按 FALLBACK_MAX_TOKENS 重建一个再问
|
||||
(要退回时才建,平常不多建),之后各批都用小的。被嫌大过的按 (接入, 模型, 额度) 记在进程里:之后每份报告、每次
|
||||
点击直接用小的,核查(verify_agent._with_fallback)同一种记法、互相认。经本模块的 get_chat_model 建:测试换成剧本
|
||||
时,小额度的那个也是剧本。"""
|
||||
|
||||
async def build() -> Any:
|
||||
async with SessionLocal() as session:
|
||||
return (await get_chat_model(session, spec.model_copy(update={"max_tokens": FALLBACK_MAX_TOKENS})))[0]
|
||||
|
||||
return BudgetFallback(model, build=build, limits=(spec.max_tokens, FALLBACK_MAX_TOKENS), rejected=budget_refused,
|
||||
key=(spec.provider, model_id, spec.max_tokens))
|
||||
|
||||
|
||||
def _unjudged(reason: str, rationale: str, *, judge: str | None, post_seal: bool) -> dict[str, Any]:
|
||||
return {"status": "unjudged", "rationale": rationale, "judge": judge, "post_seal": post_seal, "reason": reason}
|
||||
|
||||
@@ -1116,6 +1167,8 @@ def _why_unjudged(reason: str, budget: Budget, detail: str = "", *, click: bool
|
||||
return f"裁判失败:{detail}"[:120]
|
||||
if reason == "format":
|
||||
return detail or "裁判返回的结果格式有误,这句未裁判"
|
||||
if reason == "truncated":
|
||||
return f"{detail or cut_words('裁判模型', None)},这句未裁判。{TRUNCATED_ADVICE}"
|
||||
if reason == "missing":
|
||||
return "裁判未对这句给出判定"
|
||||
return "探索运行按需裁判:点开这句并请模型判断后才会裁判"
|
||||
@@ -1149,6 +1202,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
model, model_id = await get_chat_model(session, spec)
|
||||
model = _with_fallback(model, spec, model_id)
|
||||
except Exception as e: # noqa: BLE001 - 裁判模型没配好:这回判不了,报告照常
|
||||
why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e)
|
||||
details["error"] = f"裁判模型不可用({why})"
|
||||
@@ -1185,6 +1239,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
|
||||
|
||||
spent = 0.0
|
||||
width = 1
|
||||
cut_limit: int | None = None # 被截断的那次调用的额度:judge_truncated 日志照它写
|
||||
while pending:
|
||||
remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started)
|
||||
if remaining is not None and remaining <= 0:
|
||||
@@ -1261,6 +1316,14 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
|
||||
end = {"model": model_id, "purpose": "judge", "duration_ms": int((time.perf_counter() - began) * 1000),
|
||||
"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out,
|
||||
"cost_usd": round(actual, 6), "calls": calls, "units": len(batch)}
|
||||
if problem == _TRUNCATED:
|
||||
# 输出到了上限被截断:这一批记未裁判,原因照实说是额度,不说「格式有误」「调用失败」。别的批照常问
|
||||
# (同格式不对):截断取决于这一批要想多久,不像鉴权、网关出错那样后面几批多半一样
|
||||
cut_limit = model.max_tokens
|
||||
details["truncated"] = cut_words("裁判模型", cut_limit)
|
||||
emit(EventType.LLM_END, **end, error=details["truncated"], truncated=True, max_tokens=cut_limit)
|
||||
stop(batch, "truncated", details["truncated"])
|
||||
continue
|
||||
if problem is not None and problem != "format":
|
||||
reason = "timeout_s" if problem == "timeout_s" else "error"
|
||||
detail = problem.partition(":")[2]
|
||||
@@ -1297,7 +1360,12 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
|
||||
n = sum(outcome["unjudged"].get(r, 0) for r in hit)
|
||||
emit(EventType.LOG, level="warn", code="judge_limit", limits=hit, unjudged=n,
|
||||
message=f"结论句裁判已达上限({labels}):{n} 句未裁判,已裁判的结果保留")
|
||||
if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS]:
|
||||
# 输出被截断单独记一条(带那次的额度、几句未裁判):judge_failed 在时间线上的下一步是检查接入、重新运行,截断时接入
|
||||
# 是好的,同一个模型重新运行多半还是截断,该换的是模型
|
||||
if truncated := [g for r, g in outcome["_gaps"] if r == "truncated"]:
|
||||
emit(EventType.LOG, level="warn", code="judge_truncated", max_tokens=cut_limit,
|
||||
unjudged=outcome["unjudged"]["truncated"], message="结论句裁判未完成:" + ";".join(truncated))
|
||||
if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS and r != "truncated"]:
|
||||
emit(EventType.LOG, level="warn", code="judge_failed", message="结论句裁判未完成:" + ";".join(failed))
|
||||
outcome.pop("_gaps")
|
||||
return outcome
|
||||
@@ -1394,6 +1462,8 @@ def _outcome(request: JudgeRequest, verdicts: dict[str, dict[str, Any]], budget:
|
||||
text = f"有 {n} 句结论未裁判:裁判调用失败({details.get('error') or '原因不明'})"
|
||||
elif reason == "format":
|
||||
text = f"有 {n} 句结论未裁判:裁判返回的结果格式有误"
|
||||
elif reason == "truncated":
|
||||
text = f"有 {n} 句结论未裁判:{details.get('truncated') or cut_words('裁判模型', None)},{TRUNCATED_ADVICE}"
|
||||
else:
|
||||
text = f"有 {n} 句结论未获得裁判结果"
|
||||
gaps.append((reason, text))
|
||||
|
||||
@@ -104,6 +104,9 @@ WARN_CODES: dict[str, tuple[str, str]] = {
|
||||
# 结论句裁判(claims: judge):触顶、没跑成的句子记为未裁判,出具不能判完整——是缺口,要说明
|
||||
"judge_limit": ("judge_limit", DEGRADED),
|
||||
"judge_failed": ("judge_failed", DEGRADED),
|
||||
# 裁判模型的输出到了上限被截断(推理模型的思考也算在额度里):同 judge_failed 是缺口,记成它自己的 kind——说得清
|
||||
# 是输出额度的问题,该换的是模型
|
||||
"judge_truncated": ("judge_truncated", DEGRADED),
|
||||
# 证据不支持的句子交回写作者改过一次(和 report_repair 一样只记缺口);改写稿冒出新问题没采用的,
|
||||
# 原稿里那几句仍是证据不支持
|
||||
"report_rewrite": ("report_rewrite", DEGRADED),
|
||||
@@ -128,6 +131,9 @@ WARN_CODES: dict[str, tuple[str, str]] = {
|
||||
# 核查 agent 自己触顶、按已有结果收尾:多半和节点自己的 *_limit 说的是同一次触顶,照样是缺口,记成它自己的 kind,
|
||||
# 不落成笼统的 warn——步数上限(max_steps)只有它说
|
||||
"verify_limit": ("verify_limit", DEGRADED),
|
||||
# 核查 agent 的输出到了上限被截断(推理模型的思考也算在额度里):没交成结果,由节点按无法复算、未能反查记缺口。
|
||||
# 同 verify_limit,是缺口不是断裂,记成它自己的 kind:说得清是输出额度的问题,换个模型就好
|
||||
"verify_truncated": ("verify_truncated", DEGRADED),
|
||||
}
|
||||
|
||||
# 这些 kind 重跑一次大概率能好:原因说得清,且不是「问题本身没问明白」。
|
||||
|
||||
@@ -25,6 +25,15 @@ query_artifact 和 role: verify——随运行封存、证据面板能下钻。
|
||||
每日上限不传就按设置里的「核查每日金额上限」,明确传 None 才是不限。模型不在价格目录里时估不出金额,金额上限
|
||||
对它无效,照裁判的说法记一条说明。每次调用都记进核查的每日账(engine/spend.py)。
|
||||
|
||||
**输出额度。** 经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考也算在输出额度里:额度按 spec(缺省 judge.MAX_TOKENS),
|
||||
服务方嫌大时(或者 anthropic SDK 在本地就拒了)退回 judge.FALLBACK_MAX_TOKENS 再试一次,之后工具循环、收尾都用小的;
|
||||
同一接入、模型被嫌大过,之后的核查一开始就用小的(providers/budget.py)。每次调用都看有没有到了上限被截断,截断的回复
|
||||
一律不采信:工具循环里截断的回复不算模型自己停下,带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input
|
||||
是空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾——同时长到点一样算半路被掐断,按截断前查到的收尾
|
||||
交出去的结果说不清完整不完整,还会照常参与比对和处置。收尾的强制工具调用被截断时,解析出来的空对象、半截对象都不算
|
||||
交了结果,也不再退回普通调用多花一次额度。因为截断没交成结果的,error 照实说是输出额度(TRUNCATED_ERROR),不说成
|
||||
「没查到」「没按结构交」;节点里调用时另记一条 verify_truncated。
|
||||
|
||||
永远不抛:模型没配好、调用失败、格式不对、触顶、库写不进去,都落在返回值里(error / limit)。模型调用都经
|
||||
get_chat_model,测试换成剧本。
|
||||
"""
|
||||
@@ -49,11 +58,12 @@ from app.db.base import SessionLocal
|
||||
from app.engine import spend
|
||||
from app.engine.evidence import build_catalog, cell_eid, compose_doc, query_entry_fields, schema_ledger_entry
|
||||
from app.engine.expressions import CellError, column_kind, locate_cell, numeric_text
|
||||
from app.engine.judge import _json_in, _tokens, _usd
|
||||
from app.engine.judge import FALLBACK_MAX_TOKENS, MAX_TOKENS, _json_in, _tokens, _usd
|
||||
from app.engine.nodes.llm import _DIGEST_BUDGET, _digest, _split_structured, _typed, explain_model_error
|
||||
from app.engine.state import message_text
|
||||
from app.engine.toolcalls import ToolTimeout, limit_fields, limit_of, run_bounded
|
||||
from app.providers import catalog as pricing
|
||||
from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error
|
||||
from app.providers.factory import ModelSpec, ProviderNotConfigured, bind_tools_safely, get_chat_model
|
||||
from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX, build_datasource_tools
|
||||
from app.tools.registry import ToolArgsError, ToolContext, Versions, args_model_of, prepare_args
|
||||
@@ -79,6 +89,14 @@ _REF_DEPTH = 8
|
||||
RATIONALE = "rationale"
|
||||
#: 触顶原因
|
||||
LIMITS = ("max_queries", "timeout_s", "max_cost_usd", "daily_max_usd", "max_steps")
|
||||
#: 核查模型的输出到了上限被截断、没交成结果时给人看的话:双盲复算的无法复算、反方审稿的「反查未完成」、质询的报错都照
|
||||
#: 它说,也是 verify_truncated 日志的原话。cut 是 budget.cut_words 的「核查模型的输出达到上限(N token)被截断」。开头只说
|
||||
#: 「未能给出结果」:质询里提问人看到的就是这句,它问的是问题,不是在核查
|
||||
TRUNCATED_ERROR = "{cut},未能给出结果。{advice}"
|
||||
#: 截断的建议:该换的是模型。节点里(双盲复算、反方审稿)节点上指定的模型优先于设置(api/settings.py 的
|
||||
#: verify_model_spec),两处都说,和时间线上的下一步一致;不在节点里(审批质询)核查模型只看设置
|
||||
TRUNCATED_ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型"
|
||||
TRUNCATED_ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型"
|
||||
|
||||
VERIFY_SYSTEM = (
|
||||
"你是数据核查员:只读地查库,用查询结果完成交给你的任务。\n"
|
||||
@@ -129,7 +147,8 @@ class VerifyOutcome:
|
||||
source, tool, call_id}。
|
||||
rationale:理由组装成的报告文档(evidence.compose_doc 的形状,violations 里有裸数字)。usage:{input_tokens,
|
||||
output_tokens, total_tokens, cost_usd, calls, unpriced_calls}。limit:触顶原因(LIMITS 之一),没触顶为 None。
|
||||
answered:模型是不是自己停下来的(没被上限掐断)。
|
||||
answered:模型是不是自己停下来的(没被上限掐断,也没因为输出到了上限被截断)。被截断的一律没有结果,error 是
|
||||
TRUNCATED_ERROR。
|
||||
"""
|
||||
|
||||
result: dict[str, Any] | None
|
||||
@@ -686,13 +705,16 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
def left() -> float | None:
|
||||
return None if deadline is None else deadline - time.monotonic()
|
||||
|
||||
# ---- 模型
|
||||
# ---- 模型。额度没写的按 MAX_TOKENS:推理模型的思考算在输出额度里,4096 常常想完就没额度发查询、交结果了
|
||||
spec = spec.model_copy(update={"max_tokens": spec.max_tokens or MAX_TOKENS})
|
||||
small = FALLBACK_MAX_TOKENS if spec.max_tokens > FALLBACK_MAX_TOKENS else None
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
base_model, model_id = await get_chat_model(session, spec)
|
||||
first, model_id = await get_chat_model(session, spec)
|
||||
except Exception as e: # noqa: BLE001 - 核查模型没配好:这回核不了,照实交回
|
||||
why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e)
|
||||
return outcome(error=f"核查模型不可用({why})")
|
||||
base_model = _with_fallback(first, spec, small, model_id)
|
||||
state.model = model_id
|
||||
if daily_max_usd is Default.SETTINGS:
|
||||
from app.api.settings import verify_settings # 在这里导入:api.settings 导入了 engine 里的模块
|
||||
@@ -743,7 +765,8 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
raw = got if isinstance(got, BaseMessage) else _split_structured(got)[0]
|
||||
except (asyncio.TimeoutError, Exception) as e: # noqa: BLE001 - 调用失败落在返回值里
|
||||
failure = e
|
||||
inp, out = _usage(raw)
|
||||
# openai 的结构化输出被截断时抛异常,用量在异常里:额度是真花出去了
|
||||
inp, out = _usage(raw) if raw is not None else error_usage(failure)
|
||||
cost = await budget.settle(day, estimate, inp, out)
|
||||
usage["input_tokens"] += inp
|
||||
usage["output_tokens"] += out
|
||||
@@ -756,9 +779,23 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
if failure is not None:
|
||||
end["error"] = "超过核查的时长上限,已放弃这次调用" if isinstance(failure, asyncio.TimeoutError) \
|
||||
else explain_model_error(failure, model_id)
|
||||
if truncated_error(failure) or cut_off(raw):
|
||||
# 到了上限被截断:不是「调用失败,请稍后重试」,重试解决不了额度问题
|
||||
limit = base_model.max_tokens
|
||||
end.update(truncated=True, max_tokens=limit, error=cut_words("核查模型", limit))
|
||||
await _announce(emit, EventType.LLM_END, **end)
|
||||
return got, raw, failure
|
||||
|
||||
# 截断的建议按模型从哪来的说:节点里节点上指定的优先,不在节点里(质询)只看设置
|
||||
advice = TRUNCATED_ADVICE if node_id else TRUNCATED_ADVICE_SETTINGS
|
||||
|
||||
async def report_cut(purpose: str, limit: int | None) -> VerifyOutcome:
|
||||
"""被截断、没交成结果:记一条 verify_truncated(purpose 是哪一步,limit 是那次的额度),原因照截断说。查过的照记。"""
|
||||
text = TRUNCATED_ERROR.format(cut=cut_words("核查模型", limit), advice=advice)
|
||||
await _announce(emit, EventType.LOG, level="warn", code="verify_truncated", purpose=purpose, max_tokens=limit,
|
||||
message=text)
|
||||
return outcome(limit=stopped, answered=answered, error=text)
|
||||
|
||||
# ---- 查库的循环
|
||||
stopped: str | None = None
|
||||
answered = False
|
||||
@@ -781,6 +818,12 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
if isinstance(failure, asyncio.TimeoutError):
|
||||
stopped = "timeout_s"
|
||||
break
|
||||
if truncated_error(failure) or cut_off(response):
|
||||
# 到了上限被截断(推理模型想完了额度也用完了):话没说完,不是模型自己停下来的;带着的工具调用是写到一半的
|
||||
# (Anthropic 的 tool_use 截在半路时 input 是空的,有的网关给半截的 SQL),不执行。也不收尾:一条都没查到的,
|
||||
# 收尾只能交「查不到」(双盲复算会记成「盲算没有查到这个数」);查到过的,截断前查到的不一定是要的那个数
|
||||
# (模型可能正想改查询),同时长到点一样算半路被掐断。收尾那次多半同样被推理占满,也不为它多花一次额度
|
||||
return await report_cut("verify", base_model.max_tokens)
|
||||
if failure is not None or not isinstance(response, BaseMessage):
|
||||
return outcome(error=explain_model_error(failure, model_id) if failure else "核查模型没有给出回复")
|
||||
messages.append(response)
|
||||
@@ -850,12 +893,15 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
await _announce(emit, EventType.LOG, level="warn", code="verify_limit", limit=stopped,
|
||||
message=f"核查已达上限({words}),已停止查询,按已有结果收尾")
|
||||
|
||||
# ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容
|
||||
# ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容(经 OpenAI 兼容接入的推理模型
|
||||
# 关不掉,所以额度和工具循环一样,退回小额度也和工具循环共用一个开关)
|
||||
settle_spec = spec.model_copy(update={"thinking": "off"})
|
||||
try:
|
||||
async with SessionLocal() as session:
|
||||
settler, _ = await get_chat_model(session, spec.model_copy(update={"thinking": "off"}))
|
||||
settle_model, _ = await get_chat_model(session, settle_spec)
|
||||
except Exception as e: # noqa: BLE001
|
||||
return outcome(limit=stopped, answered=answered, error=f"核查模型不可用({describe_exception(e)})")
|
||||
settler = _with_fallback(settle_model, settle_spec, small, model_id, shared_with=base_model)
|
||||
final = _final_messages(head, task, queries, said, words)
|
||||
try:
|
||||
structured = settler.with_structured_output(schema, include_raw=True)
|
||||
@@ -870,21 +916,27 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
|
||||
parsed: dict[str, Any] | None = None
|
||||
failure: BaseException | None = None
|
||||
settle_cut = False # 收尾那次到了上限被截断
|
||||
if structured is not None:
|
||||
estimate = budget.estimate(_text_of(final) + schema_text, _OUT_FINAL)
|
||||
day, refused = await budget.reserve(estimate, kept=kept)
|
||||
if refused:
|
||||
return outcome(limit=stopped or refused, answered=answered, error=f"已达{words_of(refused)},没能交结果")
|
||||
got, raw, failure = await ask(final, structured.ainvoke, estimate, day or "", "verify_result", until_finish())
|
||||
parsed = None if failure is not None else _parsed(_split_structured(got)[1])
|
||||
if parsed is None and raw is not None:
|
||||
# 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用
|
||||
for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]:
|
||||
if (parsed := _parsed(candidate)) is not None:
|
||||
break
|
||||
if parsed is None and not isinstance(failure, asyncio.TimeoutError):
|
||||
# 强制工具调用写到一半到了上限:Anthropic 截在 tool_use 里时 input 是空的,有的网关给半截对象(反方审稿的结论
|
||||
# 写完了、理由写到一半)。解析出来的、能从回复里抠出来的都说不清写完了没有,一律不采信
|
||||
settle_cut = truncated_error(failure) or bool(cut_off(raw))
|
||||
if failure is None and not settle_cut:
|
||||
parsed = _parsed(_split_structured(got)[1])
|
||||
if parsed is None and raw is not None:
|
||||
# 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用
|
||||
for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]:
|
||||
if (parsed := _parsed(candidate)) is not None:
|
||||
break
|
||||
if parsed is None and not isinstance(failure, asyncio.TimeoutError) and not settle_cut:
|
||||
# 不支持结构化输出的模型:普通调用一次,结构写进消息,从回复里把 JSON 抠出来。结构化那次已经做过的,
|
||||
# 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱
|
||||
# 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱。结构化那次是被截断的不补救:
|
||||
# 推理占满额度时普通调用多半一样被截断,白花一次额度
|
||||
plain = _plain_messages(final, schema)
|
||||
remaining = until_finish()
|
||||
if remaining is None or remaining > 0:
|
||||
@@ -895,8 +947,13 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
error=f"已达{words_of(refused)},没能交结果")
|
||||
if not refused:
|
||||
got, _, failure = await ask(plain, settler.ainvoke, estimate, day or "", "verify_result", remaining)
|
||||
parsed = None if failure is not None else _parsed(message_text(got) if got is not None else None)
|
||||
# 同上:截断的回复里就算抠得出一段 JSON,后面还在写,说不清写完了没有
|
||||
settle_cut = truncated_error(failure) or bool(cut_off(got))
|
||||
if failure is None and not settle_cut:
|
||||
parsed = _parsed(message_text(got) if got is not None else None)
|
||||
if parsed is None:
|
||||
if settle_cut:
|
||||
return await report_cut("verify_result", settler.max_tokens)
|
||||
why = "收尾时超过时长上限,没能交结果" if isinstance(failure, asyncio.TimeoutError) else \
|
||||
f"核查模型没有按结构交结果({explain_model_error(failure, model_id)})" if failure is not None else \
|
||||
"核查模型没有按结构交结果"
|
||||
@@ -919,6 +976,22 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
return outcome(result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered)
|
||||
|
||||
|
||||
def _with_fallback(model: Any, spec: ModelSpec, small: int | None, model_id: str, *,
|
||||
shared_with: BudgetFallback | None = None) -> BudgetFallback:
|
||||
"""按 spec 建好的模型包一层:服务方嫌额度大(或者 SDK 在本地拒了)时按 small 重建一个再调,要退回时才建,平常一个
|
||||
都不多建。small 为 None 是额度本来就不比它大,没有可退的。被嫌大过的按 (接入, 模型, 额度) 记在进程里,证据裁判
|
||||
(judge._with_fallback)用的是同一种记法:同一个模型在那边被嫌大过,这边也直接用小的。经本模块的 get_chat_model
|
||||
建,测试换成剧本时小额度的那个也是剧本。"""
|
||||
|
||||
async def build() -> Any:
|
||||
async with SessionLocal() as session:
|
||||
return (await get_chat_model(session, spec.model_copy(update={"max_tokens": small})))[0]
|
||||
|
||||
return BudgetFallback(model, build=build if small else None, limits=(spec.max_tokens, small or spec.max_tokens),
|
||||
shared_with=shared_with, rejected=budget_refused,
|
||||
key=(spec.provider, model_id, spec.max_tokens))
|
||||
|
||||
|
||||
async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, remaining: float | None,
|
||||
emit: Emit | None, queries: list[dict[str, Any]], ledger: list[dict[str, Any]],
|
||||
owner: str) -> str | None:
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
"""模型调用的输出额度:认出「输出到了上限被截断」,服务方嫌额度太大时退回小额度重来。
|
||||
|
||||
助手改图(api/copilot.py)、核查(engine/verify_agent.py:双盲复算、反方审稿、审批质询共用)和证据裁判(engine/judge.py)
|
||||
共用这一份。会思考的模型思考也算在输出额度里:经 OpenAI 兼容接入的推理模型(DeepSeek、GLM、Kimi 这一类)收不到
|
||||
thinking=off,Claude 开了 adaptive thinking 也一样。额度小了,想完就没额度写正文——回复是空的,或者 JSON 只写了一半。
|
||||
这时要照实说是输出额度不够,不能当成「模型没按格式交」「没查到」。
|
||||
|
||||
- cut_off:这一块(流式)或这条回复是不是因为额度用完停下的
|
||||
- truncated_error:这个异常是不是在说同一件事(openai 结构化输出解析时抛的 LengthFinishReasonError)
|
||||
- budget_rejected:服务方一开口就嫌额度太大(助手改图按它认)
|
||||
- budget_refused:核查、证据裁判按它认,多认自建服务说的 max_new_tokens 和 anthropic SDK 在本地的拒绝
|
||||
- error_usage:截断的那次调用花掉的用量
|
||||
- cut_words:给人看的「某某模型的输出达到上限(N token)被截断」
|
||||
- BudgetFallback:先按大额度调,被嫌大时按小额度重来,之后都用小的;给了 key 的,被嫌大过就记在进程里
|
||||
|
||||
engine/nodes/llm.py(cite_fields 的抽取)、data/recipe_ai.py 各有一份同样的判断,是在这里之前写的,行为相同。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable, Hashable
|
||||
from typing import Any
|
||||
|
||||
#: 服务方嫌额度太大时报错里会出现的字样:超过模型的输出上限(DeepSeek chat 系只到 8K),
|
||||
#: 或者输入加额度超过上下文长度(本地 vLLM)。助手改图按这一份认,data/recipe_ai.py 的测试拿它比对
|
||||
BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length")
|
||||
#: 核查、证据裁判另外认的:text-generation-inference 一类自建服务说的是 max_new_tokens(「`inputs` tokens +
|
||||
#: `max_new_tokens` must be <= 8192」)。这两处的额度从 4096 提到 16384 之后才容易撞上;助手改图的认法不动
|
||||
WIDER_BUDGET_WORDS = (*BUDGET_WORDS, "max_new_tokens")
|
||||
#: anthropic SDK 在本地拒绝时的原话:客户端没设超时(Bearer 鉴权的接入,factory 换上的就是这种)、非流式调用、额度
|
||||
#: 超过这个模型不流式能给的上限(Opus 4、4.1 这一代只到 8192)或估计要写十分钟以上,请求没发出去就抛 ValueError
|
||||
LOCAL_REFUSAL = "streaming is required for operations that may take longer than 10 minutes"
|
||||
#: 截断类报错的原话:openai 的 LengthFinishReasonError(结构化输出解析时 finish_reason=length)
|
||||
LENGTH_WORDS = ("length limit was reached", "finish_reason=length", "finish_reason='length'")
|
||||
|
||||
|
||||
def cut_off(chunk: Any) -> str:
|
||||
"""这一块是不是在说「额度用完了」:OpenAI 系是 finish_reason=length,Anthropic 是 stop_reason=max_tokens。"""
|
||||
meta = getattr(chunk, "response_metadata", None) or {}
|
||||
reason = meta.get("finish_reason") or meta.get("stop_reason")
|
||||
return str(reason) if reason in ("length", "max_tokens") else ""
|
||||
|
||||
|
||||
def truncated_error(error: BaseException | None) -> bool:
|
||||
"""这个异常是不是在说「输出被截断」:openai 的 LengthFinishReasonError,或者 langchain、网关转述的同一件事。
|
||||
|
||||
服务方嫌额度太大的 400(报错里也有 max_tokens)不算:那是请求被拒,模型一个字都没写。
|
||||
"""
|
||||
if error is None:
|
||||
return False
|
||||
if type(error).__name__ == "LengthFinishReasonError":
|
||||
return True
|
||||
text = str(error).lower()
|
||||
return any(word in text for word in LENGTH_WORDS)
|
||||
|
||||
|
||||
def budget_rejected(e: BaseException, words: tuple[str, ...] = BUDGET_WORDS) -> bool:
|
||||
status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None)
|
||||
return status in (400, 422) and any(w in str(e).lower() for w in words)
|
||||
|
||||
|
||||
def budget_refused(e: BaseException) -> bool:
|
||||
"""核查、证据裁判按这个认「额度被挡下了,换小的重来」:服务方嫌额度太大(多认 max_new_tokens),或者 anthropic SDK
|
||||
在本地就拒了。两种都是一个字还没写、请求要么被拒要么根本没发出去,重来不会重复输出,也没花钱。助手改图走流式,
|
||||
SDK 不在本地拦,仍按 budget_rejected 认。"""
|
||||
if isinstance(e, ValueError) and LOCAL_REFUSAL in str(e).lower():
|
||||
return True
|
||||
return budget_rejected(e, WIDER_BUDGET_WORDS)
|
||||
|
||||
|
||||
def error_usage(error: BaseException | None) -> tuple[int, int]:
|
||||
"""失败的那次调用花掉的 (输入, 输出) token。截断时 openai 的异常带着 completion.usage:额度是真花出去了,要记账,
|
||||
不然每日上限会漏掉最贵的那几次。别的异常没有用量,是 (0, 0)。"""
|
||||
usage = getattr(getattr(error, "completion", None), "usage", None)
|
||||
return int(getattr(usage, "prompt_tokens", 0) or 0), int(getattr(usage, "completion_tokens", 0) or 0)
|
||||
|
||||
|
||||
def cut_words(who: str, limit: int | None) -> str:
|
||||
"""「核查模型的输出达到上限(16384 token)被截断」。额度不知道(没给 max_tokens、按接入的默认值)就不写数。"""
|
||||
return f"{who}的输出达到上限{f'({limit} token)' if limit else ''}被截断"
|
||||
|
||||
|
||||
#: 嫌过大额度的接入:BudgetFallback 的 key(核查、证据裁判给的是 (接入, 模型, 大额度))。双盲复算的几路盲算、反方
|
||||
#: 审稿的每一句、每次质询、每次点击裁判各是一组新建的调用,开关只在一组里共用;记在这里,之后新建的直接用小的,
|
||||
#: 不再每组先撞一次。只记被拒过的;进程内有效,重启后再按大的试(服务方可能调高了上限),同 factory 的 _NO_STREAM_USAGE
|
||||
_REJECTED: set[Hashable] = set()
|
||||
|
||||
|
||||
class _Switch:
|
||||
"""几次调用共用的「退回小额度了没有」:核查的工具循环和收尾、裁判的各批。rejected 是怎么认「额度被挡下了」;
|
||||
key 给了,开关一开始就照 _REJECTED 定,退回时记进去。"""
|
||||
|
||||
__slots__ = ("key", "rejected", "small")
|
||||
|
||||
def __init__(self, key: Hashable | None, rejected: Callable[[BaseException], bool]) -> None:
|
||||
self.key = key
|
||||
self.rejected = rejected
|
||||
self.small = key is not None and key in _REJECTED
|
||||
|
||||
def fall_back(self) -> None:
|
||||
self.small = True
|
||||
if self.key is not None:
|
||||
_REJECTED.add(self.key)
|
||||
|
||||
|
||||
class BudgetFallback:
|
||||
"""先按大额度调;服务方一开口就嫌额度太大时按小额度重来,之后的调用都用小的。
|
||||
|
||||
只在还没吐出任何东西时重来:这类拒绝都发生在请求刚发出去的时候,重来不会重复输出。流式(astream)、非流式
|
||||
(ainvoke)都认。bind_tools、with_structured_output 照样包一层,包出来的和原来的共用同一个开关;shared_with 给了,
|
||||
另一份配置建的模型(核查收尾时关掉思考的那个)也和它共用:工具循环里退回过,收尾那次直接用小的,不再先撞一次。
|
||||
|
||||
small 是现成的小额度模型;也可以不给,给 build(无参的协程函数,返回小额度的模型),要退回时才建,平常一个都不多建。
|
||||
limits 是 (大额度, 小额度):给了,max_tokens 就按它报(包过一层的模型身上不一定读得到额度);不给就读眼下那个模型的
|
||||
max_tokens。rejected 是怎么认「额度被挡下了」,缺省 budget_rejected(助手改图),核查、证据裁判给 budget_refused。
|
||||
key 给了(核查、证据裁判给 (接入, 模型, 大额度)),被嫌大一次就记在进程里,之后同一个 key 新建的一开始就用小的;
|
||||
不给(助手改图)不记也不查。shared_with 给了,rejected、key 跟着它。
|
||||
"""
|
||||
|
||||
def __init__(self, big: Any, small: Any = None, *, build: Callable[[], Awaitable[Any]] | None = None,
|
||||
limits: tuple[int | None, int | None] | None = None,
|
||||
shared_with: BudgetFallback | None = None,
|
||||
rejected: Callable[[BaseException], bool] = budget_rejected,
|
||||
key: Hashable | None = None) -> None:
|
||||
self._big = big
|
||||
self._small = small
|
||||
self._build = build
|
||||
self._limits = limits
|
||||
self._switch = shared_with._switch if shared_with is not None else _Switch(key, rejected)
|
||||
|
||||
@property
|
||||
def model(self) -> Any:
|
||||
"""眼下用的那个:退回过、小额度的也建好了就是小的,否则是大的。"""
|
||||
return self._small if self._switch.small and self._small is not None else self._big
|
||||
|
||||
@property
|
||||
def max_tokens(self) -> Any:
|
||||
if self._limits is not None:
|
||||
return self._limits[1] if self._switch.small else self._limits[0]
|
||||
return getattr(self.model, "max_tokens", None)
|
||||
|
||||
def _can_fall_back(self, used: Any) -> bool:
|
||||
"""刚才那次用的是大额度、又有小额度可退。大小是同一个模型时重来也是同样的额度,不算可退。"""
|
||||
if used is not self._big:
|
||||
return False
|
||||
return self._build is not None if self._small is None else self._small is not self._big
|
||||
|
||||
async def _small_model(self) -> Any:
|
||||
if self._small is None and self._build is not None:
|
||||
built = await self._build()
|
||||
if self._small is None: # 并发的几次调用同时要建时,留先建好的那个
|
||||
self._small = built
|
||||
return self._small
|
||||
|
||||
async def _current(self) -> Any:
|
||||
if self._switch.small and (small := await self._small_model()) is not None:
|
||||
return small
|
||||
return self._big
|
||||
|
||||
async def ainvoke(self, messages: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
used = await self._current()
|
||||
try:
|
||||
return await used.ainvoke(messages, *args, **kwargs)
|
||||
except Exception as e:
|
||||
if not self._can_fall_back(used) or not self._switch.rejected(e):
|
||||
raise
|
||||
self._switch.fall_back()
|
||||
return await (await self._current()).ainvoke(messages, *args, **kwargs)
|
||||
|
||||
async def astream(self, messages: Any, *args: Any, **kwargs: Any):
|
||||
used = await self._current()
|
||||
started = False
|
||||
try:
|
||||
async for chunk in used.astream(messages, *args, **kwargs):
|
||||
started = True
|
||||
yield chunk
|
||||
return
|
||||
except Exception as e:
|
||||
if started or not self._can_fall_back(used) or not self._switch.rejected(e):
|
||||
raise
|
||||
self._switch.fall_back()
|
||||
async for chunk in (await self._current()).astream(messages, *args, **kwargs):
|
||||
yield chunk
|
||||
|
||||
def _derived(self, make: Callable[[Any], Any]) -> BudgetFallback:
|
||||
"""对大小两个模型做同一件事(绑工具、要结构化输出),包成共用开关的一层。小的还没建时,建好再做。"""
|
||||
small, build = None, None
|
||||
if self._small is not None:
|
||||
small = make(self._small)
|
||||
elif self._build is not None:
|
||||
async def build() -> Any:
|
||||
return make(await self._small_model())
|
||||
return BudgetFallback(make(self._big), small, build=build, limits=self._limits, shared_with=self)
|
||||
|
||||
def bind_tools(self, tools: Any, **kwargs: Any) -> BudgetFallback:
|
||||
return self._derived(lambda m: m.bind_tools(tools, **kwargs))
|
||||
|
||||
def with_structured_output(self, schema: Any, **kwargs: Any) -> BudgetFallback:
|
||||
return self._derived(lambda m: m.with_structured_output(schema, **kwargs))
|
||||
|
||||
|
||||
__all__ = ["BUDGET_WORDS", "LENGTH_WORDS", "LOCAL_REFUSAL", "WIDER_BUDGET_WORDS", "BudgetFallback", "budget_refused",
|
||||
"budget_rejected", "cut_off", "cut_words", "error_usage", "truncated_error"]
|
||||
@@ -6,7 +6,7 @@
|
||||
- 一份报告合并成一次调用(超过每批的句数就分批),每句给 supported / partial / contradicted / insufficient /
|
||||
not_a_claim
|
||||
- 模型不支持结构化输出、或者给的不是合法结构时退回 JSON 解析,再失败就记「未裁判」,不编判定
|
||||
- 裁判模型关掉 thinking,max_tokens 4096
|
||||
- 裁判模型关掉 thinking,输出额度 16384(推理模型的思考也算在里面);服务方嫌大时退回 4096,截断照实说(见 test_judge_truncation)
|
||||
- 预算(每份报告的句数、金额、时长,全局每日金额)每一项都能写 null 表示不限;触顶就停、已判的保留、
|
||||
没判的记未裁判并说明是哪个上限——不报错、不让报告失败。金额超了按优先级截断:先判挂了依据的,
|
||||
再判有方向词、因果词的
|
||||
@@ -207,7 +207,7 @@ async def test_the_judge_model_runs_without_thinking(monkeypatch, stored):
|
||||
max_tokens=64000),
|
||||
budget=Budget(max_claims=None, max_cost_usd=None, timeout_s=30, daily_max_usd=None))
|
||||
[spec] = judge.specs
|
||||
assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 4096
|
||||
assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 16384
|
||||
assert spec.model == PRICED and spec.timeout and spec.timeout <= 30
|
||||
|
||||
|
||||
|
||||
@@ -350,7 +350,7 @@ async def test_the_judge_model_follows_node_then_settings(client, monkeypatch):
|
||||
row = await finish(weekly(claims="judge"))
|
||||
await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])
|
||||
[spec] = models.specs
|
||||
assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 4096
|
||||
assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 16384
|
||||
|
||||
row = await finish(weekly(claims="judge", judge={"model": "judge-from-node"}))
|
||||
body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json()
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.db.base import SessionLocal
|
||||
from app.db.models import Setting
|
||||
from app.engine.judge import (
|
||||
JUDGE_DEFAULTS,
|
||||
MAX_TOKENS,
|
||||
SETTING_GROUP,
|
||||
SPEND_KEY,
|
||||
Budget,
|
||||
@@ -166,7 +167,7 @@ async def test_the_judge_model_priority():
|
||||
async with SessionLocal() as session:
|
||||
spec = await judge_model_spec(session, None)
|
||||
assert (spec.provider, spec.model) == (None, None), "什么都没配:默认 provider"
|
||||
assert spec.thinking == "off" and spec.max_tokens == 4096
|
||||
assert spec.thinking == "off" and spec.max_tokens == MAX_TOKENS == 16384
|
||||
|
||||
session.add(Setting(key="copilot", value={"provider": "", "model": "copilot-model"}))
|
||||
await session.commit()
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
"""结论句裁判(engine/judge.py)的输出额度:截断照实说,服务方嫌额度大时退回小额度。
|
||||
|
||||
裁判原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考就能把 4096 用光:
|
||||
结构化调用写不出判定(openai 抛 LengthFinishReasonError,Anthropic 的强制工具调用写到一半),退回纯文本调用也一样,
|
||||
最后记成「裁判返回的结果格式有误」——没说出真正的原因。这里守几件事:
|
||||
|
||||
- 额度:16384(MAX_TOKENS);服务方嫌大时退回 4096(FALLBACK_MAX_TOKENS)再问一次,之后各批都用小的,只撞一次
|
||||
- 输出到了上限被截断(stop_reason=max_tokens、finish_reason=length、LengthFinishReasonError):这一批记未裁判,原因
|
||||
是 truncated,理由、缺口、llm.end 都写明「裁判模型的输出达到上限(N token)被截断」;截断的回复整批不采信(有的网关
|
||||
把截断的强制工具调用解析成半截清单:最后一条只写了一半);结构化调用被截断的不再退回纯文本多花一次额度;截断那次
|
||||
的用量照记
|
||||
- 时间线上另记一条 judge_truncated(带额度、几句未裁判),不混进 judge_failed:judge_failed 的下一步是「检查接入、
|
||||
重新运行」,截断时接入是好的,同一个模型重新运行多半还是截断
|
||||
- 被嫌大过的接入、模型记在进程里,之后的裁判直接用小的;自建服务说的 max_new_tokens 也认
|
||||
- 没有截断时原有说法不变(格式不对仍是 format)
|
||||
|
||||
模型都是剧本;证据借 test_evidence_judge 的虚构夹具(口径卡、一次查询、一次检索)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
from openai import LengthFinishReasonError
|
||||
from openai.types import CompletionUsage
|
||||
from openai.types.chat import ChatCompletion
|
||||
|
||||
from app.core.events import EventType
|
||||
from app.db.models import Provider
|
||||
from app.engine import judge as judge_mod
|
||||
from app.engine import review as rv
|
||||
from app.engine.evidence import compose_doc
|
||||
from app.engine.judge import BATCH_SIZE, FALLBACK_MAX_TOKENS, MAX_TOKENS, REASONS, judge_doc
|
||||
from app.providers import budget
|
||||
from app.providers.factory import build_chat_model
|
||||
from tests.test_evidence_judge import PRICED, RAW, SPEC, UNLIMITED, clean_settings, make_catalog, stored # noqa: F401
|
||||
from tests.test_verify_truncation import messages_api
|
||||
|
||||
try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上
|
||||
from langchain_openai._compat import httpx
|
||||
except ImportError: # pragma: no cover - 老版本的 langchain_openai
|
||||
import httpx
|
||||
|
||||
CUT_16K = "裁判模型的输出达到上限(16384 token)被截断"
|
||||
ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def forget_rejections():
|
||||
"""被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。"""
|
||||
budget._REJECTED.clear()
|
||||
yield
|
||||
budget._REJECTED.clear()
|
||||
|
||||
|
||||
def length_error(prompt: int = 3000, completion: int = 16384) -> LengthFinishReasonError:
|
||||
usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion)
|
||||
return LengthFinishReasonError(completion=ChatCompletion.model_construct(
|
||||
id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage))
|
||||
|
||||
|
||||
def anthropic_cut(units) -> dict:
|
||||
"""强制工具调用写到一半到了上限:原始回复 stop_reason=max_tokens,解析不出判定。"""
|
||||
raw = AIMessage(content=[], response_metadata={"stop_reason": "max_tokens"},
|
||||
usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384})
|
||||
return {"raw": raw, "parsed": None, "parsing_error": None}
|
||||
|
||||
|
||||
def thinking_only(units) -> AIMessage:
|
||||
"""纯文本调用:额度全用在思考上,正文为空,finish_reason=length(推理模型经 OpenAI 兼容接入的样子)。"""
|
||||
return AIMessage(content="", additional_kwargs={"reasoning_content": "逐句看证据……"},
|
||||
response_metadata={"finish_reason": "length"},
|
||||
usage_metadata={"input_tokens": 3000, "output_tokens": 4096, "total_tokens": 7096})
|
||||
|
||||
|
||||
def half_list(units) -> dict:
|
||||
"""有的网关把截断的强制工具调用解析成半截对象:第一条判定写完了,第二条只写了编号,后面的没写到。"""
|
||||
items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []},
|
||||
{"unit": units[1]}]
|
||||
raw = AIMessage(content=[{"type": "tool_use", "id": "toolu_1", "name": "judge", "input": {"items": items}}],
|
||||
tool_calls=[{"name": "judge", "args": {"items": items}, "id": "toolu_1"}],
|
||||
response_metadata={"stop_reason": "max_tokens"},
|
||||
usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384})
|
||||
return {"raw": raw, "parsed": {"items": items}, "parsing_error": None}
|
||||
|
||||
|
||||
def supported(units) -> dict:
|
||||
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
|
||||
return {"raw": AIMessage(content="", response_metadata={"finish_reason": "stop"}), "parsed": {"items": items}}
|
||||
|
||||
|
||||
class BudgetRejected(Exception):
|
||||
"""服务方嫌输出额度太大时的 400。"""
|
||||
|
||||
status_code = 400
|
||||
|
||||
|
||||
class Reasoner:
|
||||
"""裁判模型的剧本。structured(units) / plain(units) 给出结构化调用、退回纯文本调用的回复(返回异常就抛它);
|
||||
unstructured 为真时不支持结构化输出。calls 记下每次调用的种类和判的句子。"""
|
||||
|
||||
def __init__(self, *, structured=supported, plain=None, unstructured: bool = False) -> None:
|
||||
self.structured, self.plain, self.unstructured = structured, plain, unstructured
|
||||
self.calls: list[tuple[str, list[str]]] = []
|
||||
|
||||
def with_structured_output(self, schema, **kwargs):
|
||||
if self.unstructured:
|
||||
raise NotImplementedError("这个模型不支持结构化输出")
|
||||
return _View(self)
|
||||
|
||||
async def ainvoke(self, messages, *args, **kwargs):
|
||||
return self.answer("plain", messages)
|
||||
|
||||
def answer(self, kind: str, messages):
|
||||
units = re.findall(r"^\[(u\d+)\]", str(messages[-1].content), re.M)
|
||||
self.calls.append((kind, units))
|
||||
make = self.structured if kind == "structured" else self.plain
|
||||
reply = make(units)
|
||||
if isinstance(reply, BaseException):
|
||||
raise reply
|
||||
return reply
|
||||
|
||||
|
||||
class _View:
|
||||
def __init__(self, model: Reasoner) -> None:
|
||||
self.model = model
|
||||
|
||||
async def ainvoke(self, messages, *args, **kwargs):
|
||||
return self.model.answer("structured", messages)
|
||||
|
||||
|
||||
class Rejecting:
|
||||
def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens",
|
||||
status_code: int = 400) -> None:
|
||||
self.message, self.status_code = message, status_code
|
||||
self.calls = 0
|
||||
|
||||
def with_structured_output(self, schema, **kwargs):
|
||||
return self
|
||||
|
||||
async def ainvoke(self, messages, *args, **kwargs):
|
||||
self.calls += 1
|
||||
error = BudgetRejected(self.message)
|
||||
error.status_code = self.status_code
|
||||
raise error
|
||||
|
||||
|
||||
class Models:
|
||||
def __init__(self, monkeypatch, model, *, rejects_above: int | None = None,
|
||||
rejecting: Rejecting | None = None) -> None:
|
||||
self.specs = []
|
||||
self.rejecting = rejecting or Rejecting()
|
||||
|
||||
async def fake(session, spec):
|
||||
self.specs.append(spec)
|
||||
if rejects_above is not None and (spec.max_tokens or 0) > rejects_above:
|
||||
return self.rejecting, PRICED
|
||||
return model, PRICED
|
||||
|
||||
monkeypatch.setattr(judge_mod, "get_chat_model", fake)
|
||||
|
||||
|
||||
async def judged(stored, raw: str = RAW):
|
||||
catalog = make_catalog(stored)
|
||||
events: list[tuple] = []
|
||||
outcome = await judge_doc(compose_doc(raw, catalog), catalog, spec=SPEC, budget=UNLIMITED,
|
||||
emit=lambda kind, **data: events.append((kind, data)))
|
||||
return outcome, events
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 截断
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_cut_off_structured_reply_is_unjudged_as_truncation_without_a_plain_retry(monkeypatch, stored):
|
||||
model = Reasoner(structured=anthropic_cut, plain=lambda units: AIMessage(content=json.dumps({"items": []})))
|
||||
Models(monkeypatch, model)
|
||||
outcome, events = await judged(stored)
|
||||
assert [kind for kind, _ in model.calls] == ["structured"], "判定写到一半到了上限:不再退回纯文本多花一次额度"
|
||||
verdicts = list(outcome["verdicts"].values())
|
||||
assert {(v["status"], v["reason"]) for v in verdicts} == {("unjudged", "truncated")}
|
||||
assert verdicts[0]["rationale"] == f"{CUT_16K},这句未裁判。{ADVICE}"
|
||||
assert outcome["unjudged"] == {"truncated": 4} and outcome["limits_hit"] == []
|
||||
assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"]
|
||||
[end] = [d for k, d in events if k == EventType.LLM_END]
|
||||
assert end["purpose"] == "judge" and end["error"] == CUT_16K and end["truncated"] is True
|
||||
assert end["max_tokens"] == 16384 and end["output_tokens"] == 16384
|
||||
[log] = logs(events, "judge_truncated")
|
||||
assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["unjudged"] == 4
|
||||
assert log["message"] == f"结论句裁判未完成:有 4 句结论未裁判:{CUT_16K},{ADVICE}"
|
||||
assert logs(events, "judge_failed") == [], "不混进 judge_failed:那一行的下一步是检查接入、重新运行"
|
||||
|
||||
|
||||
def logs(events, code: str) -> list[dict]:
|
||||
return [d for k, d in events if k == EventType.LOG and d.get("code") == code]
|
||||
|
||||
|
||||
async def test_a_half_written_list_in_a_cut_off_reply_is_not_taken(monkeypatch, stored):
|
||||
"""截断的回复整批不采信:写完的那条也可能只是理由写完了一半;以前第一条记成「证据支持」、第二条「判定无法识别」、
|
||||
后面的「未给出判定」,llm.end 也没标截断。"""
|
||||
model = Reasoner(structured=half_list, plain=lambda units: AIMessage(content=json.dumps({"items": []})))
|
||||
Models(monkeypatch, model)
|
||||
outcome, events = await judged(stored)
|
||||
assert [kind for kind, _ in model.calls] == ["structured"]
|
||||
assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0
|
||||
[end] = [d for k, d in events if k == EventType.LLM_END]
|
||||
assert end["truncated"] is True and end["error"] == CUT_16K
|
||||
[log] = logs(events, "judge_truncated")
|
||||
assert log["unjudged"] == 4
|
||||
|
||||
|
||||
async def test_a_plain_reply_cut_after_the_list_is_not_taken_either(monkeypatch, stored):
|
||||
"""纯文本调用:清单后面还在写、回复是截断的,说不清写完了没有,同样整批记截断。"""
|
||||
def cut(units):
|
||||
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
|
||||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False) + "\n补充:第二句的依据还要",
|
||||
response_metadata={"finish_reason": "length"})
|
||||
|
||||
Models(monkeypatch, Reasoner(plain=cut, unstructured=True))
|
||||
outcome, _ = await judged(stored)
|
||||
assert outcome["unjudged"] == {"truncated": 4}
|
||||
|
||||
|
||||
async def test_an_openai_length_error_is_truncation_and_its_tokens_are_counted(monkeypatch, stored):
|
||||
model = Reasoner(structured=lambda units: length_error(completion=16384))
|
||||
Models(monkeypatch, model)
|
||||
outcome, events = await judged(stored)
|
||||
assert [kind for kind, _ in model.calls] == ["structured"]
|
||||
assert outcome["unjudged"] == {"truncated": 4}
|
||||
assert outcome["usage"]["output_tokens"] == 16384 and outcome["cost_usd"] > 0, "截断那次的额度是真花出去了"
|
||||
end = [d for k, d in events if k == EventType.LLM_END][-1]
|
||||
assert "调用" not in end["error"] and end["error"] == CUT_16K
|
||||
|
||||
|
||||
async def test_a_plain_reply_with_only_thinking_is_truncation(monkeypatch, stored):
|
||||
"""不支持结构化输出的模型直接退回纯文本调用:回复只有思考、finish_reason=length。"""
|
||||
model = Reasoner(plain=thinking_only, unstructured=True)
|
||||
Models(monkeypatch, model)
|
||||
outcome, _ = await judged(stored)
|
||||
assert [kind for kind, _ in model.calls] == ["plain"] and outcome["calls"] == 2
|
||||
assert outcome["unjudged"] == {"truncated": 4}
|
||||
assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"]
|
||||
|
||||
|
||||
async def test_a_truncated_batch_does_not_stop_the_others(monkeypatch, stored):
|
||||
"""只有一批被截断:别的批照常判(和格式不对一样,不像调用失败那样后面的都不再问)。"""
|
||||
first: list[bool] = []
|
||||
|
||||
def once(units):
|
||||
if not first:
|
||||
first.append(True)
|
||||
return anthropic_cut(units)
|
||||
return supported(units)
|
||||
|
||||
Models(monkeypatch, Reasoner(structured=once))
|
||||
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3))
|
||||
outcome, _ = await judged(stored, raw)
|
||||
assert outcome["unjudged"] == {"truncated": BATCH_SIZE}
|
||||
assert outcome["counts"]["supported"] == 3
|
||||
|
||||
|
||||
async def test_truncation_and_other_failures_are_logged_apart(monkeypatch, stored):
|
||||
"""一批被截断、另一批格式不对:截断记 judge_truncated,格式不对照旧记 judge_failed,各说各的。"""
|
||||
first: list[bool] = []
|
||||
|
||||
def once(units):
|
||||
if not first:
|
||||
first.append(True)
|
||||
return anthropic_cut(units)
|
||||
return {"raw": AIMessage(content="不是 JSON", response_metadata={"finish_reason": "stop"}), "parsed": None}
|
||||
|
||||
Models(monkeypatch, Reasoner(structured=once, plain=lambda units: AIMessage(content="还是不是")))
|
||||
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3))
|
||||
outcome, events = await judged(stored, raw)
|
||||
assert outcome["unjudged"] == {"format": 3, "truncated": BATCH_SIZE}
|
||||
[cut] = logs(events, "judge_truncated")
|
||||
[failed] = logs(events, "judge_failed")
|
||||
assert cut["unjudged"] == BATCH_SIZE and CUT_16K in cut["message"]
|
||||
assert failed["message"] == "结论句裁判未完成:有 3 句结论未裁判:裁判返回的结果格式有误"
|
||||
|
||||
|
||||
def test_the_review_layer_knows_the_truncation_log():
|
||||
"""跑完之后的复核层认得 judge_truncated:同 judge_failed,是缺口(要说明),kind 是它自己。"""
|
||||
assert rv.WARN_CODES["judge_truncated"] == ("judge_truncated", rv.DEGRADED)
|
||||
event = {"type": str(EventType.LOG), "node_id": "write",
|
||||
"data": {"level": "warn", "code": "judge_truncated", "message": f"有 4 句结论未裁判:{CUT_16K}"}}
|
||||
[signal] = rv.scan([event], {"answer": "本周销售额 45,678.5元。"})
|
||||
assert (signal.kind, signal.severity) == ("judge_truncated", rv.DEGRADED) and not rv.should_retry([signal])
|
||||
|
||||
|
||||
async def test_without_truncation_format_problems_keep_their_wording(monkeypatch, stored):
|
||||
model = Reasoner(structured=lambda units: {"raw": AIMessage(content="不是 JSON",
|
||||
response_metadata={"finish_reason": "stop"}),
|
||||
"parsed": None},
|
||||
plain=lambda units: AIMessage(content="还是不是", response_metadata={"finish_reason": "stop"}))
|
||||
Models(monkeypatch, model)
|
||||
outcome, events = await judged(stored)
|
||||
assert [kind for kind, _ in model.calls] == ["structured", "plain"]
|
||||
assert outcome["unjudged"] == {"format": 4}
|
||||
assert outcome["gaps"] == ["有 4 句结论未裁判:裁判返回的结果格式有误"]
|
||||
assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END)
|
||||
assert logs(events, "judge_truncated") == [] and len(logs(events, "judge_failed")) == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 额度与退回
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_judge_asks_for_the_larger_budget(monkeypatch, stored):
|
||||
models = Models(monkeypatch, Reasoner())
|
||||
outcome, _ = await judged(stored)
|
||||
assert outcome["counts"]["supported"] == 4
|
||||
assert [s.max_tokens for s in models.specs] == [MAX_TOKENS] == [16384], "平常只建一个模型"
|
||||
assert "truncated" in REASONS and FALLBACK_MAX_TOKENS == 4096
|
||||
|
||||
|
||||
async def test_a_rejected_budget_falls_back_to_4096_once_for_all_batches(monkeypatch, stored):
|
||||
model = Reasoner()
|
||||
models = Models(monkeypatch, model, rejects_above=4096)
|
||||
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE * 2 + 3))
|
||||
outcome, _ = await judged(stored, raw)
|
||||
assert outcome["counts"]["supported"] == BATCH_SIZE * 2 + 3 and outcome["gaps"] == []
|
||||
assert [s.max_tokens for s in models.specs] == [16384, 4096]
|
||||
assert models.rejecting.calls == 1, "第一批退回过,后面几批直接用小的"
|
||||
assert [kind for kind, _ in model.calls] == ["structured"] * 3
|
||||
|
||||
|
||||
async def test_a_rejected_budget_is_remembered_for_the_next_reports(monkeypatch, stored):
|
||||
"""每份报告、每次点击各问一次裁判:同一接入、模型被嫌大过一次,之后直接用小的,不再每次先撞一次。"""
|
||||
model = Reasoner()
|
||||
models = Models(monkeypatch, model, rejects_above=4096)
|
||||
first, _ = await judged(stored)
|
||||
second, _ = await judged(stored)
|
||||
assert first["counts"]["supported"] == 4 and second["counts"]["supported"] == 4
|
||||
assert models.rejecting.calls == 1, "第二份报告没有再按 16384 撞一次"
|
||||
|
||||
|
||||
async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, stored):
|
||||
tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. "
|
||||
"Given: 3000 `inputs` tokens and 16384 `max_new_tokens`", status_code=422)
|
||||
models = Models(monkeypatch, Reasoner(), rejects_above=4096, rejecting=tgi)
|
||||
outcome, _ = await judged(stored)
|
||||
assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == []
|
||||
assert [s.max_tokens for s in models.specs] == [16384, 4096] and tgi.calls == 1
|
||||
|
||||
|
||||
async def test_a_truncation_after_the_fallback_names_the_small_budget(monkeypatch, stored):
|
||||
Models(monkeypatch, Reasoner(plain=thinking_only, unstructured=True), rejects_above=4096)
|
||||
outcome, _ = await judged(stored)
|
||||
[verdict, *_] = outcome["verdicts"].values()
|
||||
assert verdict["reason"] == "truncated"
|
||||
assert verdict["rationale"].startswith("裁判模型的输出达到上限(4096 token)被截断")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 真实的 ChatOpenAI 和 openai SDK:只有 HTTP 层是假的(经 OpenAI 兼容网关接入的推理模型就是这一路)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Gateway:
|
||||
"""OpenAI 兼容网关:reply(这一批的句子编号) 给出 (message, finish_reason, 输出 token 数)。cap 给了,请求的额度超过它
|
||||
就回 400。bodies 记下每个请求体。"""
|
||||
|
||||
def __init__(self, reply, *, cap: int | None = None) -> None:
|
||||
self.reply, self.cap = reply, cap
|
||||
self.bodies: list[dict] = []
|
||||
|
||||
async def handle(self, request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content or b"{}")
|
||||
self.bodies.append(body)
|
||||
budget = body.get("max_completion_tokens") or body.get("max_tokens")
|
||||
if self.cap and budget and budget > self.cap:
|
||||
message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens"
|
||||
return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}},
|
||||
request=request)
|
||||
units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M)
|
||||
message, finish, spent = self.reply(units)
|
||||
return httpx.Response(200, request=request, json={
|
||||
"id": "c-1", "object": "chat.completion", "created": 1, "model": PRICED,
|
||||
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
|
||||
"usage": {"prompt_tokens": 3000, "completion_tokens": spent, "total_tokens": 3000 + spent}})
|
||||
|
||||
|
||||
def gateway(monkeypatch, reply, *, cap: int | None = None) -> Gateway:
|
||||
fake = Gateway(reply, cap=cap)
|
||||
monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle)
|
||||
provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1",
|
||||
api_key=None, models=[{"id": PRICED}], default_model=PRICED, enabled=True, extra={})
|
||||
|
||||
async def build(session, spec):
|
||||
return build_chat_model(provider, spec), PRICED
|
||||
|
||||
monkeypatch.setattr(judge_mod, "get_chat_model", build)
|
||||
return fake
|
||||
|
||||
|
||||
async def test_a_real_client_cut_off_in_the_json_schema_call(monkeypatch, stored):
|
||||
"""json_schema 的结构化输出:finish_reason=length 时 openai SDK 抛 LengthFinishReasonError,不再退回纯文本。"""
|
||||
fake = gateway(monkeypatch, lambda units: ({"role": "assistant", "content": '{"items": [{"unit": "u'}, "length", 16384))
|
||||
outcome, _ = await judged(stored)
|
||||
assert outcome["unjudged"] == {"truncated": 4}
|
||||
assert outcome["gaps"] == [f"有 4 句结论未裁判:{CUT_16K},{ADVICE}"]
|
||||
assert len(fake.bodies) == 1 and "response_format" in fake.bodies[0]
|
||||
assert fake.bodies[0]["max_completion_tokens"] == 16384
|
||||
assert outcome["usage"]["output_tokens"] == 16384, "异常里带着的那次用量照记"
|
||||
|
||||
|
||||
async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, stored):
|
||||
def answer(units):
|
||||
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
|
||||
return {"role": "assistant", "content": json.dumps({"items": items}, ensure_ascii=False)}, "stop", 80
|
||||
|
||||
fake = gateway(monkeypatch, answer, cap=8192)
|
||||
outcome, _ = await judged(stored)
|
||||
assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == []
|
||||
assert [b.get("max_completion_tokens") for b in fake.bodies] == [16384, 4096]
|
||||
|
||||
|
||||
async def test_a_real_anthropic_cut_with_a_half_written_list(monkeypatch, stored):
|
||||
"""真实的 ChatAnthropic:强制工具调用写到第二条判定时到了上限,网关给的是半截清单。整批记截断,llm.end 标出来。"""
|
||||
def half(body):
|
||||
units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M)
|
||||
items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []},
|
||||
{"unit": units[1]}]
|
||||
return [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": {"items": items}}]
|
||||
|
||||
fake = messages_api(monkeypatch, [(half, "max_tokens", 16384)], model=PRICED, module=judge_mod)
|
||||
outcome, events = await judged(stored)
|
||||
assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0
|
||||
assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384, "没有再退回纯文本"
|
||||
[end] = [d for k, d in events if k == EventType.LLM_END]
|
||||
assert end["truncated"] is True and end["error"] == CUT_16K
|
||||
@@ -0,0 +1,344 @@
|
||||
"""模型调用的输出额度(providers/budget.py):助手改图、核查、证据裁判共用的那几样。
|
||||
|
||||
推理模型的思考也算在输出额度里,额度小了,想完就没额度写正文:回复是空的,或者 JSON 只写了一半。这里守几件事:
|
||||
- 认得出「输出到了上限被截断」:OpenAI 系 finish_reason=length、Anthropic stop_reason=max_tokens,以及 openai 结构化
|
||||
输出解析时抛的 LengthFinishReasonError;服务方嫌额度太大的 400 不算截断(那是请求被拒,一个字都没写)
|
||||
- 认得出「服务方嫌额度太大」:400 / 422,报错里有 max_tokens 一类字样;别的 400 不算
|
||||
- 截断的那次调用花掉的用量从异常里取得到(额度是真花出去了)
|
||||
- BudgetFallback:先按大额度调,被嫌大时按小额度重来一次,之后的调用都用小的;流式、非流式都认;bind_tools、
|
||||
with_structured_output 包出来的和原来的共用同一个开关;只在还没吐出任何东西时重来;小额度的模型要退回时才建
|
||||
- 核查、证据裁判另认两种「额度被挡下」(budget_refused):自建服务(text-generation-inference 一类)说的
|
||||
max_new_tokens,和 anthropic SDK 在本地就拒的额度(请求没发出去);助手改图的认法(budget_rejected)不变
|
||||
- 给了 key(接入、模型、大额度)的,被嫌大一次就记在进程里:之后新建的调用直接用小的,不再每次先撞一次;没给 key 的
|
||||
(助手改图)不记也不查
|
||||
|
||||
模型都是剧本,不碰真接口。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
from openai import LengthFinishReasonError
|
||||
from openai.types import CompletionUsage
|
||||
from openai.types.chat import ChatCompletion
|
||||
|
||||
from app.providers import budget
|
||||
from app.providers.budget import (
|
||||
BUDGET_WORDS,
|
||||
BudgetFallback,
|
||||
budget_refused,
|
||||
budget_rejected,
|
||||
cut_off,
|
||||
cut_words,
|
||||
error_usage,
|
||||
truncated_error,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def forget_rejections():
|
||||
"""被嫌大过的接入记在进程里:每个测试从没记过开始,也不留给后面的测试。"""
|
||||
budget._REJECTED.clear()
|
||||
yield
|
||||
budget._REJECTED.clear()
|
||||
|
||||
|
||||
def length_error(prompt: int = 9000, completion: int = 16384) -> LengthFinishReasonError:
|
||||
"""openai 的结构化输出在 finish_reason=length 时抛的那个错(原话和真实运行里一样)。"""
|
||||
usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion)
|
||||
return LengthFinishReasonError(completion=ChatCompletion.model_construct(
|
||||
id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage))
|
||||
|
||||
|
||||
class Rejected(Exception):
|
||||
"""仿 openai / anthropic SDK 的 400:一发请求就被拒,还没吐出任何东西。"""
|
||||
|
||||
def __init__(self, message: str, status_code: int = 400) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
TOO_LARGE = "max_tokens: 16384 > 8192, which is the maximum allowed number of output tokens"
|
||||
#: text-generation-inference 的 OpenAI 兼容接口:输入加额度超过 max_total_tokens 时的 422(原话的样子)
|
||||
TGI_TOO_LARGE = ("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. "
|
||||
"Given: 1200 `inputs` tokens and 16384 `max_new_tokens`")
|
||||
#: anthropic SDK 在本地拒绝时的原话:没设超时的客户端非流式调用、额度超过这个模型不流式能给的上限
|
||||
SDK_REFUSED = ("Streaming is required for operations that may take longer than 10 minutes. See "
|
||||
"https://platform.claude.com/docs/en/cli-sdks-libraries/sdks/python#long-requests for more details")
|
||||
|
||||
|
||||
class Model:
|
||||
"""记下每次调用的剧本模型。fail:每次调用都抛它;stream:流式时吐的块,吐到 boom 处抛 fail。"""
|
||||
|
||||
def __init__(self, name: str, *, fail: Exception | None = None, stream: list | None = None,
|
||||
boom: int | None = None, max_tokens: int | None = None) -> None:
|
||||
self.name = name
|
||||
self.fail = fail
|
||||
self.stream = stream if stream is not None else [AIMessageChunk(content=name)]
|
||||
self.boom = boom
|
||||
self.max_tokens = max_tokens
|
||||
self.calls: list[tuple[str, object]] = []
|
||||
self.bound: list[tuple[str, object, dict]] = []
|
||||
|
||||
async def ainvoke(self, messages, *args, **kwargs):
|
||||
self.calls.append(("ainvoke", messages))
|
||||
if self.fail is not None:
|
||||
raise self.fail
|
||||
return AIMessage(content=self.name)
|
||||
|
||||
async def astream(self, messages, *args, **kwargs):
|
||||
self.calls.append(("astream", messages))
|
||||
for i, chunk in enumerate(self.stream):
|
||||
if self.boom is not None and i == self.boom:
|
||||
raise self.fail
|
||||
yield chunk
|
||||
if self.fail is not None and self.boom is None:
|
||||
raise self.fail
|
||||
|
||||
def bind_tools(self, tools, **kwargs):
|
||||
self.bound.append(("tools", tools, kwargs))
|
||||
return self
|
||||
|
||||
def with_structured_output(self, schema, **kwargs):
|
||||
self.bound.append(("structured", schema, kwargs))
|
||||
return self
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 认出截断、认出嫌额度太大
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cut_off_replies_are_recognised_for_both_families():
|
||||
assert cut_off(AIMessage(content="", response_metadata={"finish_reason": "length"})) == "length"
|
||||
assert cut_off(AIMessageChunk(content=[], response_metadata={"stop_reason": "max_tokens"})) == "max_tokens"
|
||||
assert cut_off(AIMessage(content="好了", response_metadata={"finish_reason": "stop"})) == ""
|
||||
assert cut_off(AIMessage(content="", response_metadata={"stop_reason": "end_turn"})) == ""
|
||||
assert cut_off(None) == "" and cut_off({"finish_reason": "length"}) == ""
|
||||
|
||||
|
||||
def test_length_errors_are_recognised_but_a_rejected_budget_is_not():
|
||||
assert truncated_error(length_error())
|
||||
assert truncated_error(RuntimeError("Could not parse response content as the length limit was reached"))
|
||||
assert not truncated_error(None)
|
||||
assert not truncated_error(RuntimeError("网关超时"))
|
||||
assert not truncated_error(Rejected(TOO_LARGE)), "嫌额度太大是请求被拒,模型一个字都没写,不是截断"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("message", [
|
||||
"Invalid max_tokens value, the valid range of max_tokens is [1, 8192]",
|
||||
TOO_LARGE,
|
||||
"max_completion_tokens is too large: 16384. This model supports at most 8192 completion tokens",
|
||||
"This model's maximum context length is 32768 tokens. However, you requested 40369 tokens "
|
||||
"(7601 in the messages, 32768 in the completion).",
|
||||
])
|
||||
def test_a_rejected_budget_is_recognised(message):
|
||||
assert budget_rejected(Rejected(message))
|
||||
assert budget_rejected(Rejected(message, status_code=422))
|
||||
|
||||
|
||||
def test_other_rejections_are_not_a_budget_problem():
|
||||
assert not budget_rejected(Rejected("model 'no-such-model' does not exist"))
|
||||
assert not budget_rejected(Rejected(TOO_LARGE, status_code=500)), "服务方自己出错不是嫌额度大"
|
||||
assert not budget_rejected(RuntimeError(TOO_LARGE)), "没有状态码的不认"
|
||||
assert "max_tokens" in BUDGET_WORDS
|
||||
|
||||
|
||||
def test_verify_and_judge_also_recognise_tgi_and_a_local_sdk_refusal():
|
||||
"""核查、证据裁判的额度从 4096 提到了 16384,才撞得上这两种:自建服务说的是 max_new_tokens;Bearer 鉴权的 Anthropic
|
||||
接入(客户端没设超时)调 Opus 4、4.1 这一代时,SDK 嫌 16384 太大、请求没发出去就抛 ValueError。"""
|
||||
assert budget_refused(Rejected(TGI_TOO_LARGE, status_code=422))
|
||||
assert budget_refused(ValueError(SDK_REFUSED))
|
||||
assert budget_refused(Rejected(TOO_LARGE)), "原来认的照样认"
|
||||
assert not budget_refused(ValueError("别的本地错误"))
|
||||
assert not budget_refused(RuntimeError(SDK_REFUSED)), "只认 SDK 抛的 ValueError"
|
||||
assert not budget_refused(Rejected("model 'no-such-model' does not exist"))
|
||||
assert not budget_refused(Rejected(TGI_TOO_LARGE, status_code=500))
|
||||
|
||||
|
||||
def test_the_copilot_wording_list_stays_as_it_was():
|
||||
"""助手改图按 budget_rejected 认(流式调用,SDK 不在本地拦):认法不变,data/recipe_ai.py 的测试拿 BUDGET_WORDS 比对。"""
|
||||
assert BUDGET_WORDS == ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length")
|
||||
assert not budget_rejected(Rejected(TGI_TOO_LARGE, status_code=422))
|
||||
assert not budget_rejected(ValueError(SDK_REFUSED))
|
||||
|
||||
|
||||
def test_the_tokens_spent_by_a_cut_off_call_come_from_the_error():
|
||||
assert error_usage(length_error(prompt=9000, completion=16384)) == (9000, 16384)
|
||||
assert error_usage(RuntimeError("网关超时")) == (0, 0) and error_usage(None) == (0, 0)
|
||||
|
||||
|
||||
def test_the_words_name_who_was_cut_off_and_the_limit():
|
||||
assert cut_words("核查模型", 16384) == "核查模型的输出达到上限(16384 token)被截断"
|
||||
assert cut_words("裁判模型", None) == "裁判模型的输出达到上限被截断", "额度不知道就不写数"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 退回小额度
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_rejected_budget_is_retried_once_with_the_small_one_and_stays_small():
|
||||
big, small = Model("big", fail=Rejected(TOO_LARGE)), Model("small")
|
||||
model = BudgetFallback(big, small)
|
||||
assert (await model.ainvoke(["问"])).content == "small"
|
||||
assert (await model.ainvoke(["再问"])).content == "small", "之后的调用都用小的,不再先撞一次"
|
||||
assert len(big.calls) == 1 and len(small.calls) == 2
|
||||
assert model.model is small
|
||||
|
||||
|
||||
async def test_other_errors_are_not_retried():
|
||||
big, small = Model("big", fail=Rejected("model 'no-such-model' does not exist")), Model("small")
|
||||
with pytest.raises(Rejected):
|
||||
await BudgetFallback(big, small).ainvoke(["问"])
|
||||
assert small.calls == []
|
||||
|
||||
|
||||
async def test_the_small_model_is_only_built_when_needed():
|
||||
built: list[str] = []
|
||||
small = Model("small")
|
||||
|
||||
async def build():
|
||||
built.append("small")
|
||||
return small
|
||||
|
||||
fine = BudgetFallback(Model("big"), build=build, limits=(16384, 4096))
|
||||
assert (await fine.ainvoke(["问"])).content == "big"
|
||||
assert built == [] and fine.max_tokens == 16384, "平常一个小额度的模型都不多建"
|
||||
|
||||
rejecting = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=build, limits=(16384, 4096))
|
||||
assert (await rejecting.ainvoke(["问"])).content == "small"
|
||||
assert (await rejecting.ainvoke(["再问"])).content == "small"
|
||||
assert built == ["small"], "退回之后建一次,之后沿用"
|
||||
assert rejecting.max_tokens == 4096
|
||||
|
||||
|
||||
async def test_without_a_small_model_the_rejection_is_raised():
|
||||
with pytest.raises(Rejected):
|
||||
await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE))).ainvoke(["问"])
|
||||
same = Model("same", fail=Rejected(TOO_LARGE))
|
||||
with pytest.raises(Rejected):
|
||||
await BudgetFallback(same, same).ainvoke(["问"])
|
||||
assert len(same.calls) == 1, "大小是同一个模型:重来也是同样的额度,不白撞第二次"
|
||||
|
||||
|
||||
async def test_bound_and_structured_views_share_the_switch():
|
||||
"""核查的工具循环里退回过小额度,收尾那次直接用小的;包出来的视图各自转给对应的小模型。"""
|
||||
big, small = Model("big", fail=Rejected(TOO_LARGE)), Model("small")
|
||||
model = BudgetFallback(big, small, limits=(16384, 4096))
|
||||
tools = model.bind_tools(["db_query"], parallel_tool_calls=False)
|
||||
assert big.bound == [("tools", ["db_query"], {"parallel_tool_calls": False})]
|
||||
assert small.bound == [("tools", ["db_query"], {"parallel_tool_calls": False})]
|
||||
assert (await tools.ainvoke(["查"])).content == "small"
|
||||
structured = model.with_structured_output({"type": "object"}, include_raw=True)
|
||||
assert (await structured.ainvoke(["交结果"])).content == "small"
|
||||
assert len(big.calls) == 1, "收尾那次不再先按大额度撞一次"
|
||||
assert structured.max_tokens == 4096 and model.max_tokens == 4096
|
||||
|
||||
|
||||
async def test_a_sibling_shares_the_switch_and_builds_its_own_small_model():
|
||||
"""收尾用的是另一份配置(关掉思考)建的模型:和工具循环的模型共用开关,要退回时按自己的配置建小的。"""
|
||||
built: list[str] = []
|
||||
loop_small, settle_small = Model("loop-small"), Model("settle-small")
|
||||
|
||||
async def build_loop():
|
||||
built.append("loop")
|
||||
return loop_small
|
||||
|
||||
async def build_settle():
|
||||
built.append("settle")
|
||||
return settle_small
|
||||
|
||||
loop = BudgetFallback(Model("loop-big", fail=Rejected(TOO_LARGE)), build=build_loop, limits=(16384, 4096))
|
||||
assert (await loop.ainvoke(["查"])).content == "loop-small"
|
||||
settle_big = Model("settle-big")
|
||||
settle = BudgetFallback(settle_big, build=build_settle, limits=(16384, 4096), shared_with=loop)
|
||||
assert (await settle.with_structured_output({"type": "object"}).ainvoke(["交结果"])).content == "settle-small"
|
||||
assert settle_big.calls == [] and built == ["loop", "settle"]
|
||||
|
||||
|
||||
async def test_streaming_falls_back_only_before_anything_came_out():
|
||||
big = Model("big", fail=Rejected(TOO_LARGE), stream=[])
|
||||
small = Model("small", stream=[AIMessageChunk(content="a"), AIMessageChunk(content="b")])
|
||||
model = BudgetFallback(big, small)
|
||||
assert [c.content async for c in model.astream(["问"])] == ["a", "b"]
|
||||
assert len(big.calls) == 1 and len(small.calls) == 1
|
||||
|
||||
started = Model("big", fail=Rejected(TOO_LARGE), stream=[AIMessageChunk(content="x"), AIMessageChunk(content="y")],
|
||||
boom=1)
|
||||
unused = Model("small")
|
||||
got: list[str] = []
|
||||
with pytest.raises(Rejected):
|
||||
async for chunk in BudgetFallback(started, unused).astream(["问"]):
|
||||
got.append(chunk.content)
|
||||
assert got == ["x"] and unused.calls == [], "已经吐出东西了:重来会重复输出,原样抛出"
|
||||
|
||||
|
||||
async def test_without_limits_max_tokens_reads_the_current_model():
|
||||
big = Model("big", fail=Rejected(TOO_LARGE), max_tokens=32768)
|
||||
small = Model("small", max_tokens=8192)
|
||||
model = BudgetFallback(big, small)
|
||||
assert model.max_tokens == 32768
|
||||
await model.ainvoke(["问"])
|
||||
assert model.max_tokens == 8192
|
||||
|
||||
|
||||
async def test_a_wider_check_can_be_given():
|
||||
"""核查、证据裁判按 budget_refused 认:服务方说的是 max_new_tokens 也退回小额度。缺省(助手改图)不认。"""
|
||||
tgi = Rejected(TGI_TOO_LARGE, status_code=422)
|
||||
model = BudgetFallback(Model("big", fail=tgi), Model("small"), rejected=budget_refused)
|
||||
assert (await model.ainvoke(["问"])).content == "small"
|
||||
with pytest.raises(Rejected):
|
||||
await BudgetFallback(Model("big", fail=tgi), Model("small")).ainvoke(["问"])
|
||||
local = BudgetFallback(Model("big", fail=ValueError(SDK_REFUSED)), Model("small"), rejected=budget_refused)
|
||||
structured = local.with_structured_output({"type": "object"})
|
||||
assert (await structured.ainvoke(["交结果"])).content == "small", "包出来的视图照样按这个认"
|
||||
|
||||
|
||||
async def test_a_rejection_is_remembered_for_later_calls_to_the_same_model():
|
||||
"""双盲复算的几路盲算、反方审稿的每一句、每次质询、每次点击裁判,各是一组新建的调用:同一接入、模型、额度被嫌大
|
||||
过一次,之后新建的直接用小的,不再先撞一次。别的模型、别的额度照旧先按大的试。"""
|
||||
built: list[str] = []
|
||||
|
||||
def builder(name: str):
|
||||
async def build():
|
||||
built.append(name)
|
||||
return Model(name)
|
||||
return build
|
||||
|
||||
key = ("gw", "reasoner-x", 16384)
|
||||
first = BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), build=builder("small-1"), limits=(16384, 4096),
|
||||
key=key)
|
||||
assert first.max_tokens == 16384
|
||||
assert (await first.ainvoke(["问"])).content == "small-1"
|
||||
later_big = Model("big", fail=Rejected(TOO_LARGE))
|
||||
later = BudgetFallback(later_big, build=builder("small-2"), limits=(16384, 4096), key=key)
|
||||
assert later.max_tokens == 4096, "还没调用就知道要用小的:截断时报的额度对得上"
|
||||
assert (await later.bind_tools(["db_query"]).ainvoke(["再问"])).content == "small-2"
|
||||
assert later_big.calls == [], "不再先按大额度撞一次"
|
||||
other = Model("other-big")
|
||||
assert (await BudgetFallback(other, build=builder("x"), key=("gw", "other", 16384)).ainvoke(["问"])).content \
|
||||
== "other-big"
|
||||
bigger = Model("bigger")
|
||||
assert (await BudgetFallback(bigger, build=builder("y"), key=("gw", "reasoner-x", 32768)).ainvoke(["问"])).content \
|
||||
== "bigger"
|
||||
assert built == ["small-1", "small-2"]
|
||||
|
||||
|
||||
async def test_without_a_key_nothing_is_remembered():
|
||||
"""助手改图不给 key:每次改图照旧先按大额度试(行为和挪过来之前一样)。"""
|
||||
await BudgetFallback(Model("big", fail=Rejected(TOO_LARGE)), Model("small")).ainvoke(["问"])
|
||||
assert budget._REJECTED == set()
|
||||
fresh = Model("big")
|
||||
assert (await BudgetFallback(fresh, Model("small")).ainvoke(["问"])).content == "big"
|
||||
|
||||
|
||||
async def test_a_streaming_rejection_is_remembered_too():
|
||||
key = ("gw", "streamer", 16384)
|
||||
big = Model("big", fail=Rejected(TOO_LARGE), stream=[])
|
||||
model = BudgetFallback(big, Model("small", stream=[AIMessageChunk(content="a")]), key=key)
|
||||
assert [c.content async for c in model.astream(["问"])] == ["a"]
|
||||
again = Model("big", stream=[AIMessageChunk(content="b")])
|
||||
assert [c.content async for c in BudgetFallback(again, Model("small", stream=[AIMessageChunk(content="c")]),
|
||||
key=key).astream(["问"])] == ["c"]
|
||||
assert again.calls == []
|
||||
@@ -0,0 +1,680 @@
|
||||
"""核查 agent(engine/verify_agent.py)的输出额度:截断照实说,服务方嫌额度大时退回小额度。
|
||||
|
||||
核查(双盲复算、反方审稿、审批质询共用)原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到
|
||||
thinking=off,思考就能把 4096 用光:这次调用 finish_reason=length,没有工具调用也没有 JSON。于是工具循环把它当成
|
||||
「模型说完了」去收尾,双盲复算记成「盲算没有查到这个数」;收尾被截断时记成「核查模型没有按结构交结果」;质询回答失败
|
||||
——都没说出真正的原因。这里守几件事:
|
||||
|
||||
- 额度:默认 16384(judge.MAX_TOKENS,核查和裁判共用);服务方嫌大时退回 4096 再试一次,之后工具循环、收尾都用小的,
|
||||
只在还没有任何输出时重来;别的 400 不重试。被嫌大过的接入、模型记在进程里,之后的核查直接用小的;自建服务说的
|
||||
max_new_tokens、anthropic SDK 在本地就拒的额度也认
|
||||
- 被截断的回复一律不采信:工具循环里截断的回复带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input 是
|
||||
空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾(同时长到点,算半路被掐断);收尾的强制工具调用被
|
||||
截断时,解析出来的空对象、半截对象都不算交了结果,也不再退回普通调用多花一次额度。截断那次的用量照记
|
||||
- 时间线上记一条 verify_truncated(节点里调用时),llm.end 标出这次被截断
|
||||
- 原因经现有的渠道交出去:双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错。开头只说「未能给出结果」,
|
||||
建议按模型从哪来的说:节点里节点上指定的模型优先,两处都说;质询只看设置。没有截断时原有说法不变,触顶原因的先后
|
||||
也不变
|
||||
|
||||
模型是剧本,另有真实的 ChatOpenAI、ChatAnthropic(只有 HTTP 层是假的);数据源是现造的 SQLite 文件(借
|
||||
test_verify_agent 的夹具),表名、数字都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, BaseMessage
|
||||
from openai import LengthFinishReasonError
|
||||
from openai.types import CompletionUsage
|
||||
from openai.types.chat import ChatCompletion
|
||||
|
||||
from app.core.events import EventType
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Provider, Setting
|
||||
from app.engine import judge as judging
|
||||
from app.engine import review as rv
|
||||
from app.engine import spend, verify_agent
|
||||
from app.engine.inquiry import INQUIRY_SCHEMA, _error_words
|
||||
from app.engine.nodes.challenge import CHALLENGE_SCHEMA, _claim_of
|
||||
from app.engine.nodes.dual_check import _slim
|
||||
from app.providers import budget
|
||||
from app.providers.factory import build_chat_model
|
||||
from tests.test_verify_agent import VALID_SUM, Script, clean_spend, priced, q, run, shop # noqa: F401 - 夹具按名字引入
|
||||
|
||||
try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上
|
||||
from langchain_openai._compat import httpx
|
||||
except ImportError: # pragma: no cover - 老版本的 langchain_openai
|
||||
import httpx
|
||||
|
||||
CUT_16K = "核查模型的输出达到上限(16384 token)被截断"
|
||||
#: 节点里(双盲复算、反方审稿):节点上指定的模型优先于设置,两处都说
|
||||
ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型"
|
||||
#: 不在节点里(审批质询):只看设置
|
||||
ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型"
|
||||
FAILED = f"{CUT_16K},未能给出结果。{ADVICE}"
|
||||
FAILED_SETTINGS = f"{CUT_16K},未能给出结果。{ADVICE_SETTINGS}"
|
||||
FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致",
|
||||
"rationale": "未作废订单的金额合计是 [[v:Q1.r0.total]]。"}
|
||||
#: 反方审稿那种结构写到一半:结论写完了,理由写到一半
|
||||
HALF_CHALLENGE = {"verdict": "stands", "rationale": "合计是 [[v:Q1.r0.total]],与报告"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def forget_rejections():
|
||||
"""被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。"""
|
||||
budget._REJECTED.clear()
|
||||
yield
|
||||
budget._REJECTED.clear()
|
||||
|
||||
|
||||
def cut_reply(output: int = 16384) -> AIMessage:
|
||||
"""推理模型把额度全用在思考上:正文为空、没有工具调用,finish_reason=length(DeepSeek 经 LiteLLM 的样子)。"""
|
||||
return AIMessage(content="", additional_kwargs={"reasoning_content": "先想想该查哪张表……"},
|
||||
response_metadata={"finish_reason": "length"},
|
||||
usage_metadata={"input_tokens": 900, "output_tokens": output, "total_tokens": 900 + output})
|
||||
|
||||
|
||||
def cut_call(name: str, args: dict) -> AIMessage:
|
||||
"""工具调用写到一半到了上限(ChatAnthropic 的样子):tool_use 截在半路时 input 是空的,有的网关给半截对象;
|
||||
stop_reason=max_tokens。"""
|
||||
block = {"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": args}
|
||||
return AIMessage(content=[{"type": "thinking", "thinking": "想了很久……", "signature": "s"}, block],
|
||||
tool_calls=[{"name": block["name"], "args": args, "id": block["id"]}],
|
||||
response_metadata={"stop_reason": "max_tokens"},
|
||||
usage_metadata={"input_tokens": 900, "output_tokens": 16384, "total_tokens": 17284})
|
||||
|
||||
|
||||
def anthropic_cut(args: dict | None = None) -> dict:
|
||||
"""收尾的强制工具调用写到一半到了上限(结构化输出走 function_calling,ChatAnthropic 的缺省):原始回复
|
||||
stop_reason=max_tokens。args 为 None 是截在思考里、一个工具调用都没写出来;给了就是截在 tool_use 里——input 是
|
||||
空的(Anthropic 非流式的样子)或半截对象(有的网关),解析出来的就是它。"""
|
||||
content = [] if args is None else [{"type": "tool_use", "id": "toolu_1", "name": "verify_result", "input": args}]
|
||||
calls = [] if args is None else [{"name": "verify_result", "args": args, "id": "toolu_1"}]
|
||||
raw = AIMessage(content=content, tool_calls=calls, response_metadata={"stop_reason": "max_tokens"},
|
||||
usage_metadata={"input_tokens": 700, "output_tokens": 16384, "total_tokens": 17084})
|
||||
return {"raw": raw, "parsed": args, "parsing_error": None}
|
||||
|
||||
|
||||
def length_error(prompt: int = 800, completion: int = 16384) -> LengthFinishReasonError:
|
||||
"""openai 的结构化输出(json_schema)在 finish_reason=length 时抛的那个错。"""
|
||||
usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion)
|
||||
return LengthFinishReasonError(completion=ChatCompletion.model_construct(
|
||||
id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage))
|
||||
|
||||
|
||||
def query_step(name: str, sql: str = VALID_SUM, call_id: str = "c1") -> AIMessage:
|
||||
return AIMessage(content="查一下。", tool_calls=[{"name": n, "args": a, "id": i} for n, a, i in q(name, sql, call_id)])
|
||||
|
||||
|
||||
class BudgetRejected(Exception):
|
||||
"""服务方嫌输出额度太大时的 400(DeepSeek chat 系只收 8K)。"""
|
||||
|
||||
def __init__(self, message: str, status_code: int = 400) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class Reasoner:
|
||||
"""核查模型的剧本。steps:工具循环里每一轮的回复,用完了回「查完了。」;structured / plain:收尾时结构化调用、
|
||||
普通调用的回复(AIMessage、结构化结果,或者一个异常就抛它)。calls 按顺序记下每次调用是哪一种。"""
|
||||
|
||||
def __init__(self, steps=(), *, structured=None, plain=None, unstructured: bool = False) -> None:
|
||||
self.steps = list(steps)
|
||||
self.structured = structured
|
||||
self.plain = plain
|
||||
self.unstructured = unstructured
|
||||
self.calls: list[str] = []
|
||||
|
||||
def bind_tools(self, tools, **kwargs):
|
||||
return _View(self, "step")
|
||||
|
||||
def with_structured_output(self, schema, **kwargs):
|
||||
if self.unstructured:
|
||||
raise NotImplementedError("这个模型不支持结构化输出")
|
||||
return _View(self, "final")
|
||||
|
||||
async def ainvoke(self, messages, *args, **kwargs):
|
||||
return self.answer("plain")
|
||||
|
||||
def answer(self, kind: str):
|
||||
self.calls.append(kind)
|
||||
if kind == "step":
|
||||
reply = self.steps.pop(0) if self.steps else AIMessage(content="查完了。")
|
||||
else:
|
||||
reply = self.structured if kind == "final" else self.plain
|
||||
if isinstance(reply, BaseException):
|
||||
raise reply
|
||||
return reply
|
||||
|
||||
|
||||
class _View:
|
||||
def __init__(self, model: Reasoner, kind: str) -> None:
|
||||
self.model, self.kind = model, kind
|
||||
|
||||
async def ainvoke(self, messages, *args, **kwargs):
|
||||
return self.model.answer(self.kind)
|
||||
|
||||
|
||||
class Rejecting:
|
||||
"""一开口就被拒的模型:工具循环、收尾都一样。"""
|
||||
|
||||
def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens",
|
||||
status_code: int = 400):
|
||||
self.message, self.status_code = message, status_code
|
||||
self.calls = 0
|
||||
|
||||
def bind_tools(self, tools, **kwargs):
|
||||
return self
|
||||
|
||||
def with_structured_output(self, schema, **kwargs):
|
||||
return self
|
||||
|
||||
async def ainvoke(self, messages, *args, **kwargs):
|
||||
self.calls += 1
|
||||
raise BudgetRejected(self.message, self.status_code)
|
||||
|
||||
|
||||
class Models:
|
||||
"""按额度给模型:rejects_above 给了,额度比它大的一律交 Rejecting。specs 记下每次建模型的配置。"""
|
||||
|
||||
def __init__(self, monkeypatch, model, *, rejects_above: int | None = None, rejecting: Rejecting | None = None):
|
||||
self.specs = []
|
||||
self.rejecting = rejecting or Rejecting()
|
||||
|
||||
async def fake(session, spec):
|
||||
self.specs.append(spec)
|
||||
if rejects_above is not None and (spec.max_tokens or 0) > rejects_above:
|
||||
return self.rejecting, "reasoner-x"
|
||||
return model, "reasoner-x"
|
||||
|
||||
monkeypatch.setattr(verify_agent, "get_chat_model", fake)
|
||||
|
||||
|
||||
def collector():
|
||||
events: list[tuple] = []
|
||||
return events, (lambda kind, **data: events.append((kind, data)))
|
||||
|
||||
|
||||
def truncated_logs(events) -> list[dict]:
|
||||
return [d for k, d in events if k == EventType.LOG and d.get("code") == "verify_truncated"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 工具循环里被截断
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_thinking_that_eats_the_budget_in_the_loop_is_reported_as_truncation(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
model = Reasoner([cut_reply()], structured={"value": None, "verdict": "无法判断", "rationale": "没有查询结果"})
|
||||
Models(monkeypatch, model)
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc")
|
||||
assert out.result is None and out.queries == []
|
||||
assert out.error == FAILED
|
||||
assert model.calls == ["step"], "一条查询都没有:收尾只能交「查不到」,不再为它多花一次额度"
|
||||
assert out.answered is False, "被截断不是模型自己停下来的"
|
||||
assert out.usage["output_tokens"] == 16384, "截断那次的额度是真花出去了"
|
||||
[log] = truncated_logs(events)
|
||||
assert log["level"] == "warn" and log["max_tokens"] == 16384 and "delivered" not in log
|
||||
assert log["purpose"] == "verify" and log["message"] == out.error
|
||||
[end] = [d for k, d in events if k == EventType.LLM_END]
|
||||
assert end["truncated"] is True and end["max_tokens"] == 16384 and CUT_16K in end["error"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args", [{}, {"sql": "SELECT SUM(amount) AS total FROM orders WHERE is_del"}],
|
||||
ids=["empty-input", "half-sql"])
|
||||
async def test_a_tool_call_cut_off_halfway_is_not_run(monkeypatch, shop, args):
|
||||
"""Anthropic 系:工具调用写到一半到了上限。写到一半的调用不能执行——空参数只会报「缺少必填参数」,半截的 SQL
|
||||
查出来的数是错的——也不按它收尾,原因照截断说。"""
|
||||
name, _ = shop
|
||||
model = Reasoner([cut_call(name, args)], structured=FOUND)
|
||||
Models(monkeypatch, model)
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc")
|
||||
assert model.calls == ["step"], "不执行、不收尾,也不再问一轮"
|
||||
assert out.result is None and out.queries == [] and out.error == FAILED
|
||||
assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_END, EventType.TOOL_ERROR)]
|
||||
[log] = truncated_logs(events)
|
||||
assert log["purpose"] == "verify" and log["max_tokens"] == 16384
|
||||
|
||||
|
||||
async def test_a_cut_after_a_query_is_reported_without_settling(monkeypatch, shop):
|
||||
"""查过一次、下一轮被截断:截断前查到的不一定是要的那个数(模型可能正想改查询),按它收尾交出去的结果说不清完整
|
||||
不完整,会照常参与双盲复算的比对、反方审稿的处置。同时长到点一样算半路被掐断:不收尾,原因照截断说,查过的照记。"""
|
||||
name, _ = shop
|
||||
model = Reasoner([query_step(name), cut_reply()], structured=FOUND)
|
||||
Models(monkeypatch, model)
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc")
|
||||
assert model.calls == ["step", "step"], "不再为收尾多花一次额度"
|
||||
assert out.result is None and out.error == FAILED and out.answered is False
|
||||
assert [q["alias"] for q in out.queries] == ["Q1"], "查过的照记"
|
||||
[log] = truncated_logs(events)
|
||||
assert log["purpose"] == "verify" and log["max_tokens"] == 16384
|
||||
assert not [n for n in out.notes if "截断" in n], "没有「按截断前查到的结果收尾」这一说"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 收尾被截断
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_cut_off_settle_is_reported_and_not_retried_as_plain(monkeypatch, shop):
|
||||
"""Anthropic 系:强制工具调用写出来之前就到了上限。退回普通调用也多半被推理占满,不再白花一次额度。"""
|
||||
name, _ = shop
|
||||
model = Reasoner([query_step(name)], structured=anthropic_cut(), plain=AIMessage(content=json.dumps(FOUND)))
|
||||
Models(monkeypatch, model)
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc")
|
||||
assert model.calls == ["step", "step", "final"], "没有再退回普通调用"
|
||||
assert out.result is None and out.error == FAILED
|
||||
[log] = truncated_logs(events)
|
||||
assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384
|
||||
end = [d for k, d in events if k == EventType.LLM_END][-1]
|
||||
assert end["purpose"] == "verify_result" and end["truncated"] is True and CUT_16K in end["error"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("schema", "args"), [
|
||||
(None, {}),
|
||||
(None, {"value": {"alias": "Q1"}}),
|
||||
(CHALLENGE_SCHEMA, HALF_CHALLENGE),
|
||||
(INQUIRY_SCHEMA, {"rationale": "合计是 [[v:Q1.r0.total]],比报告里写的"}),
|
||||
], ids=["empty-input", "half-cell", "half-challenge", "half-inquiry"])
|
||||
async def test_a_settle_cut_inside_the_forced_tool_call_is_not_taken(monkeypatch, shop, schema, args):
|
||||
"""强制工具调用写到一半到了上限:解析出来的是空对象(Anthropic)或半截对象(有的网关)。空对象以前记成「没有按结构
|
||||
交结果」,半截的格子引用记成「引用不可用」,反方审稿的半截对象(结论写完了、理由写到一半)照单全收成「经受住」,
|
||||
质询把半句话当成回答——截断的回复一律不采信,原因照截断说,也不退回普通调用。"""
|
||||
name, _ = shop
|
||||
model = Reasoner([query_step(name)], structured=anthropic_cut(args), plain=AIMessage(content=json.dumps(FOUND)))
|
||||
Models(monkeypatch, model)
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {}))
|
||||
assert model.calls == ["step", "step", "final"]
|
||||
assert out.result is None and out.error == FAILED and out.cells == []
|
||||
[log] = truncated_logs(events)
|
||||
assert log["purpose"] == "verify_result"
|
||||
|
||||
|
||||
async def test_an_openai_length_error_in_the_settle_is_truncation_not_a_failed_call(monkeypatch, shop):
|
||||
"""openai 的结构化输出解析时抛 LengthFinishReasonError:不是「调用失败,请稍后重试」,是额度被推理占满。"""
|
||||
name, _ = shop
|
||||
model = Reasoner([query_step(name)], structured=length_error(completion=16384))
|
||||
Models(monkeypatch, model)
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc")
|
||||
assert model.calls == ["step", "step", "final"]
|
||||
assert out.error == FAILED
|
||||
assert out.usage["output_tokens"] == 16384 and out.usage["input_tokens"] >= 800, "异常里带着的用量照记"
|
||||
end = [d for k, d in events if k == EventType.LLM_END][-1]
|
||||
assert "请稍后重试" not in end["error"] and CUT_16K in end["error"] and end["output_tokens"] == 16384
|
||||
|
||||
|
||||
async def test_a_plain_settle_cut_off_is_reported(monkeypatch, shop):
|
||||
"""不支持结构化输出的模型:收尾直接走普通调用,回复只有思考、finish_reason=length。"""
|
||||
name, _ = shop
|
||||
model = Reasoner([query_step(name)], plain=cut_reply(), unstructured=True)
|
||||
Models(monkeypatch, model)
|
||||
out = await run(name)
|
||||
assert model.calls == ["step", "step", "plain"]
|
||||
assert out.result is None and out.error == FAILED_SETTINGS
|
||||
|
||||
|
||||
async def test_a_plain_settle_cut_after_the_json_is_not_taken_either(monkeypatch, shop):
|
||||
"""普通调用的正文里有一段 JSON、回复却是截断的:JSON 后面还在写(多半是理由的补充),说不清写完了没有,同样不采信。"""
|
||||
name, _ = shop
|
||||
cut = AIMessage(content=json.dumps(FOUND, ensure_ascii=False) + "\n补充说明:合计里还应该",
|
||||
response_metadata={"finish_reason": "length"})
|
||||
model = Reasoner([query_step(name)], plain=cut, unstructured=True)
|
||||
Models(monkeypatch, model)
|
||||
out = await run(name)
|
||||
assert out.result is None and out.error == FAILED_SETTINGS
|
||||
|
||||
|
||||
async def test_without_truncation_the_old_wording_stays(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
model = Reasoner([query_step(name)], structured="不是 JSON", plain=AIMessage(content="还是不是 JSON"))
|
||||
Models(monkeypatch, model)
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc")
|
||||
assert model.calls == ["step", "step", "final", "plain"], "格式不对照旧退回普通调用补救一次"
|
||||
assert out.error == "核查模型没有按结构交结果"
|
||||
assert truncated_logs(events) == []
|
||||
assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END)
|
||||
|
||||
|
||||
async def test_an_empty_object_without_truncation_keeps_the_shape_wording(monkeypatch, shop):
|
||||
"""没被截断、交回的是空对象:照旧是「没有按结构交结果」,不说成截断。"""
|
||||
name, _ = shop
|
||||
whole = {"raw": AIMessage(content="", response_metadata={"stop_reason": "tool_use"}), "parsed": {}}
|
||||
Models(monkeypatch, Reasoner([query_step(name)], structured=whole))
|
||||
out = await run(name)
|
||||
assert out.error == "核查模型没有按结构交结果(交回的内容里没有要求的字段:value、verdict)"
|
||||
|
||||
|
||||
async def test_the_first_limit_hit_is_still_the_one_reported(monkeypatch, shop, priced):
|
||||
"""循环先因时长到点停下(一分钱还没留),收尾又被每日金额上限拦下:触顶原因照旧报先撞上的时长。双盲复算按它写
|
||||
无法复算的原因,反方审稿按「每日上限」决定还派不派后面的句子——截断的改动不能把这个先后换掉。"""
|
||||
name, _ = shop
|
||||
async with SessionLocal() as session:
|
||||
session.add(Setting(key=spend.SPEND_KEY, value={"date": spend._today(), "usd": 1.995, "calls": 9}))
|
||||
await session.commit()
|
||||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], FOUND, model_id="priced-model")
|
||||
out = await run(name, daily_max_usd=2.0, timeout_s=0)
|
||||
assert out.limit == "timeout_s"
|
||||
assert out.error == "已达全局每日核查金额上限 $2.00,没能交结果"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 额度与退回
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_default_budget_leaves_room_for_reasoning(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
models = Models(monkeypatch, Reasoner([query_step(name)], structured=FOUND))
|
||||
out = await run(name)
|
||||
assert out.error is None and out.result["value"] == 230
|
||||
assert [s.max_tokens for s in models.specs] == [16384, 16384], "工具循环、收尾都按 16384,不多建模型"
|
||||
assert models.specs[1].thinking == "off"
|
||||
|
||||
|
||||
async def test_a_rejected_budget_falls_back_to_4096_for_the_loop_and_the_settle(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
model = Reasoner([query_step(name)], structured=FOUND)
|
||||
models = Models(monkeypatch, model, rejects_above=4096)
|
||||
out = await run(name)
|
||||
assert out.error is None and out.result["value"] == 230
|
||||
assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096]
|
||||
assert [s.thinking for s in models.specs][2:] == ["off", "off"], "收尾的小额度模型同样关掉思考"
|
||||
assert models.rejecting.calls == 1, "退回过一次之后收尾直接用小的,不再先撞一次"
|
||||
assert model.calls == ["step", "step", "final"]
|
||||
|
||||
|
||||
async def test_a_rejected_budget_is_remembered_for_the_next_runs(monkeypatch, shop):
|
||||
"""双盲复算的几路盲算、反方审稿的每一句、每次质询各是一次核查:同一接入、模型被嫌大过一次,之后的核查直接用小的,
|
||||
不再每次先撞一次。"""
|
||||
name, _ = shop
|
||||
model = Reasoner([query_step(name), AIMessage(content="查完了。"), query_step(name, call_id="c2")],
|
||||
structured=FOUND)
|
||||
models = Models(monkeypatch, model, rejects_above=4096)
|
||||
first, second = await run(name), await run(name)
|
||||
assert first.error is None and second.error is None and second.result["value"] == 230
|
||||
assert models.rejecting.calls == 1, "第二次核查没有再按 16384 撞一次"
|
||||
|
||||
|
||||
async def test_a_cut_after_the_fallback_names_the_small_budget(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
Models(monkeypatch, Reasoner([cut_reply(output=4096)]), rejects_above=4096)
|
||||
out = await run(name)
|
||||
assert out.error.startswith("核查模型的输出达到上限(4096 token)被截断")
|
||||
|
||||
|
||||
async def test_other_rejections_are_not_retried_with_the_small_budget(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
models = Models(monkeypatch, Reasoner(), rejects_above=4096,
|
||||
rejecting=Rejecting("model 'reasoner-x' does not exist"))
|
||||
out = await run(name)
|
||||
assert out.result is None and "reasoner-x" in out.error
|
||||
assert [s.max_tokens for s in models.specs] == [16384], "不是额度的问题:不拿小额度再撞一次"
|
||||
|
||||
|
||||
async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, shop):
|
||||
"""text-generation-inference 一类自建服务:输入加额度超过 max_total_tokens 时回 422,说的是 max_new_tokens。"""
|
||||
name, _ = shop
|
||||
tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. "
|
||||
"Given: 1200 `inputs` tokens and 16384 `max_new_tokens`", status_code=422)
|
||||
model = Reasoner([query_step(name)], structured=FOUND)
|
||||
models = Models(monkeypatch, model, rejects_above=4096, rejecting=tgi)
|
||||
out = await run(name)
|
||||
assert out.error is None and out.result["value"] == 230
|
||||
assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096] and tgi.calls == 1
|
||||
|
||||
|
||||
async def test_the_verify_spec_asks_for_the_larger_budget():
|
||||
"""核查模型的配置(api/settings.py 的 verify_model_spec 沿用证据裁判的额度):16384,思考照旧关掉。"""
|
||||
assert judging.MAX_TOKENS == 16384 and judging.FALLBACK_MAX_TOKENS == 4096
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 原因经现有的渠道交出去
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_reason_reaches_dual_check_and_challenge(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
Models(monkeypatch, Reasoner([cut_reply()]))
|
||||
out = await run(name, node_id="dc")
|
||||
# 双盲复算:无法复算的原因
|
||||
slim = _slim(out)
|
||||
assert slim["value"] is None and slim["reason"] == FAILED
|
||||
# 反方审稿:这句记为无法判断,理由写「反查未完成」,没能反查的原因就是截断
|
||||
cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="",
|
||||
aliases=("Q1",))
|
||||
claim, why = _claim_of(cand, out)
|
||||
assert claim["verdict"] == "inconclusive" and claim["rationale"] == f"反查未完成:{FAILED}" and why == FAILED
|
||||
|
||||
|
||||
async def test_an_inquiry_names_the_settings_and_not_a_check(monkeypatch, shop):
|
||||
"""审批质询不在节点里(核查模型只看设置):建议只说设置;开头只说「未能给出结果」——提问人问的是问题,不是在核查。"""
|
||||
name, _ = shop
|
||||
Models(monkeypatch, Reasoner([cut_reply()]))
|
||||
out = await run(name, output_schema=INQUIRY_SCHEMA)
|
||||
words = _error_words(out, timeout_s=60, daily_max_usd=1.0)
|
||||
assert words == FAILED_SETTINGS
|
||||
assert "核查未能完成" not in words and "节点" not in words
|
||||
|
||||
|
||||
def test_the_review_layer_knows_the_truncation_log():
|
||||
"""跑完之后的复核层(engine/review.py)认得 verify_truncated:同 verify_limit,是缺口不是断裂,kind 是它自己,
|
||||
不落成笼统的 warn;不触发「重跑一次大概率能好」——同样的模型再跑一遍还是撞同一堵墙。"""
|
||||
assert rv.WARN_CODES["verify_truncated"] == ("verify_truncated", rv.DEGRADED)
|
||||
event = {"type": str(EventType.LOG), "node_id": "dc",
|
||||
"data": {"level": "warn", "code": "verify_truncated", "message": FAILED}}
|
||||
[signal] = rv.scan([event], {"结论": "本周订单 120 单。"})
|
||||
assert (signal.kind, signal.severity) == ("verify_truncated", rv.DEGRADED) and not rv.should_retry([signal])
|
||||
|
||||
|
||||
def test_the_reply_shapes_used_here_are_messages():
|
||||
assert isinstance(cut_reply(), BaseMessage) and isinstance(anthropic_cut()["raw"], BaseMessage)
|
||||
assert isinstance(cut_call("x", {}), BaseMessage)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 真实的 ChatOpenAI 和 openai SDK:只有 HTTP 层是假的(经 OpenAI 兼容网关接入的推理模型就是这一路)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
GATEWAY_MODEL = "reasoner-v4"
|
||||
|
||||
|
||||
class Gateway:
|
||||
"""OpenAI 兼容网关:按顺序回 chat.completion(最后一个重复用)。cap 给了,请求的额度超过它就回 400(DeepSeek chat 系
|
||||
只收 8K 那样)。bodies 记下每个请求体。"""
|
||||
|
||||
def __init__(self, replies: list, *, cap: int | None = None) -> None:
|
||||
self.replies, self.cap = list(replies), cap
|
||||
self.bodies: list[dict] = []
|
||||
|
||||
async def handle(self, request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content or b"{}")
|
||||
self.bodies.append(body)
|
||||
budget = body.get("max_completion_tokens") or body.get("max_tokens")
|
||||
if self.cap and budget and budget > self.cap:
|
||||
message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens"
|
||||
return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}},
|
||||
request=request)
|
||||
message, finish, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0]
|
||||
return httpx.Response(200, request=request, json={
|
||||
"id": "c-1", "object": "chat.completion", "created": 1, "model": GATEWAY_MODEL,
|
||||
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
|
||||
"usage": {"prompt_tokens": 900, "completion_tokens": spent, "total_tokens": 900 + spent}})
|
||||
|
||||
|
||||
def gateway(monkeypatch, replies: list, *, cap: int | None = None) -> Gateway:
|
||||
fake = Gateway(replies, cap=cap)
|
||||
monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle)
|
||||
provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1",
|
||||
api_key=None, models=[{"id": GATEWAY_MODEL}], default_model=GATEWAY_MODEL, enabled=True, extra={})
|
||||
|
||||
async def build(session, spec):
|
||||
return build_chat_model(provider, spec), GATEWAY_MODEL
|
||||
|
||||
monkeypatch.setattr(verify_agent, "get_chat_model", build)
|
||||
return fake
|
||||
|
||||
|
||||
THINKING_ONLY = ({"role": "assistant", "content": "", "reasoning_content": "先看看有哪些表……"}, "length", 16384)
|
||||
|
||||
|
||||
def tool_call(name: str) -> tuple:
|
||||
return ({"role": "assistant", "content": None, "tool_calls": [{
|
||||
"id": "c1", "type": "function",
|
||||
"function": {"name": f"db_query__{name}", "arguments": json.dumps({"sql": VALID_SUM})}}]}, "tool_calls", 120)
|
||||
|
||||
|
||||
async def test_a_real_client_cut_off_in_the_loop(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
fake = gateway(monkeypatch, [THINKING_ONLY])
|
||||
out = await run(name)
|
||||
assert out.error == FAILED_SETTINGS
|
||||
assert len(fake.bodies) == 1 and fake.bodies[0]["max_completion_tokens"] == 16384
|
||||
assert "tools" in fake.bodies[0], "工具循环那次带着只读的查询工具"
|
||||
|
||||
|
||||
async def test_a_real_client_cut_off_with_a_half_written_tool_call(monkeypatch, shop):
|
||||
"""OpenAI 系:工具调用的参数写到一半到了上限(finish_reason=length,arguments 是半截 JSON)。不执行、不收尾。"""
|
||||
name, _ = shop
|
||||
half = ({"role": "assistant", "content": None, "tool_calls": [{
|
||||
"id": "c1", "type": "function",
|
||||
"function": {"name": f"db_query__{name}", "arguments": '{"sql": "SELECT SUM(amount) AS total FROM ord'}}]},
|
||||
"length", 16384)
|
||||
fake = gateway(monkeypatch, [half])
|
||||
out = await run(name)
|
||||
assert out.error == FAILED_SETTINGS and out.queries == [] and len(fake.bodies) == 1
|
||||
|
||||
|
||||
async def test_a_real_client_cut_off_in_the_json_schema_settle(monkeypatch, shop):
|
||||
"""收尾走 openai 的结构化输出(json_schema):finish_reason=length 时 SDK 抛 LengthFinishReasonError。"""
|
||||
name, _ = shop
|
||||
half = ({"role": "assistant", "content": '{"value": {"alias": "Q1", "row'}, "length", 16384)
|
||||
fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), half])
|
||||
out = await run(name)
|
||||
assert out.error == FAILED_SETTINGS and len(out.queries) == 1
|
||||
assert len(fake.bodies) == 3 and "response_format" in fake.bodies[2], "收尾被截断:没有再退回普通调用"
|
||||
assert out.usage["output_tokens"] == 120 + 20 + 16384, "异常里带着的那次用量照记"
|
||||
|
||||
|
||||
async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
done = ({"role": "assistant", "content": json.dumps(FOUND, ensure_ascii=False)}, "stop", 60)
|
||||
fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), done],
|
||||
cap=8192)
|
||||
out = await run(name)
|
||||
assert out.error is None and out.result["value"] == 230
|
||||
budgets = [b.get("max_completion_tokens") for b in fake.bodies]
|
||||
assert budgets == [16384, 4096, 4096, 4096], "第一次被拒后退回 4096,之后工具循环和收尾都用小的,不再先撞一次"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 真实的 ChatAnthropic 和 anthropic SDK:只有 HTTP 层是假的
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
CLAUDE = "claude-x"
|
||||
|
||||
|
||||
class Messages:
|
||||
"""Anthropic 的 Messages 接口:按顺序回 message(最后一个重复用)。每个回复是 (content, stop_reason, 输出 token 数),
|
||||
content 可以是按请求体算的函数(收尾那次强制调用的工具名照请求里的写)。bodies 记下每个请求体。"""
|
||||
|
||||
def __init__(self, replies: list) -> None:
|
||||
self.replies = list(replies)
|
||||
self.bodies: list[dict] = []
|
||||
|
||||
async def handle(self, request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content or b"{}")
|
||||
self.bodies.append(body)
|
||||
content, stop, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0]
|
||||
return httpx.Response(200, request=request, json={
|
||||
"id": "msg_1", "type": "message", "role": "assistant", "model": body.get("model"),
|
||||
"content": content(body) if callable(content) else content, "stop_reason": stop, "stop_sequence": None,
|
||||
"usage": {"input_tokens": 700, "output_tokens": spent}})
|
||||
|
||||
|
||||
def messages_api(monkeypatch, replies: list, *, model: str = CLAUDE, bearer: bool = False,
|
||||
module=verify_agent) -> Messages:
|
||||
"""接上一个 Anthropic 接入(bearer 为真时按 Authorization: Bearer 鉴权,factory 会换上没设超时的 SDK 客户端)。"""
|
||||
fake = Messages(replies)
|
||||
monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle)
|
||||
provider = Provider(name=f"an-{uuid.uuid4().hex[:8]}", kind="anthropic", base_url="http://an.example.test",
|
||||
api_key="test-only-key", models=[{"id": model}], default_model=model, enabled=True,
|
||||
extra={"auth_style": "bearer"} if bearer else {})
|
||||
|
||||
async def build(session, spec):
|
||||
# 按这个接入的模型建:调用方的 spec 写的是别的模型名(mock-fast),SDK 在本地拦不拦要看真正发出去的 id
|
||||
return build_chat_model(provider, spec.model_copy(update={"model": model})), model
|
||||
|
||||
monkeypatch.setattr(module, "get_chat_model", build)
|
||||
return fake
|
||||
|
||||
|
||||
def forced(args: dict):
|
||||
"""收尾那次强制调用的工具:名字照请求里的写,input 是 args。"""
|
||||
return lambda body: [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": args}]
|
||||
|
||||
|
||||
def query_use(name: str) -> list:
|
||||
return [{"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {"sql": VALID_SUM}}]
|
||||
|
||||
|
||||
DONE = ([{"type": "text", "text": "查完了。"}], "end_turn", 10)
|
||||
|
||||
|
||||
async def test_a_real_anthropic_cut_in_the_middle_of_a_tool_call(monkeypatch, shop):
|
||||
"""思考占满了额度、tool_use 截在半路(input 是空的):以前每一轮都执行一遍空参数的查询(报「缺少必填参数 sql」),
|
||||
撞到步数上限,最后记成「没有按结构交结果」。"""
|
||||
name, _ = shop
|
||||
cut = ([{"type": "thinking", "thinking": "想了很久……", "signature": "s"},
|
||||
{"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {}}], "max_tokens", 16384)
|
||||
fake = messages_api(monkeypatch, [cut])
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc", max_queries=2)
|
||||
assert out.error == FAILED and out.queries == [] and out.limit is None
|
||||
assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384
|
||||
assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_ERROR)]
|
||||
[log] = truncated_logs(events)
|
||||
assert log["purpose"] == "verify"
|
||||
signals = rv.scan([{"type": str(k), "node_id": "dc", "data": d} for k, d in events], {"结论": "本周订单 120 单。"})
|
||||
assert [s.kind for s in signals] == ["verify_truncated"], "不是工具出错、也不是触顶"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("schema", "args"), [(None, {}), (CHALLENGE_SCHEMA, HALF_CHALLENGE)],
|
||||
ids=["empty-input", "half-challenge"])
|
||||
async def test_a_real_anthropic_settle_cut_inside_the_forced_tool_call(monkeypatch, shop, schema, args):
|
||||
name, _ = shop
|
||||
fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE, (forced(args), "max_tokens", 16384)])
|
||||
events, emit = collector()
|
||||
out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {}))
|
||||
assert out.error == FAILED and out.result is None and len(out.queries) == 1
|
||||
assert len(fake.bodies) == 3 and fake.bodies[2]["tool_choice"]["type"] == "tool", "没有再退回普通调用"
|
||||
[log] = truncated_logs(events)
|
||||
assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384
|
||||
if schema is CHALLENGE_SCHEMA:
|
||||
cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="",
|
||||
aliases=("Q1",))
|
||||
claim, why = _claim_of(cand, out)
|
||||
assert claim["verdict"] == "inconclusive" and why == FAILED, "半截的「经受住」不能当真"
|
||||
|
||||
|
||||
async def test_an_anthropic_client_refusing_the_budget_locally_falls_back(monkeypatch, shop):
|
||||
"""Bearer 鉴权的 Anthropic 接入(factory 换上的 SDK 客户端没设超时):Opus 4、4.1 这一代的 id(经网关用的 Vertex、
|
||||
Bedrock 写法也是)非流式只能给 8192,16384 的请求 anthropic SDK 在本地就拒了(ValueError,请求没发出去)。照服务方
|
||||
嫌额度大处理,退回 4096。这条守的是 SDK 的行为:anthropic 以后不再在本地拦这个 id,第一个请求就会按 16384 发出去,
|
||||
到时改期望。"""
|
||||
name, _ = shop
|
||||
fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE,
|
||||
(forced(FOUND), "tool_use", 60)], model="claude-opus-4-1@20250805", bearer=True)
|
||||
out = await run(name)
|
||||
assert out.error is None and out.result["value"] == 230
|
||||
assert [b["max_tokens"] for b in fake.bodies] == [4096, 4096, 4096], "16384 的请求一个都没发出去"
|
||||
@@ -1039,6 +1039,34 @@ export const AGENT_GUARD_TEXT = {
|
||||
unlimitedShort: '不限',
|
||||
} as const
|
||||
|
||||
/** 「(16,384 token)」:输出额度,没给就不写 */
|
||||
const tokenCap = (maxTokens?: number) => (maxTokens ? `(${formatNumber(maxTokens)} token)` : '')
|
||||
|
||||
/**
|
||||
* 时间线上「核查模型的输出到了上限被截断」那一行(engine/verify_agent.py 的 verify_truncated 日志,双盲复算、反方审稿的
|
||||
* 核查 Agent 发)。推理模型的思考也算在输出额度里,额度用完时回复是空的或只写了一半:照实说是输出额度不够,不说成
|
||||
* 「没查到」「格式不对」,下一步指到换模型。截断的回复一律不采信,这次核查没有结果,节点按无法复算、未能反查记缺口
|
||||
*/
|
||||
export const VERIFY_TRUNCATED_TEXT = {
|
||||
title: (maxTokens?: number) => `核查模型的输出达到上限${tokenCap(maxTokens)}被截断`,
|
||||
sub: '这次核查未能给出结果',
|
||||
next: '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型',
|
||||
} as const
|
||||
|
||||
/**
|
||||
* 时间线上结论句裁判的输出到了上限被截断(engine/judge.py):被截断的那次调用(llm.end 带 truncated),和
|
||||
* judge_truncated 日志那一行。不用「裁判未能执行」「检查接入后重新运行」的说法:截断时接入是好的,同一个模型重新运行
|
||||
* 多半还是截断,该换的是模型
|
||||
*/
|
||||
export const JUDGE_TRUNCATED_TEXT = {
|
||||
/** 被截断的那次调用:原来叫「请裁判模型判断 N 句结论」的那一行 */
|
||||
call: (maxTokens?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断`,
|
||||
title: (maxTokens?: number, unjudged?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断`
|
||||
+ (unjudged ? `,${formatNumber(unjudged)} 句结论未裁判` : ''),
|
||||
next: '未裁判的结论句记为缺口,不能完整出具。通常是推理占满了输出额度:请在报告撰写节点的「结论句裁判」中指定不带推理的裁判模型,'
|
||||
+ '或在「设置 → 偏好设置 → 证据裁判」中更换,再重新运行',
|
||||
} as const
|
||||
|
||||
/**
|
||||
* 上传表格:数据页「表格」标签的上传弹窗、导入回执和卡片。种类的说法和后端 data/tabular.py 的
|
||||
* CONVERSION_KINDS、SHAPE_KINDS 对得上;后端以后多出来的种类,界面上只显示它给的那句话
|
||||
|
||||
@@ -3,7 +3,8 @@ import type { NodeState, RunPhase } from './trace'
|
||||
import { formatDuration, formatNumber } from '../lib/format'
|
||||
import {
|
||||
ABSENCE_CLAIM_TEXT, CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, SCHEMA_PARTIAL_TEXT, SQL_CHECK_TEXT, TYPE_LABEL,
|
||||
CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, NODE_TYPE_LABEL,
|
||||
CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, JUDGE_TRUNCATED_TEXT, NODE_TYPE_LABEL,
|
||||
VERIFY_TRUNCATED_TEXT,
|
||||
claimTally,
|
||||
evidenceTally, issuanceLabel, nodeTypeLabel,
|
||||
} from '../lib/terms'
|
||||
@@ -765,17 +766,19 @@ function gateTitle(tool: string, verdict: 'allow' | 'escalate' | 'team', reason:
|
||||
* 双盲复算、反方审稿的日志代码(期 3 前后端共用的约定)。这几条按 code 和结构化字段解读,不看原话;后端发成 info 的
|
||||
* (一致、裁决、记入数据目录、审稿完成)也上时间线,不然这两种节点就是一个跑了几十秒、什么都没说的黑盒。
|
||||
* 无法复算(dual_check_unchecked)、未能反查(challenge_unreviewed)、只查了部分数据源(challenge_sources_lost)是
|
||||
* 期 4 补上的:以前认不得,只能照原话显示
|
||||
* 期 4 补上的:以前认不得,只能照原话显示。核查模型的输出到了上限被截断(verify_truncated,两种节点共用的核查 Agent 发)
|
||||
* 是期 5 验收前补上的
|
||||
*/
|
||||
const VERIFY_LOG_CODES = new Set([
|
||||
'dual_check_skipped', 'dual_check_same_model', 'dual_check_agreed', 'dual_check_disagreed', 'dual_check_resolved',
|
||||
'dual_check_catalog', 'dual_check_limit', 'dual_check_unchecked',
|
||||
'challenge_skipped', 'challenge_same_model', 'challenge_done', 'challenge_refuted', 'challenge_limit',
|
||||
'challenge_unreviewed', 'challenge_sources_lost',
|
||||
'challenge_unreviewed', 'challenge_sources_lost', 'verify_truncated',
|
||||
])
|
||||
|
||||
const DC = DUAL_CHECK_TEXT
|
||||
const CL = CHALLENGE_TEXT.log
|
||||
const VT = VERIFY_TRUNCATED_TEXT
|
||||
|
||||
/**
|
||||
* 出具缺口里哪几条是双盲复算、反方审稿报上来的:出具契约给上游节点报的缺口加了「节点类型「节点名」:」的前缀
|
||||
@@ -1010,6 +1013,15 @@ function explainLog(code: string | undefined, message: string, d: Record<string,
|
||||
fix: 'canvas',
|
||||
}
|
||||
}
|
||||
case 'judge_truncated': {
|
||||
// 裁判模型的输出到了上限被截断(judge.py run_request):max_tokens 是那次调用的额度,unjudged 是因此未裁判的句数。
|
||||
// 读结构化字段,不从原话里认;缺了字段只给不带数的标题。下一步指到换模型:同一个模型重新运行多半还是截断
|
||||
const limit = typeof d.max_tokens === 'number' && d.max_tokens > 0 ? d.max_tokens : undefined
|
||||
const n = typeof d.unjudged === 'number' && d.unjudged > 0 ? Math.round(d.unjudged) : undefined
|
||||
return {
|
||||
title: JUDGE_TRUNCATED_TEXT.title(limit, n), next: JUDGE_TRUNCATED_TEXT.next, fix: 'canvas', fixField: 'judge',
|
||||
}
|
||||
}
|
||||
case 'judge_failed': {
|
||||
// 「结论句裁判没跑完:有 2 句结论没裁判:裁判调用失败(…)」
|
||||
return {
|
||||
@@ -1214,6 +1226,19 @@ function explainLog(code: string | undefined, message: string, d: Record<string,
|
||||
level: 'warn',
|
||||
}
|
||||
}
|
||||
case 'verify_truncated': {
|
||||
// 核查 Agent 的输出到了上限被截断(verify_agent.py):max_tokens 是那次调用的输出额度。截断的回复一律不采信,这次核查
|
||||
// 没有结果。读结构化字段,不从原话里认;缺了字段(别的服务端版本)只给不带数的标题。入口落到节点的模型:该换的是模型
|
||||
const limit = typeof d.max_tokens === 'number' && d.max_tokens > 0 ? d.max_tokens : undefined
|
||||
return {
|
||||
title: VT.title(limit),
|
||||
sub: VT.sub,
|
||||
next: VT.next,
|
||||
fix: 'canvas',
|
||||
fixField: 'model',
|
||||
level: 'warn',
|
||||
}
|
||||
}
|
||||
default:
|
||||
return null
|
||||
}
|
||||
@@ -1912,9 +1937,11 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt
|
||||
step.detail = [String(d.error), step.detail].filter(Boolean).join('\n')
|
||||
}
|
||||
if (d.purpose === 'judge' && d.error) {
|
||||
// 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败
|
||||
// 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败。输出到了上限被截断的
|
||||
// 照实说是截断(不是「未能执行」:接入是好的,该换的是模型,下一步见 judge_truncated 那一行)
|
||||
const cap = num(d.max_tokens)
|
||||
step.level = 'warn'
|
||||
step.title = JUDGE_FAILED_TITLE
|
||||
step.title = d.truncated === true ? JUDGE_TRUNCATED_TEXT.call(cap && cap > 0 ? cap : undefined) : JUDGE_FAILED_TITLE
|
||||
step.detail = [String(d.error), step.detail].filter(Boolean).join('\n')
|
||||
}
|
||||
pendingLlm.delete(nodeId ?? '_')
|
||||
|
||||
@@ -2319,6 +2319,103 @@ await section('核查类日志与出具缺口(期 4):无法复算、未能
|
||||
rows.map((r) => `${r?.title}|${r?.sub ?? ''}`).join(' ; '))
|
||||
})
|
||||
|
||||
await section('核查模型的输出到了上限被截断(verify_truncated):说清额度、这次核查没有结果,下一步指到换模型', async () => {
|
||||
// 推理模型的思考也算在输出额度里:核查 Agent 的一次调用 finish_reason=length、没有工具调用也没有 JSON 时,以前双盲复算
|
||||
// 记成「盲算没有查到这个数」、反方审稿记成「核查模型没有按结构交结果」。现在截断的回复一律不采信,核查 Agent 另发
|
||||
// verify_truncated(max_tokens、purpose),时间线按结构化字段说人话,原话留在展开区。先核对后端源码还是这么发的
|
||||
const verifyPy = readFileSync(`${root}backend/app/engine/verify_agent.py`, 'utf8')
|
||||
check('后端发截断日志的格式还是夹具里这一种', verifyPy.includes('code="verify_truncated"') && verifyPy.includes('max_tokens=limit')
|
||||
&& verifyPy.includes('purpose=purpose') && !verifyPy.includes('delivered=')
|
||||
&& verifyPy.includes('TRUNCATED_ERROR = "{cut},未能给出结果。{advice}"'),
|
||||
'改了 verify_agent.py 的字段或说法就同步改 decode.ts 的 explainLog 和这里的夹具')
|
||||
const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data })
|
||||
const ADVICE = '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型'
|
||||
const FAILED = `核查模型的输出达到上限(16384 token)被截断,未能给出结果。${ADVICE}`
|
||||
const SMALL = `核查模型的输出达到上限(4096 token)被截断,未能给出结果。${ADVICE}`
|
||||
const all = flatten(mod.decodeRun([
|
||||
E(1, 'run.started', null, { total: 2 }),
|
||||
E(2, 'node.started', 'dc', { node_type: 'dual_check', label: '复核销售额' }),
|
||||
E(3, 'log', 'dc', { level: 'warn', code: 'verify_truncated', purpose: 'verify', max_tokens: 16384, message: FAILED }),
|
||||
E(4, 'node.finished', 'dc', { duration_ms: 20 }),
|
||||
E(5, 'node.started', 'ch', { node_type: 'challenge', label: '审月报' }),
|
||||
// 退回小额度之后在收尾时被截断
|
||||
E(6, 'log', 'ch', { level: 'warn', code: 'verify_truncated', purpose: 'verify_result', max_tokens: 4096, message: SMALL }),
|
||||
// 缺字段、字段类型不对(别的服务端版本):只给不带数的标题,不瞎填
|
||||
E(7, 'log', 'ch', { level: 'warn', code: 'verify_truncated', max_tokens: '16384', message: FAILED }),
|
||||
E(8, 'node.finished', 'ch', { duration_ms: 30 }),
|
||||
E(9, 'run.finished', null, { status: 'succeeded' }),
|
||||
]))
|
||||
const rows = all.filter((s) => s.code === 'verify_truncated')
|
||||
const nodes = all.filter((s) => s.kind === 'node')
|
||||
check('每条一行,挂在自己的节点下', rows.length === 3 && nodes.find((s) => s.nodeId === 'dc')?.children?.filter((x) => x.code === 'verify_truncated').length === 1
|
||||
&& nodes.find((s) => s.nodeId === 'ch')?.children?.filter((x) => x.code === 'verify_truncated').length === 2, `${rows.length} 行`)
|
||||
const [failed, small, bare] = rows
|
||||
check('标题说输出到了上限被截断、带额度,副标题说这次核查没有结果', failed?.title === '核查模型的输出达到上限(16,384 token)被截断'
|
||||
&& failed.sub === '这次核查未能给出结果', `${failed?.title} / ${failed?.sub}`)
|
||||
check('额度照实写(退回小额度之后是 4,096)', small?.title === '核查模型的输出达到上限(4,096 token)被截断' && small.sub === '这次核查未能给出结果',
|
||||
`${small?.title} / ${small?.sub}`)
|
||||
check('……下一步指到换不带推理的模型(节点上指定的优先,设置里的也说),入口落到节点的模型,提醒色',
|
||||
rows.every((s) => !!s.next?.includes('在节点上指定不带推理的模型') && s.next.includes('设置 → 偏好设置 → 核查')
|
||||
&& s.fix === 'canvas' && s.fixField === 'model' && s.level === 'warn'),
|
||||
`${failed?.next} / ${failed?.fix} / ${failed?.fixField} / ${failed?.level}`)
|
||||
check('……没有「已按截断前查到的结果收尾」这一说', !rows.some((s) => /收尾|可能不完整/.test(`${s.title}${s.sub ?? ''}`)))
|
||||
check('……原话留在展开区', failed?.detail === FAILED && small?.detail === SMALL)
|
||||
check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '核查模型的输出达到上限被截断'
|
||||
&& !/undefined|NaN/.test(`${bare?.title}${bare?.sub ?? ''}${bare?.next ?? ''}`), `${bare?.title} / ${bare?.sub}`)
|
||||
check('标题、副标题里没有内部代码', !rows.some((s) => /verify_truncated|max_tokens|purpose/.test(`${s.title}${s.sub ?? ''}${s.next ?? ''}`)))
|
||||
check('节点照常完成', nodes.every((s) => s.status === 'done'))
|
||||
})
|
||||
|
||||
await section('结论句裁判的输出到了上限被截断(judge_truncated):调用那一行说截断,日志另起一行,下一步指到换模型', async () => {
|
||||
// 以前截断的那一批只发 llm.end(标题沿用「结论句裁判未能执行」)和 judge_failed(下一步是「检查接入后重新运行」):
|
||||
// 截断时接入是好的,同一个模型重新运行多半还是截断。现在调用那一行照 truncated、max_tokens 说截断,另发 judge_truncated
|
||||
// (max_tokens、unjudged),judge_failed 只说别的失败。先核对后端源码还是这么发的
|
||||
const judgePy = readFileSync(`${root}backend/app/engine/judge.py`, 'utf8')
|
||||
check('后端发裁判截断的格式还是夹具里这一种', judgePy.includes('code="judge_truncated", max_tokens=cut_limit')
|
||||
&& judgePy.includes('unjudged=outcome["unjudged"]["truncated"]') && judgePy.includes('truncated=True, max_tokens=cut_limit')
|
||||
&& judgePy.includes('r not in LIMITS and r != "truncated"'),
|
||||
'改了 judge.py 的字段或说法就同步改 decode.ts 的 explainLog、llm.end 和这里的夹具')
|
||||
const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data })
|
||||
const CUT = '裁判模型的输出达到上限(16384 token)被截断'
|
||||
const TRUNCATED = `结论句裁判未完成:有 4 句结论未裁判:${CUT},通常是推理占满了输出额度,可换用不带推理的裁判模型`
|
||||
const all = flatten(mod.decodeRun([
|
||||
E(1, 'run.started', null, { total: 1 }),
|
||||
E(2, 'node.started', 'write', { node_type: 'report', label: '写周报' }),
|
||||
E(3, 'llm.start', 'write', { model: 'jm-r', structured: true, purpose: 'judge', message_count: 2, units: 4 }),
|
||||
E(4, 'llm.end', 'write', { model: 'jm-r', purpose: 'judge', duration_ms: 41000, units: 4, input_tokens: 3000,
|
||||
output_tokens: 16384, error: CUT, truncated: true, max_tokens: 16384 }),
|
||||
E(5, 'log', 'write', { level: 'warn', code: 'judge_truncated', max_tokens: 16384, unjudged: 4, message: TRUNCATED }),
|
||||
E(6, 'node.finished', 'write', { duration_ms: 43000 }),
|
||||
E(7, 'run.finished', null, { status: 'succeeded' }),
|
||||
]))
|
||||
const call = all.find((s) => s.kind === 'llm' && s.code === 'judge')
|
||||
check('调用那一行:说输出到了上限被截断、带额度,不说「未能执行」;提醒色、照常完成,原话在展开区',
|
||||
call?.title === '裁判模型的输出达到上限(16,384 token)被截断' && call.level === 'warn' && call.status === 'done'
|
||||
&& !!call.detail?.includes(CUT), `${call?.title} / ${call?.level} / ${call?.status}`)
|
||||
const row = all.find((s) => s.code === 'judge_truncated')
|
||||
check('judge_truncated:标题说截断、带额度和几句未裁判', row?.title === '裁判模型的输出达到上限(16,384 token)被截断,4 句结论未裁判',
|
||||
row?.title)
|
||||
check('……下一步指到换不带推理的裁判模型(节点的「结论句裁判」、设置都说),不说检查接入', !!row?.next?.includes('不带推理的裁判模型')
|
||||
&& row.next.includes('结论句裁判') && row.next.includes('设置 → 偏好设置 → 证据裁判') && !row.next.includes('接入')
|
||||
&& row.fix === 'canvas' && row.fixField === 'judge' && row.level === 'warn', `${row?.next} / ${row?.fix} / ${row?.fixField}`)
|
||||
check('……原话留在展开区,标题里没有内部代码', row?.detail === TRUNCATED && !/judge_truncated|max_tokens|unjudged/.test(`${row?.title}${row?.next}`))
|
||||
check('没有混出一条 judge_failed', !all.some((s) => s.code === 'judge_failed'))
|
||||
// 没截断的失败照旧:「未能执行」那一行和 judge_failed 的说法不变
|
||||
const other = flatten(mod.decodeRun([
|
||||
E(1, 'node.started', 'write', { node_type: 'report' }),
|
||||
E(2, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 2 }),
|
||||
E(3, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 300, units: 2, error: 'ConnectError: 连不上' }),
|
||||
E(4, 'node.finished', 'write', { duration_ms: 400 }),
|
||||
]))
|
||||
check('没截断的调用失败照旧叫「结论句裁判未能执行」', other.find((s) => s.kind === 'llm')?.title === '结论句裁判未能执行',
|
||||
other.find((s) => s.kind === 'llm')?.title)
|
||||
const bare = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report' }),
|
||||
E(2, 'log', 'w', { level: 'warn', code: 'judge_truncated', max_tokens: 'x', message: '裁判被截断了' })]))
|
||||
.find((s) => s.code === 'judge_truncated')
|
||||
check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '裁判模型的输出达到上限被截断'
|
||||
&& !/undefined|NaN/.test(`${bare?.title}${bare?.next ?? ''}`), bare?.title)
|
||||
})
|
||||
|
||||
await section('审批质询 approval.inquiry(期 4):一行「审批人质询」,标题带问题前 40 字,副标题说回答是否完成、查了几次', async () => {
|
||||
// 前后端共用的约定:运行停在审批上时服务端追加 approval.inquiry,node_id 为空;data 里有问题、划选的原文、回答正文、
|
||||
// 回答文档、成没成、触顶原因、查询(queries 和 inquiry_artifacts 两份)。以前这类事件落进「未识别的事件」
|
||||
|
||||
Reference in New Issue
Block a user