From 4905de728c4f8c47284022c77326b43dd90234dd Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 14:30:03 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A0=B8=E6=9F=A5=E6=88=AA=E6=96=AD=E7=9A=84?= =?UTF-8?q?=E5=9B=9E=E5=A4=8D=E4=B8=80=E5=BE=8B=E4=B8=8D=E9=87=87=E4=BF=A1?= =?UTF-8?q?=EF=BC=8C=E8=A7=A6=E9=A1=B6=E5=8E=9F=E5=9B=A0=E7=9A=84=E5=85=88?= =?UTF-8?q?=E5=90=8E=E7=85=A7=E6=97=A7=EF=BC=8C=E6=88=AA=E6=96=AD=E7=9A=84?= =?UTF-8?q?=E5=BB=BA=E8=AE=AE=E6=8C=89=E6=A8=A1=E5=9E=8B=E4=BB=8E=E5=93=AA?= =?UTF-8?q?=E6=9D=A5=E7=9A=84=E8=AF=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复核第 1 轮的发现(主要一条):Anthropic 的工具调用写到一半到了上限时,回复照样带着 tool_use(input 是空的,有的网关给半截对象)。工具循环只在没有工具调用时才看截断,于是一轮轮执行空参数的查询,撞到步数上限,最后记成「没有按结构交结果」;收尾的强制工具调用截在半路时解析出空对象或半截对象,空对象记成没按结构交,反方审稿的半截对象(结论写完、理由写到一半)被当成「经受住」,质询把半句话当成回答。 - 工具循环:回复被截断就停,带着的工具调用不执行(半截的 SQL 查出来的数是错的),也不收尾 - 截断前查到过东西的也不再按查到的收尾:截断前查到的不一定是要的那个数,交出去的结果只在 notes 里写一句「可能不完整」,审批卡、反方结论、质询回答都看不到,残缺的结果还照常参与比对和处置。同时长到点一样算半路被掐断,原因照截断说,查过的照记;TRUNCATED_SETTLED 和日志里的 delivered 字段去掉 - 收尾:回复被截断时,解析出来的、从回复里抠出来的都不采信(结构化、普通调用都是),也不退回普通调用 - 上一轮的 give_up 把触顶原因的先后从「先 stopped」换成了「先 refused」:去掉 give_up,收尾被上限拦下时照旧报先撞上的那个(双盲复算按它写无法复算的原因,反方审稿按每日上限决定还派不派后面的句子) - 截断的原因开头只说「未能给出结果」(质询里提问人看到的就是这句,不说「核查未能完成」);建议在节点里说「在节点上指定不带推理的模型,或在设置里更换核查模型」(节点上的模型优先于设置,和时间线上的下一步一致),质询只说设置 - 退回小额度按 budget_refused 认,按 (接入, 模型, 额度) 记在进程里 - 时间线:verify_truncated 的副标题固定说「这次核查未能给出结果」,check-decode 跟上;review.py 里 verify_truncated 的注释跟上 测试补上真实 ChatAnthropic(只有 HTTP 层是假的)的循环截断、收尾截断(空对象、半截对象),真实 ChatOpenAI 写到一半的工具调用参数,Bearer 鉴权下 SDK 在本地拒绝时退回 4096,自建服务的 max_new_tokens,跨次核查记住被嫌大,触顶原因的先后 --- backend/app/engine/review.py | 4 +- backend/app/engine/verify_agent.py | 139 +++++---- backend/tests/test_verify_truncation.py | 360 ++++++++++++++++++++---- frontend/src/lib/terms.ts | 11 +- frontend/src/run/decode.ts | 6 +- scripts/check-decode.mjs | 49 ++-- 6 files changed, 405 insertions(+), 164 deletions(-) diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index c497c95e..bae1b65b 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -128,8 +128,8 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 核查 agent 自己触顶、按已有结果收尾:多半和节点自己的 *_limit 说的是同一次触顶,照样是缺口,记成它自己的 kind, # 不落成笼统的 warn——步数上限(max_steps)只有它说 "verify_limit": ("verify_limit", DEGRADED), - # 核查 agent 的输出到了上限被截断(推理模型的思考也算在额度里):没交成结果的由节点按无法复算、未能反查记缺口,交了的 - # 可能不完整。同 verify_limit,是缺口不是断裂,记成它自己的 kind:说得清是输出额度的问题,换个模型就好 + # 核查 agent 的输出到了上限被截断(推理模型的思考也算在额度里):没交成结果,由节点按无法复算、未能反查记缺口。 + # 同 verify_limit,是缺口不是断裂,记成它自己的 kind:说得清是输出额度的问题,换个模型就好 "verify_truncated": ("verify_truncated", DEGRADED), } diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index 022706c9..b127eaef 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -26,10 +26,13 @@ query_artifact 和 role: verify——随运行封存、证据面板能下钻。 对它无效,照裁判的说法记一条说明。每次调用都记进核查的每日账(engine/spend.py)。 **输出额度。** 经 OpenAI 兼容接入的推理模型收不到 thinking=off,思考也算在输出额度里:额度按 spec(缺省 judge.MAX_TOKENS), -服务方嫌大时退回 judge.FALLBACK_MAX_TOKENS 再试一次,之后工具循环、收尾都用小的(providers/budget.py)。每次调用都看 -有没有到了上限被截断:工具循环里被截断(话没说完、也没发查询)不算模型自己停下,一条查询都没成功时不再收尾;收尾被截断 -时不再退回普通调用多花一次额度。因为截断没交成结果的,error 照实说是输出额度(TRUNCATED_ERROR),不说成「没查到」 -「没按结构交」;节点里调用时另记一条 verify_truncated。 +服务方嫌大时(或者 anthropic SDK 在本地就拒了)退回 judge.FALLBACK_MAX_TOKENS 再试一次,之后工具循环、收尾都用小的; +同一接入、模型被嫌大过,之后的核查一开始就用小的(providers/budget.py)。每次调用都看有没有到了上限被截断,截断的回复 +一律不采信:工具循环里截断的回复不算模型自己停下,带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input +是空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾——同时长到点一样算半路被掐断,按截断前查到的收尾 +交出去的结果说不清完整不完整,还会照常参与比对和处置。收尾的强制工具调用被截断时,解析出来的空对象、半截对象都不算 +交了结果,也不再退回普通调用多花一次额度。因为截断没交成结果的,error 照实说是输出额度(TRUNCATED_ERROR),不说成 +「没查到」「没按结构交」;节点里调用时另记一条 verify_truncated。 永远不抛:模型没配好、调用失败、格式不对、触顶、库写不进去,都落在返回值里(error / limit)。模型调用都经 get_chat_model,测试换成剧本。 @@ -60,7 +63,7 @@ from app.engine.nodes.llm import _DIGEST_BUDGET, _digest, _split_structured, _ty from app.engine.state import message_text from app.engine.toolcalls import ToolTimeout, limit_fields, limit_of, run_bounded from app.providers import catalog as pricing -from app.providers.budget import BudgetFallback, cut_off, cut_words, error_usage, truncated_error +from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error from app.providers.factory import ModelSpec, ProviderNotConfigured, bind_tools_safely, get_chat_model from app.tools.datasource import QUERY_PREFIX, SCHEMA_PREFIX, build_datasource_tools from app.tools.registry import ToolArgsError, ToolContext, Versions, args_model_of, prepare_args @@ -86,12 +89,14 @@ _REF_DEPTH = 8 RATIONALE = "rationale" #: 触顶原因 LIMITS = ("max_queries", "timeout_s", "max_cost_usd", "daily_max_usd", "max_steps") -#: 核查模型的输出到了上限被截断时给人看的话。TRUNCATED_ERROR 是没交成结果的原因:双盲复算的无法复算、反方审稿的「反查 -#: 未完成」、质询的报错都照它说,也是 verify_truncated 日志的原话。TRUNCATED_SETTLED 是截断前查到过东西、照常收尾了的, -#: 记进 notes 和日志。cut 是 budget.cut_words 的「核查模型的输出达到上限(N token)被截断」 -TRUNCATED_ERROR = "{cut},核查未能完成。{advice}" -TRUNCATED_SETTLED = "{cut},已按截断前查到的结果收尾,结果可能不完整。{advice}" -TRUNCATED_ADVICE = "通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型" +#: 核查模型的输出到了上限被截断、没交成结果时给人看的话:双盲复算的无法复算、反方审稿的「反查未完成」、质询的报错都照 +#: 它说,也是 verify_truncated 日志的原话。cut 是 budget.cut_words 的「核查模型的输出达到上限(N token)被截断」。开头只说 +#: 「未能给出结果」:质询里提问人看到的就是这句,它问的是问题,不是在核查 +TRUNCATED_ERROR = "{cut},未能给出结果。{advice}" +#: 截断的建议:该换的是模型。节点里(双盲复算、反方审稿)节点上指定的模型优先于设置(api/settings.py 的 +#: verify_model_spec),两处都说,和时间线上的下一步一致;不在节点里(审批质询)核查模型只看设置 +TRUNCATED_ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型" +TRUNCATED_ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型" VERIFY_SYSTEM = ( "你是数据核查员:只读地查库,用查询结果完成交给你的任务。\n" @@ -142,8 +147,8 @@ class VerifyOutcome: source, tool, call_id}。 rationale:理由组装成的报告文档(evidence.compose_doc 的形状,violations 里有裸数字)。usage:{input_tokens, output_tokens, total_tokens, cost_usd, calls, unpriced_calls}。limit:触顶原因(LIMITS 之一),没触顶为 None。 - answered:模型是不是自己停下来的(没被上限掐断,也没因为输出到了上限被截断)。因为截断没交成结果的,error 是 - TRUNCATED_ERROR;截断前查到过东西、照常交了结果的,error 为 None,notes 里记一句 TRUNCATED_SETTLED。 + answered:模型是不是自己停下来的(没被上限掐断,也没因为输出到了上限被截断)。被截断的一律没有结果,error 是 + TRUNCATED_ERROR。 """ result: dict[str, Any] | None @@ -709,7 +714,7 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou except Exception as e: # noqa: BLE001 - 核查模型没配好:这回核不了,照实交回 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) return outcome(error=f"核查模型不可用({why})") - base_model = _with_fallback(first, spec, small) + base_model = _with_fallback(first, spec, small, model_id) state.model = model_id if daily_max_usd is Default.SETTINGS: from app.api.settings import verify_settings # 在这里导入:api.settings 导入了 engine 里的模块 @@ -781,6 +786,16 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou await _announce(emit, EventType.LLM_END, **end) return got, raw, failure + # 截断的建议按模型从哪来的说:节点里节点上指定的优先,不在节点里(质询)只看设置 + advice = TRUNCATED_ADVICE if node_id else TRUNCATED_ADVICE_SETTINGS + + async def report_cut(purpose: str, limit: int | None) -> VerifyOutcome: + """被截断、没交成结果:记一条 verify_truncated(purpose 是哪一步,limit 是那次的额度),原因照截断说。查过的照记。""" + text = TRUNCATED_ERROR.format(cut=cut_words("核查模型", limit), advice=advice) + await _announce(emit, EventType.LOG, level="warn", code="verify_truncated", purpose=purpose, max_tokens=limit, + message=text) + return outcome(limit=stopped, answered=answered, error=text) + # ---- 查库的循环 stopped: str | None = None answered = False @@ -788,7 +803,6 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou said = "" attempts = 0 done: dict[str, str] = {} # 成功过的查询(工具名 + 参数)→ 编号:一模一样的不再执行 - cut: tuple[str, int | None] | None = None # 最近一次因为输出到了上限被截断的调用:(哪一步, 那次的额度) for _ in range(steps): remaining = left() if remaining is not None and remaining <= 0: @@ -804,18 +818,17 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou if isinstance(failure, asyncio.TimeoutError): stopped = "timeout_s" break - if truncated_error(failure): - cut = ("verify", base_model.max_tokens) - break + if truncated_error(failure) or cut_off(response): + # 到了上限被截断(推理模型想完了额度也用完了):话没说完,不是模型自己停下来的;带着的工具调用是写到一半的 + # (Anthropic 的 tool_use 截在半路时 input 是空的,有的网关给半截的 SQL),不执行。也不收尾:一条都没查到的, + # 收尾只能交「查不到」(双盲复算会记成「盲算没有查到这个数」);查到过的,截断前查到的不一定是要的那个数 + # (模型可能正想改查询),同时长到点一样算半路被掐断。收尾那次多半同样被推理占满,也不为它多花一次额度 + return await report_cut("verify", base_model.max_tokens) if failure is not None or not isinstance(response, BaseMessage): return outcome(error=explain_model_error(failure, model_id) if failure else "核查模型没有给出回复") messages.append(response) calls = getattr(response, "tool_calls", None) or [] if not calls: - if cut_off(response): - # 话没说完、也没发查询就到了上限(推理模型想完了额度也用完了):不是模型自己停下来的,不能当成说完了去收尾 - cut, said = ("verify", base_model.max_tokens), message_text(response) or said - break answered, said = True, message_text(response) break said = message_text(response) or said @@ -880,26 +893,6 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou await _announce(emit, EventType.LOG, level="warn", code="verify_limit", limit=stopped, message=f"核查已达上限({words}),已停止查询,按已有结果收尾") - async def report_cut(delivered: bool) -> str: - """被截断过:记一条 verify_truncated,返回给人看的话(没交成结果的原因,或者交了结果时记进 notes 的说明)。""" - purpose, limit = cut or ("verify", None) - text = (TRUNCATED_SETTLED if delivered else TRUNCATED_ERROR).format(cut=cut_words("核查模型", limit), - advice=TRUNCATED_ADVICE) - await _announce(emit, EventType.LOG, level="warn", code="verify_truncated", purpose=purpose, max_tokens=limit, - delivered=delivered, message=text) - return text - - async def give_up(error: str, limit: str | None = None) -> VerifyOutcome: - """没交成结果,原因照 error 说。工具循环里被截断过的,时间线上照记一条:收尾失败可能就是它带出来的。""" - if cut is not None: - await report_cut(False) - return outcome(limit=limit or stopped, answered=answered, error=error) - - if cut is not None and not queries: - # 一条查询都没成功就被截断:收尾只能交「查不到」,双盲复算会记成「盲算没有查到这个数」,原因却是输出额度。 - # 收尾那次多半同样被推理占满,不再为它多花一次额度,直接交回截断的原因 - return outcome(limit=stopped, answered=answered, error=await report_cut(False)) - # ---- 收尾:按结构交结果。关掉 thinking:结构化输出靠强制工具调用,和思考模式不兼容(经 OpenAI 兼容接入的推理模型 # 关不掉,所以额度和工具循环一样,退回小额度也和工具循环共用一个开关) settle_spec = spec.model_copy(update={"thinking": "off"}) @@ -907,8 +900,8 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou async with SessionLocal() as session: settle_model, _ = await get_chat_model(session, settle_spec) except Exception as e: # noqa: BLE001 - return await give_up(f"核查模型不可用({describe_exception(e)})") - settler = _with_fallback(settle_model, settle_spec, small, shared_with=base_model) + return outcome(limit=stopped, answered=answered, error=f"核查模型不可用({describe_exception(e)})") + settler = _with_fallback(settle_model, settle_spec, small, model_id, shared_with=base_model) final = _final_messages(head, task, queries, said, words) try: structured = settler.with_structured_output(schema, include_raw=True) @@ -928,15 +921,18 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou estimate = budget.estimate(_text_of(final) + schema_text, _OUT_FINAL) day, refused = await budget.reserve(estimate, kept=kept) if refused: - return await give_up(f"已达{words_of(refused)},没能交结果", refused) + return outcome(limit=stopped or refused, answered=answered, error=f"已达{words_of(refused)},没能交结果") got, raw, failure = await ask(final, structured.ainvoke, estimate, day or "", "verify_result", until_finish()) - parsed = None if failure is not None else _parsed(_split_structured(got)[1]) - if parsed is None and raw is not None: - # 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用 - for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: - if (parsed := _parsed(candidate)) is not None: - break - settle_cut = parsed is None and (truncated_error(failure) or bool(cut_off(raw))) + # 强制工具调用写到一半到了上限:Anthropic 截在 tool_use 里时 input 是空的,有的网关给半截对象(反方审稿的结论 + # 写完了、理由写到一半)。解析出来的、能从回复里抠出来的都说不清写完了没有,一律不采信 + settle_cut = truncated_error(failure) or bool(cut_off(raw)) + if failure is None and not settle_cut: + parsed = _parsed(_split_structured(got)[1]) + if parsed is None and raw is not None: + # 有的网关把 JSON 写在正文里、或者塞进了一个没按结构解析出来的工具调用 + for candidate in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: + if (parsed := _parsed(candidate)) is not None: + break if parsed is None and not isinstance(failure, asyncio.TimeoutError) and not settle_cut: # 不支持结构化输出的模型:普通调用一次,结构写进消息,从回复里把 JSON 抠出来。结构化那次已经做过的, # 这一次是补救,照常按上限拦;没做过的,这一次就是收尾,循环里留过钱。结构化那次是被截断的不补救: @@ -947,54 +943,53 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou estimate = budget.estimate(_text_of(plain), _OUT_FINAL) day, refused = await budget.reserve(estimate, kept=kept and structured is None) if refused and structured is None: - return await give_up(f"已达{words_of(refused)},没能交结果", refused) + return outcome(limit=stopped or refused, answered=answered, + error=f"已达{words_of(refused)},没能交结果") if not refused: got, _, failure = await ask(plain, settler.ainvoke, estimate, day or "", "verify_result", remaining) - parsed = None if failure is not None else _parsed(message_text(got) if got is not None else None) - settle_cut = parsed is None and (truncated_error(failure) or bool(cut_off(got))) + # 同上:截断的回复里就算抠得出一段 JSON,后面还在写,说不清写完了没有 + settle_cut = truncated_error(failure) or bool(cut_off(got)) + if failure is None and not settle_cut: + parsed = _parsed(message_text(got) if got is not None else None) if parsed is None: if settle_cut: - cut = ("verify_result", settler.max_tokens) - return outcome(limit=stopped, answered=answered, error=await report_cut(False)) + return await report_cut("verify_result", settler.max_tokens) why = "收尾时超过时长上限,没能交结果" if isinstance(failure, asyncio.TimeoutError) else \ f"核查模型没有按结构交结果({explain_model_error(failure, model_id)})" if failure is not None else \ "核查模型没有按结构交结果" - return await give_up(why) + return outcome(limit=stopped, answered=answered, error=why) declared = list((resolved.get("properties") or {}) if _is_object(resolved) else {}) if declared and not any(k in parsed for k in declared): # 一个要求的字段都没交:不是「查不到」,是没按结构交。不这么分,字段名对不上的回复会被当成什么都没查到 - return await give_up("核查模型没有按结构交结果(交回的内容里没有要求的字段:" + "、".join(declared[:8]) - + ("等" if len(declared) > 8 else "") + ")") + return outcome(limit=stopped, answered=answered, + error="核查模型没有按结构交结果(交回的内容里没有要求的字段:" + + "、".join(declared[:8]) + ("等" if len(declared) > 8 else "") + ")") # ---- 取值、组装理由 cells = _Cells(queries) result = cells.walk(resolved, parsed) if _is_object(resolved) else {} - if cut is not None and any(c["status"] != "resolved" for c in cells.cells): - # 工具循环被截断,截断前查到的里面没有要的数(或者引用指不到):话没说完就停了,说「没查到」会让人以为数据里 - # 没有。不交这份残缺的结果,原因照截断说;查过的照记 - return outcome(cells=cells.cells, limit=stopped, answered=answered, error=await report_cut(False)) rationale = parsed.get(RATIONALE) doc = None if isinstance(rationale, str) and rationale.strip(): doc = compose_doc(rationale, build_catalog(nodes={}, ledger=ledger), node_id=owner, run_id=run_id or None) result[RATIONALE] = doc["markdown"] if doc else "" - if cut is not None: - # 截断前查到的够用:照常交结果,说明记进 notes,时间线上照记一条(结果可能不完整) - notes.append(await report_cut(True)) return outcome(result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered) -def _with_fallback(model: Any, spec: ModelSpec, small: int | None, *, shared_with: BudgetFallback | None = None - ) -> BudgetFallback: - """按 spec 建好的模型包一层:服务方嫌额度大时按 small 重建一个再调,要退回时才建,平常一个都不多建。small 为 None - 是额度本来就不比它大,没有可退的。经本模块的 get_chat_model 建,测试换成剧本时小额度的那个也是剧本。""" +def _with_fallback(model: Any, spec: ModelSpec, small: int | None, model_id: str, *, + shared_with: BudgetFallback | None = None) -> BudgetFallback: + """按 spec 建好的模型包一层:服务方嫌额度大(或者 SDK 在本地拒了)时按 small 重建一个再调,要退回时才建,平常一个 + 都不多建。small 为 None 是额度本来就不比它大,没有可退的。被嫌大过的按 (接入, 模型, 额度) 记在进程里,证据裁判 + (judge._with_fallback)用的是同一种记法:同一个模型在那边被嫌大过,这边也直接用小的。经本模块的 get_chat_model + 建,测试换成剧本时小额度的那个也是剧本。""" async def build() -> Any: async with SessionLocal() as session: return (await get_chat_model(session, spec.model_copy(update={"max_tokens": small})))[0] return BudgetFallback(model, build=build if small else None, limits=(spec.max_tokens, small or spec.max_tokens), - shared_with=shared_with) + shared_with=shared_with, rejected=budget_refused, + key=(spec.provider, model_id, spec.max_tokens)) async def _run_tool(tool: Any, name: str, args: dict[str, Any], call_id: str, remaining: float | None, diff --git a/backend/tests/test_verify_truncation.py b/backend/tests/test_verify_truncation.py index 2d237caf..733a0066 100644 --- a/backend/tests/test_verify_truncation.py +++ b/backend/tests/test_verify_truncation.py @@ -6,36 +6,42 @@ thinking=off,思考就能把 4096 用光:这次调用 finish_reason=length ——都没说出真正的原因。这里守几件事: - 额度:默认 16384(judge.MAX_TOKENS,核查和裁判共用);服务方嫌大时退回 4096 再试一次,之后工具循环、收尾都用小的, - 只在还没有任何输出时重来;别的 400 不重试 -- 工具循环里被截断(没有工具调用、话没说完)不算模型自己停下:一条查询都没成功时不再收尾,直接交回截断的原因; - 查到过东西的照常按查到的收尾,取不到要的数时同样交回截断的原因 -- 收尾被截断(Anthropic 的 stop_reason=max_tokens、openai 结构化输出抛的 LengthFinishReasonError、普通调用的 - finish_reason=length):交回截断的原因,不再退回普通调用多花一次额度;截断那次的用量照记 + 只在还没有任何输出时重来;别的 400 不重试。被嫌大过的接入、模型记在进程里,之后的核查直接用小的;自建服务说的 + max_new_tokens、anthropic SDK 在本地就拒的额度也认 +- 被截断的回复一律不采信:工具循环里截断的回复带着的工具调用是写到一半的(Anthropic 的 tool_use 截在半路时 input 是 + 空的,有的网关给半截对象),不执行;截断前查到过东西的也不收尾(同时长到点,算半路被掐断);收尾的强制工具调用被 + 截断时,解析出来的空对象、半截对象都不算交了结果,也不再退回普通调用多花一次额度。截断那次的用量照记 - 时间线上记一条 verify_truncated(节点里调用时),llm.end 标出这次被截断 -- 原因经现有的渠道交出去:双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错;没有截断时原有说法不变 +- 原因经现有的渠道交出去:双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错。开头只说「未能给出结果」, + 建议按模型从哪来的说:节点里节点上指定的模型优先,两处都说;质询只看设置。没有截断时原有说法不变,触顶原因的先后 + 也不变 -模型是剧本,数据源是现造的 SQLite 文件(借 test_verify_agent 的夹具),表名、数字都是虚构的。 +模型是剧本,另有真实的 ChatOpenAI、ChatAnthropic(只有 HTTP 层是假的);数据源是现造的 SQLite 文件(借 +test_verify_agent 的夹具),表名、数字都是虚构的。 """ from __future__ import annotations import json import uuid +import pytest from langchain_core.messages import AIMessage, BaseMessage from openai import LengthFinishReasonError from openai.types import CompletionUsage from openai.types.chat import ChatCompletion from app.core.events import EventType -from app.db.models import Provider +from app.db.base import SessionLocal +from app.db.models import Provider, Setting from app.engine import judge as judging from app.engine import review as rv -from app.engine import verify_agent -from app.engine.inquiry import _error_words -from app.engine.nodes.challenge import _claim_of +from app.engine import spend, verify_agent +from app.engine.inquiry import INQUIRY_SCHEMA, _error_words +from app.engine.nodes.challenge import CHALLENGE_SCHEMA, _claim_of from app.engine.nodes.dual_check import _slim +from app.providers import budget from app.providers.factory import build_chat_model -from tests.test_verify_agent import VALID_SUM, clean_spend, q, run, shop # noqa: F401 - 夹具按名字引入 +from tests.test_verify_agent import VALID_SUM, Script, clean_spend, priced, q, run, shop # noqa: F401 - 夹具按名字引入 try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上 from langchain_openai._compat import httpx @@ -43,9 +49,24 @@ except ImportError: # pragma: no cover - 老版本的 langchain_openai import httpx CUT_16K = "核查模型的输出达到上限(16384 token)被截断" -ADVICE = "通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型" +#: 节点里(双盲复算、反方审稿):节点上指定的模型优先于设置,两处都说 +ADVICE = "通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型" +#: 不在节点里(审批质询):只看设置 +ADVICE_SETTINGS = "通常是推理占满了输出额度。请在「设置 → 偏好设置 → 核查」中换用不带推理的核查模型" +FAILED = f"{CUT_16K},未能给出结果。{ADVICE}" +FAILED_SETTINGS = f"{CUT_16K},未能给出结果。{ADVICE_SETTINGS}" FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致", "rationale": "未作废订单的金额合计是 [[v:Q1.r0.total]]。"} +#: 反方审稿那种结构写到一半:结论写完了,理由写到一半 +HALF_CHALLENGE = {"verdict": "stands", "rationale": "合计是 [[v:Q1.r0.total]],与报告"} + + +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() def cut_reply(output: int = 16384) -> AIMessage: @@ -55,11 +76,25 @@ def cut_reply(output: int = 16384) -> AIMessage: usage_metadata={"input_tokens": 900, "output_tokens": output, "total_tokens": 900 + output}) -def anthropic_cut() -> dict: - """结构化输出走强制工具调用(Anthropic)时被截断:原始回复 stop_reason=max_tokens,解析不出结果。""" - raw = AIMessage(content=[], response_metadata={"stop_reason": "max_tokens"}, +def cut_call(name: str, args: dict) -> AIMessage: + """工具调用写到一半到了上限(ChatAnthropic 的样子):tool_use 截在半路时 input 是空的,有的网关给半截对象; + stop_reason=max_tokens。""" + block = {"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": args} + return AIMessage(content=[{"type": "thinking", "thinking": "想了很久……", "signature": "s"}, block], + tool_calls=[{"name": block["name"], "args": args, "id": block["id"]}], + response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 900, "output_tokens": 16384, "total_tokens": 17284}) + + +def anthropic_cut(args: dict | None = None) -> dict: + """收尾的强制工具调用写到一半到了上限(结构化输出走 function_calling,ChatAnthropic 的缺省):原始回复 + stop_reason=max_tokens。args 为 None 是截在思考里、一个工具调用都没写出来;给了就是截在 tool_use 里——input 是 + 空的(Anthropic 非流式的样子)或半截对象(有的网关),解析出来的就是它。""" + content = [] if args is None else [{"type": "tool_use", "id": "toolu_1", "name": "verify_result", "input": args}] + calls = [] if args is None else [{"name": "verify_result", "args": args, "id": "toolu_1"}] + raw = AIMessage(content=content, tool_calls=calls, response_metadata={"stop_reason": "max_tokens"}, usage_metadata={"input_tokens": 700, "output_tokens": 16384, "total_tokens": 17084}) - return {"raw": raw, "parsed": None, "parsing_error": None} + return {"raw": raw, "parsed": args, "parsing_error": None} def length_error(prompt: int = 800, completion: int = 16384) -> LengthFinishReasonError: @@ -76,7 +111,9 @@ def query_step(name: str, sql: str = VALID_SUM, call_id: str = "c1") -> AIMessag class BudgetRejected(Exception): """服务方嫌输出额度太大时的 400(DeepSeek chat 系只收 8K)。""" - status_code = 400 + def __init__(self, message: str, status_code: int = 400) -> None: + super().__init__(message) + self.status_code = status_code class Reasoner: @@ -123,8 +160,9 @@ class _View: class Rejecting: """一开口就被拒的模型:工具循环、收尾都一样。""" - def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens"): - self.message = message + def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens", + status_code: int = 400): + self.message, self.status_code = message, status_code self.calls = 0 def bind_tools(self, tools, **kwargs): @@ -135,7 +173,7 @@ class Rejecting: async def ainvoke(self, messages, *args, **kwargs): self.calls += 1 - raise BudgetRejected(self.message) + raise BudgetRejected(self.message, self.status_code) class Models: @@ -175,43 +213,48 @@ async def test_thinking_that_eats_the_budget_in_the_loop_is_reported_as_truncati events, emit = collector() out = await run(name, emit=emit, node_id="dc") assert out.result is None and out.queries == [] - assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.error == FAILED assert model.calls == ["step"], "一条查询都没有:收尾只能交「查不到」,不再为它多花一次额度" assert out.answered is False, "被截断不是模型自己停下来的" assert out.usage["output_tokens"] == 16384, "截断那次的额度是真花出去了" [log] = truncated_logs(events) - assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["delivered"] is False + assert log["level"] == "warn" and log["max_tokens"] == 16384 and "delivered" not in log assert log["purpose"] == "verify" and log["message"] == out.error [end] = [d for k, d in events if k == EventType.LLM_END] assert end["truncated"] is True and end["max_tokens"] == 16384 and CUT_16K in end["error"] -async def test_a_cut_after_a_query_still_settles_on_what_was_found(monkeypatch, shop): +@pytest.mark.parametrize("args", [{}, {"sql": "SELECT SUM(amount) AS total FROM orders WHERE is_del"}], + ids=["empty-input", "half-sql"]) +async def test_a_tool_call_cut_off_halfway_is_not_run(monkeypatch, shop, args): + """Anthropic 系:工具调用写到一半到了上限。写到一半的调用不能执行——空参数只会报「缺少必填参数」,半截的 SQL + 查出来的数是错的——也不按它收尾,原因照截断说。""" + name, _ = shop + model = Reasoner([cut_call(name, args)], structured=FOUND) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc") + assert model.calls == ["step"], "不执行、不收尾,也不再问一轮" + assert out.result is None and out.queries == [] and out.error == FAILED + assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_END, EventType.TOOL_ERROR)] + [log] = truncated_logs(events) + assert log["purpose"] == "verify" and log["max_tokens"] == 16384 + + +async def test_a_cut_after_a_query_is_reported_without_settling(monkeypatch, shop): + """查过一次、下一轮被截断:截断前查到的不一定是要的那个数(模型可能正想改查询),按它收尾交出去的结果说不清完整 + 不完整,会照常参与双盲复算的比对、反方审稿的处置。同时长到点一样算半路被掐断:不收尾,原因照截断说,查过的照记。""" name, _ = shop model = Reasoner([query_step(name), cut_reply()], structured=FOUND) Models(monkeypatch, model) events, emit = collector() out = await run(name, emit=emit, node_id="dc") - assert model.calls == ["step", "step", "final"] - assert out.error is None and out.result["value"] == 230, "查到的数照常交:截断前的查询是真查过的" - assert out.answered is False - assert any(CUT_16K in n and "已按截断前查到的结果收尾" in n for n in out.notes) + assert model.calls == ["step", "step"], "不再为收尾多花一次额度" + assert out.result is None and out.error == FAILED and out.answered is False + assert [q["alias"] for q in out.queries] == ["Q1"], "查过的照记" [log] = truncated_logs(events) - assert log["delivered"] is True and log["purpose"] == "verify" and log["max_tokens"] == 16384 - - -async def test_a_cut_that_leaves_the_number_unfound_is_reported_as_truncation(monkeypatch, shop): - """查过、可要的数没查到就被截断了:说「没查到」会让人以为数据里没有,原因要写截断。""" - name, _ = shop - model = Reasoner([query_step(name, "SELECT COUNT(*) AS n FROM orders"), cut_reply()], - structured={"value": None, "verdict": "无法判断", "rationale": "还没查到合计"}) - Models(monkeypatch, model) - events, emit = collector() - out = await run(name, emit=emit, node_id="dc") - assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" - assert [c["status"] for c in out.cells] == ["missing"] and len(out.queries) == 1, "查过的照记" - [log] = truncated_logs(events) - assert log["delivered"] is False and log["purpose"] == "verify" + assert log["purpose"] == "verify" and log["max_tokens"] == 16384 + assert not [n for n in out.notes if "截断" in n], "没有「按截断前查到的结果收尾」这一说" # -------------------------------------------------------------------------- @@ -220,20 +263,41 @@ async def test_a_cut_that_leaves_the_number_unfound_is_reported_as_truncation(mo async def test_a_cut_off_settle_is_reported_and_not_retried_as_plain(monkeypatch, shop): - """Anthropic 系:强制工具调用写到一半到了上限。退回普通调用也多半被推理占满,不再白花一次额度。""" + """Anthropic 系:强制工具调用写出来之前就到了上限。退回普通调用也多半被推理占满,不再白花一次额度。""" name, _ = shop model = Reasoner([query_step(name)], structured=anthropic_cut(), plain=AIMessage(content=json.dumps(FOUND))) Models(monkeypatch, model) events, emit = collector() out = await run(name, emit=emit, node_id="dc") assert model.calls == ["step", "step", "final"], "没有再退回普通调用" - assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.result is None and out.error == FAILED [log] = truncated_logs(events) - assert log["purpose"] == "verify_result" and log["delivered"] is False + assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384 end = [d for k, d in events if k == EventType.LLM_END][-1] assert end["purpose"] == "verify_result" and end["truncated"] is True and CUT_16K in end["error"] +@pytest.mark.parametrize(("schema", "args"), [ + (None, {}), + (None, {"value": {"alias": "Q1"}}), + (CHALLENGE_SCHEMA, HALF_CHALLENGE), + (INQUIRY_SCHEMA, {"rationale": "合计是 [[v:Q1.r0.total]],比报告里写的"}), +], ids=["empty-input", "half-cell", "half-challenge", "half-inquiry"]) +async def test_a_settle_cut_inside_the_forced_tool_call_is_not_taken(monkeypatch, shop, schema, args): + """强制工具调用写到一半到了上限:解析出来的是空对象(Anthropic)或半截对象(有的网关)。空对象以前记成「没有按结构 + 交结果」,半截的格子引用记成「引用不可用」,反方审稿的半截对象(结论写完了、理由写到一半)照单全收成「经受住」, + 质询把半句话当成回答——截断的回复一律不采信,原因照截断说,也不退回普通调用。""" + name, _ = shop + model = Reasoner([query_step(name)], structured=anthropic_cut(args), plain=AIMessage(content=json.dumps(FOUND))) + Models(monkeypatch, model) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {})) + assert model.calls == ["step", "step", "final"] + assert out.result is None and out.error == FAILED and out.cells == [] + [log] = truncated_logs(events) + assert log["purpose"] == "verify_result" + + async def test_an_openai_length_error_in_the_settle_is_truncation_not_a_failed_call(monkeypatch, shop): """openai 的结构化输出解析时抛 LengthFinishReasonError:不是「调用失败,请稍后重试」,是额度被推理占满。""" name, _ = shop @@ -242,7 +306,7 @@ async def test_an_openai_length_error_in_the_settle_is_truncation_not_a_failed_c events, emit = collector() out = await run(name, emit=emit, node_id="dc") assert model.calls == ["step", "step", "final"] - assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.error == FAILED assert out.usage["output_tokens"] == 16384 and out.usage["input_tokens"] >= 800, "异常里带着的用量照记" end = [d for k, d in events if k == EventType.LLM_END][-1] assert "请稍后重试" not in end["error"] and CUT_16K in end["error"] and end["output_tokens"] == 16384 @@ -255,7 +319,18 @@ async def test_a_plain_settle_cut_off_is_reported(monkeypatch, shop): Models(monkeypatch, model) out = await run(name) assert model.calls == ["step", "step", "plain"] - assert out.result is None and out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.result is None and out.error == FAILED_SETTINGS + + +async def test_a_plain_settle_cut_after_the_json_is_not_taken_either(monkeypatch, shop): + """普通调用的正文里有一段 JSON、回复却是截断的:JSON 后面还在写(多半是理由的补充),说不清写完了没有,同样不采信。""" + name, _ = shop + cut = AIMessage(content=json.dumps(FOUND, ensure_ascii=False) + "\n补充说明:合计里还应该", + response_metadata={"finish_reason": "length"}) + model = Reasoner([query_step(name)], plain=cut, unstructured=True) + Models(monkeypatch, model) + out = await run(name) + assert out.result is None and out.error == FAILED_SETTINGS async def test_without_truncation_the_old_wording_stays(monkeypatch, shop): @@ -270,6 +345,28 @@ async def test_without_truncation_the_old_wording_stays(monkeypatch, shop): assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END) +async def test_an_empty_object_without_truncation_keeps_the_shape_wording(monkeypatch, shop): + """没被截断、交回的是空对象:照旧是「没有按结构交结果」,不说成截断。""" + name, _ = shop + whole = {"raw": AIMessage(content="", response_metadata={"stop_reason": "tool_use"}), "parsed": {}} + Models(monkeypatch, Reasoner([query_step(name)], structured=whole)) + out = await run(name) + assert out.error == "核查模型没有按结构交结果(交回的内容里没有要求的字段:value、verdict)" + + +async def test_the_first_limit_hit_is_still_the_one_reported(monkeypatch, shop, priced): + """循环先因时长到点停下(一分钱还没留),收尾又被每日金额上限拦下:触顶原因照旧报先撞上的时长。双盲复算按它写 + 无法复算的原因,反方审稿按「每日上限」决定还派不派后面的句子——截断的改动不能把这个先后换掉。""" + name, _ = shop + async with SessionLocal() as session: + session.add(Setting(key=spend.SPEND_KEY, value={"date": spend._today(), "usd": 1.995, "calls": 9})) + await session.commit() + Script(monkeypatch, [q(name, VALID_SUM, "c1")], FOUND, model_id="priced-model") + out = await run(name, daily_max_usd=2.0, timeout_s=0) + assert out.limit == "timeout_s" + assert out.error == "已达全局每日核查金额上限 $2.00,没能交结果" + + # -------------------------------------------------------------------------- # 额度与退回 # -------------------------------------------------------------------------- @@ -296,6 +393,18 @@ async def test_a_rejected_budget_falls_back_to_4096_for_the_loop_and_the_settle( assert model.calls == ["step", "step", "final"] +async def test_a_rejected_budget_is_remembered_for_the_next_runs(monkeypatch, shop): + """双盲复算的几路盲算、反方审稿的每一句、每次质询各是一次核查:同一接入、模型被嫌大过一次,之后的核查直接用小的, + 不再每次先撞一次。""" + name, _ = shop + model = Reasoner([query_step(name), AIMessage(content="查完了。"), query_step(name, call_id="c2")], + structured=FOUND) + models = Models(monkeypatch, model, rejects_above=4096) + first, second = await run(name), await run(name) + assert first.error is None and second.error is None and second.result["value"] == 230 + assert models.rejecting.calls == 1, "第二次核查没有再按 16384 撞一次" + + async def test_a_cut_after_the_fallback_names_the_small_budget(monkeypatch, shop): name, _ = shop Models(monkeypatch, Reasoner([cut_reply(output=4096)]), rejects_above=4096) @@ -312,6 +421,18 @@ async def test_other_rejections_are_not_retried_with_the_small_budget(monkeypatc assert [s.max_tokens for s in models.specs] == [16384], "不是额度的问题:不拿小额度再撞一次" +async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, shop): + """text-generation-inference 一类自建服务:输入加额度超过 max_total_tokens 时回 422,说的是 max_new_tokens。""" + name, _ = shop + tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 1200 `inputs` tokens and 16384 `max_new_tokens`", status_code=422) + model = Reasoner([query_step(name)], structured=FOUND) + models = Models(monkeypatch, model, rejects_above=4096, rejecting=tgi) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [s.max_tokens for s in models.specs] == [16384, 4096, 16384, 4096] and tgi.calls == 1 + + async def test_the_verify_spec_asks_for_the_larger_budget(): """核查模型的配置(api/settings.py 的 verify_model_spec 沿用证据裁判的额度):16384,思考照旧关掉。""" assert judging.MAX_TOKENS == 16384 and judging.FALLBACK_MAX_TOKENS == 4096 @@ -322,21 +443,28 @@ async def test_the_verify_spec_asks_for_the_larger_budget(): # -------------------------------------------------------------------------- -async def test_the_reason_reaches_dual_check_challenge_and_inquiry(monkeypatch, shop): +async def test_the_reason_reaches_dual_check_and_challenge(monkeypatch, shop): name, _ = shop Models(monkeypatch, Reasoner([cut_reply()])) - out = await run(name) - expected = f"{CUT_16K},核查未能完成。{ADVICE}" + out = await run(name, node_id="dc") # 双盲复算:无法复算的原因 slim = _slim(out) - assert slim["value"] is None and slim["reason"] == expected + assert slim["value"] is None and slim["reason"] == FAILED # 反方审稿:这句记为无法判断,理由写「反查未完成」,没能反查的原因就是截断 cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="", aliases=("Q1",)) claim, why = _claim_of(cand, out) - assert claim["verdict"] == "inconclusive" and claim["rationale"] == f"反查未完成:{expected}" and why == expected - # 审批质询:提问人看到的报错照原话 - assert _error_words(out, timeout_s=60, daily_max_usd=1.0) == expected + assert claim["verdict"] == "inconclusive" and claim["rationale"] == f"反查未完成:{FAILED}" and why == FAILED + + +async def test_an_inquiry_names_the_settings_and_not_a_check(monkeypatch, shop): + """审批质询不在节点里(核查模型只看设置):建议只说设置;开头只说「未能给出结果」——提问人问的是问题,不是在核查。""" + name, _ = shop + Models(monkeypatch, Reasoner([cut_reply()])) + out = await run(name, output_schema=INQUIRY_SCHEMA) + words = _error_words(out, timeout_s=60, daily_max_usd=1.0) + assert words == FAILED_SETTINGS + assert "核查未能完成" not in words and "节点" not in words def test_the_review_layer_knows_the_truncation_log(): @@ -344,13 +472,14 @@ def test_the_review_layer_knows_the_truncation_log(): 不落成笼统的 warn;不触发「重跑一次大概率能好」——同样的模型再跑一遍还是撞同一堵墙。""" assert rv.WARN_CODES["verify_truncated"] == ("verify_truncated", rv.DEGRADED) event = {"type": str(EventType.LOG), "node_id": "dc", - "data": {"level": "warn", "code": "verify_truncated", "message": f"{CUT_16K},核查未能完成。{ADVICE}"}} + "data": {"level": "warn", "code": "verify_truncated", "message": FAILED}} [signal] = rv.scan([event], {"结论": "本周订单 120 单。"}) assert (signal.kind, signal.severity) == ("verify_truncated", rv.DEGRADED) and not rv.should_retry([signal]) def test_the_reply_shapes_used_here_are_messages(): assert isinstance(cut_reply(), BaseMessage) and isinstance(anthropic_cut()["raw"], BaseMessage) + assert isinstance(cut_call("x", {}), BaseMessage) # -------------------------------------------------------------------------- @@ -409,18 +538,30 @@ async def test_a_real_client_cut_off_in_the_loop(monkeypatch, shop): name, _ = shop fake = gateway(monkeypatch, [THINKING_ONLY]) out = await run(name) - assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" + assert out.error == FAILED_SETTINGS assert len(fake.bodies) == 1 and fake.bodies[0]["max_completion_tokens"] == 16384 assert "tools" in fake.bodies[0], "工具循环那次带着只读的查询工具" +async def test_a_real_client_cut_off_with_a_half_written_tool_call(monkeypatch, shop): + """OpenAI 系:工具调用的参数写到一半到了上限(finish_reason=length,arguments 是半截 JSON)。不执行、不收尾。""" + name, _ = shop + half = ({"role": "assistant", "content": None, "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": f"db_query__{name}", "arguments": '{"sql": "SELECT SUM(amount) AS total FROM ord'}}]}, + "length", 16384) + fake = gateway(monkeypatch, [half]) + out = await run(name) + assert out.error == FAILED_SETTINGS and out.queries == [] and len(fake.bodies) == 1 + + async def test_a_real_client_cut_off_in_the_json_schema_settle(monkeypatch, shop): """收尾走 openai 的结构化输出(json_schema):finish_reason=length 时 SDK 抛 LengthFinishReasonError。""" name, _ = shop half = ({"role": "assistant", "content": '{"value": {"alias": "Q1", "row'}, "length", 16384) fake = gateway(monkeypatch, [tool_call(name), ({"role": "assistant", "content": "查完了。"}, "stop", 20), half]) out = await run(name) - assert out.error == f"{CUT_16K},核查未能完成。{ADVICE}" and len(out.queries) == 1 + assert out.error == FAILED_SETTINGS and len(out.queries) == 1 assert len(fake.bodies) == 3 and "response_format" in fake.bodies[2], "收尾被截断:没有再退回普通调用" assert out.usage["output_tokens"] == 120 + 20 + 16384, "异常里带着的那次用量照记" @@ -434,3 +575,106 @@ async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch assert out.error is None and out.result["value"] == 230 budgets = [b.get("max_completion_tokens") for b in fake.bodies] assert budgets == [16384, 4096, 4096, 4096], "第一次被拒后退回 4096,之后工具循环和收尾都用小的,不再先撞一次" + + +# -------------------------------------------------------------------------- +# 真实的 ChatAnthropic 和 anthropic SDK:只有 HTTP 层是假的 +# -------------------------------------------------------------------------- + +CLAUDE = "claude-x" + + +class Messages: + """Anthropic 的 Messages 接口:按顺序回 message(最后一个重复用)。每个回复是 (content, stop_reason, 输出 token 数), + content 可以是按请求体算的函数(收尾那次强制调用的工具名照请求里的写)。bodies 记下每个请求体。""" + + def __init__(self, replies: list) -> None: + self.replies = list(replies) + self.bodies: list[dict] = [] + + async def handle(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content or b"{}") + self.bodies.append(body) + content, stop, spent = self.replies.pop(0) if len(self.replies) > 1 else self.replies[0] + return httpx.Response(200, request=request, json={ + "id": "msg_1", "type": "message", "role": "assistant", "model": body.get("model"), + "content": content(body) if callable(content) else content, "stop_reason": stop, "stop_sequence": None, + "usage": {"input_tokens": 700, "output_tokens": spent}}) + + +def messages_api(monkeypatch, replies: list, *, model: str = CLAUDE, bearer: bool = False, + module=verify_agent) -> Messages: + """接上一个 Anthropic 接入(bearer 为真时按 Authorization: Bearer 鉴权,factory 会换上没设超时的 SDK 客户端)。""" + fake = Messages(replies) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle) + provider = Provider(name=f"an-{uuid.uuid4().hex[:8]}", kind="anthropic", base_url="http://an.example.test", + api_key="test-only-key", models=[{"id": model}], default_model=model, enabled=True, + extra={"auth_style": "bearer"} if bearer else {}) + + async def build(session, spec): + # 按这个接入的模型建:调用方的 spec 写的是别的模型名(mock-fast),SDK 在本地拦不拦要看真正发出去的 id + return build_chat_model(provider, spec.model_copy(update={"model": model})), model + + monkeypatch.setattr(module, "get_chat_model", build) + return fake + + +def forced(args: dict): + """收尾那次强制调用的工具:名字照请求里的写,input 是 args。""" + return lambda body: [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": args}] + + +def query_use(name: str) -> list: + return [{"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {"sql": VALID_SUM}}] + + +DONE = ([{"type": "text", "text": "查完了。"}], "end_turn", 10) + + +async def test_a_real_anthropic_cut_in_the_middle_of_a_tool_call(monkeypatch, shop): + """思考占满了额度、tool_use 截在半路(input 是空的):以前每一轮都执行一遍空参数的查询(报「缺少必填参数 sql」), + 撞到步数上限,最后记成「没有按结构交结果」。""" + name, _ = shop + cut = ([{"type": "thinking", "thinking": "想了很久……", "signature": "s"}, + {"type": "tool_use", "id": "toolu_q", "name": f"db_query__{name}", "input": {}}], "max_tokens", 16384) + fake = messages_api(monkeypatch, [cut]) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", max_queries=2) + assert out.error == FAILED and out.queries == [] and out.limit is None + assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384 + assert not [k for k, _ in events if k in (EventType.TOOL_START, EventType.TOOL_ERROR)] + [log] = truncated_logs(events) + assert log["purpose"] == "verify" + signals = rv.scan([{"type": str(k), "node_id": "dc", "data": d} for k, d in events], {"结论": "本周订单 120 单。"}) + assert [s.kind for s in signals] == ["verify_truncated"], "不是工具出错、也不是触顶" + + +@pytest.mark.parametrize(("schema", "args"), [(None, {}), (CHALLENGE_SCHEMA, HALF_CHALLENGE)], + ids=["empty-input", "half-challenge"]) +async def test_a_real_anthropic_settle_cut_inside_the_forced_tool_call(monkeypatch, shop, schema, args): + name, _ = shop + fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE, (forced(args), "max_tokens", 16384)]) + events, emit = collector() + out = await run(name, emit=emit, node_id="dc", **({"output_schema": schema} if schema else {})) + assert out.error == FAILED and out.result is None and len(out.queries) == 1 + assert len(fake.bodies) == 3 and fake.bodies[2]["tool_choice"]["type"] == "tool", "没有再退回普通调用" + [log] = truncated_logs(events) + assert log["purpose"] == "verify_result" and log["max_tokens"] == 16384 + if schema is CHALLENGE_SCHEMA: + cand = judging.Candidate(unit="u1", priority=1, order=0, text="华南每周的销售额都高于华东", section="", + aliases=("Q1",)) + claim, why = _claim_of(cand, out) + assert claim["verdict"] == "inconclusive" and why == FAILED, "半截的「经受住」不能当真" + + +async def test_an_anthropic_client_refusing_the_budget_locally_falls_back(monkeypatch, shop): + """Bearer 鉴权的 Anthropic 接入(factory 换上的 SDK 客户端没设超时):Opus 4、4.1 这一代的 id(经网关用的 Vertex、 + Bedrock 写法也是)非流式只能给 8192,16384 的请求 anthropic SDK 在本地就拒了(ValueError,请求没发出去)。照服务方 + 嫌额度大处理,退回 4096。这条守的是 SDK 的行为:anthropic 以后不再在本地拦这个 id,第一个请求就会按 16384 发出去, + 到时改期望。""" + name, _ = shop + fake = messages_api(monkeypatch, [(query_use(name), "tool_use", 50), DONE, + (forced(FOUND), "tool_use", 60)], model="claude-opus-4-1@20250805", bearer=True) + out = await run(name) + assert out.error is None and out.result["value"] == 230 + assert [b["max_tokens"] for b in fake.bodies] == [4096, 4096, 4096], "16384 的请求一个都没发出去" diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index cdd1af77..7e0cfb9a 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1039,16 +1039,17 @@ export const AGENT_GUARD_TEXT = { unlimitedShort: '不限', } as const +/** 「(16,384 token)」:输出额度,没给就不写 */ +const tokenCap = (maxTokens?: number) => (maxTokens ? `(${formatNumber(maxTokens)} token)` : '') + /** * 时间线上「核查模型的输出到了上限被截断」那一行(engine/verify_agent.py 的 verify_truncated 日志,双盲复算、反方审稿的 * 核查 Agent 发)。推理模型的思考也算在输出额度里,额度用完时回复是空的或只写了一半:照实说是输出额度不够,不说成 - * 「没查到」「格式不对」,下一步指到换模型。delivered:截断发生在查询过程中、已按截断前查到的结果收尾(结果可能不完整); - * undelivered:这次核查没有给出结果,节点按无法复算、未能反查处理 + * 「没查到」「格式不对」,下一步指到换模型。截断的回复一律不采信,这次核查没有结果,节点按无法复算、未能反查记缺口 */ export const VERIFY_TRUNCATED_TEXT = { - title: (maxTokens?: number) => `核查模型的输出达到上限${maxTokens ? `(${formatNumber(maxTokens)} token)` : ''}被截断`, - delivered: '已按截断前查到的结果收尾,结果可能不完整', - undelivered: '这次核查未能完成', + title: (maxTokens?: number) => `核查模型的输出达到上限${tokenCap(maxTokens)}被截断`, + sub: '这次核查未能给出结果', next: '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型', } as const diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index 8a3a92ce..8a80a716 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -1217,12 +1217,12 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.max_tokens : undefined return { title: VT.title(limit), - sub: d.delivered === true ? VT.delivered : d.delivered === false ? VT.undelivered : undefined, + sub: VT.sub, next: VT.next, fix: 'canvas', fixField: 'model', diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index 3804887b..fa9b0093 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -2319,29 +2319,29 @@ await section('核查类日志与出具缺口(期 4):无法复算、未能 rows.map((r) => `${r?.title}|${r?.sub ?? ''}`).join(' ; ')) }) -await section('核查模型的输出到了上限被截断(verify_truncated):说清额度、交没交出结果,下一步指到换模型', async () => { +await section('核查模型的输出到了上限被截断(verify_truncated):说清额度、这次核查没有结果,下一步指到换模型', async () => { // 推理模型的思考也算在输出额度里:核查 Agent 的一次调用 finish_reason=length、没有工具调用也没有 JSON 时,以前双盲复算 - // 记成「盲算没有查到这个数」、反方审稿记成「核查模型没有按结构交结果」。现在核查 Agent 另发 verify_truncated(max_tokens、 - // delivered、purpose),时间线按结构化字段说人话,原话留在展开区。先核对后端源码还是这么发的 + // 记成「盲算没有查到这个数」、反方审稿记成「核查模型没有按结构交结果」。现在截断的回复一律不采信,核查 Agent 另发 + // verify_truncated(max_tokens、purpose),时间线按结构化字段说人话,原话留在展开区。先核对后端源码还是这么发的 const verifyPy = readFileSync(`${root}backend/app/engine/verify_agent.py`, 'utf8') check('后端发截断日志的格式还是夹具里这一种', verifyPy.includes('code="verify_truncated"') && verifyPy.includes('max_tokens=limit') - && verifyPy.includes('delivered=delivered') && verifyPy.includes('purpose=purpose') - && verifyPy.includes('TRUNCATED_ERROR = "{cut},核查未能完成。{advice}"'), + && verifyPy.includes('purpose=purpose') && !verifyPy.includes('delivered=') + && verifyPy.includes('TRUNCATED_ERROR = "{cut},未能给出结果。{advice}"'), '改了 verify_agent.py 的字段或说法就同步改 decode.ts 的 explainLog 和这里的夹具') const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data }) - const ADVICE = '通常是推理占满了输出额度,可在「设置 → 偏好设置 → 核查」中换用不带推理的模型' - const FAILED = `核查模型的输出达到上限(16384 token)被截断,核查未能完成。${ADVICE}` - const SETTLED = `核查模型的输出达到上限(16384 token)被截断,已按截断前查到的结果收尾,结果可能不完整。${ADVICE}` + const ADVICE = '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型' + const FAILED = `核查模型的输出达到上限(16384 token)被截断,未能给出结果。${ADVICE}` + const SMALL = `核查模型的输出达到上限(4096 token)被截断,未能给出结果。${ADVICE}` const all = flatten(mod.decodeRun([ E(1, 'run.started', null, { total: 2 }), E(2, 'node.started', 'dc', { node_type: 'dual_check', label: '复核销售额' }), - E(3, 'log', 'dc', { level: 'warn', code: 'verify_truncated', purpose: 'verify', max_tokens: 16384, delivered: false, message: FAILED }), + E(3, 'log', 'dc', { level: 'warn', code: 'verify_truncated', purpose: 'verify', max_tokens: 16384, message: FAILED }), E(4, 'node.finished', 'dc', { duration_ms: 20 }), E(5, 'node.started', 'ch', { node_type: 'challenge', label: '审月报' }), - E(6, 'log', 'ch', { level: 'warn', code: 'verify_truncated', purpose: 'verify_result', max_tokens: 4096, delivered: true, - message: SETTLED }), + // 退回小额度之后在收尾时被截断 + E(6, 'log', 'ch', { level: 'warn', code: 'verify_truncated', purpose: 'verify_result', max_tokens: 4096, message: SMALL }), // 缺字段、字段类型不对(别的服务端版本):只给不带数的标题,不瞎填 - E(7, 'log', 'ch', { level: 'warn', code: 'verify_truncated', max_tokens: '16384', delivered: 'no', message: FAILED }), + E(7, 'log', 'ch', { level: 'warn', code: 'verify_truncated', max_tokens: '16384', message: FAILED }), E(8, 'node.finished', 'ch', { duration_ms: 30 }), E(9, 'run.finished', null, { status: 'succeeded' }), ])) @@ -2349,19 +2349,20 @@ await section('核查模型的输出到了上限被截断(verify_truncated) const nodes = all.filter((s) => s.kind === 'node') check('每条一行,挂在自己的节点下', rows.length === 3 && nodes.find((s) => s.nodeId === 'dc')?.children?.filter((x) => x.code === 'verify_truncated').length === 1 && nodes.find((s) => s.nodeId === 'ch')?.children?.filter((x) => x.code === 'verify_truncated').length === 2, `${rows.length} 行`) - const [failed, settled, bare] = rows - check('没交出结果:标题说输出到了上限被截断、带额度,副标题说核查未能完成', failed?.title === '核查模型的输出达到上限(16,384 token)被截断' - && failed.sub === '这次核查未能完成', `${failed?.title} / ${failed?.sub}`) - check('交了结果:额度照实写(退回小额度之后是 4,096),副标题说按截断前查到的结果收尾、可能不完整', - settled?.title === '核查模型的输出达到上限(4,096 token)被截断' && settled.sub === '已按截断前查到的结果收尾,结果可能不完整', - `${settled?.title} / ${settled?.sub}`) - check('……下一步指到换不带推理的模型,入口落到节点的模型,提醒色', rows.every((s) => !!s.next?.includes('不带推理的模型') - && s.next.includes('设置 → 偏好设置 → 核查') && s.fix === 'canvas' && s.fixField === 'model' && s.level === 'warn'), + const [failed, small, bare] = rows + check('标题说输出到了上限被截断、带额度,副标题说这次核查没有结果', failed?.title === '核查模型的输出达到上限(16,384 token)被截断' + && failed.sub === '这次核查未能给出结果', `${failed?.title} / ${failed?.sub}`) + check('额度照实写(退回小额度之后是 4,096)', small?.title === '核查模型的输出达到上限(4,096 token)被截断' && small.sub === '这次核查未能给出结果', + `${small?.title} / ${small?.sub}`) + check('……下一步指到换不带推理的模型(节点上指定的优先,设置里的也说),入口落到节点的模型,提醒色', + rows.every((s) => !!s.next?.includes('在节点上指定不带推理的模型') && s.next.includes('设置 → 偏好设置 → 核查') + && s.fix === 'canvas' && s.fixField === 'model' && s.level === 'warn'), `${failed?.next} / ${failed?.fix} / ${failed?.fixField} / ${failed?.level}`) - check('……原话留在展开区', failed?.detail === FAILED && settled?.detail === SETTLED) - check('缺字段、类型不对:标题不带数,不写副标题,不出 undefined、NaN', bare?.title === '核查模型的输出达到上限被截断' && bare.sub === undefined - && !/undefined|NaN/.test(`${bare?.title}${bare?.next ?? ''}`), `${bare?.title} / ${bare?.sub}`) - check('标题、副标题里没有内部代码', !rows.some((s) => /verify_truncated|max_tokens|delivered/.test(`${s.title}${s.sub ?? ''}${s.next ?? ''}`))) + check('……没有「已按截断前查到的结果收尾」这一说', !rows.some((s) => /收尾|可能不完整/.test(`${s.title}${s.sub ?? ''}`))) + check('……原话留在展开区', failed?.detail === FAILED && small?.detail === SMALL) + check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '核查模型的输出达到上限被截断' + && !/undefined|NaN/.test(`${bare?.title}${bare?.sub ?? ''}${bare?.next ?? ''}`), `${bare?.title} / ${bare?.sub}`) + check('标题、副标题里没有内部代码', !rows.some((s) => /verify_truncated|max_tokens|purpose/.test(`${s.title}${s.sub ?? ''}${s.next ?? ''}`))) check('节点照常完成', nodes.every((s) => s.status === 'done')) })