From 48f656f3628cac4ea7a55f14660e11336fbd0ae9 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 14:30:15 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=81=E6=8D=AE=E8=A3=81=E5=88=A4=E6=88=AA?= =?UTF-8?q?=E6=96=AD=E7=9A=84=E5=9B=9E=E5=A4=8D=E6=95=B4=E6=89=B9=E4=B8=8D?= =?UTF-8?q?=E9=87=87=E4=BF=A1=EF=BC=8C=E6=88=AA=E6=96=AD=E5=8F=A6=E8=AE=B0?= =?UTF-8?q?=20judge=5Ftruncated=EF=BC=8C=E6=97=B6=E9=97=B4=E7=BA=BF?= =?UTF-8?q?=E7=9A=84=E4=B8=8B=E4=B8=80=E6=AD=A5=E6=8C=87=E5=88=B0=E6=8D=A2?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复核第 1 轮的发现: - 有的网关把截断的强制工具调用解析成半截清单:第一条判定写完、第二条只写了编号。_ask 拿到清单就交出去,不再看截断,于是记成一句支持、一句格式不对、两句未给出判定,llm.end 也不标截断。现在回复被截断就整批记 truncated(最后一条可能只写了一半,也可能判定写完、理由写到一半,分不清哪条完整),纯文本调用同理 - 截断的那一批原来只发 llm.end(时间线标题「结论句裁判未能执行」)和 judge_failed(下一步「检查裁判模型的接入后重新运行」):截断时接入是好的,同一个模型重新运行多半还是截断。现在另发 judge_truncated(max_tokens、unjudged),judge_failed 不再带截断;时间线上调用那一行说「裁判模型的输出达到上限(N token)被截断」,judge_truncated 那一行的下一步指到在报告撰写节点的「结论句裁判」或设置里换不带推理的裁判模型,入口落到节点的结论句裁判;check-decode 加一节 - 复核层(review.py)认得 judge_truncated:是缺口,记成它自己的 kind - 退回小额度按 budget_refused 认,和核查同一种记法记在进程里:之后每份报告、每次点击直接用小的 测试补上半截清单(剧本和真实 ChatAnthropic)、纯文本清单后面还在写、截断和格式不对各记各的日志、跨报告记住被嫌大、自建服务的 max_new_tokens --- backend/app/engine/judge.py | 46 +++++--- backend/app/engine/review.py | 3 + backend/tests/test_judge_truncation.py | 145 +++++++++++++++++++++++-- frontend/src/lib/terms.ts | 14 +++ frontend/src/run/decode.ts | 18 ++- scripts/check-decode.mjs | 50 +++++++++ 6 files changed, 248 insertions(+), 28 deletions(-) diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index d4c3903d..25b03568 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -13,7 +13,8 @@ 查询在 SQL 之后另有来历行(来源、口径、已接受、区域外文字,_provenance_lines) 3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析, 再失败就记未裁判。thinking 关掉,输出额度 MAX_TOKENS(推理模型的思考也算在里面),服务方嫌大时退回 - FALLBACK_MAX_TOKENS;输出到了上限被截断的记未裁判(truncated),照实说是额度不够、不再退回纯文本多花一次 + FALLBACK_MAX_TOKENS,被嫌大过的接入、模型之后直接用小的;输出到了上限被截断的整批记未裁判(truncated), + 照实说是额度不够、不再退回纯文本多花一次,时间线上另记 judge_truncated 4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算, 超了按优先级截断;触顶就停,已判的保留,没判的记未裁判并说明是哪个上限——不报错 5. 每日计数落在设置表(SPEND_KEY),按本地日期滚动。目录里没有价格的模型估不出金额,金额上限对它 @@ -70,7 +71,7 @@ from app.engine.merge_query import MERGE_SOURCE from app.engine.nodes.llm import _split_structured, _usage_of from app.engine.state import message_text from app.providers import catalog as pricing -from app.providers.budget import BudgetFallback, cut_off, cut_words, error_usage, truncated_error +from app.providers.budget import BudgetFallback, budget_refused, cut_off, cut_words, error_usage, truncated_error from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model logger = logging.getLogger(__name__) @@ -1081,8 +1082,9 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non 先走结构化输出;不成(模型不支持、给的不是合法结构)先从原始回复里找 JSON,再退回纯文本调用、 从回复里抠 JSON。原因以 error: 开头的是调用本身失败,format 是格式不对,truncated 是输出到了上限被截断 (Anthropic 的 stop_reason=max_tokens、OpenAI 系的 finish_reason=length,以及 openai 结构化输出抛的 - LengthFinishReasonError)。结构化调用被截断的不再退回纯文本:推理占满额度时纯文本调用多半一样被截断,白花一次 - 额度。超时由调用方掐。 + LengthFinishReasonError)。截断的回复整批不采信:强制工具调用截在半路时 Anthropic 给的是空对象,有的网关给半截的 + 清单——最后一条可能只写了编号,也可能判定写完了、理由写到一半,分不清哪条是完整的。结构化调用被截断的不再退回 + 纯文本:推理占满额度时纯文本调用多半一样被截断,白花一次额度。超时由调用方掐。 """ raws: list[BaseMessage] = [] try: @@ -1090,6 +1092,8 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non raw, parsed = _split_structured(got) if raw is not None: raws.append(raw) + if cut_off(raw): + return None, raws, _TRUNCATED, 1 if (items := _items_of(parsed)) is not None: return items, raws, None, 1 if raw is not None: @@ -1097,8 +1101,6 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non for found in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]: if (items := _items_of(found)) is not None: return items, raws, None, 1 - if cut_off(raw): - return None, raws, _TRUNCATED, 1 except Exception as e: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析 if truncated_error(e): return None, [*raws, *_spent(e)], _TRUNCATED, 1 @@ -1109,9 +1111,10 @@ async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | Non return None, [*raws, *_spent(e)], _TRUNCATED, 2 return None, raws, f"error:{describe_exception(e)}", 2 raws.append(reply) - items = _items_of(message_text(reply)) - if items is None and cut_off(reply): + if cut_off(reply): + # 正文里就算抠得出一段清单,后面还在写,说不清写完了没有:同样整批不采信 return None, raws, _TRUNCATED, 2 + items = _items_of(message_text(reply)) return items, raws, None if items is not None else "format", 2 @@ -1123,15 +1126,18 @@ def _spent(error: BaseException) -> list[BaseMessage]: return [AIMessage(content="", usage_metadata={"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out})] -def _with_fallback(model: Any, spec: ModelSpec) -> BudgetFallback: - """裁判模型包一层:服务方嫌 MAX_TOKENS 太大时按 FALLBACK_MAX_TOKENS 重建一个再问(要退回时才建,平常不多建),之后 - 各批都用小的。经本模块的 get_chat_model 建:测试换成剧本时,小额度的那个也是剧本。""" +def _with_fallback(model: Any, spec: ModelSpec, model_id: str) -> BudgetFallback: + """裁判模型包一层:服务方嫌 MAX_TOKENS 太大(或者 anthropic SDK 在本地就拒了)时按 FALLBACK_MAX_TOKENS 重建一个再问 + (要退回时才建,平常不多建),之后各批都用小的。被嫌大过的按 (接入, 模型, 额度) 记在进程里:之后每份报告、每次 + 点击直接用小的,核查(verify_agent._with_fallback)同一种记法、互相认。经本模块的 get_chat_model 建:测试换成剧本 + 时,小额度的那个也是剧本。""" async def build() -> Any: async with SessionLocal() as session: return (await get_chat_model(session, spec.model_copy(update={"max_tokens": FALLBACK_MAX_TOKENS})))[0] - return BudgetFallback(model, build=build, limits=(spec.max_tokens, FALLBACK_MAX_TOKENS)) + return BudgetFallback(model, build=build, limits=(spec.max_tokens, FALLBACK_MAX_TOKENS), rejected=budget_refused, + key=(spec.provider, model_id, spec.max_tokens)) def _unjudged(reason: str, rationale: str, *, judge: str | None, post_seal: bool) -> dict[str, Any]: @@ -1196,7 +1202,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, try: async with SessionLocal() as session: model, model_id = await get_chat_model(session, spec) - model = _with_fallback(model, spec) + model = _with_fallback(model, spec, model_id) except Exception as e: # noqa: BLE001 - 裁判模型没配好:这回判不了,报告照常 why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e) details["error"] = f"裁判模型不可用({why})" @@ -1233,6 +1239,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, spent = 0.0 width = 1 + cut_limit: int | None = None # 被截断的那次调用的额度:judge_truncated 日志照它写 while pending: remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started) if remaining is not None and remaining <= 0: @@ -1312,9 +1319,9 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, if problem == _TRUNCATED: # 输出到了上限被截断:这一批记未裁判,原因照实说是额度,不说「格式有误」「调用失败」。别的批照常问 # (同格式不对):截断取决于这一批要想多久,不像鉴权、网关出错那样后面几批多半一样 - limit = model.max_tokens - details["truncated"] = cut_words("裁判模型", limit) - emit(EventType.LLM_END, **end, error=details["truncated"], truncated=True, max_tokens=limit) + cut_limit = model.max_tokens + details["truncated"] = cut_words("裁判模型", cut_limit) + emit(EventType.LLM_END, **end, error=details["truncated"], truncated=True, max_tokens=cut_limit) stop(batch, "truncated", details["truncated"]) continue if problem is not None and problem != "format": @@ -1353,7 +1360,12 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, n = sum(outcome["unjudged"].get(r, 0) for r in hit) emit(EventType.LOG, level="warn", code="judge_limit", limits=hit, unjudged=n, message=f"结论句裁判已达上限({labels}):{n} 句未裁判,已裁判的结果保留") - if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS]: + # 输出被截断单独记一条(带那次的额度、几句未裁判):judge_failed 在时间线上的下一步是检查接入、重新运行,截断时接入 + # 是好的,同一个模型重新运行多半还是截断,该换的是模型 + if truncated := [g for r, g in outcome["_gaps"] if r == "truncated"]: + emit(EventType.LOG, level="warn", code="judge_truncated", max_tokens=cut_limit, + unjudged=outcome["unjudged"]["truncated"], message="结论句裁判未完成:" + ";".join(truncated)) + if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS and r != "truncated"]: emit(EventType.LOG, level="warn", code="judge_failed", message="结论句裁判未完成:" + ";".join(failed)) outcome.pop("_gaps") return outcome diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index bae1b65b..e56e4713 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -104,6 +104,9 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 结论句裁判(claims: judge):触顶、没跑成的句子记为未裁判,出具不能判完整——是缺口,要说明 "judge_limit": ("judge_limit", DEGRADED), "judge_failed": ("judge_failed", DEGRADED), + # 裁判模型的输出到了上限被截断(推理模型的思考也算在额度里):同 judge_failed 是缺口,记成它自己的 kind——说得清 + # 是输出额度的问题,该换的是模型 + "judge_truncated": ("judge_truncated", DEGRADED), # 证据不支持的句子交回写作者改过一次(和 report_repair 一样只记缺口);改写稿冒出新问题没采用的, # 原稿里那几句仍是证据不支持 "report_rewrite": ("report_rewrite", DEGRADED), diff --git a/backend/tests/test_judge_truncation.py b/backend/tests/test_judge_truncation.py index 9686bd2a..1b8a5f75 100644 --- a/backend/tests/test_judge_truncation.py +++ b/backend/tests/test_judge_truncation.py @@ -6,8 +6,12 @@ - 额度:16384(MAX_TOKENS);服务方嫌大时退回 4096(FALLBACK_MAX_TOKENS)再问一次,之后各批都用小的,只撞一次 - 输出到了上限被截断(stop_reason=max_tokens、finish_reason=length、LengthFinishReasonError):这一批记未裁判,原因 - 是 truncated,理由、缺口、llm.end、judge_failed 日志都写明「裁判模型的输出达到上限(N token)被截断」;结构化调用 - 被截断的不再退回纯文本多花一次额度;截断那次的用量照记 + 是 truncated,理由、缺口、llm.end 都写明「裁判模型的输出达到上限(N token)被截断」;截断的回复整批不采信(有的网关 + 把截断的强制工具调用解析成半截清单:最后一条只写了一半);结构化调用被截断的不再退回纯文本多花一次额度;截断那次 + 的用量照记 +- 时间线上另记一条 judge_truncated(带额度、几句未裁判),不混进 judge_failed:judge_failed 的下一步是「检查接入、 + 重新运行」,截断时接入是好的,同一个模型重新运行多半还是截断 +- 被嫌大过的接入、模型记在进程里,之后的裁判直接用小的;自建服务说的 max_new_tokens 也认 - 没有截断时原有说法不变(格式不对仍是 format) 模型都是剧本;证据借 test_evidence_judge 的虚构夹具(口径卡、一次查询、一次检索)。 @@ -18,6 +22,7 @@ import json import re import uuid +import pytest from langchain_core.messages import AIMessage from openai import LengthFinishReasonError from openai.types import CompletionUsage @@ -26,10 +31,13 @@ from openai.types.chat import ChatCompletion from app.core.events import EventType from app.db.models import Provider from app.engine import judge as judge_mod +from app.engine import review as rv from app.engine.evidence import compose_doc from app.engine.judge import BATCH_SIZE, FALLBACK_MAX_TOKENS, MAX_TOKENS, REASONS, judge_doc +from app.providers import budget from app.providers.factory import build_chat_model from tests.test_evidence_judge import PRICED, RAW, SPEC, UNLIMITED, clean_settings, make_catalog, stored # noqa: F401 +from tests.test_verify_truncation import messages_api try: # openai SDK 3 起换成了 httpx2(和 httpx 同一套接口):假服务要装在 SDK 真正用的那个库上 from langchain_openai._compat import httpx @@ -40,6 +48,14 @@ CUT_16K = "裁判模型的输出达到上限(16384 token)被截断" ADVICE = "通常是推理占满了输出额度,可换用不带推理的裁判模型" +@pytest.fixture(autouse=True) +def forget_rejections(): + """被嫌大过的接入记在进程里(providers/budget.py):每个测试从没记过开始,也不留给后面的测试。""" + budget._REJECTED.clear() + yield + budget._REJECTED.clear() + + def length_error(prompt: int = 3000, completion: int = 16384) -> LengthFinishReasonError: usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion) return LengthFinishReasonError(completion=ChatCompletion.model_construct( @@ -60,6 +76,17 @@ def thinking_only(units) -> AIMessage: usage_metadata={"input_tokens": 3000, "output_tokens": 4096, "total_tokens": 7096}) +def half_list(units) -> dict: + """有的网关把截断的强制工具调用解析成半截对象:第一条判定写完了,第二条只写了编号,后面的没写到。""" + items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []}, + {"unit": units[1]}] + raw = AIMessage(content=[{"type": "tool_use", "id": "toolu_1", "name": "judge", "input": {"items": items}}], + tool_calls=[{"name": "judge", "args": {"items": items}, "id": "toolu_1"}], + response_metadata={"stop_reason": "max_tokens"}, + usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384}) + return {"raw": raw, "parsed": {"items": items}, "parsing_error": None} + + def supported(units) -> dict: items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] return {"raw": AIMessage(content="", response_metadata={"finish_reason": "stop"}), "parsed": {"items": items}} @@ -106,7 +133,9 @@ class _View: class Rejecting: - def __init__(self) -> None: + def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens", + status_code: int = 400) -> None: + self.message, self.status_code = message, status_code self.calls = 0 def with_structured_output(self, schema, **kwargs): @@ -114,13 +143,16 @@ class Rejecting: async def ainvoke(self, messages, *args, **kwargs): self.calls += 1 - raise BudgetRejected("max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens") + error = BudgetRejected(self.message) + error.status_code = self.status_code + raise error class Models: - def __init__(self, monkeypatch, model, *, rejects_above: int | None = None) -> None: + def __init__(self, monkeypatch, model, *, rejects_above: int | None = None, + rejecting: Rejecting | None = None) -> None: self.specs = [] - self.rejecting = Rejecting() + self.rejecting = rejecting or Rejecting() async def fake(session, spec): self.specs.append(spec) @@ -157,8 +189,40 @@ async def test_a_cut_off_structured_reply_is_unjudged_as_truncation_without_a_pl [end] = [d for k, d in events if k == EventType.LLM_END] assert end["purpose"] == "judge" and end["error"] == CUT_16K and end["truncated"] is True assert end["max_tokens"] == 16384 and end["output_tokens"] == 16384 - [failed] = [d for k, d in events if k == EventType.LOG and d.get("code") == "judge_failed"] - assert failed["message"] == f"结论句裁判未完成:有 4 句结论未裁判:{CUT_16K},{ADVICE}" + [log] = logs(events, "judge_truncated") + assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["unjudged"] == 4 + assert log["message"] == f"结论句裁判未完成:有 4 句结论未裁判:{CUT_16K},{ADVICE}" + assert logs(events, "judge_failed") == [], "不混进 judge_failed:那一行的下一步是检查接入、重新运行" + + +def logs(events, code: str) -> list[dict]: + return [d for k, d in events if k == EventType.LOG and d.get("code") == code] + + +async def test_a_half_written_list_in_a_cut_off_reply_is_not_taken(monkeypatch, stored): + """截断的回复整批不采信:写完的那条也可能只是理由写完了一半;以前第一条记成「证据支持」、第二条「判定无法识别」、 + 后面的「未给出判定」,llm.end 也没标截断。""" + model = Reasoner(structured=half_list, plain=lambda units: AIMessage(content=json.dumps({"items": []}))) + Models(monkeypatch, model) + outcome, events = await judged(stored) + assert [kind for kind, _ in model.calls] == ["structured"] + assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0 + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["truncated"] is True and end["error"] == CUT_16K + [log] = logs(events, "judge_truncated") + assert log["unjudged"] == 4 + + +async def test_a_plain_reply_cut_after_the_list_is_not_taken_either(monkeypatch, stored): + """纯文本调用:清单后面还在写、回复是截断的,说不清写完了没有,同样整批记截断。""" + def cut(units): + items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units] + return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False) + "\n补充:第二句的依据还要", + response_metadata={"finish_reason": "length"}) + + Models(monkeypatch, Reasoner(plain=cut, unstructured=True)) + outcome, _ = await judged(stored) + assert outcome["unjudged"] == {"truncated": 4} async def test_an_openai_length_error_is_truncation_and_its_tokens_are_counted(monkeypatch, stored): @@ -199,6 +263,35 @@ async def test_a_truncated_batch_does_not_stop_the_others(monkeypatch, stored): assert outcome["counts"]["supported"] == 3 +async def test_truncation_and_other_failures_are_logged_apart(monkeypatch, stored): + """一批被截断、另一批格式不对:截断记 judge_truncated,格式不对照旧记 judge_failed,各说各的。""" + first: list[bool] = [] + + def once(units): + if not first: + first.append(True) + return anthropic_cut(units) + return {"raw": AIMessage(content="不是 JSON", response_metadata={"finish_reason": "stop"}), "parsed": None} + + Models(monkeypatch, Reasoner(structured=once, plain=lambda units: AIMessage(content="还是不是"))) + raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3)) + outcome, events = await judged(stored, raw) + assert outcome["unjudged"] == {"format": 3, "truncated": BATCH_SIZE} + [cut] = logs(events, "judge_truncated") + [failed] = logs(events, "judge_failed") + assert cut["unjudged"] == BATCH_SIZE and CUT_16K in cut["message"] + assert failed["message"] == "结论句裁判未完成:有 3 句结论未裁判:裁判返回的结果格式有误" + + +def test_the_review_layer_knows_the_truncation_log(): + """跑完之后的复核层认得 judge_truncated:同 judge_failed,是缺口(要说明),kind 是它自己。""" + assert rv.WARN_CODES["judge_truncated"] == ("judge_truncated", rv.DEGRADED) + event = {"type": str(EventType.LOG), "node_id": "write", + "data": {"level": "warn", "code": "judge_truncated", "message": f"有 4 句结论未裁判:{CUT_16K}"}} + [signal] = rv.scan([event], {"answer": "本周销售额 45,678.5元。"}) + assert (signal.kind, signal.severity) == ("judge_truncated", rv.DEGRADED) and not rv.should_retry([signal]) + + async def test_without_truncation_format_problems_keep_their_wording(monkeypatch, stored): model = Reasoner(structured=lambda units: {"raw": AIMessage(content="不是 JSON", response_metadata={"finish_reason": "stop"}), @@ -210,6 +303,7 @@ async def test_without_truncation_format_problems_keep_their_wording(monkeypatch assert outcome["unjudged"] == {"format": 4} assert outcome["gaps"] == ["有 4 句结论未裁判:裁判返回的结果格式有误"] assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END) + assert logs(events, "judge_truncated") == [] and len(logs(events, "judge_failed")) == 1 # -------------------------------------------------------------------------- @@ -236,6 +330,25 @@ async def test_a_rejected_budget_falls_back_to_4096_once_for_all_batches(monkeyp assert [kind for kind, _ in model.calls] == ["structured"] * 3 +async def test_a_rejected_budget_is_remembered_for_the_next_reports(monkeypatch, stored): + """每份报告、每次点击各问一次裁判:同一接入、模型被嫌大过一次,之后直接用小的,不再每次先撞一次。""" + model = Reasoner() + models = Models(monkeypatch, model, rejects_above=4096) + first, _ = await judged(stored) + second, _ = await judged(stored) + assert first["counts"]["supported"] == 4 and second["counts"]["supported"] == 4 + assert models.rejecting.calls == 1, "第二份报告没有再按 16384 撞一次" + + +async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, stored): + tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. " + "Given: 3000 `inputs` tokens and 16384 `max_new_tokens`", status_code=422) + models = Models(monkeypatch, Reasoner(), rejects_above=4096, rejecting=tgi) + outcome, _ = await judged(stored) + assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == [] + assert [s.max_tokens for s in models.specs] == [16384, 4096] and tgi.calls == 1 + + async def test_a_truncation_after_the_fallback_names_the_small_budget(monkeypatch, stored): Models(monkeypatch, Reasoner(plain=thinking_only, unstructured=True), rejects_above=4096) outcome, _ = await judged(stored) @@ -306,3 +419,19 @@ async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch outcome, _ = await judged(stored) assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == [] assert [b.get("max_completion_tokens") for b in fake.bodies] == [16384, 4096] + + +async def test_a_real_anthropic_cut_with_a_half_written_list(monkeypatch, stored): + """真实的 ChatAnthropic:强制工具调用写到第二条判定时到了上限,网关给的是半截清单。整批记截断,llm.end 标出来。""" + def half(body): + units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M) + items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []}, + {"unit": units[1]}] + return [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": {"items": items}}] + + fake = messages_api(monkeypatch, [(half, "max_tokens", 16384)], model=PRICED, module=judge_mod) + outcome, events = await judged(stored) + assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0 + assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384, "没有再退回纯文本" + [end] = [d for k, d in events if k == EventType.LLM_END] + assert end["truncated"] is True and end["error"] == CUT_16K diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 7e0cfb9a..00f352b6 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1053,6 +1053,20 @@ export const VERIFY_TRUNCATED_TEXT = { next: '通常是推理占满了输出额度。请在节点上指定不带推理的模型,或在「设置 → 偏好设置 → 核查」中更换核查模型', } as const +/** + * 时间线上结论句裁判的输出到了上限被截断(engine/judge.py):被截断的那次调用(llm.end 带 truncated),和 + * judge_truncated 日志那一行。不用「裁判未能执行」「检查接入后重新运行」的说法:截断时接入是好的,同一个模型重新运行 + * 多半还是截断,该换的是模型 + */ +export const JUDGE_TRUNCATED_TEXT = { + /** 被截断的那次调用:原来叫「请裁判模型判断 N 句结论」的那一行 */ + call: (maxTokens?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断`, + title: (maxTokens?: number, unjudged?: number) => `裁判模型的输出达到上限${tokenCap(maxTokens)}被截断` + + (unjudged ? `,${formatNumber(unjudged)} 句结论未裁判` : ''), + next: '未裁判的结论句记为缺口,不能完整出具。通常是推理占满了输出额度:请在报告撰写节点的「结论句裁判」中指定不带推理的裁判模型,' + + '或在「设置 → 偏好设置 → 证据裁判」中更换,再重新运行', +} as const + /** * 上传表格:数据页「表格」标签的上传弹窗、导入回执和卡片。种类的说法和后端 data/tabular.py 的 * CONVERSION_KINDS、SHAPE_KINDS 对得上;后端以后多出来的种类,界面上只显示它给的那句话 diff --git a/frontend/src/run/decode.ts b/frontend/src/run/decode.ts index 8a80a716..fd01b6c7 100644 --- a/frontend/src/run/decode.ts +++ b/frontend/src/run/decode.ts @@ -3,7 +3,8 @@ import type { NodeState, RunPhase } from './trace' import { formatDuration, formatNumber } from '../lib/format' import { ABSENCE_CLAIM_TEXT, CATALOG_DRIFT_TEXT, CATALOG_PATCH_TEXT, COPILOT_CONTEXT_TEXT, JUDGE_TEXT, MERGE_TEXT, RUN_SQL_CHECK_TEXT, SCHEMA_PARTIAL_TEXT, SQL_CHECK_TEXT, TYPE_LABEL, - CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, NODE_TYPE_LABEL, VERIFY_TRUNCATED_TEXT, + CATALOG_TABLE_FIELD_LABEL, CHALLENGE_TEXT, DUAL_CHECK_TEXT, INQUIRY_TEXT, JUDGE_TRUNCATED_TEXT, NODE_TYPE_LABEL, + VERIFY_TRUNCATED_TEXT, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel, } from '../lib/terms' @@ -1012,6 +1013,15 @@ function explainLog(code: string | undefined, message: string, d: Record 0 ? d.max_tokens : undefined + const n = typeof d.unjudged === 'number' && d.unjudged > 0 ? Math.round(d.unjudged) : undefined + return { + title: JUDGE_TRUNCATED_TEXT.title(limit, n), next: JUDGE_TRUNCATED_TEXT.next, fix: 'canvas', fixField: 'judge', + } + } case 'judge_failed': { // 「结论句裁判没跑完:有 2 句结论没裁判:裁判调用失败(…)」 return { @@ -1927,9 +1937,11 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt step.detail = [String(d.error), step.detail].filter(Boolean).join('\n') } if (d.purpose === 'judge' && d.error) { - // 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败 + // 裁判没跑成(超时、调用失败):这一批记为未裁判、报告照常交——同样是提醒,不是失败。输出到了上限被截断的 + // 照实说是截断(不是「未能执行」:接入是好的,该换的是模型,下一步见 judge_truncated 那一行) + const cap = num(d.max_tokens) step.level = 'warn' - step.title = JUDGE_FAILED_TITLE + step.title = d.truncated === true ? JUDGE_TRUNCATED_TEXT.call(cap && cap > 0 ? cap : undefined) : JUDGE_FAILED_TITLE step.detail = [String(d.error), step.detail].filter(Boolean).join('\n') } pendingLlm.delete(nodeId ?? '_') diff --git a/scripts/check-decode.mjs b/scripts/check-decode.mjs index fa9b0093..8e361973 100644 --- a/scripts/check-decode.mjs +++ b/scripts/check-decode.mjs @@ -2366,6 +2366,56 @@ await section('核查模型的输出到了上限被截断(verify_truncated) check('节点照常完成', nodes.every((s) => s.status === 'done')) }) +await section('结论句裁判的输出到了上限被截断(judge_truncated):调用那一行说截断,日志另起一行,下一步指到换模型', async () => { + // 以前截断的那一批只发 llm.end(标题沿用「结论句裁判未能执行」)和 judge_failed(下一步是「检查接入后重新运行」): + // 截断时接入是好的,同一个模型重新运行多半还是截断。现在调用那一行照 truncated、max_tokens 说截断,另发 judge_truncated + // (max_tokens、unjudged),judge_failed 只说别的失败。先核对后端源码还是这么发的 + const judgePy = readFileSync(`${root}backend/app/engine/judge.py`, 'utf8') + check('后端发裁判截断的格式还是夹具里这一种', judgePy.includes('code="judge_truncated", max_tokens=cut_limit') + && judgePy.includes('unjudged=outcome["unjudged"]["truncated"]') && judgePy.includes('truncated=True, max_tokens=cut_limit') + && judgePy.includes('r not in LIMITS and r != "truncated"'), + '改了 judge.py 的字段或说法就同步改 decode.ts 的 explainLog、llm.end 和这里的夹具') + const E = (seq, type, node_id, data = {}) => ({ seq, type, node_id, ts: 1790000000 + seq, data }) + const CUT = '裁判模型的输出达到上限(16384 token)被截断' + const TRUNCATED = `结论句裁判未完成:有 4 句结论未裁判:${CUT},通常是推理占满了输出额度,可换用不带推理的裁判模型` + const all = flatten(mod.decodeRun([ + E(1, 'run.started', null, { total: 1 }), + E(2, 'node.started', 'write', { node_type: 'report', label: '写周报' }), + E(3, 'llm.start', 'write', { model: 'jm-r', structured: true, purpose: 'judge', message_count: 2, units: 4 }), + E(4, 'llm.end', 'write', { model: 'jm-r', purpose: 'judge', duration_ms: 41000, units: 4, input_tokens: 3000, + output_tokens: 16384, error: CUT, truncated: true, max_tokens: 16384 }), + E(5, 'log', 'write', { level: 'warn', code: 'judge_truncated', max_tokens: 16384, unjudged: 4, message: TRUNCATED }), + E(6, 'node.finished', 'write', { duration_ms: 43000 }), + E(7, 'run.finished', null, { status: 'succeeded' }), + ])) + const call = all.find((s) => s.kind === 'llm' && s.code === 'judge') + check('调用那一行:说输出到了上限被截断、带额度,不说「未能执行」;提醒色、照常完成,原话在展开区', + call?.title === '裁判模型的输出达到上限(16,384 token)被截断' && call.level === 'warn' && call.status === 'done' + && !!call.detail?.includes(CUT), `${call?.title} / ${call?.level} / ${call?.status}`) + const row = all.find((s) => s.code === 'judge_truncated') + check('judge_truncated:标题说截断、带额度和几句未裁判', row?.title === '裁判模型的输出达到上限(16,384 token)被截断,4 句结论未裁判', + row?.title) + check('……下一步指到换不带推理的裁判模型(节点的「结论句裁判」、设置都说),不说检查接入', !!row?.next?.includes('不带推理的裁判模型') + && row.next.includes('结论句裁判') && row.next.includes('设置 → 偏好设置 → 证据裁判') && !row.next.includes('接入') + && row.fix === 'canvas' && row.fixField === 'judge' && row.level === 'warn', `${row?.next} / ${row?.fix} / ${row?.fixField}`) + check('……原话留在展开区,标题里没有内部代码', row?.detail === TRUNCATED && !/judge_truncated|max_tokens|unjudged/.test(`${row?.title}${row?.next}`)) + check('没有混出一条 judge_failed', !all.some((s) => s.code === 'judge_failed')) + // 没截断的失败照旧:「未能执行」那一行和 judge_failed 的说法不变 + const other = flatten(mod.decodeRun([ + E(1, 'node.started', 'write', { node_type: 'report' }), + E(2, 'llm.start', 'write', { model: 'jm-x', structured: true, purpose: 'judge', message_count: 2, units: 2 }), + E(3, 'llm.end', 'write', { model: 'jm-x', purpose: 'judge', duration_ms: 300, units: 2, error: 'ConnectError: 连不上' }), + E(4, 'node.finished', 'write', { duration_ms: 400 }), + ])) + check('没截断的调用失败照旧叫「结论句裁判未能执行」', other.find((s) => s.kind === 'llm')?.title === '结论句裁判未能执行', + other.find((s) => s.kind === 'llm')?.title) + const bare = flatten(mod.decodeRun([E(1, 'node.started', 'w', { node_type: 'report' }), + E(2, 'log', 'w', { level: 'warn', code: 'judge_truncated', max_tokens: 'x', message: '裁判被截断了' })])) + .find((s) => s.code === 'judge_truncated') + check('缺字段、类型不对:标题不带数,不出 undefined、NaN', bare?.title === '裁判模型的输出达到上限被截断' + && !/undefined|NaN/.test(`${bare?.title}${bare?.next ?? ''}`), bare?.title) +}) + await section('审批质询 approval.inquiry(期 4):一行「审批人质询」,标题带问题前 40 字,副标题说回答是否完成、查了几次', async () => { // 前后端共用的约定:运行停在审批上时服务端追加 approval.inquiry,node_id 为空;data 里有问题、划选的原文、回答正文、 // 回答文档、成没成、触顶原因、查询(queries 和 inquiry_artifacts 两份)。以前这类事件落进「未识别的事件」