mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 14:31:58 +00:00
ThearchyHelios
48f656f362
复核第 1 轮的发现： - 有的网关把截断的强制工具调用解析成半截清单：第一条判定写完、第二条只写了编号。_ask 拿到清单就交出去，不再看截断，于是记成一句支持、一句格式不对、两句未给出判定，llm.end 也不标截断。现在回复被截断就整批记 truncated（最后一条可能只写了一半，也可能判定写完、理由写到一半，分不清哪条完整），纯文本调用同理 - 截断的那一批原来只发 llm.end（时间线标题「结论句裁判未能执行」）和 judge_failed（下一步「检查裁判模型的接入后重新运行」）：截断时接入是好的，同一个模型重新运行多半还是截断。现在另发 judge_truncated（max_tokens、unjudged），judge_failed 不再带截断；时间线上调用那一行说「裁判模型的输出达到上限（N token）被截断」，judge_truncated 那一行的下一步指到在报告撰写节点的「结论句裁判」或设置里换不带推理的裁判模型，入口落到节点的结论句裁判；check-decode 加一节 - 复核层（review.py）认得 judge_truncated：是缺口，记成它自己的 kind - 退回小额度按 budget_refused 认，和核查同一种记法记在进程里：之后每份报告、每次点击直接用小的 测试补上半截清单（剧本和真实 ChatAnthropic）、纯文本清单后面还在写、截断和格式不对各记各的日志、跨报告记住被嫌大、自建服务的 max_new_tokens
438 lines
23 KiB
Python
438 lines
23 KiB
Python
"""结论句裁判（engine/judge.py）的输出额度：截断照实说，服务方嫌额度大时退回小额度。
|
||
|
||
裁判原来固定 max_tokens 4096、thinking off。经 OpenAI 兼容接入的推理模型收不到 thinking=off，思考就能把 4096 用光：
|
||
结构化调用写不出判定（openai 抛 LengthFinishReasonError，Anthropic 的强制工具调用写到一半），退回纯文本调用也一样，
|
||
最后记成「裁判返回的结果格式有误」——没说出真正的原因。这里守几件事：
|
||
|
||
- 额度：16384（MAX_TOKENS）；服务方嫌大时退回 4096（FALLBACK_MAX_TOKENS）再问一次，之后各批都用小的，只撞一次
|
||
- 输出到了上限被截断（stop_reason=max_tokens、finish_reason=length、LengthFinishReasonError）：这一批记未裁判，原因
|
||
是 truncated，理由、缺口、llm.end 都写明「裁判模型的输出达到上限（N token）被截断」；截断的回复整批不采信（有的网关
|
||
把截断的强制工具调用解析成半截清单：最后一条只写了一半）；结构化调用被截断的不再退回纯文本多花一次额度；截断那次
|
||
的用量照记
|
||
- 时间线上另记一条 judge_truncated（带额度、几句未裁判），不混进 judge_failed：judge_failed 的下一步是「检查接入、
|
||
重新运行」，截断时接入是好的，同一个模型重新运行多半还是截断
|
||
- 被嫌大过的接入、模型记在进程里，之后的裁判直接用小的；自建服务说的 max_new_tokens 也认
|
||
- 没有截断时原有说法不变（格式不对仍是 format）
|
||
|
||
模型都是剧本；证据借 test_evidence_judge 的虚构夹具（口径卡、一次查询、一次检索）。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
import uuid
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessage
|
||
from openai import LengthFinishReasonError
|
||
from openai.types import CompletionUsage
|
||
from openai.types.chat import ChatCompletion
|
||
|
||
from app.core.events import EventType
|
||
from app.db.models import Provider
|
||
from app.engine import judge as judge_mod
|
||
from app.engine import review as rv
|
||
from app.engine.evidence import compose_doc
|
||
from app.engine.judge import BATCH_SIZE, FALLBACK_MAX_TOKENS, MAX_TOKENS, REASONS, judge_doc
|
||
from app.providers import budget
|
||
from app.providers.factory import build_chat_model
|
||
from tests.test_evidence_judge import PRICED, RAW, SPEC, UNLIMITED, clean_settings, make_catalog, stored # noqa: F401
|
||
from tests.test_verify_truncation import messages_api
|
||
|
||
try: # openai SDK 3 起换成了 httpx2（和 httpx 同一套接口）：假服务要装在 SDK 真正用的那个库上
|
||
from langchain_openai._compat import httpx
|
||
except ImportError: # pragma: no cover - 老版本的 langchain_openai
|
||
import httpx
|
||
|
||
CUT_16K = "裁判模型的输出达到上限（16384 token）被截断"
|
||
ADVICE = "通常是推理占满了输出额度，可换用不带推理的裁判模型"
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def forget_rejections():
|
||
"""被嫌大过的接入记在进程里（providers/budget.py）：每个测试从没记过开始，也不留给后面的测试。"""
|
||
budget._REJECTED.clear()
|
||
yield
|
||
budget._REJECTED.clear()
|
||
|
||
|
||
def length_error(prompt: int = 3000, completion: int = 16384) -> LengthFinishReasonError:
|
||
usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion)
|
||
return LengthFinishReasonError(completion=ChatCompletion.model_construct(
|
||
id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage))
|
||
|
||
|
||
def anthropic_cut(units) -> dict:
|
||
"""强制工具调用写到一半到了上限：原始回复 stop_reason=max_tokens，解析不出判定。"""
|
||
raw = AIMessage(content=[], response_metadata={"stop_reason": "max_tokens"},
|
||
usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384})
|
||
return {"raw": raw, "parsed": None, "parsing_error": None}
|
||
|
||
|
||
def thinking_only(units) -> AIMessage:
|
||
"""纯文本调用：额度全用在思考上，正文为空，finish_reason=length（推理模型经 OpenAI 兼容接入的样子）。"""
|
||
return AIMessage(content="", additional_kwargs={"reasoning_content": "逐句看证据……"},
|
||
response_metadata={"finish_reason": "length"},
|
||
usage_metadata={"input_tokens": 3000, "output_tokens": 4096, "total_tokens": 7096})
|
||
|
||
|
||
def half_list(units) -> dict:
|
||
"""有的网关把截断的强制工具调用解析成半截对象：第一条判定写完了，第二条只写了编号，后面的没写到。"""
|
||
items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []},
|
||
{"unit": units[1]}]
|
||
raw = AIMessage(content=[{"type": "tool_use", "id": "toolu_1", "name": "judge", "input": {"items": items}}],
|
||
tool_calls=[{"name": "judge", "args": {"items": items}, "id": "toolu_1"}],
|
||
response_metadata={"stop_reason": "max_tokens"},
|
||
usage_metadata={"input_tokens": 3000, "output_tokens": 16384, "total_tokens": 19384})
|
||
return {"raw": raw, "parsed": {"items": items}, "parsing_error": None}
|
||
|
||
|
||
def supported(units) -> dict:
|
||
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
|
||
return {"raw": AIMessage(content="", response_metadata={"finish_reason": "stop"}), "parsed": {"items": items}}
|
||
|
||
|
||
class BudgetRejected(Exception):
|
||
"""服务方嫌输出额度太大时的 400。"""
|
||
|
||
status_code = 400
|
||
|
||
|
||
class Reasoner:
|
||
"""裁判模型的剧本。structured(units) / plain(units) 给出结构化调用、退回纯文本调用的回复（返回异常就抛它）；
|
||
unstructured 为真时不支持结构化输出。calls 记下每次调用的种类和判的句子。"""
|
||
|
||
def __init__(self, *, structured=supported, plain=None, unstructured: bool = False) -> None:
|
||
self.structured, self.plain, self.unstructured = structured, plain, unstructured
|
||
self.calls: list[tuple[str, list[str]]] = []
|
||
|
||
def with_structured_output(self, schema, **kwargs):
|
||
if self.unstructured:
|
||
raise NotImplementedError("这个模型不支持结构化输出")
|
||
return _View(self)
|
||
|
||
async def ainvoke(self, messages, *args, **kwargs):
|
||
return self.answer("plain", messages)
|
||
|
||
def answer(self, kind: str, messages):
|
||
units = re.findall(r"^\[(u\d+)\]", str(messages[-1].content), re.M)
|
||
self.calls.append((kind, units))
|
||
make = self.structured if kind == "structured" else self.plain
|
||
reply = make(units)
|
||
if isinstance(reply, BaseException):
|
||
raise reply
|
||
return reply
|
||
|
||
|
||
class _View:
|
||
def __init__(self, model: Reasoner) -> None:
|
||
self.model = model
|
||
|
||
async def ainvoke(self, messages, *args, **kwargs):
|
||
return self.model.answer("structured", messages)
|
||
|
||
|
||
class Rejecting:
|
||
def __init__(self, message: str = "max_tokens: 16384 > 4096, which is the maximum allowed number of output tokens",
|
||
status_code: int = 400) -> None:
|
||
self.message, self.status_code = message, status_code
|
||
self.calls = 0
|
||
|
||
def with_structured_output(self, schema, **kwargs):
|
||
return self
|
||
|
||
async def ainvoke(self, messages, *args, **kwargs):
|
||
self.calls += 1
|
||
error = BudgetRejected(self.message)
|
||
error.status_code = self.status_code
|
||
raise error
|
||
|
||
|
||
class Models:
|
||
def __init__(self, monkeypatch, model, *, rejects_above: int | None = None,
|
||
rejecting: Rejecting | None = None) -> None:
|
||
self.specs = []
|
||
self.rejecting = rejecting or Rejecting()
|
||
|
||
async def fake(session, spec):
|
||
self.specs.append(spec)
|
||
if rejects_above is not None and (spec.max_tokens or 0) > rejects_above:
|
||
return self.rejecting, PRICED
|
||
return model, PRICED
|
||
|
||
monkeypatch.setattr(judge_mod, "get_chat_model", fake)
|
||
|
||
|
||
async def judged(stored, raw: str = RAW):
|
||
catalog = make_catalog(stored)
|
||
events: list[tuple] = []
|
||
outcome = await judge_doc(compose_doc(raw, catalog), catalog, spec=SPEC, budget=UNLIMITED,
|
||
emit=lambda kind, **data: events.append((kind, data)))
|
||
return outcome, events
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 截断
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_a_cut_off_structured_reply_is_unjudged_as_truncation_without_a_plain_retry(monkeypatch, stored):
|
||
model = Reasoner(structured=anthropic_cut, plain=lambda units: AIMessage(content=json.dumps({"items": []})))
|
||
Models(monkeypatch, model)
|
||
outcome, events = await judged(stored)
|
||
assert [kind for kind, _ in model.calls] == ["structured"], "判定写到一半到了上限：不再退回纯文本多花一次额度"
|
||
verdicts = list(outcome["verdicts"].values())
|
||
assert {(v["status"], v["reason"]) for v in verdicts} == {("unjudged", "truncated")}
|
||
assert verdicts[0]["rationale"] == f"{CUT_16K}，这句未裁判。{ADVICE}"
|
||
assert outcome["unjudged"] == {"truncated": 4} and outcome["limits_hit"] == []
|
||
assert outcome["gaps"] == [f"有 4 句结论未裁判：{CUT_16K}，{ADVICE}"]
|
||
[end] = [d for k, d in events if k == EventType.LLM_END]
|
||
assert end["purpose"] == "judge" and end["error"] == CUT_16K and end["truncated"] is True
|
||
assert end["max_tokens"] == 16384 and end["output_tokens"] == 16384
|
||
[log] = logs(events, "judge_truncated")
|
||
assert log["level"] == "warn" and log["max_tokens"] == 16384 and log["unjudged"] == 4
|
||
assert log["message"] == f"结论句裁判未完成：有 4 句结论未裁判：{CUT_16K}，{ADVICE}"
|
||
assert logs(events, "judge_failed") == [], "不混进 judge_failed：那一行的下一步是检查接入、重新运行"
|
||
|
||
|
||
def logs(events, code: str) -> list[dict]:
|
||
return [d for k, d in events if k == EventType.LOG and d.get("code") == code]
|
||
|
||
|
||
async def test_a_half_written_list_in_a_cut_off_reply_is_not_taken(monkeypatch, stored):
|
||
"""截断的回复整批不采信：写完的那条也可能只是理由写完了一半；以前第一条记成「证据支持」、第二条「判定无法识别」、
|
||
后面的「未给出判定」，llm.end 也没标截断。"""
|
||
model = Reasoner(structured=half_list, plain=lambda units: AIMessage(content=json.dumps({"items": []})))
|
||
Models(monkeypatch, model)
|
||
outcome, events = await judged(stored)
|
||
assert [kind for kind, _ in model.calls] == ["structured"]
|
||
assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0
|
||
[end] = [d for k, d in events if k == EventType.LLM_END]
|
||
assert end["truncated"] is True and end["error"] == CUT_16K
|
||
[log] = logs(events, "judge_truncated")
|
||
assert log["unjudged"] == 4
|
||
|
||
|
||
async def test_a_plain_reply_cut_after_the_list_is_not_taken_either(monkeypatch, stored):
|
||
"""纯文本调用：清单后面还在写、回复是截断的，说不清写完了没有，同样整批记截断。"""
|
||
def cut(units):
|
||
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
|
||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False) + "\n补充：第二句的依据还要",
|
||
response_metadata={"finish_reason": "length"})
|
||
|
||
Models(monkeypatch, Reasoner(plain=cut, unstructured=True))
|
||
outcome, _ = await judged(stored)
|
||
assert outcome["unjudged"] == {"truncated": 4}
|
||
|
||
|
||
async def test_an_openai_length_error_is_truncation_and_its_tokens_are_counted(monkeypatch, stored):
|
||
model = Reasoner(structured=lambda units: length_error(completion=16384))
|
||
Models(monkeypatch, model)
|
||
outcome, events = await judged(stored)
|
||
assert [kind for kind, _ in model.calls] == ["structured"]
|
||
assert outcome["unjudged"] == {"truncated": 4}
|
||
assert outcome["usage"]["output_tokens"] == 16384 and outcome["cost_usd"] > 0, "截断那次的额度是真花出去了"
|
||
end = [d for k, d in events if k == EventType.LLM_END][-1]
|
||
assert "调用" not in end["error"] and end["error"] == CUT_16K
|
||
|
||
|
||
async def test_a_plain_reply_with_only_thinking_is_truncation(monkeypatch, stored):
|
||
"""不支持结构化输出的模型直接退回纯文本调用：回复只有思考、finish_reason=length。"""
|
||
model = Reasoner(plain=thinking_only, unstructured=True)
|
||
Models(monkeypatch, model)
|
||
outcome, _ = await judged(stored)
|
||
assert [kind for kind, _ in model.calls] == ["plain"] and outcome["calls"] == 2
|
||
assert outcome["unjudged"] == {"truncated": 4}
|
||
assert outcome["gaps"] == [f"有 4 句结论未裁判：{CUT_16K}，{ADVICE}"]
|
||
|
||
|
||
async def test_a_truncated_batch_does_not_stop_the_others(monkeypatch, stored):
|
||
"""只有一批被截断：别的批照常判（和格式不对一样，不像调用失败那样后面的都不再问）。"""
|
||
first: list[bool] = []
|
||
|
||
def once(units):
|
||
if not first:
|
||
first.append(True)
|
||
return anthropic_cut(units)
|
||
return supported(units)
|
||
|
||
Models(monkeypatch, Reasoner(structured=once))
|
||
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3))
|
||
outcome, _ = await judged(stored, raw)
|
||
assert outcome["unjudged"] == {"truncated": BATCH_SIZE}
|
||
assert outcome["counts"]["supported"] == 3
|
||
|
||
|
||
async def test_truncation_and_other_failures_are_logged_apart(monkeypatch, stored):
|
||
"""一批被截断、另一批格式不对：截断记 judge_truncated，格式不对照旧记 judge_failed，各说各的。"""
|
||
first: list[bool] = []
|
||
|
||
def once(units):
|
||
if not first:
|
||
first.append(True)
|
||
return anthropic_cut(units)
|
||
return {"raw": AIMessage(content="不是 JSON", response_metadata={"finish_reason": "stop"}), "parsed": None}
|
||
|
||
Models(monkeypatch, Reasoner(structured=once, plain=lambda units: AIMessage(content="还是不是")))
|
||
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE + 3))
|
||
outcome, events = await judged(stored, raw)
|
||
assert outcome["unjudged"] == {"format": 3, "truncated": BATCH_SIZE}
|
||
[cut] = logs(events, "judge_truncated")
|
||
[failed] = logs(events, "judge_failed")
|
||
assert cut["unjudged"] == BATCH_SIZE and CUT_16K in cut["message"]
|
||
assert failed["message"] == "结论句裁判未完成：有 3 句结论未裁判：裁判返回的结果格式有误"
|
||
|
||
|
||
def test_the_review_layer_knows_the_truncation_log():
|
||
"""跑完之后的复核层认得 judge_truncated：同 judge_failed，是缺口（要说明），kind 是它自己。"""
|
||
assert rv.WARN_CODES["judge_truncated"] == ("judge_truncated", rv.DEGRADED)
|
||
event = {"type": str(EventType.LOG), "node_id": "write",
|
||
"data": {"level": "warn", "code": "judge_truncated", "message": f"有 4 句结论未裁判：{CUT_16K}"}}
|
||
[signal] = rv.scan([event], {"answer": "本周销售额 45,678.5元。"})
|
||
assert (signal.kind, signal.severity) == ("judge_truncated", rv.DEGRADED) and not rv.should_retry([signal])
|
||
|
||
|
||
async def test_without_truncation_format_problems_keep_their_wording(monkeypatch, stored):
|
||
model = Reasoner(structured=lambda units: {"raw": AIMessage(content="不是 JSON",
|
||
response_metadata={"finish_reason": "stop"}),
|
||
"parsed": None},
|
||
plain=lambda units: AIMessage(content="还是不是", response_metadata={"finish_reason": "stop"}))
|
||
Models(monkeypatch, model)
|
||
outcome, events = await judged(stored)
|
||
assert [kind for kind, _ in model.calls] == ["structured", "plain"]
|
||
assert outcome["unjudged"] == {"format": 4}
|
||
assert outcome["gaps"] == ["有 4 句结论未裁判：裁判返回的结果格式有误"]
|
||
assert not any(d.get("truncated") for k, d in events if k == EventType.LLM_END)
|
||
assert logs(events, "judge_truncated") == [] and len(logs(events, "judge_failed")) == 1
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 额度与退回
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_the_judge_asks_for_the_larger_budget(monkeypatch, stored):
|
||
models = Models(monkeypatch, Reasoner())
|
||
outcome, _ = await judged(stored)
|
||
assert outcome["counts"]["supported"] == 4
|
||
assert [s.max_tokens for s in models.specs] == [MAX_TOKENS] == [16384], "平常只建一个模型"
|
||
assert "truncated" in REASONS and FALLBACK_MAX_TOKENS == 4096
|
||
|
||
|
||
async def test_a_rejected_budget_falls_back_to_4096_once_for_all_batches(monkeypatch, stored):
|
||
model = Reasoner()
|
||
models = Models(monkeypatch, model, rejects_above=4096)
|
||
raw = "".join(f"第 {i} 个区域的销售额增长来自新客。[[see:m:gmv]]" for i in range(BATCH_SIZE * 2 + 3))
|
||
outcome, _ = await judged(stored, raw)
|
||
assert outcome["counts"]["supported"] == BATCH_SIZE * 2 + 3 and outcome["gaps"] == []
|
||
assert [s.max_tokens for s in models.specs] == [16384, 4096]
|
||
assert models.rejecting.calls == 1, "第一批退回过，后面几批直接用小的"
|
||
assert [kind for kind, _ in model.calls] == ["structured"] * 3
|
||
|
||
|
||
async def test_a_rejected_budget_is_remembered_for_the_next_reports(monkeypatch, stored):
|
||
"""每份报告、每次点击各问一次裁判：同一接入、模型被嫌大过一次，之后直接用小的，不再每次先撞一次。"""
|
||
model = Reasoner()
|
||
models = Models(monkeypatch, model, rejects_above=4096)
|
||
first, _ = await judged(stored)
|
||
second, _ = await judged(stored)
|
||
assert first["counts"]["supported"] == 4 and second["counts"]["supported"] == 4
|
||
assert models.rejecting.calls == 1, "第二份报告没有再按 16384 撞一次"
|
||
|
||
|
||
async def test_a_self_hosted_max_new_tokens_rejection_falls_back(monkeypatch, stored):
|
||
tgi = Rejecting("Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8192. "
|
||
"Given: 3000 `inputs` tokens and 16384 `max_new_tokens`", status_code=422)
|
||
models = Models(monkeypatch, Reasoner(), rejects_above=4096, rejecting=tgi)
|
||
outcome, _ = await judged(stored)
|
||
assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == []
|
||
assert [s.max_tokens for s in models.specs] == [16384, 4096] and tgi.calls == 1
|
||
|
||
|
||
async def test_a_truncation_after_the_fallback_names_the_small_budget(monkeypatch, stored):
|
||
Models(monkeypatch, Reasoner(plain=thinking_only, unstructured=True), rejects_above=4096)
|
||
outcome, _ = await judged(stored)
|
||
[verdict, *_] = outcome["verdicts"].values()
|
||
assert verdict["reason"] == "truncated"
|
||
assert verdict["rationale"].startswith("裁判模型的输出达到上限（4096 token）被截断")
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 真实的 ChatOpenAI 和 openai SDK：只有 HTTP 层是假的（经 OpenAI 兼容网关接入的推理模型就是这一路）
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
class Gateway:
|
||
"""OpenAI 兼容网关：reply(这一批的句子编号) 给出 (message, finish_reason, 输出 token 数)。cap 给了，请求的额度超过它
|
||
就回 400。bodies 记下每个请求体。"""
|
||
|
||
def __init__(self, reply, *, cap: int | None = None) -> None:
|
||
self.reply, self.cap = reply, cap
|
||
self.bodies: list[dict] = []
|
||
|
||
async def handle(self, request: httpx.Request) -> httpx.Response:
|
||
body = json.loads(request.content or b"{}")
|
||
self.bodies.append(body)
|
||
budget = body.get("max_completion_tokens") or body.get("max_tokens")
|
||
if self.cap and budget and budget > self.cap:
|
||
message = f"max_tokens: {budget} > {self.cap}, which is the maximum allowed number of output tokens"
|
||
return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}},
|
||
request=request)
|
||
units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M)
|
||
message, finish, spent = self.reply(units)
|
||
return httpx.Response(200, request=request, json={
|
||
"id": "c-1", "object": "chat.completion", "created": 1, "model": PRICED,
|
||
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
|
||
"usage": {"prompt_tokens": 3000, "completion_tokens": spent, "total_tokens": 3000 + spent}})
|
||
|
||
|
||
def gateway(monkeypatch, reply, *, cap: int | None = None) -> Gateway:
|
||
fake = Gateway(reply, cap=cap)
|
||
monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", fake.handle)
|
||
provider = Provider(name=f"gw-{uuid.uuid4().hex[:8]}", kind="openai_compatible", base_url="http://gw.example.test/v1",
|
||
api_key=None, models=[{"id": PRICED}], default_model=PRICED, enabled=True, extra={})
|
||
|
||
async def build(session, spec):
|
||
return build_chat_model(provider, spec), PRICED
|
||
|
||
monkeypatch.setattr(judge_mod, "get_chat_model", build)
|
||
return fake
|
||
|
||
|
||
async def test_a_real_client_cut_off_in_the_json_schema_call(monkeypatch, stored):
|
||
"""json_schema 的结构化输出：finish_reason=length 时 openai SDK 抛 LengthFinishReasonError，不再退回纯文本。"""
|
||
fake = gateway(monkeypatch, lambda units: ({"role": "assistant", "content": '{"items": [{"unit": "u'}, "length", 16384))
|
||
outcome, _ = await judged(stored)
|
||
assert outcome["unjudged"] == {"truncated": 4}
|
||
assert outcome["gaps"] == [f"有 4 句结论未裁判：{CUT_16K}，{ADVICE}"]
|
||
assert len(fake.bodies) == 1 and "response_format" in fake.bodies[0]
|
||
assert fake.bodies[0]["max_completion_tokens"] == 16384
|
||
assert outcome["usage"]["output_tokens"] == 16384, "异常里带着的那次用量照记"
|
||
|
||
|
||
async def test_a_real_client_rejecting_the_budget_falls_back_to_4096(monkeypatch, stored):
|
||
def answer(units):
|
||
items = [{"unit": u, "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []} for u in units]
|
||
return {"role": "assistant", "content": json.dumps({"items": items}, ensure_ascii=False)}, "stop", 80
|
||
|
||
fake = gateway(monkeypatch, answer, cap=8192)
|
||
outcome, _ = await judged(stored)
|
||
assert outcome["counts"]["supported"] == 4 and outcome["gaps"] == []
|
||
assert [b.get("max_completion_tokens") for b in fake.bodies] == [16384, 4096]
|
||
|
||
|
||
async def test_a_real_anthropic_cut_with_a_half_written_list(monkeypatch, stored):
|
||
"""真实的 ChatAnthropic：强制工具调用写到第二条判定时到了上限，网关给的是半截清单。整批记截断，llm.end 标出来。"""
|
||
def half(body):
|
||
units = re.findall(r"^\[(u\d+)\]", str(body["messages"][-1]["content"]), re.M)
|
||
items = [{"unit": units[0], "verdict": "supported", "rationale": "证据支持", "missing": "", "used": []},
|
||
{"unit": units[1]}]
|
||
return [{"type": "tool_use", "id": "toolu_1", "name": body["tools"][0]["name"], "input": {"items": items}}]
|
||
|
||
fake = messages_api(monkeypatch, [(half, "max_tokens", 16384)], model=PRICED, module=judge_mod)
|
||
outcome, events = await judged(stored)
|
||
assert outcome["unjudged"] == {"truncated": 4} and outcome["counts"]["supported"] == 0
|
||
assert len(fake.bodies) == 1 and fake.bodies[0]["max_tokens"] == 16384, "没有再退回纯文本"
|
||
[end] = [d for k, d in events if k == EventType.LLM_END]
|
||
assert end["truncated"] is True and end["error"] == CUT_16K
|