mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 10:51:58 +00:00
ThearchyHelios
a43c530fb8
一次真实运行里 4 个开了按出处核对字段的 Agent 抽取全部失败：抽取关掉思考、额度 4096，可经 OpenAI 兼容接口接入的推理模型关不掉思考，额度在推理阶段就用完了（openai 抛 LengthFinishReasonError）；日志却叫人「稍后重试、去设置测试该模型」，两样都不解决问题。 - 额度默认至少 16384，节点配了更大的用节点的 - 认出输出被截断：openai 的 length limit 类错误、finish_reason=length、Anthropic 的 stop_reason=max_tokens；被截断时额度翻倍（最多 32768）重试一次，每次调用各记一行，被截断那次的用量照样记账 - 服务方嫌额度太大（400，报错里有 max_tokens 一类字样）时退回 8192、4096，不能因为额度提高反而抽不成 - 两次都截断：新日志 code agent_field_truncated（带 fields、max_tokens），原话说清额度在推理阶段用完、结构化结论没能写出来，下一步是换非推理模型或调大最大输出 token；字段照旧记空值、不以 0 代替。非截断类的失败照旧是 agent_field_unverified - 时间线给 agent_field_truncated 专门的标题、副标题和下一步，入口落到该 Agent 的最大输出 token；check-decode 补断言
209 lines
9.4 KiB
Python
209 lines
9.4 KiB
Python
"""cite_fields 的结构化抽取：输出被截断时加大额度重试，仍截断就照实说是额度在推理阶段用完了。
|
||
|
||
2026-10 一次运行里 4 个开了 cite_fields 的 Agent 抽取全部失败：抽取用 thinking: off、额度 4096，而经 OpenAI 兼容接口
|
||
接入的推理模型关不掉思考，4096 在推理阶段就用完了，openai 抛 LengthFinishReasonError（「Could not parse response
|
||
content as the length limit was reached」）。日志却写「请稍后重试；如持续失败，请到设置测试该模型」——重试、测试都没用。
|
||
|
||
- 额度默认至少 16384，节点配了更大的用节点的
|
||
- 认出「输出被截断」：openai 的 length limit 类错误、finish_reason=length、Anthropic 的 stop_reason=max_tokens；
|
||
被截断时额度翻倍（最多 32768）重试一次
|
||
- 仍然截断：日志 code agent_field_truncated，说清是额度在推理阶段用完了，给出能做的下一步；字段记空值，不以 0 代替
|
||
- 服务方嫌额度太大（400，报错里有 max_tokens 一类字样）：退回小一档的额度重来，不能因为额度提高反而抽不成
|
||
- 非截断类的失败照旧（agent_field_unverified）
|
||
|
||
夹具全部合成：门店、会员这类虚构的表。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessage
|
||
from openai import LengthFinishReasonError
|
||
from openai.types import CompletionUsage
|
||
from openai.types.chat import ChatCompletion
|
||
|
||
from app.engine.nodes import llm as llm_mod
|
||
from app.engine.runner import run_manager
|
||
from tests.test_agent_cited_fields import ( # noqa: F401 - 夹具按名字引入
|
||
EXTRACTED,
|
||
FINAL,
|
||
SCHEMA,
|
||
SQLS,
|
||
TOOL,
|
||
Script,
|
||
engine_up,
|
||
events,
|
||
graph,
|
||
logs,
|
||
output_of,
|
||
shop,
|
||
wait,
|
||
)
|
||
|
||
|
||
def length_error(prompt: int = 12000, completion: int = 4096) -> LengthFinishReasonError:
|
||
"""openai 的结构化输出在 finish_reason=length 时抛的那个错（和真实运行里的原话一样）。"""
|
||
usage = CompletionUsage(completion_tokens=completion, prompt_tokens=prompt, total_tokens=prompt + completion)
|
||
return LengthFinishReasonError(completion=ChatCompletion.model_construct(
|
||
id="x", choices=[], created=0, model="m", object="chat.completion", usage=usage))
|
||
|
||
|
||
class BudgetRejected(Exception):
|
||
"""服务方嫌输出额度太大时的 400（DeepSeek chat 系只收 8K）。"""
|
||
|
||
status_code = 400
|
||
|
||
|
||
class Extraction:
|
||
"""记下每次抽取调用的额度（thinking 关掉的那次 get_chat_model），并按额度决定这次怎么回。
|
||
|
||
outcome(budget) 返回 None 表示照常回 EXTRACTED，抛异常就是这次调用失败。
|
||
"""
|
||
|
||
def __init__(self, monkeypatch, outcome):
|
||
from app.providers import mock_model
|
||
|
||
self.budgets: list[int | None] = []
|
||
real = llm_mod.get_chat_model
|
||
|
||
async def spy(session, spec):
|
||
if spec.thinking == "off":
|
||
self.budgets.append(spec.max_tokens)
|
||
return await real(session, spec)
|
||
|
||
monkeypatch.setattr(llm_mod, "get_chat_model", spy)
|
||
base = Script(monkeypatch)
|
||
loop = mock_model.MockChatModel._decide
|
||
|
||
def decide(model, messages):
|
||
if model.response_format:
|
||
outcome(self.budgets[-1])
|
||
return AIMessage(content=json.dumps(EXTRACTED, ensure_ascii=False))
|
||
return loop(model, messages)
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide)
|
||
self.script = base
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 认出「输出被截断」
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_length_errors_are_recognised():
|
||
assert llm_mod.truncated_error(length_error())
|
||
assert llm_mod.truncated_error(RuntimeError("Could not parse response content as the length limit was reached"))
|
||
assert not llm_mod.truncated_error(RuntimeError("网关超时"))
|
||
assert not llm_mod.truncated_error(BudgetRejected("max_tokens is too large: 16384"))
|
||
|
||
|
||
def test_cut_off_replies_are_recognised_for_both_families():
|
||
assert llm_mod.truncated_reply(AIMessage(content="", response_metadata={"finish_reason": "length"}))
|
||
assert llm_mod.truncated_reply(AIMessage(content="", response_metadata={"stop_reason": "max_tokens"}))
|
||
assert not llm_mod.truncated_reply(AIMessage(content="", response_metadata={"finish_reason": "stop"}))
|
||
assert not llm_mod.truncated_reply(None)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 额度与重试
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_the_default_budget_leaves_room_for_reasoning(monkeypatch):
|
||
spy = Extraction(monkeypatch, lambda budget: None)
|
||
row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id)
|
||
assert row.status == "succeeded", row.error
|
||
assert spy.budgets == [16384], "推理模型的思考也算在额度里：默认至少 16384"
|
||
|
||
|
||
async def test_a_larger_node_budget_is_used(monkeypatch):
|
||
spy = Extraction(monkeypatch, lambda budget: None)
|
||
row = await wait((await run_manager.start(graph=graph(card=False, max_tokens=24000), input_payload={})).id)
|
||
assert row.status == "succeeded", row.error
|
||
assert spy.budgets == [24000]
|
||
|
||
|
||
async def test_a_cut_off_extraction_is_retried_with_a_larger_budget(monkeypatch):
|
||
def outcome(budget):
|
||
if budget < 32768:
|
||
raise length_error(completion=budget)
|
||
|
||
spy = Extraction(monkeypatch, outcome)
|
||
row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id)
|
||
assert row.status == "succeeded", row.error
|
||
assert spy.budgets == [16384, 32768], "截断后翻倍重试一次"
|
||
bot = await output_of(row.id, "bot")
|
||
assert bot["data"]["gmv"] == 600.5 and bot["data_evidence"]["gmv"]["status"] == "verified"
|
||
evts = await events(row.id)
|
||
assert not logs(evts, "agent_field_truncated")
|
||
# EXTRACTED 里 region 指向本节点没有的 Q9：只有它核对不了，别的字段都照常核对
|
||
assert [w["fields"] for w in logs(evts, "agent_field_unverified")] == [["region"]]
|
||
# 两次调用各记一行：第一次写明为什么重来
|
||
ends = [e.data for e in evts if e.type == "llm.end" and e.data.get("purpose") == "cite_fields"]
|
||
assert len(ends) == 2 and "32768" in ends[0]["error"] and not ends[1].get("error")
|
||
# 被截断那次的用量照样记账（花出去的 token 是真的）
|
||
assert ends[0]["output_tokens"] == 16384
|
||
|
||
|
||
async def test_cut_off_twice_is_reported_as_truncation_with_fields_empty(monkeypatch):
|
||
def outcome(budget):
|
||
raise length_error(completion=budget)
|
||
|
||
spy = Extraction(monkeypatch, outcome)
|
||
row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id)
|
||
assert row.status == "succeeded", row.error
|
||
assert spy.budgets == [16384, 32768], "只重试一次"
|
||
bot = await output_of(row.id, "bot")
|
||
assert bot["data"] == {k: None for k in SCHEMA["properties"]}, "记空值，不以 0 代替"
|
||
assert all(e["status"] == "unresolved" for e in bot["data_evidence"].values())
|
||
evts = await events(row.id)
|
||
assert not logs(evts, "agent_field_unverified")
|
||
[warn] = logs(evts, "agent_field_truncated")
|
||
assert warn["level"] == "warn" and warn["fields"] == list(SCHEMA["properties"])
|
||
message = warn["message"]
|
||
assert "推理阶段就用完了" in message and "结构化结论没能写出来" in message
|
||
assert "32768" in message and "非推理模型" in message and "最大输出 token" in message
|
||
assert "请稍后重试" not in message and "测试该模型" not in message, "重试、测试模型都不解决额度问题"
|
||
assert "未以 0 代替" in message
|
||
assert warn.get("max_tokens") == 32768
|
||
|
||
|
||
async def test_a_node_budget_at_the_cap_is_not_retried(monkeypatch):
|
||
def outcome(budget):
|
||
raise length_error(completion=budget)
|
||
|
||
spy = Extraction(monkeypatch, outcome)
|
||
row = await wait((await run_manager.start(graph=graph(card=False, max_tokens=40000), input_payload={})).id)
|
||
assert row.status == "succeeded", row.error
|
||
assert spy.budgets == [40000], "额度已经超过重试上限，再调一次也一样"
|
||
[warn] = logs(await events(row.id), "agent_field_truncated")
|
||
assert "40000" in warn["message"]
|
||
|
||
|
||
async def test_a_budget_the_service_rejects_falls_back(monkeypatch):
|
||
def outcome(budget):
|
||
if budget > 8192:
|
||
raise BudgetRejected(f"Invalid max_tokens value, the valid range of max_tokens is [1, 8192]: {budget}")
|
||
|
||
spy = Extraction(monkeypatch, outcome)
|
||
row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id)
|
||
assert row.status == "succeeded", row.error
|
||
assert spy.budgets == [16384, 8192]
|
||
bot = await output_of(row.id, "bot")
|
||
assert bot["data_evidence"]["gmv"]["status"] == "verified"
|
||
|
||
|
||
async def test_other_failures_are_reported_as_before(monkeypatch):
|
||
def outcome(budget):
|
||
raise RuntimeError("网关超时")
|
||
|
||
spy = Extraction(monkeypatch, outcome)
|
||
row = await wait((await run_manager.start(graph=graph(card=False), input_payload={})).id)
|
||
assert row.status == "succeeded", row.error
|
||
assert spy.budgets == [16384], "不是截断就不重试"
|
||
evts = await events(row.id)
|
||
assert not logs(evts, "agent_field_truncated")
|
||
[warn] = logs(evts, "agent_field_unverified")
|
||
assert "网关超时" in warn["message"] and "未以 0 代替" in warn["message"]
|