mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 03:01:58 +00:00
ThearchyHelios
4057cdc35f
走到步数上限一定意味着最后那步**要求了工具**（不要工具的话循环早 break 了）， 也就是说模型从来没拿到过"说结论"的那一轮——它只是被掐断在半路。交出去的 是它上一句中间过程的话，末尾补一句"步数不够，请调大"。 实测这条线是绷紧的：库里 116 个 agent 节点，87% 在 10 步内收口，11 步只有 1 个，**正好 12 步的有 11 个**。1 跳到 11 是被截断的特征，不是自然分布。 所以补上那一轮：不给工具，让它基于已经查到的收口，并说清哪些没查到、 结论因此有什么局限。用 base_model 而不是绑过工具的 model——收尾轮必须在 结构上发不出 tool_call，靠提示词请它别调是约束不住的。 两种收场发不同的 code：收上来了是 step_limit_settled（DEGRADED）， 收不上来仍是 step_limit（BROKEN）。不复用同一个 code：库里那 8 条老事件 的含义确实是硬截断，改语义等于把历史运行重新解释成另一回事。 顺带修掉一个一直在白烧钱的 bug：chat.ts:440 的注释说自动重试能"调大最大 步数"，而 copilot.py 明确禁止模型写 max_steps。两边对不上，于是 step_limit 的重试是用同一个默认 12 再跑一遍，必然撞同一堵墙。现在那条禁令改成有条件 的——只有上一次死于步数不够时才准显式写大值。改完两条路各自成立：收尾 成功走 DEGRADED，闸门不开、不重试（对的，同配置再跑还是撞墙）；收尾失败 仍是 BROKEN，而这次 Copilot 真能把步数调大。 硬顶 25 → 100，画布滑块跟着放开，**节点默认仍是 12**：要更多的人调得动， 不知情的人不会突然多花十倍。 新增 tests/test_agent_steps.py，顺带补回 agent 循环的集成测试（原来那个 文件在脱敏那轮被整个删掉了）。两条反向断言是重点：正常收口的 agent 一次 调用都不该多花，以及收尾轮发不出工具调用。两者都做过变异验证——把收尾 结果丢弃、把收尾轮换成绑工具的 model，各自都能让对应的断言变红。 273 passed（去掉一个会真连 api.openai.com 的既有慢测试是 238 passed / 9.4s）。
219 lines
8.4 KiB
Python
219 lines
8.4 KiB
Python
"""步数用尽时，agent 要收口，而不是把半截过程交出去。
|
||
|
||
走到步数上限一定意味着最后那步**要求了工具**（不要工具的话循环早 break 了），
|
||
也就是说模型从来没拿到过"说结论"的那一轮——它只是被掐断在半路。交出去的
|
||
是它上一句中间过程的话，末尾补一句"步数不够"。
|
||
|
||
所以补上那一轮：不给工具，让它基于已经查到的收口。这里守三件事——收尾轮
|
||
真的发生了、它**发不出**工具调用、以及它没有变成每次都多烧的一次调用。
|
||
|
||
顺带补回 agent 循环的集成测试：原来那个文件在脱敏那轮被整个删掉了。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessage
|
||
from sqlalchemy import select
|
||
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run, RunEvent
|
||
from app.engine.runner import run_manager
|
||
|
||
#: 收尾轮该说的话。挑一句和"中间过程"截然不同的，好分辨答案到底取自哪一轮
|
||
SETTLED = "已经查到的部分：门店 8 家。其余未查到，这个数是下限。"
|
||
MIDWAY = "中间过程：还得再查一张表。"
|
||
|
||
MAX_STEPS = 3
|
||
|
||
|
||
@pytest.fixture
|
||
async def runner():
|
||
await run_manager.setup()
|
||
yield run_manager
|
||
await run_manager.shutdown()
|
||
|
||
|
||
def _graph(*, max_steps: int = MAX_STEPS) -> dict:
|
||
return {
|
||
"nodes": [
|
||
{"id": "in", "type": "input", "data": {"config": {}}},
|
||
{"id": "work", "type": "agent", "data": {"config": {
|
||
"prompt": "{{ input.question }}",
|
||
"tools": ["calculator"],
|
||
"max_steps": max_steps,
|
||
"approval": "never",
|
||
}}},
|
||
{"id": "out", "type": "output", "data": {"config": {}}},
|
||
],
|
||
"edges": [{"source": "in", "target": "work"}, {"source": "work", "target": "out"}],
|
||
}
|
||
|
||
|
||
def never_concludes(monkeypatch, *, settle_raises: bool = False) -> None:
|
||
"""绑了工具就永远再要一次工具；没绑工具才说话。
|
||
|
||
`self.tools` 是 bind_tools 之后才有的。收尾轮走的是**没绑过工具**的
|
||
base_model，所以它必然落到下面那个分支——这正是"收尾轮发不出工具调用"
|
||
那条断言的着力点：真要是绑着工具过来的，拿到的就会是 MIDWAY。
|
||
"""
|
||
from app.providers import mock_model
|
||
|
||
def _decide(self, messages):
|
||
if self.tools:
|
||
return AIMessage(
|
||
content=MIDWAY,
|
||
tool_calls=[{"name": self.tools[0]["name"],
|
||
"args": {"expression": "1+1"},
|
||
"id": f"call_{len(messages)}"}],
|
||
)
|
||
if settle_raises:
|
||
raise RuntimeError("收尾轮炸了")
|
||
return AIMessage(content=SETTLED)
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
||
|
||
|
||
async def _run(graph: dict, question: str = "有几家门店"):
|
||
run = await run_manager.start(graph=graph, input_payload={"question": question})
|
||
for _ in range(200):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = (await session.execute(select(Run).where(Run.id == run.id))).scalar_one()
|
||
if row.status in ("succeeded", "failed", "cancelled"):
|
||
break
|
||
else:
|
||
pytest.fail("运行没有在 10 秒内结束")
|
||
async with SessionLocal() as session:
|
||
events = list((await session.execute(
|
||
select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq)
|
||
)).scalars())
|
||
return events, row
|
||
|
||
|
||
def codes(events) -> list[str]:
|
||
return [e.data.get("code") for e in events
|
||
if e.type == "log" and (e.data or {}).get("level") == "warn"]
|
||
|
||
|
||
def answer_of(row) -> str:
|
||
from app.engine import review as rv
|
||
return rv.answer_text(row.output)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_step_limit_ends_with_a_conclusion_not_a_stump(runner, monkeypatch) -> None:
|
||
"""这块改动的全部意义：交出去的是结论，不是中间过程加一句抱怨。"""
|
||
never_concludes(monkeypatch)
|
||
events, row = await _run(_graph())
|
||
assert row.status == "succeeded", row.error
|
||
|
||
text = answer_of(row)
|
||
assert SETTLED in text, text
|
||
assert MIDWAY not in text, f"上一句中间过程冒充了答案：{text}"
|
||
assert "step_limit_settled" in codes(events), codes(events)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_the_settle_round_has_no_tools_bound(runner, monkeypatch) -> None:
|
||
"""收尾轮必须在**结构上**发不出工具调用，而不是靠提示词请它别调。
|
||
|
||
mock 的判据就是"有没有绑工具"：真要是把绑过工具的 model 传进了收尾轮，
|
||
它会照常再要一次工具，答案里出现的就是 MIDWAY。
|
||
"""
|
||
never_concludes(monkeypatch)
|
||
events, row = await _run(_graph())
|
||
|
||
assert SETTLED in answer_of(row)
|
||
# 工具调用次数正好等于步数——收尾轮一次都没多调
|
||
assert len([e for e in events if e.type == "tool.start"]) == MAX_STEPS
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_the_settle_round_costs_exactly_one_extra_call(runner, monkeypatch) -> None:
|
||
"""多花的必须是可数的一次，不是一个说不清的尾巴。"""
|
||
never_concludes(monkeypatch)
|
||
events, _ = await _run(_graph())
|
||
assert len([e for e in events if e.type == "llm.start"]) == MAX_STEPS + 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_normal_agent_does_not_pay_for_a_settle_round(runner, monkeypatch) -> None:
|
||
"""反向断言：正常收口的 agent 一分钱都不该多花。
|
||
|
||
没有这条，"每次都多跑一轮"这种退化会悄无声息地通过全部其他测试。
|
||
"""
|
||
from app.providers import mock_model
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide",
|
||
lambda self, messages: AIMessage(content="直接就有结论了。"))
|
||
events, row = await _run(_graph())
|
||
|
||
assert "直接就有结论了。" in answer_of(row)
|
||
assert len([e for e in events if e.type == "llm.start"]) == 1
|
||
assert not [c for c in codes(events) if c and c.startswith("step_limit")]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_failed_settle_round_falls_back_and_keeps_what_was_found(
|
||
runner, monkeypatch
|
||
) -> None:
|
||
"""收尾轮自己挂了，不能把已经查到的一起赔进去。"""
|
||
never_concludes(monkeypatch, settle_raises=True)
|
||
events, row = await _run(_graph())
|
||
assert row.status == "succeeded", row.error
|
||
|
||
text = answer_of(row)
|
||
assert "settle_failed" in codes(events), codes(events)
|
||
# 退回原来的行为：老的 step_limit（BROKEN），而不是 settled 那个
|
||
assert "step_limit" in codes(events)
|
||
assert "step_limit_settled" not in codes(events)
|
||
# 已有成果还在，末尾挂着说明
|
||
assert MIDWAY in text and "用满了" in text, text
|
||
|
||
|
||
def test_max_steps_is_clamped_by_the_hard_cap() -> None:
|
||
"""节点上写多大都过不去硬顶——否则一张图就能把预算烧穿。"""
|
||
from app.core.config import settings
|
||
|
||
assert min(200, settings.max_agent_steps) == settings.max_agent_steps
|
||
assert settings.max_agent_steps >= 25, "抬高上限不该反而把原来能配的值挡回去"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 复核层怎么看这两种收场
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _warn(message: str, code: str) -> dict:
|
||
return {"type": "log", "node_id": "work",
|
||
"data": {"level": "warn", "message": message, "code": code}}
|
||
|
||
|
||
OK_OUTPUT = {"text": SETTLED}
|
||
|
||
|
||
def test_a_settled_run_is_a_gap_not_a_break() -> None:
|
||
"""收口过的答案是完整一段话，只是覆盖面不全——那是缺口不是断裂。"""
|
||
from app.engine import review as rv
|
||
|
||
signals = rv.scan([_warn("用满了 3 步，基于已查到的部分给出", "step_limit_settled")],
|
||
OK_OUTPUT)
|
||
assert [s.severity for s in signals] == [rv.DEGRADED]
|
||
# 连带效果正是想要的：不再触发自动重跑
|
||
assert not rv.should_retry(signals)
|
||
|
||
|
||
def test_a_hard_truncation_is_still_broken_and_still_retries() -> None:
|
||
"""收尾轮没能收上来，判定一点没变——这条是防止改动把老路一起改坏。"""
|
||
from app.engine import review as rv
|
||
|
||
signals = rv.scan([_warn("用满了 3 步还没给出结论", "step_limit")], OK_OUTPUT)
|
||
assert rv.worst(signals) == rv.BROKEN
|
||
assert rv.should_retry(signals)
|