mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-22 04:02:03 +00:00
587bf7e27c
起因是一次探索运行:Copilot 改节点时把工具绑定整份抹掉,agent 没有工具只能写「假设调用」, 校验节点的自动修复编出 total_count=0 让 schema 通过,协作成员把工具调用标记当文字输出了四轮, 轮数用完后节点把原话当成功结果交出去——运行标成已完成,数据库一次都没查。 平台可信度(先写失败测试) - Copilot 修改节点改为合并:没写的字段保留,null 才删除;协作成员按名字合并。前后端同一套规则, 工具集合变小时自查报警、改图回执列出变化 - 新校验规则:提示词点名了某个工具而节点没有绑定它,判错误,Copilot 自查会交回去修 - 校验修复只许改格式:冒出原文没有的值就判修复无效;修复这次模型调用计入用量 - 识别模型把工具调用写成文字的情况(DSML、<tool_call> 等标记):纠正一次,仍然如此就判失败并说明原因 - 协作团队用完轮数不再按成功交付:默认失败,可选降档,点出一次都没派到的成员 - 审批恢复后,同一节点里已经执行过的工具调用复用结果,不再重复执行 后端 - 查询时限交给数据库执行(Oracle call_timeout、PG statement_timeout、MySQL max_execution_time、 SQLite progress handler),可按数据源配置;超时报错与实际耗时一致 - 等审批的运行可以真正取消;会话列表带最后一轮状态;测连接结果持久化;探查结构可只看不存; 自定义工具可草稿试跑、写坏的参数定义保存时拒绝;MCP 服务改动后作废旧工具缓存; 检索命中带贡献分解;重建索引在后台跑并报进度;报错统一收进 core/errors 前端 - 公共层:管理页共用件、失败说明(lib/explain)、健康状态进公共库;目录按表各自加载并有超时, 一张表卡住不再拖垮整站;上传显示真实字节进度;离线横幅只播报不变的句子 - 画布:卡片只说游标所在的那一次执行;等审批时可放弃运行;团队收尾判定与未完成说清楚; 回放用量取那一刻的值 - 编排页:Copilot 合并语义与工具变化回执;「用完轮数时」选项;正式运行期间锁住所有改图入口 - 回答流 / 问数据:新失败类型说人话并给下一步;画布与右栏互相定位;数据源范围、回收站、重新核对 - 记录页:「航迹」页签复用画布时间轴,可拖动回放、列出关键时刻;列表行带执行 / 等人 / 失败缩略条 - 外壳:统一的离开前确认(data router),导航、⌘K、快捷键、后退都会先问 验证(沙箱):tsc 干净,pytest 838 passed(两遍),14 个检查脚本中 13 个全绿; check-ui 余 1 项旧断言(按旧日期格式找运行行),3c 更新。
235 lines
11 KiB
Python
235 lines
11 KiB
Python
"""模型把工具调用当成文字写出来,这一步没有真的调用工具——要认出来,不能当成功交出去。
|
|
|
|
一次真实运行的最终答案是一整段 `<||DSML|| invoke name="db_schema__…">` 标记:
|
|
协作成员没有绑定工具,模型把自己那套工具调用格式当正文输出了四轮,数据库一次
|
|
都没查,运行却是 succeeded。换成 agent 节点、单次模型调用也是一样。
|
|
|
|
处理:先带一句纠正提示重试一次;仍然如此就判失败,说清原因和怎么办,并发一条
|
|
code=tool_markup_leak 的警告。协作成员这样时,这一步记为失败,原因交回调度者。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage, HumanMessage
|
|
from sqlalchemy import select
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Run, RunEvent
|
|
from app.engine.runner import run_manager
|
|
from app.engine.toolcalls import leaked_markup
|
|
|
|
DSML = ('我将真正执行数据库工具,先取表结构。\n\n<||DSML|| calls>\n'
|
|
'<||DSML|| invoke name="db_schema__shop">\n'
|
|
'<||DSML|| parameter name="table">orders</||DSML|| parameter>\n'
|
|
'</||DSML|| invoke>\n</||DSML|| calls>')
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
|
"data": {"label": nid, "config": config}}
|
|
|
|
|
|
def chain(*nodes):
|
|
return {"nodes": list(nodes),
|
|
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
async def _finish(graph) -> tuple[Run, list[RunEvent]]:
|
|
run = await run_manager.start(graph=graph, input_payload={"question": "orders 有多少"})
|
|
for _ in range(300):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
if row.status in ("succeeded", "failed"):
|
|
break
|
|
async with SessionLocal() as session:
|
|
events = list((await session.execute(
|
|
select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq)
|
|
)).scalars())
|
|
return row, events
|
|
|
|
|
|
def _leaks(monkeypatch, *, until: int | None = None) -> list[list]:
|
|
"""模型一直(或前 until 次)把工具调用写成文字。返回每次收到的消息。"""
|
|
from app.providers import mock_model
|
|
|
|
seen: list[list] = []
|
|
|
|
def _decide(self, messages):
|
|
if self.response_format: # 协作团队的调度者:先派查数员,之后收工
|
|
routed = sum(1 for batch in seen if batch and batch[0] == "route")
|
|
seen.append(["route", "\n".join(str(m.content) for m in messages)])
|
|
plan = ({"assignments": [{"agent": "查数员", "instruction": "查 orders 有多少"}]}
|
|
if routed == 0 else {"assignments": [], "done": True, "reason": "够了"})
|
|
return AIMessage(content=json.dumps(plan, ensure_ascii=False))
|
|
seen.append(list(messages))
|
|
calls = sum(1 for batch in seen if batch and batch[0] != "route")
|
|
if until is None or calls <= until:
|
|
return AIMessage(content=DSML)
|
|
return AIMessage(content="orders 一共 12 条。")
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
return seen
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("text", [
|
|
DSML,
|
|
"<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>db_query__shop",
|
|
'<tool_call>\n{"name": "db_query__shop", "arguments": {"sql": "SELECT 1"}}\n</tool_call>',
|
|
'<function_call>{"name": "python_exec"}</function_call>',
|
|
'<invoke name="db_query__shop"><parameter name="sql">SELECT 1</parameter></invoke>',
|
|
'假设调用工具得到:{"tool": "db_query__shop", "sql": "SELECT COUNT(*) FROM orders"}',
|
|
])
|
|
def test_markup_is_recognised(text):
|
|
assert leaked_markup(text)
|
|
|
|
|
|
@pytest.mark.parametrize("text", [
|
|
"orders 一共 12 条。",
|
|
"查询用的是 `SELECT COUNT(*) FROM orders`,结果 12。",
|
|
'返回的 JSON 是 {"tool": "calculator", "result": 3}', # 没自称「假设调用」
|
|
"工具调用失败了,原因是连接超时。",
|
|
])
|
|
def test_ordinary_answers_are_not(text):
|
|
assert not leaked_markup(text)
|
|
|
|
|
|
async def test_an_agent_that_keeps_writing_markup_fails(monkeypatch):
|
|
_leaks(monkeypatch)
|
|
row, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("query", "agent", prompt="查 orders 有多少", tools=[], max_steps=4),
|
|
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.query.text }}"}]),
|
|
))
|
|
assert row.status == "failed", f"一堆标记被当成了答案:{row.output}"
|
|
assert "工具调用的原始标记" in (row.error or "") and "没有真正调用工具" in (row.error or "")
|
|
assert row.error_node_id == "query"
|
|
assert "tool_markup_leak" in [e.data.get("code") for e in events if e.type == "log"]
|
|
|
|
|
|
async def test_an_agent_gets_one_corrective_retry(monkeypatch):
|
|
seen = _leaks(monkeypatch, until=1)
|
|
row, _ = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("query", "agent", prompt="查 orders 有多少", tools=[], max_steps=4),
|
|
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.query.text }}"}]),
|
|
))
|
|
assert row.status == "succeeded", row.error
|
|
assert row.output["r"] == "orders 一共 12 条。"
|
|
retry = seen[1]
|
|
assert isinstance(retry[-1], HumanMessage) and "没有" in str(retry[-1].content)
|
|
|
|
|
|
async def test_a_plain_model_call_that_writes_markup_fails(monkeypatch):
|
|
_leaks(monkeypatch)
|
|
row, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("say", "llm", prompt="回答 {{ input.question }}"),
|
|
node("out", "output", fields=[]),
|
|
))
|
|
assert row.status == "failed"
|
|
assert "工具调用的原始标记" in (row.error or "")
|
|
ends = [e for e in events if e.type == "llm.end" and e.node_id == "say"]
|
|
assert len(ends) == 2, "重试那一次也是一次模型调用,要记账"
|
|
|
|
|
|
async def test_a_team_member_writing_markup_fails_its_step_and_the_coordinator_is_told(monkeypatch):
|
|
seen = _leaks(monkeypatch)
|
|
row, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("team", "supervisor", goal="查 orders 有多少", max_rounds=3, agents=[
|
|
{"name": "查数员", "system": "你负责查库", "max_steps": 3}]),
|
|
node("out", "output", fields=[]),
|
|
))
|
|
step = next(e.data for e in events if e.type == "agent.step.end")
|
|
assert step.get("failed") is True and "原始标记" in step["preview"], step
|
|
assert "tool_markup_leak" in [e.data.get("code") for e in events if e.type == "log"]
|
|
# 调度者下一轮读到的进展里有这条失败的原因
|
|
routes = [batch[1] for batch in seen if batch and batch[0] == "route"]
|
|
assert len(routes) == 2 and "原始标记" in routes[1], routes
|
|
assert "<||DSML" not in json.dumps(row.output, ensure_ascii=False)
|
|
|
|
|
|
async def test_markup_in_the_settle_round_keeps_what_the_agent_found(monkeypatch):
|
|
"""步数用完后的收尾轮(不带工具)写出调用标记:不是「没绑工具」,不能判失败、把查到的丢掉。"""
|
|
from langchain_core.messages import ToolMessage
|
|
|
|
from app.providers import mock_model
|
|
|
|
def _decide(self, messages):
|
|
if self.tools: # 前面的步数:真的在调工具
|
|
done = sum(isinstance(m, ToolMessage) for m in messages)
|
|
return AIMessage(content=f"先算第 {done + 1} 步。", tool_calls=[
|
|
{"name": "calculator", "args": {"expression": f"{done} + 1"}, "id": f"c{done}"}])
|
|
return AIMessage(content=DSML) # 收尾轮:还想接着查
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
row, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("query", "agent", prompt="算一下", tools=["calculator"], max_steps=2),
|
|
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.query.text }}"}]),
|
|
))
|
|
assert row.status == "succeeded", row.error
|
|
assert [e.type for e in events].count("tool.end") == 2
|
|
assert "<||DSML" not in row.output["r"]
|
|
assert "先算第 2 步。" in row.output["r"] and "用满了 2 步" in row.output["r"], row.output
|
|
leak = [e.data for e in events if e.type == "log" and e.data.get("code") == "tool_markup_leak"]
|
|
assert leak and "收尾轮" in leak[0]["message"], leak
|
|
|
|
|
|
async def test_markup_to_the_very_end_without_any_real_call_still_fails(monkeypatch):
|
|
"""纠正之后步数就用完了、收尾轮还是标记:一次真调用都没有,仍是「调不了工具」。"""
|
|
_leaks(monkeypatch)
|
|
row, _ = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("query", "agent", prompt="查 orders 有多少", tools=["calculator"], max_steps=1),
|
|
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.query.text }}"}]),
|
|
))
|
|
assert row.status == "failed" and "工具调用的原始标记" in (row.error or ""), row.error
|
|
|
|
|
|
async def test_a_member_whose_settle_round_writes_markup_keeps_its_findings(monkeypatch):
|
|
from langchain_core.messages import ToolMessage
|
|
|
|
from app.providers import mock_model
|
|
|
|
routed = {"n": 0}
|
|
|
|
def _decide(self, messages):
|
|
if self.response_format:
|
|
routed["n"] += 1
|
|
plan = ({"assignments": [{"agent": "查数员", "instruction": "算"}]} if routed["n"] == 1
|
|
else {"assignments": [], "done": True, "reason": "够了"})
|
|
return AIMessage(content=json.dumps(plan, ensure_ascii=False))
|
|
if self.tools:
|
|
done = sum(isinstance(m, ToolMessage) for m in messages)
|
|
return AIMessage(content=f"第 {done + 1} 步:先算一下。", tool_calls=[
|
|
{"name": "calculator", "args": {"expression": "1 + 1"}, "id": f"c{done}"}])
|
|
return AIMessage(content=DSML)
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
row, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("team", "supervisor", goal="算一下", max_rounds=3, agents=[
|
|
{"name": "查数员", "system": "你负责算", "tools": ["calculator"], "max_steps": 2}]),
|
|
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.team.text }}"}]),
|
|
))
|
|
assert row.status == "succeeded", row.error
|
|
step = next(e.data for e in events if e.type == "agent.step.end")
|
|
assert not step.get("failed"), step
|
|
assert "第 2 步" in row.output["r"] and "<||DSML" not in row.output["r"], row.output
|
|
assert "tool_markup_leak" in [e.data.get("code") for e in events if e.type == "log"]
|