mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 05:41:57 +00:00
ThearchyHelios
587bf7e27c
起因是一次探索运行：Copilot 改节点时把工具绑定整份抹掉，agent 没有工具只能写「假设调用」， 校验节点的自动修复编出 total_count=0 让 schema 通过，协作成员把工具调用标记当文字输出了四轮， 轮数用完后节点把原话当成功结果交出去——运行标成已完成，数据库一次都没查。 平台可信度（先写失败测试） - Copilot 修改节点改为合并：没写的字段保留，null 才删除；协作成员按名字合并。前后端同一套规则， 工具集合变小时自查报警、改图回执列出变化 - 新校验规则：提示词点名了某个工具而节点没有绑定它，判错误，Copilot 自查会交回去修 - 校验修复只许改格式：冒出原文没有的值就判修复无效；修复这次模型调用计入用量 - 识别模型把工具调用写成文字的情况（DSML、<tool_call> 等标记）：纠正一次，仍然如此就判失败并说明原因 - 协作团队用完轮数不再按成功交付：默认失败，可选降档，点出一次都没派到的成员 - 审批恢复后，同一节点里已经执行过的工具调用复用结果，不再重复执行 后端 - 查询时限交给数据库执行（Oracle call_timeout、PG statement_timeout、MySQL max_execution_time、 SQLite progress handler），可按数据源配置；超时报错与实际耗时一致 - 等审批的运行可以真正取消；会话列表带最后一轮状态；测连接结果持久化；探查结构可只看不存； 自定义工具可草稿试跑、写坏的参数定义保存时拒绝；MCP 服务改动后作废旧工具缓存； 检索命中带贡献分解；重建索引在后台跑并报进度；报错统一收进 core/errors 前端 - 公共层：管理页共用件、失败说明（lib/explain）、健康状态进公共库；目录按表各自加载并有超时， 一张表卡住不再拖垮整站；上传显示真实字节进度；离线横幅只播报不变的句子 - 画布：卡片只说游标所在的那一次执行；等审批时可放弃运行；团队收尾判定与未完成说清楚； 回放用量取那一刻的值 - 编排页：Copilot 合并语义与工具变化回执；「用完轮数时」选项；正式运行期间锁住所有改图入口 - 回答流 / 问数据：新失败类型说人话并给下一步；画布与右栏互相定位；数据源范围、回收站、重新核对 - 记录页：「航迹」页签复用画布时间轴，可拖动回放、列出关键时刻；列表行带执行 / 等人 / 失败缩略条 - 外壳：统一的离开前确认（data router），导航、⌘K、快捷键、后退都会先问 验证（沙箱）：tsc 干净，pytest 838 passed（两遍），14 个检查脚本中 13 个全绿； check-ui 余 1 项旧断言（按旧日期格式找运行行），3c 更新。
308 lines
13 KiB
Python
308 lines
13 KiB
Python
"""自定义工具的参数定义写坏了：保存时就拒掉；库里已经存着的坏工具，运行时说清是哪个、坏在哪。
|
|
|
|
参数定义是用户在编辑器里手写的 JSON Schema。以前保存照单全收，写成
|
|
{"city": "string"} 或 {"type": "int"} 都存得进去；到了工作流绑定这个工具的那一刻，
|
|
schema_to_model 在 .get 上炸掉，运行记录里只剩一句「执行出错：'str' object has no
|
|
attribute 'get'」——既不知道是哪个工具，也不知道该去哪里改。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import uuid
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from sqlalchemy import func, select
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import CustomTool, Run, RunEvent
|
|
from app.engine.runner import run_manager
|
|
from app.main import app
|
|
|
|
GOOD = {"type": "object", "properties": {"store": {"type": "string", "description": "门店"}},
|
|
"required": ["store"]}
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def _name() -> str:
|
|
return f"lookup_{uuid.uuid4().hex[:6]}"
|
|
|
|
|
|
async def _count() -> int:
|
|
async with SessionLocal() as session:
|
|
return int((await session.execute(select(func.count(CustomTool.id)))).scalar_one())
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 保存时拒掉
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("parameters, words", [
|
|
({"type": "object", "properties": {"city": "string"}}, ["city", '{"type": "string"}']),
|
|
({"type": "object", "properties": {"n": {"type": "int"}}}, ["n", "「int」", "integer"]),
|
|
({"type": "object", "properties": {"n": {"type": "integer"}}, "required": "n"}, ["required"]),
|
|
({"type": "object", "properties": ["city"]}, ["properties"]),
|
|
({"type": "object", "properties": {"_id": {"type": "string"}}}, ["_id", "下划线"]),
|
|
({"type": "object", "properties": {"model_config": {"type": "string"}}}, ["model_config"]),
|
|
({"type": "object", "properties": {"": {"type": "string"}}}, ["参数名"]),
|
|
])
|
|
async def test_a_malformed_schema_is_refused_with_words(client, parameters, words):
|
|
before = await _count()
|
|
r = await client.post("/api/custom-tools", json={
|
|
"name": _name(), "kind": "http", "parameters": parameters,
|
|
"config": {"url": "https://example.com/api"}})
|
|
assert r.status_code == 422, r.text
|
|
detail = r.json()["detail"]
|
|
assert isinstance(detail, str), f"detail 要是一句话，不是 pydantic 的错误列表：{detail!r}"
|
|
for word in words:
|
|
assert word in detail, (word, detail)
|
|
assert "Error" not in detail and "Traceback" not in detail
|
|
assert await _count() == before, "写坏的工具不该落库"
|
|
|
|
|
|
@pytest.mark.parametrize("spec, fix, not_this", [
|
|
# 照抄报错里的写法就能改好：写的是 "string"，就别让人改成 integer
|
|
("string", '{"type": "string"}', "integer"),
|
|
("int", '{"type": "integer"}', None),
|
|
("Number", '{"type": "number"}', None),
|
|
("bool", '{"type": "boolean"}', None),
|
|
(5, '{"type": "string"}', None),
|
|
])
|
|
def test_the_example_in_the_message_is_the_type_they_meant(spec, fix, not_this):
|
|
from app.tools.custom import schema_problem
|
|
|
|
problem = schema_problem({"type": "object", "properties": {"store": spec}})
|
|
assert problem and fix in problem, problem
|
|
if not_this:
|
|
assert not_this not in problem, problem
|
|
|
|
|
|
@pytest.mark.parametrize("kind, guess", [("int", "integer"), ("String", "string"), ("float", "number"),
|
|
("dict", "object")])
|
|
def test_a_near_miss_type_gets_a_suggestion(kind, guess):
|
|
from app.tools.custom import schema_problem
|
|
|
|
problem = schema_problem({"type": "object", "properties": {"n": {"type": kind}}})
|
|
assert problem and f"是不是想写 {guess}" in problem, problem
|
|
|
|
|
|
def test_an_unrecognisable_type_gets_no_guess():
|
|
from app.tools.custom import schema_problem
|
|
|
|
problem = schema_problem({"type": "object", "properties": {"n": {"type": "timestamp"}}})
|
|
assert problem and "「timestamp」" in problem and "是不是想写" not in problem, problem
|
|
|
|
|
|
async def test_parameters_that_are_not_an_object_are_refused_in_words_too(client):
|
|
r = await client.post("/api/custom-tools", json={
|
|
"name": _name(), "kind": "http", "parameters": ["city"], "config": {}})
|
|
assert r.status_code == 422
|
|
assert r.json()["detail"] == "参数定义要是一个 JSON 对象"
|
|
|
|
|
|
async def test_editing_a_tool_into_a_broken_schema_is_refused_and_nothing_changes(client):
|
|
r = await client.post("/api/custom-tools", json={
|
|
"name": _name(), "kind": "http", "parameters": GOOD,
|
|
"config": {"url": "https://example.com/api"}})
|
|
assert r.status_code == 201, r.text
|
|
tool = r.json()
|
|
try:
|
|
broken = {**tool, "parameters": {"type": "object", "properties": {"store": "string"}}}
|
|
r = await client.patch(f"/api/custom-tools/{tool['id']}", json=broken)
|
|
assert r.status_code == 422, r.text
|
|
assert "store" in r.json()["detail"]
|
|
async with SessionLocal() as session:
|
|
row = await session.get(CustomTool, tool["id"])
|
|
assert row.parameters == GOOD, "被拒的修改不能落一半"
|
|
finally:
|
|
await client.delete(f"/api/custom-tools/{tool['id']}")
|
|
|
|
|
|
async def test_a_good_schema_still_saves(client):
|
|
r = await client.post("/api/custom-tools", json={
|
|
"name": _name(), "kind": "http", "parameters": GOOD,
|
|
"config": {"url": "https://example.com/api"}})
|
|
assert r.status_code == 201, r.text
|
|
assert r.json()["problem"] is None
|
|
await client.delete(f"/api/custom-tools/{r.json()['id']}")
|
|
|
|
|
|
@pytest.mark.filterwarnings("ignore:Field name .* shadows an attribute")
|
|
def test_every_schema_the_check_lets_through_really_builds():
|
|
"""schema_problem 放行的，schema_to_model 一定建得出来：它是两者之间唯一的关卡。"""
|
|
from app.tools.custom import schema_problem, schema_to_model
|
|
|
|
for name in ("store", "城市", "my-field", "json", "copy", "model_name"):
|
|
schema = {"type": "object", "properties": {name: {"type": "string"}}}
|
|
assert schema_problem(schema) is None, name
|
|
schema_to_model("t", schema).model_json_schema()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 库里已经存着的坏工具
|
|
# --------------------------------------------------------------------------
|
|
|
|
BROKEN = {"type": "object", "properties": {"store": "string"}}
|
|
|
|
|
|
@pytest.fixture
|
|
async def broken_tool():
|
|
"""绕过接口直接写库：模拟这次改动之前就存进去的坏工具。"""
|
|
name = _name()
|
|
async with SessionLocal() as session:
|
|
row = CustomTool(name=name, kind="http", parameters=BROKEN,
|
|
config={"url": "https://example.com/api"}, description="查门店")
|
|
session.add(row)
|
|
await session.commit()
|
|
tool_id = row.id
|
|
yield name
|
|
async with SessionLocal() as session:
|
|
row = await session.get(CustomTool, tool_id)
|
|
if row:
|
|
await session.delete(row)
|
|
await session.commit()
|
|
|
|
|
|
@pytest.fixture
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
|
"data": {"label": nid, "config": config}}
|
|
|
|
|
|
def chain(*nodes):
|
|
return {"nodes": list(nodes),
|
|
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
async def _finish(graph) -> tuple[Run, list[RunEvent]]:
|
|
run = await run_manager.start(graph=graph, input_payload={"question": "北区门店"})
|
|
for _ in range(300):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
if row.status in ("succeeded", "failed", "cancelled", "interrupted"):
|
|
break
|
|
async with SessionLocal() as session:
|
|
events = list((await session.execute(
|
|
select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq)
|
|
)).scalars())
|
|
return row, events
|
|
|
|
|
|
def _clear(text: str | None, name: str) -> None:
|
|
text = text or ""
|
|
assert name in text, f"没说是哪个工具：{text}"
|
|
assert "参数定义" in text and "store" in text, f"没说坏在哪：{text}"
|
|
assert "工具" in text and "改" in text, f"没说去哪里改：{text}"
|
|
for noise in ("执行出错", "内部", "attribute", "object has no"):
|
|
assert noise not in text, f"还是笼统的内部错误：{text}"
|
|
|
|
|
|
async def test_an_agent_bound_to_a_broken_tool_fails_saying_which_and_why(engine_up, broken_tool):
|
|
row, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("ask", "agent", prompt="查一下门店", tools=[broken_tool], max_steps=2),
|
|
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.ask.text }}"}]),
|
|
))
|
|
assert row.status == "failed"
|
|
assert row.error_node_id == "ask"
|
|
_clear(row.error, broken_tool)
|
|
failed = next(e.data for e in events if e.type == "node.failed")
|
|
_clear(failed["error"], broken_tool)
|
|
|
|
|
|
async def test_a_tool_node_bound_to_a_broken_tool_fails_saying_which_and_why(engine_up, broken_tool):
|
|
row, _ = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("call", "tool", tool=broken_tool, args={"store": "north"}, approval="never"),
|
|
node("out", "output", fields=[]),
|
|
))
|
|
assert row.status == "failed"
|
|
assert row.error_node_id == "call"
|
|
_clear(row.error, broken_tool)
|
|
|
|
|
|
async def test_a_team_member_with_a_broken_tool_fails_that_step_with_the_reason(
|
|
engine_up, broken_tool, monkeypatch,
|
|
):
|
|
import json
|
|
|
|
from langchain_core.messages import AIMessage
|
|
|
|
from app.providers import mock_model
|
|
|
|
routed: list[str] = []
|
|
|
|
def _decide(self, messages):
|
|
if self.response_format: # 调度者：派一次，看到结果就收工
|
|
plan = ({"assignments": [{"agent": "查数员", "instruction": "查门店"}]} if not routed
|
|
else {"assignments": [], "done": True, "reason": "查数员的工具坏了，先收工"})
|
|
routed.append("x")
|
|
return AIMessage(content=json.dumps(plan, ensure_ascii=False))
|
|
return AIMessage(content="北区 3 家")
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
_, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("team", "supervisor", goal="查门店", max_rounds=2, approval="never", agents=[
|
|
{"name": "查数员", "system": "你负责查", "tools": [broken_tool], "max_steps": 2}]),
|
|
node("out", "output", fields=[]),
|
|
))
|
|
step = next(e.data for e in events if e.type == "agent.step.end")
|
|
assert step.get("failed") is True, step
|
|
_clear(step.get("error"), broken_tool)
|
|
|
|
|
|
async def test_the_tool_library_marks_a_stored_broken_tool(client, broken_tool):
|
|
listed = next(t for t in (await client.get("/api/custom-tools")).json() if t["name"] == broken_tool)
|
|
assert listed["problem"] and "store" in listed["problem"]
|
|
item = next(t for t in (await client.get("/api/tools")).json() if t["id"] == broken_tool)
|
|
assert item["problem"] == listed["problem"]
|
|
good = [t for t in (await client.get("/api/tools")).json() if t["source"] == "builtin"]
|
|
assert good and all("problem" not in t for t in good), "内置工具不用带这个字段"
|
|
|
|
|
|
async def test_running_a_broken_tool_from_the_library_says_why(client, broken_tool):
|
|
r = await client.post(f"/api/tools/{broken_tool}/run", json={"args": {"store": "north"},
|
|
"confirm": True})
|
|
body = r.json()
|
|
assert body["ok"] is False
|
|
_clear(body["error"], broken_tool)
|
|
|
|
|
|
async def test_a_broken_tool_does_not_take_its_neighbours_down(broken_tool):
|
|
"""同一个节点绑了好几个工具时，报错只点坏的那一个。"""
|
|
from app.tools.registry import ToolBuildError, ToolContext, build_tools
|
|
|
|
good = _name()
|
|
async with SessionLocal() as session:
|
|
session.add(CustomTool(name=good, kind="http", parameters=GOOD,
|
|
config={"url": "https://example.com/api"}))
|
|
await session.commit()
|
|
try:
|
|
async with SessionLocal() as session:
|
|
ctx = ToolContext(run_id="t", node_id="n")
|
|
built = await build_tools([good], ctx, session=session)
|
|
assert [t.name for t in built] == [good]
|
|
with pytest.raises(ToolBuildError) as caught:
|
|
await build_tools([good, broken_tool], ctx, session=session)
|
|
assert broken_tool in str(caught.value) and good not in str(caught.value)
|
|
finally:
|
|
async with SessionLocal() as session:
|
|
row = (await session.execute(select(CustomTool).where(CustomTool.name == good))).scalar_one()
|
|
await session.delete(row)
|
|
await session.commit()
|