mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 09:21:58 +00:00
ThearchyHelios
587bf7e27c
起因是一次探索运行：Copilot 改节点时把工具绑定整份抹掉，agent 没有工具只能写「假设调用」， 校验节点的自动修复编出 total_count=0 让 schema 通过，协作成员把工具调用标记当文字输出了四轮， 轮数用完后节点把原话当成功结果交出去——运行标成已完成，数据库一次都没查。 平台可信度（先写失败测试） - Copilot 修改节点改为合并：没写的字段保留，null 才删除；协作成员按名字合并。前后端同一套规则， 工具集合变小时自查报警、改图回执列出变化 - 新校验规则：提示词点名了某个工具而节点没有绑定它，判错误，Copilot 自查会交回去修 - 校验修复只许改格式：冒出原文没有的值就判修复无效；修复这次模型调用计入用量 - 识别模型把工具调用写成文字的情况（DSML、<tool_call> 等标记）：纠正一次，仍然如此就判失败并说明原因 - 协作团队用完轮数不再按成功交付：默认失败，可选降档，点出一次都没派到的成员 - 审批恢复后，同一节点里已经执行过的工具调用复用结果，不再重复执行 后端 - 查询时限交给数据库执行（Oracle call_timeout、PG statement_timeout、MySQL max_execution_time、 SQLite progress handler），可按数据源配置；超时报错与实际耗时一致 - 等审批的运行可以真正取消；会话列表带最后一轮状态；测连接结果持久化；探查结构可只看不存； 自定义工具可草稿试跑、写坏的参数定义保存时拒绝；MCP 服务改动后作废旧工具缓存； 检索命中带贡献分解；重建索引在后台跑并报进度；报错统一收进 core/errors 前端 - 公共层：管理页共用件、失败说明（lib/explain）、健康状态进公共库；目录按表各自加载并有超时， 一张表卡住不再拖垮整站；上传显示真实字节进度；离线横幅只播报不变的句子 - 画布：卡片只说游标所在的那一次执行；等审批时可放弃运行；团队收尾判定与未完成说清楚； 回放用量取那一刻的值 - 编排页：Copilot 合并语义与工具变化回执；「用完轮数时」选项；正式运行期间锁住所有改图入口 - 回答流 / 问数据：新失败类型说人话并给下一步；画布与右栏互相定位；数据源范围、回收站、重新核对 - 记录页：「航迹」页签复用画布时间轴，可拖动回放、列出关键时刻；列表行带执行 / 等人 / 失败缩略条 - 外壳：统一的离开前确认（data router），导航、⌘K、快捷键、后退都会先问 验证（沙箱）：tsc 干净，pytest 838 passed（两遍），14 个检查脚本中 13 个全绿； check-ui 余 1 项旧断言（按旧日期格式找运行行），3c 更新。
253 lines
11 KiB
Python
253 lines
11 KiB
Python
"""校验要指到字段上；提示词点名要用的工具，节点得真的绑了它。
|
|
|
|
field：检查器要把问题落到具体的输入框下面。以前 ValidationIssue 只有 node_id，前端
|
|
只能拿消息原文去猜是哪个字段（认 {{ path }}、认几句固定说法），认不出就只挂在
|
|
面板顶上。
|
|
|
|
工具绑定：一次真实的运行里，助手改图时把三个查库 agent 的 tools 丢了，提示词里却
|
|
还写着「用 db_query__… 查」。模型拿不到工具，只能写一句「假设调用工具」，再往下
|
|
编。这种图在跑之前就能认出来：提示词点名了工具、节点却没绑定，判 error——
|
|
Copilot 的自查读的是同一份 error，搭图时就会交回模型去修。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from app.engine.schema import GraphSpec, validate_graph
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
|
"data": {"label": nid, "config": config}}
|
|
|
|
|
|
def graph(*nodes, edges=None):
|
|
ids = [n["id"] for n in nodes]
|
|
return GraphSpec.model_validate({
|
|
"nodes": list(nodes),
|
|
"edges": edges if edges is not None else
|
|
[{"source": a, "target": b} for a, b in zip(ids, ids[1:])],
|
|
})
|
|
|
|
|
|
def issues_of(spec, node_id=None):
|
|
return [i for i in validate_graph(spec).issues if node_id is None or i.node_id == node_id]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 提示词点名的工具
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def test_a_named_tool_that_is_not_bound_is_an_error():
|
|
spec = graph(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("query", "agent", model="m", system="你负责取数。",
|
|
prompt="用 db_query__shop 查 orders 表的总数：{{ input.question }}", tools=[]),
|
|
node("out", "output"),
|
|
)
|
|
errors = [i for i in issues_of(spec, "query") if i.level == "error"]
|
|
assert errors, issues_of(spec)
|
|
assert "提示词要求用「db_query__shop」，但节点没有绑定它" in errors[0].message
|
|
assert errors[0].field == "prompt"
|
|
assert not validate_graph(spec).ok
|
|
|
|
|
|
def test_builtin_tool_names_are_recognised_too():
|
|
spec = graph(
|
|
node("start", "input"),
|
|
node("calc", "agent", model="m", system="算完用 python_exec 核对一遍", tools=["calculator"]),
|
|
)
|
|
hit = [i for i in issues_of(spec, "calc") if "python_exec" in i.message]
|
|
assert hit and hit[0].level == "error" and hit[0].field == "system"
|
|
|
|
|
|
def test_a_bound_tool_is_fine():
|
|
spec = graph(
|
|
node("start", "input"),
|
|
node("query", "agent", model="m", prompt="用 db_query__shop 查一下", tools=["db_query__shop"]),
|
|
)
|
|
assert not [i for i in issues_of(spec, "query") if "提示词要求用" in i.message]
|
|
|
|
|
|
def test_saying_not_to_use_a_tool_is_not_asking_for_it():
|
|
spec = graph(
|
|
node("start", "input"),
|
|
node("query", "agent", model="m", prompt="不要用 python_exec，直接心算", tools=[]),
|
|
)
|
|
assert not [i for i in issues_of(spec, "query") if "python_exec" in i.message]
|
|
|
|
|
|
def test_a_vague_request_for_tools_without_any_bound_is_a_warning():
|
|
spec = graph(
|
|
node("start", "input"),
|
|
node("query", "agent", model="m", prompt="调用工具查一下数据库，再回答", tools=[]),
|
|
)
|
|
vague = [i for i in issues_of(spec, "query") if "没有绑定任何工具" in i.message]
|
|
assert vague and vague[0].level == "warning" and vague[0].field == "prompt"
|
|
assert validate_graph(spec).ok
|
|
|
|
|
|
def test_team_members_are_checked_against_their_own_tools():
|
|
spec = graph(
|
|
node("start", "input"),
|
|
node("team", "supervisor", model="m", goal="核对迁移记录", agents=[
|
|
{"name": "核对员", "system": "用 db_query__shop 独立重查一遍条数", "tools": []},
|
|
{"name": "撰稿员", "system": "根据复核结果写结论"},
|
|
]),
|
|
)
|
|
errors = [i for i in issues_of(spec, "team") if i.level == "error"]
|
|
assert errors and "核对员" in errors[0].message and "db_query__shop" in errors[0].message
|
|
assert errors[0].field == "agents[0].system"
|
|
|
|
|
|
def test_a_team_goal_naming_a_tool_nobody_has():
|
|
spec = graph(
|
|
node("start", "input"),
|
|
node("team", "supervisor", model="m", goal="先用 db_schema__shop 看表结构", agents=[
|
|
{"name": "甲", "system": "你负责查", "tools": ["db_query__shop"]}]),
|
|
)
|
|
errors = [i for i in issues_of(spec, "team") if i.level == "error"]
|
|
assert errors and errors[0].field == "goal", issues_of(spec)
|
|
|
|
|
|
# 误报：这条 error 会挡住运行，还会作为自查的阻断项交回 Copilot（模型可能反过来把
|
|
# 禁用的工具绑上去），宁可漏判不可误判
|
|
|
|
|
|
def _errors_about(tool, **config):
|
|
spec = graph(node("start", "input"), node("bot", "agent", model="m", **config))
|
|
return [i for i in issues_of(spec, "bot") if i.level == "error" and tool in i.message]
|
|
|
|
|
|
@pytest.mark.parametrize("prompt", [
|
|
"Never call web_search; answer from the notes only.",
|
|
"Avoid web_fetch.",
|
|
"Do not use web_search.",
|
|
"Don't use the web_search tool, and never call web_fetch or http_request.",
|
|
"You must not call web_search under any circumstances.",
|
|
"You cannot use web_fetch here, and no http_request either.",
|
|
])
|
|
def test_english_negation_is_not_a_request(prompt):
|
|
for tool in ("web_search", "web_fetch", "http_request"):
|
|
assert not _errors_about(tool, prompt=prompt, tools=[]), (prompt, tool)
|
|
|
|
|
|
@pytest.mark.parametrize("prompt", [
|
|
"查过一次就不再调用 python_exec，直接给结论",
|
|
"表结构已经给你了，无须再调用 db_schema__shop",
|
|
"拿到结果后别再用 db_query__shop",
|
|
"毋须使用 web_search",
|
|
])
|
|
def test_more_chinese_negations(prompt):
|
|
for tool in ("python_exec", "db_schema__shop", "db_query__shop", "web_search"):
|
|
assert not _errors_about(tool, prompt=prompt, tools=[]), (prompt, tool)
|
|
|
|
|
|
def test_an_mcp_wildcard_covers_its_tools():
|
|
assert not _errors_about("mcp:fs/read_file", prompt="用 mcp:fs/read_file 读 notes.md",
|
|
tools=["mcp:fs/*"])
|
|
assert not _errors_about("mcp:fs/read_file", prompt="用 mcp:fs/read_file 读 notes.md",
|
|
tools=["mcp:fs"])
|
|
# 别的 server 的通配不算
|
|
assert _errors_about("mcp:fs/read_file", prompt="用 mcp:fs/read_file 读 notes.md",
|
|
tools=["mcp:web/*"])
|
|
|
|
|
|
def test_a_team_goal_is_covered_by_a_member_wildcard():
|
|
spec = graph(
|
|
node("start", "input"),
|
|
node("team", "supervisor", model="m", goal="先用 mcp:fs/list_dir 看目录", agents=[
|
|
{"name": "甲", "system": "你负责查", "tools": ["mcp:fs/*"]}]),
|
|
)
|
|
assert not [i for i in issues_of(spec, "team") if i.level == "error"], issues_of(spec)
|
|
|
|
|
|
@pytest.mark.parametrize("prompt", [
|
|
"You may use a calculator if needed.",
|
|
"Work it out like a calculator would, step by step.",
|
|
"如有需要可以用 calculator 核对",
|
|
"必要时用 python_exec 验算一下",
|
|
"You can call db_query__shop when needed.",
|
|
])
|
|
def test_plain_words_and_optional_mentions_are_only_warnings(prompt):
|
|
spec = graph(node("start", "input"), node("bot", "agent", model="m", prompt=prompt, tools=[]))
|
|
found = issues_of(spec, "bot")
|
|
assert not [i for i in found if i.level == "error"], (prompt, found)
|
|
|
|
|
|
def test_a_plain_word_tool_still_counts_when_clearly_asked_for():
|
|
for prompt in ("用 calculator 算出总价", "Use calculator to add them up.", "call `calculator` twice"):
|
|
errors = _errors_about("calculator", prompt=prompt, tools=[])
|
|
assert errors and errors[0].field == "prompt", prompt
|
|
# 提到了但没明确要求：给一条 warning，不挡运行
|
|
spec = graph(node("start", "input"),
|
|
node("bot", "agent", model="m", prompt="You may use a calculator if needed.", tools=[]))
|
|
warned = [i for i in issues_of(spec, "bot") if "calculator" in i.message]
|
|
assert warned and warned[0].level == "warning", warned
|
|
assert validate_graph(spec).ok
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# field：指到字段
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def test_issues_carry_the_field_they_are_about():
|
|
spec = graph(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("route", "branch", cases=[
|
|
{"key": "a", "condition": "vars.x == 1"},
|
|
{"key": "b", "condition": "vars.y ==="},
|
|
]),
|
|
node("shape", "transform", mode="expression", expression=""),
|
|
node("say", "llm", model="m", prompt="回答 {{ input.questoin }}"),
|
|
edges=[{"source": "start", "target": "route"},
|
|
{"source": "route", "target": "shape", "sourceHandle": "a"},
|
|
{"source": "shape", "target": "say"}],
|
|
)
|
|
by = {(i.node_id, i.field) for i in validate_graph(spec).issues}
|
|
assert ("route", "cases[1].condition") in by, by
|
|
assert ("route", "cases[1].key") in by or ("route", "cases") in by, by # b 没有连出去的边
|
|
assert ("shape", "expression") in by, by
|
|
assert ("say", "prompt") in by, by # 变量拼错
|
|
|
|
|
|
def test_graph_level_issues_have_no_field():
|
|
spec = GraphSpec.model_validate({"nodes": []})
|
|
assert all(i.field is None for i in validate_graph(spec).issues)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 报错的人话规则只有一份
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def test_one_set_of_words_for_both_layers():
|
|
import importlib.util
|
|
|
|
import httpx
|
|
|
|
from app.core.errors import describe_exception, explain
|
|
|
|
for old in ("app.engine.errors", "app.api.errors"):
|
|
assert importlib.util.find_spec(old) is None, f"{old} 不该再有自己的一份"
|
|
req = httpx.Request("GET", "https://x/v1/models")
|
|
denied = httpx.HTTPStatusError("401", request=req, response=httpx.Response(401, request=req))
|
|
assert describe_exception(denied) == explain(denied)[0]
|
|
assert describe_exception(TimeoutError()) == explain(TimeoutError())[0]
|
|
# 分叉只在程序自己的毛病上：接口里是后端 bug，节点里多半是参数传错了
|
|
wrong = TypeError("_run() got an unexpected keyword argument 'query'")
|
|
assert "内部" in explain(wrong)[0]
|
|
assert "query" in describe_exception(wrong)
|
|
|
|
|
|
def test_a_list_of_forbidden_tools_is_not_a_request():
|
|
spec = graph(
|
|
node("start", "input"),
|
|
node("team", "supervisor", model="m", goal="只用内部资料，严禁调用 web_search、web_fetch、http_request 等联网工具",
|
|
agents=[{"name": "甲", "system": "禁止使用 web_fetch 和 http_request，只查知识库",
|
|
"tools": ["kb_search"]}]),
|
|
)
|
|
assert not [i for i in issues_of(spec, "team") if i.level == "error"], issues_of(spec)
|