mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:41:59 +00:00
ThearchyHelios
03c18c4d78
一、文案（前端约 1600 处，后端传到界面的约 880 处） - 术语统一：工作流层面用「运行」（继续运行、重新运行、试运行），节点和工具层面用「执行」，不再用「跑」； 总时长 / 执行时长 / 等待审批；服务端；助手；Agent；token；工作流；固定版本；模型接入；系统提示；批准 / 驳回；受管级别 - 口语改书面：取不到、没取回来、读不懂、连不上、多半、照样、兜底、揉成这类说法改成「无法 / 未能 / 失败 / 请」；去掉拟人写法 - 不再露出实现细节：界面上不写 checkpoint、同一拍、哈希（凭证处除外），不写改版历史（「以前…」「老数据」）； 报错、校验、门禁、自动修复里裸写的配置键名和枚举值换成界面上的中文标签，由新的 backend/app/engine/labels.py 统一提供； 发布弹窗里结论句的选项不再是 off / require_citation - 同一件事只留一种说法：生成阶段文案、出具档位说明、画布锁定提示各合成一份 - 去掉界面上的工业场景示例（MES、ERP、「张工」「王工」）和真实产品名，换成通用示例 二、说错的提示（都有测试证明） - 数据库测连接失败时的专用提示以前从不出现（按报错文字全等比较，永远比不上）：改为按错误类型判断，不再拼出重复的前缀 - 「设置 → 供应商」改指「设置 → 模型接入」；「在设置里建数据源」改指「数据」页 - 缺少 API Key、模型接入已停用的报错不再夹英文，写明去哪里填 - 两处原因说错（「这种引用在后续版本支持」「这一版还不支持」）改成真实原因 - 节点说明里不准确的说法改正：代码节点在 microVM 或系统沙箱中执行；报告撰写、口径卡的说明不再说过头 - 导出的证据 CSV 里两列同名，第三列改为「片段编号」 三、前后端对齐 - 前端按后端原文识别日志、报错的地方，新旧两种写法都认得（历史运行里存的是旧文字），逐条补了新旧原文的核对 - 后端交回模型的原指令放进单独的 for_model 字段，前端在解析响应时去掉，不会显示到界面上 四、检查 - 检查脚本里按原文匹配的断言改成新文案；改完会变成永远成立的反向断言逐条修正，并做了变异验证 - 新增 check-copy（静态扫描前端界面字符串和后端上界面的文案）：再出现「跑」「后端」「多半」「令牌」、 界面上的 Copilot、裸写的配置键名、半角标点等就报错；确需保留的写进 scripts/copy-allow.json 并注明理由
308 lines
13 KiB
Python
308 lines
13 KiB
Python
"""自定义工具的参数定义写坏了：保存时就拒掉；库里已经存着的坏工具，运行时说清是哪个、坏在哪。
|
|
|
|
参数定义是用户在编辑器里手写的 JSON Schema。以前保存照单全收，写成
|
|
{"city": "string"} 或 {"type": "int"} 都存得进去；到了工作流绑定这个工具的那一刻，
|
|
schema_to_model 在 .get 上炸掉，运行记录里只剩一句「执行出错：'str' object has no
|
|
attribute 'get'」——既不知道是哪个工具，也不知道该去哪里改。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import uuid
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from sqlalchemy import func, select
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import CustomTool, Run, RunEvent
|
|
from app.engine.runner import run_manager
|
|
from app.main import app
|
|
|
|
GOOD = {"type": "object", "properties": {"store": {"type": "string", "description": "门店"}},
|
|
"required": ["store"]}
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def _name() -> str:
|
|
return f"lookup_{uuid.uuid4().hex[:6]}"
|
|
|
|
|
|
async def _count() -> int:
|
|
async with SessionLocal() as session:
|
|
return int((await session.execute(select(func.count(CustomTool.id)))).scalar_one())
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 保存时拒掉
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("parameters, words", [
|
|
({"type": "object", "properties": {"city": "string"}}, ["city", '{"type": "string"}']),
|
|
({"type": "object", "properties": {"n": {"type": "int"}}}, ["n", "「int」", "integer"]),
|
|
({"type": "object", "properties": {"n": {"type": "integer"}}, "required": "n"}, ["required"]),
|
|
({"type": "object", "properties": ["city"]}, ["properties"]),
|
|
({"type": "object", "properties": {"_id": {"type": "string"}}}, ["_id", "下划线"]),
|
|
({"type": "object", "properties": {"model_config": {"type": "string"}}}, ["model_config"]),
|
|
({"type": "object", "properties": {"": {"type": "string"}}}, ["参数名"]),
|
|
])
|
|
async def test_a_malformed_schema_is_refused_with_words(client, parameters, words):
|
|
before = await _count()
|
|
r = await client.post("/api/custom-tools", json={
|
|
"name": _name(), "kind": "http", "parameters": parameters,
|
|
"config": {"url": "https://example.com/api"}})
|
|
assert r.status_code == 422, r.text
|
|
detail = r.json()["detail"]
|
|
assert isinstance(detail, str), f"detail 要是一句话，不是 pydantic 的错误列表：{detail!r}"
|
|
for word in words:
|
|
assert word in detail, (word, detail)
|
|
assert "Error" not in detail and "Traceback" not in detail
|
|
assert await _count() == before, "写坏的工具不该落库"
|
|
|
|
|
|
@pytest.mark.parametrize("spec, fix, not_this", [
|
|
# 照抄报错里的写法就能改好：写的是 "string"，就别让人改成 integer
|
|
("string", '{"type": "string"}', "integer"),
|
|
("int", '{"type": "integer"}', None),
|
|
("Number", '{"type": "number"}', None),
|
|
("bool", '{"type": "boolean"}', None),
|
|
(5, '{"type": "string"}', None),
|
|
])
|
|
def test_the_example_in_the_message_is_the_type_they_meant(spec, fix, not_this):
|
|
from app.tools.custom import schema_problem
|
|
|
|
problem = schema_problem({"type": "object", "properties": {"store": spec}})
|
|
assert problem and fix in problem, problem
|
|
if not_this:
|
|
assert not_this not in problem, problem
|
|
|
|
|
|
@pytest.mark.parametrize("kind, guess", [("int", "integer"), ("String", "string"), ("float", "number"),
|
|
("dict", "object")])
|
|
def test_a_near_miss_type_gets_a_suggestion(kind, guess):
|
|
from app.tools.custom import schema_problem
|
|
|
|
problem = schema_problem({"type": "object", "properties": {"n": {"type": kind}}})
|
|
assert problem and f"是否应为 {guess}" in problem, problem
|
|
|
|
|
|
def test_an_unrecognisable_type_gets_no_guess():
|
|
from app.tools.custom import schema_problem
|
|
|
|
problem = schema_problem({"type": "object", "properties": {"n": {"type": "timestamp"}}})
|
|
assert problem and "「timestamp」" in problem and "是否应为" not in problem, problem
|
|
|
|
|
|
async def test_parameters_that_are_not_an_object_are_refused_in_words_too(client):
|
|
r = await client.post("/api/custom-tools", json={
|
|
"name": _name(), "kind": "http", "parameters": ["city"], "config": {}})
|
|
assert r.status_code == 422
|
|
assert r.json()["detail"] == "参数定义必须是一个 JSON 对象"
|
|
|
|
|
|
async def test_editing_a_tool_into_a_broken_schema_is_refused_and_nothing_changes(client):
|
|
r = await client.post("/api/custom-tools", json={
|
|
"name": _name(), "kind": "http", "parameters": GOOD,
|
|
"config": {"url": "https://example.com/api"}})
|
|
assert r.status_code == 201, r.text
|
|
tool = r.json()
|
|
try:
|
|
broken = {**tool, "parameters": {"type": "object", "properties": {"store": "string"}}}
|
|
r = await client.patch(f"/api/custom-tools/{tool['id']}", json=broken)
|
|
assert r.status_code == 422, r.text
|
|
assert "store" in r.json()["detail"]
|
|
async with SessionLocal() as session:
|
|
row = await session.get(CustomTool, tool["id"])
|
|
assert row.parameters == GOOD, "被拒的修改不能落一半"
|
|
finally:
|
|
await client.delete(f"/api/custom-tools/{tool['id']}")
|
|
|
|
|
|
async def test_a_good_schema_still_saves(client):
|
|
r = await client.post("/api/custom-tools", json={
|
|
"name": _name(), "kind": "http", "parameters": GOOD,
|
|
"config": {"url": "https://example.com/api"}})
|
|
assert r.status_code == 201, r.text
|
|
assert r.json()["problem"] is None
|
|
await client.delete(f"/api/custom-tools/{r.json()['id']}")
|
|
|
|
|
|
@pytest.mark.filterwarnings("ignore:Field name .* shadows an attribute")
|
|
def test_every_schema_the_check_lets_through_really_builds():
|
|
"""schema_problem 放行的，schema_to_model 一定建得出来：它是两者之间唯一的关卡。"""
|
|
from app.tools.custom import schema_problem, schema_to_model
|
|
|
|
for name in ("store", "城市", "my-field", "json", "copy", "model_name"):
|
|
schema = {"type": "object", "properties": {name: {"type": "string"}}}
|
|
assert schema_problem(schema) is None, name
|
|
schema_to_model("t", schema).model_json_schema()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 库里已经存着的坏工具
|
|
# --------------------------------------------------------------------------
|
|
|
|
BROKEN = {"type": "object", "properties": {"store": "string"}}
|
|
|
|
|
|
@pytest.fixture
|
|
async def broken_tool():
|
|
"""绕过接口直接写库：模拟这次改动之前就存进去的坏工具。"""
|
|
name = _name()
|
|
async with SessionLocal() as session:
|
|
row = CustomTool(name=name, kind="http", parameters=BROKEN,
|
|
config={"url": "https://example.com/api"}, description="查门店")
|
|
session.add(row)
|
|
await session.commit()
|
|
tool_id = row.id
|
|
yield name
|
|
async with SessionLocal() as session:
|
|
row = await session.get(CustomTool, tool_id)
|
|
if row:
|
|
await session.delete(row)
|
|
await session.commit()
|
|
|
|
|
|
@pytest.fixture
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
|
"data": {"label": nid, "config": config}}
|
|
|
|
|
|
def chain(*nodes):
|
|
return {"nodes": list(nodes),
|
|
"edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
async def _finish(graph) -> tuple[Run, list[RunEvent]]:
|
|
run = await run_manager.start(graph=graph, input_payload={"question": "北区门店"})
|
|
for _ in range(300):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
if row.status in ("succeeded", "failed", "cancelled", "interrupted"):
|
|
break
|
|
async with SessionLocal() as session:
|
|
events = list((await session.execute(
|
|
select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq)
|
|
)).scalars())
|
|
return row, events
|
|
|
|
|
|
def _clear(text: str | None, name: str) -> None:
|
|
text = text or ""
|
|
assert name in text, f"没说是哪个工具：{text}"
|
|
assert "参数定义" in text and "store" in text, f"没说坏在哪：{text}"
|
|
assert "工具" in text and "改" in text, f"没说去哪里改：{text}"
|
|
for noise in ("执行出错", "内部", "attribute", "object has no"):
|
|
assert noise not in text, f"还是笼统的内部错误：{text}"
|
|
|
|
|
|
async def test_an_agent_bound_to_a_broken_tool_fails_saying_which_and_why(engine_up, broken_tool):
|
|
row, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("ask", "agent", prompt="查一下门店", tools=[broken_tool], max_steps=2),
|
|
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.ask.text }}"}]),
|
|
))
|
|
assert row.status == "failed"
|
|
assert row.error_node_id == "ask"
|
|
_clear(row.error, broken_tool)
|
|
failed = next(e.data for e in events if e.type == "node.failed")
|
|
_clear(failed["error"], broken_tool)
|
|
|
|
|
|
async def test_a_tool_node_bound_to_a_broken_tool_fails_saying_which_and_why(engine_up, broken_tool):
|
|
row, _ = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("call", "tool", tool=broken_tool, args={"store": "north"}, approval="never"),
|
|
node("out", "output", fields=[]),
|
|
))
|
|
assert row.status == "failed"
|
|
assert row.error_node_id == "call"
|
|
_clear(row.error, broken_tool)
|
|
|
|
|
|
async def test_a_team_member_with_a_broken_tool_fails_that_step_with_the_reason(
|
|
engine_up, broken_tool, monkeypatch,
|
|
):
|
|
import json
|
|
|
|
from langchain_core.messages import AIMessage
|
|
|
|
from app.providers import mock_model
|
|
|
|
routed: list[str] = []
|
|
|
|
def _decide(self, messages):
|
|
if self.response_format: # 调度者：派一次，看到结果就收工
|
|
plan = ({"assignments": [{"agent": "查数员", "instruction": "查门店"}]} if not routed
|
|
else {"assignments": [], "done": True, "reason": "查数员的工具坏了，先收工"})
|
|
routed.append("x")
|
|
return AIMessage(content=json.dumps(plan, ensure_ascii=False))
|
|
return AIMessage(content="北区 3 家")
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
_, events = await _finish(chain(
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("team", "supervisor", goal="查门店", max_rounds=2, approval="never", agents=[
|
|
{"name": "查数员", "system": "你负责查", "tools": [broken_tool], "max_steps": 2}]),
|
|
node("out", "output", fields=[]),
|
|
))
|
|
step = next(e.data for e in events if e.type == "agent.step.end")
|
|
assert step.get("failed") is True, step
|
|
_clear(step.get("error"), broken_tool)
|
|
|
|
|
|
async def test_the_tool_library_marks_a_stored_broken_tool(client, broken_tool):
|
|
listed = next(t for t in (await client.get("/api/custom-tools")).json() if t["name"] == broken_tool)
|
|
assert listed["problem"] and "store" in listed["problem"]
|
|
item = next(t for t in (await client.get("/api/tools")).json() if t["id"] == broken_tool)
|
|
assert item["problem"] == listed["problem"]
|
|
good = [t for t in (await client.get("/api/tools")).json() if t["source"] == "builtin"]
|
|
assert good and all("problem" not in t for t in good), "内置工具不用带这个字段"
|
|
|
|
|
|
async def test_running_a_broken_tool_from_the_library_says_why(client, broken_tool):
|
|
r = await client.post(f"/api/tools/{broken_tool}/run", json={"args": {"store": "north"},
|
|
"confirm": True})
|
|
body = r.json()
|
|
assert body["ok"] is False
|
|
_clear(body["error"], broken_tool)
|
|
|
|
|
|
async def test_a_broken_tool_does_not_take_its_neighbours_down(broken_tool):
|
|
"""同一个节点绑了好几个工具时，报错只点坏的那一个。"""
|
|
from app.tools.registry import ToolBuildError, ToolContext, build_tools
|
|
|
|
good = _name()
|
|
async with SessionLocal() as session:
|
|
session.add(CustomTool(name=good, kind="http", parameters=GOOD,
|
|
config={"url": "https://example.com/api"}))
|
|
await session.commit()
|
|
try:
|
|
async with SessionLocal() as session:
|
|
ctx = ToolContext(run_id="t", node_id="n")
|
|
built = await build_tools([good], ctx, session=session)
|
|
assert [t.name for t in built] == [good]
|
|
with pytest.raises(ToolBuildError) as caught:
|
|
await build_tools([good, broken_tool], ctx, session=session)
|
|
assert broken_tool in str(caught.value) and good not in str(caught.value)
|
|
finally:
|
|
async with SessionLocal() as session:
|
|
row = (await session.execute(select(CustomTool).where(CustomTool.name == good))).scalar_one()
|
|
await session.delete(row)
|
|
await session.commit()
|