mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 02:11:58 +00:00
ThearchyHelios
03c18c4d78
一、文案（前端约 1600 处，后端传到界面的约 880 处） - 术语统一：工作流层面用「运行」（继续运行、重新运行、试运行），节点和工具层面用「执行」，不再用「跑」； 总时长 / 执行时长 / 等待审批；服务端；助手；Agent；token；工作流；固定版本；模型接入；系统提示；批准 / 驳回；受管级别 - 口语改书面：取不到、没取回来、读不懂、连不上、多半、照样、兜底、揉成这类说法改成「无法 / 未能 / 失败 / 请」；去掉拟人写法 - 不再露出实现细节：界面上不写 checkpoint、同一拍、哈希（凭证处除外），不写改版历史（「以前…」「老数据」）； 报错、校验、门禁、自动修复里裸写的配置键名和枚举值换成界面上的中文标签，由新的 backend/app/engine/labels.py 统一提供； 发布弹窗里结论句的选项不再是 off / require_citation - 同一件事只留一种说法：生成阶段文案、出具档位说明、画布锁定提示各合成一份 - 去掉界面上的工业场景示例（MES、ERP、「张工」「王工」）和真实产品名，换成通用示例 二、说错的提示（都有测试证明） - 数据库测连接失败时的专用提示以前从不出现（按报错文字全等比较，永远比不上）：改为按错误类型判断，不再拼出重复的前缀 - 「设置 → 供应商」改指「设置 → 模型接入」；「在设置里建数据源」改指「数据」页 - 缺少 API Key、模型接入已停用的报错不再夹英文，写明去哪里填 - 两处原因说错（「这种引用在后续版本支持」「这一版还不支持」）改成真实原因 - 节点说明里不准确的说法改正：代码节点在 microVM 或系统沙箱中执行；报告撰写、口径卡的说明不再说过头 - 导出的证据 CSV 里两列同名，第三列改为「片段编号」 三、前后端对齐 - 前端按后端原文识别日志、报错的地方，新旧两种写法都认得（历史运行里存的是旧文字），逐条补了新旧原文的核对 - 后端交回模型的原指令放进单独的 for_model 字段，前端在解析响应时去掉，不会显示到界面上 四、检查 - 检查脚本里按原文匹配的断言改成新文案；改完会变成永远成立的反向断言逐条修正，并做了变异验证 - 新增 check-copy（静态扫描前端界面字符串和后端上界面的文案）：再出现「跑」「后端」「多半」「令牌」、 界面上的 Copilot、裸写的配置键名、半角标点等就报错；确需保留的写进 scripts/copy-allow.json 并注明理由
282 lines
13 KiB
Python
282 lines
13 KiB
Python
"""协作团队用完轮数、调度者始终没判定完成：不许把成员原话当作成功结果交出去。
|
|
|
|
一次真实运行：成员没有工具，模型把工具调用标记当文字写了四轮；调度者每轮都说
|
|
「核对员还没查到数据，本轮继续派核对员」，四轮用完，节点把核对员最后那段标记
|
|
当作结论交了出去，运行 succeeded。负责写结论的成员一次都没被派到。
|
|
|
|
新配置 on_exhausted：fail（默认）判失败，说清调度者最后一轮的理由、谁从没被派到；
|
|
degrade 照样交付，但产出里带 exhausted: true、发 team_exhausted 警告，复核和出具
|
|
都据此降档。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage
|
|
from sqlalchemy import select
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Run, RunEvent
|
|
from app.engine import review
|
|
from app.engine.runner import run_manager
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
|
|
"data": {"label": nid, "config": config}}
|
|
|
|
|
|
def _never_done(monkeypatch) -> None:
|
|
"""调度者每轮都继续派核对员，理由一样；核对员每次都交不出东西。"""
|
|
from app.providers import mock_model
|
|
|
|
def _decide(self, messages):
|
|
if self.response_format:
|
|
return AIMessage(content=json.dumps({
|
|
"assignments": [{"agent": "核对员", "instruction": "独立重查一遍"}],
|
|
"done": False, "reason": "核对员还没拿到真实条数，定稿要等复核结果",
|
|
}, ensure_ascii=False))
|
|
return AIMessage(content="我还需要再查一次。")
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
|
|
|
|
def _graph(**team):
|
|
return {
|
|
"nodes": [
|
|
node("start", "input", fields=[{"name": "question"}]),
|
|
node("team", "supervisor", goal="核对订单条数", agents=[
|
|
{"name": "核对员", "system": "你负责复核", "max_steps": 1},
|
|
{"name": "撰稿员", "system": "你负责写结论", "max_steps": 1},
|
|
], **{"max_rounds": 3, **team}),
|
|
node("out", "output", fields=[{"name": "r", "value": "{{ nodes.team.text }}"}],
|
|
contract={"metrics_from": "card", "narrative": "{{ nodes.team.text }}"}),
|
|
node("card", "metrics", metrics=[{"id": "n", "expression": "1"}]),
|
|
],
|
|
"edges": [{"source": "start", "target": "card"}, {"source": "card", "target": "team"},
|
|
{"source": "team", "target": "out"}],
|
|
}
|
|
|
|
|
|
async def _finish(graph) -> tuple[Run, list[RunEvent]]:
|
|
run = await run_manager.start(graph=graph, input_payload={"question": "x"})
|
|
for _ in range(300):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
if row.status in ("succeeded", "failed"):
|
|
break
|
|
async with SessionLocal() as session:
|
|
events = list((await session.execute(
|
|
select(RunEvent).where(RunEvent.run_id == run.id).order_by(RunEvent.seq)
|
|
)).scalars())
|
|
return row, events
|
|
|
|
|
|
async def test_running_out_of_rounds_fails_by_default(monkeypatch):
|
|
_never_done(monkeypatch)
|
|
row, events = await _finish(_graph())
|
|
assert row.status == "failed", f"成员原话被当成了结论：{row.output}"
|
|
assert "协作团队用完 3 轮仍未完成：核对员还没拿到真实条数，定稿要等复核结果" in row.error
|
|
assert "撰稿员" in row.error, "从没被派过的成员要点出来"
|
|
assert row.error_node_id == "team"
|
|
|
|
|
|
async def test_degrade_delivers_but_says_so(monkeypatch):
|
|
_never_done(monkeypatch)
|
|
row, events = await _finish(_graph(on_exhausted="degrade"))
|
|
assert row.status == "succeeded", row.error
|
|
|
|
finished = next(e.data for e in events if e.type == "node.finished" and e.node_id == "team")
|
|
assert finished["preview"]["exhausted"] is True
|
|
warn = next(e.data for e in events if e.type == "log" and e.data.get("code") == "team_exhausted")
|
|
assert "撰稿员" in warn["message"] and "3 轮" in warn["message"]
|
|
|
|
# 出具降档，理由写在 gaps 里
|
|
issuance = row.output["_issuance"]
|
|
assert issuance["tier"] == "degraded"
|
|
assert any("用完 3 轮" in g for g in issuance["gaps"]), issuance["gaps"]
|
|
# 复核也认这条警告
|
|
signals = review.scan(events, row.output)
|
|
assert any(s.kind == "team_exhausted" for s in signals), signals
|
|
|
|
|
|
async def test_a_team_that_finishes_is_unaffected(monkeypatch):
|
|
from app.providers import mock_model
|
|
|
|
turns = {"n": 0}
|
|
|
|
def _decide(self, messages):
|
|
if self.response_format:
|
|
turns["n"] += 1
|
|
plan = ({"assignments": [{"agent": "核对员", "instruction": "查"}]} if turns["n"] == 1
|
|
else {"assignments": [], "done": True, "reason": "查清楚了"})
|
|
return AIMessage(content=json.dumps(plan, ensure_ascii=False))
|
|
return AIMessage(content="一共 1 条。")
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
row, events = await _finish(_graph())
|
|
assert row.status == "succeeded", row.error
|
|
assert not [e for e in events if e.data.get("code") == "team_exhausted"]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 最后一轮派出去的成员交回来之后，调度者还要看一眼
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_a_pipeline_that_needs_exactly_max_rounds_succeeds(monkeypatch):
|
|
"""2 轮的流水线（查数 → 撰稿），最多轮数也是 2：第二轮的撰稿员交回来之后，调度者
|
|
以前再没机会说「完成」，轮数一到就算用完，缺省判失败。"""
|
|
from app.providers import mock_model
|
|
|
|
routed = {"n": 0}
|
|
|
|
def _decide(self, messages):
|
|
if self.response_format:
|
|
text = "\n".join(str(m.content) for m in messages)
|
|
if "轮数已经用完" in text:
|
|
plan = {"assignments": [], "done": "撰稿员" in text and "结论" in text,
|
|
"reason": "结论已经写好"}
|
|
else:
|
|
routed["n"] += 1
|
|
who = "核对员" if routed["n"] == 1 else "撰稿员"
|
|
plan = {"assignments": [{"agent": who, "instruction": "做你那部分"}],
|
|
"done": False, "reason": f"交给{who}"}
|
|
return AIMessage(content=json.dumps(plan, ensure_ascii=False))
|
|
return AIMessage(content="结论：一共 1 条。")
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
row, events = await _finish(_graph(max_rounds=2))
|
|
assert row.status == "succeeded", row.error
|
|
assert row.output["r"] == "结论：一共 1 条。"
|
|
closing = [e.data for e in events if e.type == "agent.route.end" and e.data.get("closing")]
|
|
assert len(closing) == 1 and closing[0]["done"] is True, closing
|
|
assert not [e for e in events if e.data.get("code") == "team_exhausted"]
|
|
|
|
|
|
async def test_the_closing_look_can_still_say_not_done(monkeypatch):
|
|
_never_done(monkeypatch)
|
|
row, events = await _finish(_graph(max_rounds=2))
|
|
assert row.status == "failed"
|
|
closing = [e.data for e in events if e.type == "agent.route.end" and e.data.get("closing")]
|
|
assert len(closing) == 1 and closing[0]["done"] is False, closing
|
|
# 收尾那一次也是调度者的一次模型调用，要记账
|
|
coordinator = [e for e in events if e.type == "llm.end" and e.data.get("agent") == "调度者"]
|
|
assert len(coordinator) == 3
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 接着跑一个失败的团队节点：前几轮取自断点，时间线上不再出现一遍
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_continuing_a_failed_team_does_not_replay_its_rounds(monkeypatch):
|
|
_never_done(monkeypatch)
|
|
row, events = await _finish(_graph(max_rounds=2))
|
|
assert row.status == "failed"
|
|
|
|
def tally(evs):
|
|
kinds = ("agent.route.start", "agent.route.end", "agent.step.start", "agent.step.end",
|
|
"llm.end")
|
|
counts = {k: sum(1 for e in evs if e.type == k and e.node_id == "team") for k in kinds}
|
|
counts["调度"] = sum(1 for e in evs if e.type == "log" and e.node_id == "team"
|
|
and str(e.data.get("message", "")).startswith("调度 →"))
|
|
return counts
|
|
|
|
before = tally(events)
|
|
assert before["agent.step.start"] == 2 and before["agent.route.start"] == 3, before
|
|
|
|
graph = _graph(max_rounds=2, on_exhausted="degrade")
|
|
await run_manager.continue_failed(row.id, graph=graph)
|
|
for _ in range(300):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, row.id)
|
|
if row.status in ("succeeded", "failed"):
|
|
break
|
|
assert row.status == "succeeded", row.error
|
|
async with SessionLocal() as session:
|
|
after_events = list((await session.execute(
|
|
select(RunEvent).where(RunEvent.run_id == row.id).order_by(RunEvent.seq)
|
|
)).scalars())
|
|
assert tally(after_events) == before, "接着跑把前几轮的调度和成员又发了一遍"
|
|
# 前几轮取自断点，用量不能跟着丢：成员的调用也要算进去
|
|
assert row.usage["calls"] == before["llm.end"], (row.usage, before)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 调度者说完成了，可派出去的成员一个都没交回结果
|
|
# --------------------------------------------------------------------------
|
|
|
|
DSML = ('<｜｜DSML｜｜ calls>\n<｜｜DSML｜｜ invoke name="db_query__shop">\n'
|
|
'<｜｜DSML｜｜ parameter name="sql">SELECT COUNT(*) FROM orders</｜｜DSML｜｜ parameter>\n'
|
|
'</｜｜DSML｜｜ invoke>\n</｜｜DSML｜｜ calls>')
|
|
|
|
|
|
def _gives_up_after(monkeypatch, *plan: str, writes: dict[str, str]) -> None:
|
|
"""调度者按 plan 依次派人，派完就判完成；成员按 system 认人，照 writes 回话。"""
|
|
from app.providers import mock_model
|
|
|
|
routed = {"n": 0}
|
|
|
|
def _decide(self, messages):
|
|
if self.response_format:
|
|
routed["n"] += 1
|
|
if routed["n"] <= len(plan):
|
|
who = plan[routed["n"] - 1]
|
|
decision = {"assignments": [{"agent": who, "instruction": "做你那部分"}],
|
|
"done": False, "reason": f"交给{who}"}
|
|
else:
|
|
decision = {"assignments": [], "done": True, "reason": "查不到了，先这样"}
|
|
return AIMessage(content=json.dumps(decision, ensure_ascii=False))
|
|
role = str(messages[0].content)
|
|
return AIMessage(content=next(v for k, v in writes.items() if k in role))
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
|
|
|
|
async def test_nobody_delivering_is_not_a_finished_team(monkeypatch):
|
|
"""验收时的一次：成员没绑工具，两次都把工具调用写成文字；调度者读到失败说明，
|
|
判定完成。团队把「（<成员> 这一步失败：…）」当成果交了出去，运行 succeeded，
|
|
画布头部是绿色的已完成——一条数据都没查到。"""
|
|
_gives_up_after(monkeypatch, "核对员", writes={"复核": DSML, "结论": "不该轮到我"})
|
|
row, events = await _finish(_graph())
|
|
assert row.status == "failed", f"失败说明被当成了成果：{row.output}"
|
|
assert row.error_node_id == "team"
|
|
assert "未交出结论" in row.error and "查不到了，先这样" in row.error, row.error
|
|
# 失败的原因一起带上：界面照它说「模型没有真正调用工具」、指到画布上去绑工具
|
|
assert "工具调用的原始标记" in row.error, row.error
|
|
assert "撰稿员" in row.error, "从没被派过的成员要点出来"
|
|
|
|
|
|
async def test_nobody_delivering_fails_even_when_set_to_degrade(monkeypatch):
|
|
"""「用完轮数时 · 降档交付」交的是成员原话。没人交回结果时手上只有失败说明，
|
|
交下去不是降档、是冒充：照样判失败，也不借用「用完 N 轮」的说法。"""
|
|
_gives_up_after(monkeypatch, "核对员", writes={"复核": DSML, "结论": "不该轮到我"})
|
|
row, events = await _finish(_graph(on_exhausted="degrade"))
|
|
assert row.status == "failed", f"失败说明被当成了成果：{row.output}"
|
|
assert "未交出结论" in row.error and "用完" not in row.error, row.error
|
|
assert not [e for e in events if e.data.get("code") == "team_exhausted"]
|
|
|
|
|
|
async def test_a_failed_last_step_is_not_handed_over_as_the_result(monkeypatch):
|
|
"""核对员查到了，撰稿员失败，调度者判完成：成果是核对员查到的，不是撰稿员的失败说明。"""
|
|
_gives_up_after(monkeypatch, "核对员", "撰稿员", writes={"复核": "一共 1 条。", "结论": DSML})
|
|
row, events = await _finish(_graph())
|
|
assert row.status == "succeeded", row.error
|
|
assert row.output["r"] == "一共 1 条。", row.output
|
|
warn = [e.data for e in events if e.type == "log" and e.data.get("code") == "team_last_failed"]
|
|
assert warn and "撰稿员" in warn[0]["message"] and "核对员" in warn[0]["message"], warn
|
|
signals = review.scan(events, row.output)
|
|
assert any(s.kind == "team_last_failed" for s in signals), signals
|