mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 02:21:58 +00:00
ThearchyHelios
6b68961c65
部署上画布助手连着几轮报「助手本轮未完成：模型没有给出任何修改，也没有说明原因」，又试了一次才成。本地用同一网关、同一模型、同一底图和需求重放复现：模型思考里写着「先输出 look」，正文却是一段 DeepSeek 的 DSML 工具调用（invoke name="look"，参数 need、targets），意图和参数都对，只是不是约定的单行 JSON。读流只认 JSON 和原生工具调用，于是一个操作都没认出、整轮报错；画布不处理 next=retry，点重试原样再发，几率不变。提示词里「输出这一行就停下，系统把结果交回来」很像工具调用，却从没说这里没有工具调用接口。
- engine/toolcalls：take_leaked_calls 读写成文字的工具调用（DSML 竖线一到两个、带不带空格，<function_calls><invoke>，<tool_call>{name, arguments}，DeepSeek 特殊 token 两种排法），按出现先后交出已写完的调用和没写完的那一段；final 时参数写完整、只差结尾的补上结尾再读，写到一半的不硬读。_MARKUP 补认 DSML 结尾标签和 function_calls / tool_calls 包装（agent 节点的识别跟着受益）。has_markup 只认标记本身
- copilot_gen._iter_ops：从有标记的那一行起攒，一次调用写完就按名字读成操作（名字是操作名的才认，规矩同原生工具调用 _call_op；op 写在最前，接着写时交回模型的是约定写法），读到 look 照样停下去查看；读不出的记「工具调用标记的名字不认识 / 参数解析不了 / 没写完」；raw 交出正文开头一段
- Generation：起草时一次调用读完、到这时还没有能落地的操作（plan、look 不算——查看完接着写什么都没认出的整轮照样是空的），把原话交回去、附 StreamDiagnosis.correction 那句纠正，当场重写一次（FORMAT_NUDGE_PROMPT，每轮一次）；什么都没写的把纠正接在最后那条消息后面。诊断按次收（absorb），记 nudged。写成标记被读出、当场纠正都打 info 日志
- 报错说法：Generation.empty 默认用 StreamDiagnosis.headline（写成工具调用格式 / 只写了思考 / 什么都没有输出 / 格式不对 / 只说了打算），不再一律「也没有说明原因」；摘要注明当场纠正过
- 提示词：look 用法（四个入口共用）和画布、应用、蓝图的输出格式都写明这里没有工具调用接口
- 问数据失败原因 meta.cause（kind=copilot）带上 code、diagnostics
验证：本地真模型重放，修后 DSML 没再出现、都正常交付；把第一次调用换成读不出的 DSML 后交给真模型，都被当场纠正救回来（先 look 再出操作）；原先抓到的真实 DSML 原文离线读成 look。测试：新增 tests/test_copilot_markup_ops.py（各写法、流式切碎、和 JSON 行混写、读不出的诊断、DSML look 驱动真查看、说明文字 / 只写思考 / 只写 plan / 查看后空续写的纠正、只纠正一次、画布流不报错、报错说法、提示词）；空生成相关的旧测试按「一次生成先纠正一次」更新（基线夹具 no_ops 的说法）。后端全量通过。
277 lines
15 KiB
Python
277 lines
15 KiB
Python
"""问数据一轮的「决定」（app/chat/turn.py；细案 01 做法 1）。
|
||
|
||
要不要运行、输入怎么填、要不要带着原图重来、重来时说什么，从问数据页下沉到后端：final 带 next，复核带 retry_instruction，
|
||
问数据的一轮后来整个搬到了服务端（chat/drive.py、chat/job.py），页面只发起、只播放。这里守：
|
||
|
||
- 各条决定的分支（next_step、fill_input、retry_after_review、retry_instruction 的两种措辞）；
|
||
- 页面上不再留一份同样的决定（一轮整个搬到服务端之后，frontend/src/store/chat.ts 只发起、只播放）；用户点重试本轮时的措辞；
|
||
- /copilot/review 按这一轮试过几次给 retry_instruction：可以重来时是那句话，不重来是 null。
|
||
|
||
问题、字段名都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
|
||
from app.chat import turn
|
||
from app.main import app
|
||
|
||
CHAT_TS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "store" / "chat.ts"
|
||
|
||
|
||
def _final(*, issues=(), autorun=True, **extra) -> dict:
|
||
return {"op": "final", "graph": {"nodes": [], "edges": []}, "issues": list(issues), "autorun": autorun, **extra}
|
||
|
||
|
||
ERROR = {"level": "error", "message": "图是空的"}
|
||
WARNING = {"level": "warning", "message": "多余的 {{ }}"}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# next_step
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_next_step_branches():
|
||
assert turn.next_step(_final()) == "run"
|
||
assert turn.next_step(_final(issues=[WARNING])) == "run", "warning 不挡运行"
|
||
assert turn.next_step(_final(autorun=False)) == "ready", "模型说不运行：只建不跑"
|
||
assert turn.next_step(_final(autorun=False, asked={"question": "哪个月？", "options": []})) == "ready"
|
||
assert turn.next_step(_final(issues=[ERROR])) == "fail", "自查过还有 error：判失败，不再整轮重来"
|
||
assert turn.next_step(_final(issues=[ERROR]), self_checked=False) == "retry", "老服务端没自查：重来一次"
|
||
assert turn.next_step(_final(issues=[ERROR], autorun=False)) == "fail", "没过检查比只建不跑优先"
|
||
assert turn.next_step({"op": "reply", "text": "已经查过了"}) == "reply"
|
||
assert turn.next_step({"op": "error", "message": "x"}) == "fail"
|
||
empty = {"op": "error", "message": "助手本轮未完成：模型没有给出任何修改，也没有说明原因",
|
||
"code": turn.EMPTY_GENERATION, "reason": "模型没有给出任何修改，也没有说明原因"}
|
||
assert turn.next_step(empty) == "retry", "只写了思考、一个操作都没给：按重来一次的规矩自动重来"
|
||
assert turn.error_reason(empty) == "模型没有给出任何修改，也没有说明原因"
|
||
assert turn.error_reason({"op": "error", "message": "助手本轮未完成：接入返回了 502"}) == "接入返回了 502"
|
||
assert turn.next_step(None) == "asked", "流结束时只有反问"
|
||
assert set(turn.NEXT_STEPS) == {"run", "ready", "fail"}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# fill_input
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _graph(*fields: dict) -> dict:
|
||
return {"nodes": [{"id": "start", "type": "input", "data": {"label": "输入", "config": {"fields": list(fields)}}},
|
||
{"id": "out", "type": "output", "data": {"label": "成果", "config": {}}}], "edges": []}
|
||
|
||
|
||
def test_fill_input_rules():
|
||
assert turn.fill_input(_graph(), "上周销售额", "问：…") == {"question": "上周销售额", "history": "问：…"}
|
||
assert turn.fill_input({"nodes": []}, "上周销售额") == {"question": "上周销售额", "history": ""}
|
||
# 名字像「问题」的优先，其余字段填默认值（没有就空串）
|
||
got = turn.fill_input(_graph({"name": "region", "default": "华东"}, {"name": "topic"}, {"name": "limit"}), "退款率")
|
||
assert got == {"history": "", "region": "华东", "topic": "退款率", "limit": ""}
|
||
got = turn.fill_input(_graph({"name": "门店"}, {"name": "需求说明"}), "退款率")
|
||
assert got["需求说明"] == "退款率" and got["门店"] == ""
|
||
# 没有像问题的：第一个必填的；再没有：第一个
|
||
got = turn.fill_input(_graph({"name": "a"}, {"name": "b", "required": True}), "退款率")
|
||
assert got["b"] == "退款率" and got["a"] == ""
|
||
got = turn.fill_input(_graph({"name": "a"}, {"name": "b"}), "退款率")
|
||
assert got["a"] == "退款率"
|
||
# 大小写不敏感
|
||
assert turn.fill_input(_graph({"name": "UserQuery"}), "退款率")["UserQuery"] == "退款率"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 复核之后：要不要重来、说什么
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
BROKEN = {"retry": True, "severity": "broken", "note": "结论不可用",
|
||
"signals": [{"kind": "step_limit", "detail": "agent 用满了 8 步"}, {"kind": "tool_error", "detail": "表不存在"}]}
|
||
|
||
|
||
def test_retry_after_review():
|
||
assert turn.retry_after_review(BROKEN, 0) is True
|
||
assert turn.retry_after_review(BROKEN, 1) is False, "一轮最多自动再试一次"
|
||
assert turn.retry_after_review({**BROKEN, "severity": "degraded"}, 0) is False
|
||
assert turn.retry_after_review({**BROKEN, "retry": False}, 0) is False
|
||
assert turn.retry_after_review(None, 0) is False
|
||
assert turn.review_reason(BROKEN) == "agent 用满了 8 步；表不存在"
|
||
assert turn.review_reason({"signals": [], "note": "结论不可用"}) == "结论不可用"
|
||
|
||
|
||
def test_retry_instruction_two_wordings():
|
||
with_base = turn.retry_instruction("上周销售额", "表不存在", True)
|
||
assert with_base.startswith("这个工作流上一次运行没有成功：表不存在。\n\n请针对这个原因改它")
|
||
assert with_base.endswith("改完继续回答原来的问题：上周销售额")
|
||
scratch = turn.retry_instruction("上周销售额", "表不存在", False)
|
||
assert scratch == "上周销售额\n\n（上一次没有成功：表不存在。请针对这个原因调整，重新给出完整的工作流。）"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 页面不再留一份同样的决定
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _ts() -> str:
|
||
return CHAT_TS.read_text(encoding="utf-8")
|
||
|
||
|
||
def test_the_page_no_longer_keeps_its_own_copy_of_the_decisions():
|
||
"""一轮整个搬到服务端之后（docs/design/orchestration-round §1），问数据页只发起、只播放：要不要运行、要不要重来、重来说
|
||
什么、入口怎么填都只在这里（和 chat/drive.py），页面上那份退路（MAX_ATTEMPTS、fillInput、retryInstruction）去掉了——
|
||
两份并存迟早走样，界面层也不放逻辑。以前这里逐字对照两边的常量和措辞，现在守的是页面上没有这几样。"""
|
||
ts = _ts()
|
||
for name in ("const MAX_ATTEMPTS", "function fillInput", "function retryInstruction", "streamCopilot(",
|
||
"api.copilot.review(", "api.runs.start(", "patchTurn("):
|
||
assert name not in ts, f"chat.ts 里又出现了 {name}：这一轮的决定、发起、落库应在服务端"
|
||
|
||
|
||
def test_user_retry_wordings():
|
||
"""用户点「重试本轮」时交给助手的话（以前在页面 retryTurn 里拼）：有图时说「没有给出可用的结果」，没图的同从零重建。"""
|
||
with_base = turn.user_retry_instruction("上周销售额", "已手动停止", True)
|
||
assert with_base == ("这个工作流上一次没有给出可用的结果：已手动停止。\n\n请针对这个原因改它（比如调大最大步数、改正工具参数、"
|
||
"换个查询方式），改完继续回答原来的问题：上周销售额")
|
||
assert turn.user_retry_instruction("上周销售额", "已手动停止", False) == turn.retry_instruction(
|
||
"上周销售额", "已手动停止", False)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# /copilot/review 给 retry_instruction
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
async def _run(output: dict, graph: dict, code: str) -> str:
|
||
from app.core.events import EventType
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run, RunEvent
|
||
|
||
async with SessionLocal() as session:
|
||
run = Run(id=uuid.uuid4().hex, status="succeeded", output=output, graph=graph)
|
||
session.add(run)
|
||
await session.flush()
|
||
session.add(RunEvent(run_id=run.id, seq=1, type=EventType.LOG, node_id="a", ts=0.0,
|
||
data={"level": "warn", "code": code, "message": "agent 用满了 8 步还没给出结论"}))
|
||
await session.commit()
|
||
return run.id
|
||
|
||
|
||
def test_the_page_says_the_server_is_retrying_an_empty_generation():
|
||
"""只写了思考、一个操作都没给（error.next=retry）时，重来一次是服务端定的（chat/drive.py 照 next_step 走，问数据的一轮记一条
|
||
turn.retry，kind=empty）；页面照它把这一轮重置、写「助手上一次没有给出任何操作，正在重新生成工作流…」。读 chat.ts 里
|
||
turn.retry 那一支（行为另由 tests/test_chat_turns.py 和 scripts/check-chat.mjs 的 emptyretry 一节守）。"""
|
||
branch = re.search(r"function retryPatch\((.*?)\n\}", _ts(), re.S)
|
||
assert branch, "chat.ts 里找不到 retryPatch"
|
||
assert re.search(r"d\.kind === 'empty' \? CHAT_RETRY_TEXT\.empty", branch.group(1)), branch.group(1)
|
||
|
||
|
||
async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch):
|
||
"""生成过程一个操作都没给（deepseek 偶发只出思考）：error 操作带机读码 copilot_empty、原因、next=retry。"""
|
||
import json
|
||
|
||
import app.api.copilot as copilot
|
||
from tests.test_usage_meter import Scripted
|
||
|
||
model = Scripted(model="claude-opus-5", replies=[[""]])
|
||
|
||
async def _model(*_a, **_k):
|
||
return model, "claude-opus-5"
|
||
|
||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||
events = []
|
||
async with client.stream("POST", "/api/copilot/generate-stream",
|
||
json={"instruction": "上周各地区销售额", "intent": "answer"}) as r:
|
||
async for line in r.aiter_lines():
|
||
if line.startswith("data: "):
|
||
events.append(json.loads(line[6:]))
|
||
[err] = [e for e in events if e["op"] == "error"]
|
||
assert err["code"] == turn.EMPTY_GENERATION and err["next"] == "retry", err
|
||
# 原因是重来时交给助手的那句纠正（这次连思考都没有），诊断写在 detail（不含原文）
|
||
assert err["reason"].startswith("什么都没有输出") and "look" in err["reason"], err
|
||
# 说法按诊断照实写；起草时已经当场纠正过一次（第二次还是空的），整轮才交 error
|
||
assert err["message"] == "助手本轮未完成：模型什么都没有输出"
|
||
assert "正文 0 字，思考 0 字" in err["detail"] and err["diagnostics"]["ops"] == 0
|
||
assert err["diagnostics"]["nudged"] is True and model.calls == 2, err
|
||
|
||
|
||
async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
|
||
"""评测的作答（app/eval/answer.Driver）走同一个决定：第一次一个操作都没给就带着原因重来一次，原因照实记在 retries；
|
||
重来还不行才照实报建图失败（助手调用失败），不再试第三次。每一次生成里起草时已经当场纠正过一次（copilot_gen
|
||
FORMAT_NUDGE_PROMPT），所以一次生成算两次调用。"""
|
||
import json
|
||
|
||
import app.api.copilot as copilot
|
||
from app.eval import answer
|
||
from tests.test_usage_meter import Scripted
|
||
|
||
reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False)
|
||
for replies, path, calls in (([[""], [""], [reply]], "reply", 3), ([[""]], "build_failed", 4)):
|
||
model = Scripted(model="claude-opus-5", replies=replies)
|
||
seen: list[str] = []
|
||
|
||
async def _model(*_a, _m=model, **_k):
|
||
return _m, "claude-opus-5"
|
||
|
||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||
real = answer.Driver._generate
|
||
|
||
async def spy(self, instruction, base_graph, _seen=seen, _real=real):
|
||
_seen.append(instruction)
|
||
return await _real(self, instruction, base_graph)
|
||
|
||
monkeypatch.setattr(answer.Driver, "_generate", spy)
|
||
driver = answer.Driver("上周华中区销售额", datasource_ids=[], ctx=answer.TurnContext(
|
||
suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None))
|
||
t = await driver.drive()
|
||
assert t.path == path and t.attempts == 1 and model.calls == calls, (t.path, t.attempts, model.calls, t.reason)
|
||
assert len(t.retries) == 1 and t.retries[0].startswith("什么都没有输出"), t.retries
|
||
# 重来时问题后面附那句纠正，不原样重发
|
||
assert seen[1] == turn.empty_retry_instruction("上周华中区销售额", t.retries[0]) != seen[0], seen
|
||
assert "什么都没有输出" in seen[1]
|
||
if path == "build_failed":
|
||
assert t.copilot_failed and t.reason.startswith("助手本轮未完成"), t.reason
|
||
assert t.usage["assistant"]["calls"] == calls, "两次生成（连同各自当场纠正的那一次）都是这一题花的"
|
||
|
||
|
||
async def test_review_says_whether_and_how_to_retry(client, monkeypatch):
|
||
import app.api.copilot as copilot
|
||
from app.providers.factory import ProviderNotConfigured
|
||
|
||
async def unconfigured(*_a, **_k):
|
||
raise ProviderNotConfigured("没有可用的 provider")
|
||
|
||
# 没配模型：规则层照样判得出不可用、该重来
|
||
monkeypatch.setattr(copilot, "get_chat_model", unconfigured)
|
||
graph = _graph({"name": "question"})
|
||
run_id = await _run({"answer": "大概 3 家"}, graph, "step_limit")
|
||
body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "有几家门店"})).json()
|
||
assert body["severity"] == "broken" and body["retry"] is True
|
||
assert body["retry_instruction"] == turn.retry_instruction("有几家门店", turn.review_reason(body), True)
|
||
# 这一轮已经重来过一次：不再给
|
||
body = (await client.post("/api/copilot/review",
|
||
json={"run_id": run_id, "question": "有几家门店", "attempt": 1})).json()
|
||
assert body["retry_instruction"] is None
|
||
# 运行的图是空的：措辞换成从零再建
|
||
bare = await _run({"answer": "大概 3 家"}, {}, "step_limit")
|
||
body = (await client.post("/api/copilot/review", json={"run_id": bare, "question": "有几家门店"})).json()
|
||
assert body["retry_instruction"].startswith("有几家门店\n\n（上一次没有成功：")
|
||
# 干净的运行：不重来
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run
|
||
|
||
async with SessionLocal() as session:
|
||
clean = Run(status="succeeded", output={"answer": "3 家"}, graph=graph)
|
||
session.add(clean)
|
||
await session.commit()
|
||
clean_id = clean.id
|
||
body = (await client.post("/api/copilot/review", json={"run_id": clean_id, "question": "有几家门店"})).json()
|
||
assert body["verdict"] == "ok" and body["retry_instruction"] is None
|