mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
ThearchyHelios
6b68961c65
部署上画布助手连着几轮报「助手本轮未完成：模型没有给出任何修改，也没有说明原因」，又试了一次才成。本地用同一网关、同一模型、同一底图和需求重放复现：模型思考里写着「先输出 look」，正文却是一段 DeepSeek 的 DSML 工具调用（invoke name="look"，参数 need、targets），意图和参数都对，只是不是约定的单行 JSON。读流只认 JSON 和原生工具调用，于是一个操作都没认出、整轮报错；画布不处理 next=retry，点重试原样再发，几率不变。提示词里「输出这一行就停下，系统把结果交回来」很像工具调用，却从没说这里没有工具调用接口。
- engine/toolcalls：take_leaked_calls 读写成文字的工具调用（DSML 竖线一到两个、带不带空格，<function_calls><invoke>，<tool_call>{name, arguments}，DeepSeek 特殊 token 两种排法），按出现先后交出已写完的调用和没写完的那一段；final 时参数写完整、只差结尾的补上结尾再读，写到一半的不硬读。_MARKUP 补认 DSML 结尾标签和 function_calls / tool_calls 包装（agent 节点的识别跟着受益）。has_markup 只认标记本身
- copilot_gen._iter_ops：从有标记的那一行起攒，一次调用写完就按名字读成操作（名字是操作名的才认，规矩同原生工具调用 _call_op；op 写在最前，接着写时交回模型的是约定写法），读到 look 照样停下去查看；读不出的记「工具调用标记的名字不认识 / 参数解析不了 / 没写完」；raw 交出正文开头一段
- Generation：起草时一次调用读完、到这时还没有能落地的操作（plan、look 不算——查看完接着写什么都没认出的整轮照样是空的），把原话交回去、附 StreamDiagnosis.correction 那句纠正，当场重写一次（FORMAT_NUDGE_PROMPT，每轮一次）；什么都没写的把纠正接在最后那条消息后面。诊断按次收（absorb），记 nudged。写成标记被读出、当场纠正都打 info 日志
- 报错说法：Generation.empty 默认用 StreamDiagnosis.headline（写成工具调用格式 / 只写了思考 / 什么都没有输出 / 格式不对 / 只说了打算），不再一律「也没有说明原因」；摘要注明当场纠正过
- 提示词：look 用法（四个入口共用）和画布、应用、蓝图的输出格式都写明这里没有工具调用接口
- 问数据失败原因 meta.cause（kind=copilot）带上 code、diagnostics
验证：本地真模型重放，修后 DSML 没再出现、都正常交付；把第一次调用换成读不出的 DSML 后交给真模型，都被当场纠正救回来（先 look 再出操作）；原先抓到的真实 DSML 原文离线读成 look。测试：新增 tests/test_copilot_markup_ops.py（各写法、流式切碎、和 JSON 行混写、读不出的诊断、DSML look 驱动真查看、说明文字 / 只写思考 / 只写 plan / 查看后空续写的纠正、只纠正一次、画布流不报错、报错说法、提示词）；空生成相关的旧测试按「一次生成先纠正一次」更新（基线夹具 no_ops 的说法）。后端全量通过。
313 lines
15 KiB
Python
313 lines
15 KiB
Python
"""Copilot 没说完就不能按「无需修改」交付。
|
||
|
||
2026-09 部署实例上，复杂的改图需求（并行算同比、环比，再加一个单独的 agent）连续两次得到
|
||
「已检查，画布无需修改 · 自查通过」。按同一模型、同一张底图、同一句指令重放：输出额度 8192
|
||
全部用在思考上（finish_reason=length），正文一个字都没有。读流时不看 finish_reason，也不要求
|
||
收到 done，于是自查对着原图报「通过」，final 交回原图，前端比出来没变化就说「无需修改」。
|
||
|
||
这里守几件事：
|
||
- 输出被截断（OpenAI 系的 finish_reason=length、Anthropic 的 stop_reason=max_tokens）要报错，
|
||
不跑自查、不交 final；
|
||
- 流正常结束却一个操作都没给、也没有 done（格式跑偏，全被跳过了）同样报错；
|
||
- 模型明确说完了（有 done）但没改任何东西，照旧交付——那才是真的无需修改；
|
||
- 改图的额度要够思考之后还写得下操作；服务方一开口就嫌额度太大时退回 8192 重来。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessageChunk
|
||
|
||
from app.api import copilot as cp
|
||
from app.api.copilot import _iter_ops
|
||
from app.main import app
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
def _node(nid: str, ntype: str, config: dict) -> dict:
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
#: 一张本来就能过自查的图：截断时自查对着它报「通过」，正是那次的样子
|
||
BASE = {
|
||
"nodes": [
|
||
_node("start", "input", {"fields": [{"name": "question"}]}),
|
||
_node("answer", "llm", {"prompt": "{{ input.question }}", "assign_to": "answer"}),
|
||
_node("out", "output", {"fields": [{"name": "答案", "value": "{{ vars.answer }}"}]}),
|
||
],
|
||
"edges": [{"source": "start", "target": "answer"}, {"source": "answer", "target": "out"}],
|
||
}
|
||
|
||
|
||
def _line(op: dict) -> AIMessageChunk:
|
||
return AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n")
|
||
|
||
|
||
def _thinking(text: str) -> AIMessageChunk:
|
||
# OpenAI 兼容系（DeepSeek 经 LiteLLM）的思考走 reasoning_content
|
||
return AIMessageChunk(content="", additional_kwargs={"reasoning_content": text})
|
||
|
||
|
||
def _openai_cut() -> AIMessageChunk:
|
||
return AIMessageChunk(content="", response_metadata={"finish_reason": "length"})
|
||
|
||
|
||
def _anthropic_cut() -> AIMessageChunk:
|
||
return AIMessageChunk(content=[], response_metadata={"stop_reason": "max_tokens"})
|
||
|
||
|
||
class _Scripted:
|
||
"""按剧本吐 chunk 的模型。each 是每次调用吐的那一串；调用次数多于剧本时重复最后一串。"""
|
||
|
||
def __init__(self, *each: list[AIMessageChunk], max_tokens: int | None = None) -> None:
|
||
self.each = each
|
||
self.calls = 0
|
||
self.max_tokens = max_tokens
|
||
|
||
async def astream(self, messages):
|
||
chunks = self.each[min(self.calls, len(self.each) - 1)]
|
||
self.calls += 1
|
||
for chunk in chunks:
|
||
yield chunk
|
||
|
||
|
||
async def _events(client, monkeypatch, model, base: dict | None = BASE) -> list[dict]:
|
||
async def _model(*_a, **_k):
|
||
return model, "mock-fast"
|
||
|
||
monkeypatch.setattr(cp, "get_chat_model", _model)
|
||
out: list[dict] = []
|
||
async with client.stream("POST", "/api/copilot/generate-stream",
|
||
json={"instruction": "并行算同比和环比", "base_graph": base}) as r:
|
||
async for line in r.aiter_lines():
|
||
# 末尾那条 usage 是这一轮助手的用量（answer_ops），不是生成的操作
|
||
if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'):
|
||
out.append(json.loads(line[6:]))
|
||
return out
|
||
|
||
|
||
def _kinds(events: list[dict]) -> list[str]:
|
||
return [e["op"] for e in events if e["op"] not in ("thinking", "heartbeat", "model")]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 读流这一层
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@pytest.mark.parametrize("cut", [_openai_cut, _anthropic_cut], ids=["finish_reason", "stop_reason"])
|
||
async def test_a_cut_off_stream_raises(cut):
|
||
model = _Scripted([_thinking("先想想怎么拆成并行分支……"), _line({"op": "plan", "summary": "拆成三支"}),
|
||
AIMessageChunk(content='{"op":"add_node","node":{"id":"yo'), cut()])
|
||
got: list[dict] = []
|
||
with pytest.raises(cp.CopilotIncomplete) as info:
|
||
async for op in _iter_ops(model, []):
|
||
got.append(op)
|
||
assert [o["op"] for o in got] == ["thinking", "plan"], "截断之前的操作照常转出去"
|
||
assert "长度上限" in str(info.value)
|
||
assert info.value.hint
|
||
|
||
|
||
async def test_done_that_lands_exactly_at_the_limit_is_complete():
|
||
"""最后一行 done 刚好写完（没来得及换行）就到了上限：模型已经说完了，不算截断。"""
|
||
model = _Scripted([_line({"op": "plan", "summary": "x"}),
|
||
AIMessageChunk(content='{"op":"done","explanation":"好了"}'), _openai_cut()])
|
||
ops = [o async for o in _iter_ops(model, [])]
|
||
assert [o["op"] for o in ops] == ["plan", "done"]
|
||
|
||
|
||
async def test_a_normal_finish_is_not_a_cut():
|
||
model = _Scripted([_line({"op": "plan", "summary": "x"}), _line({"op": "done", "explanation": "y"}),
|
||
AIMessageChunk(content="", response_metadata={"finish_reason": "stop"})])
|
||
ops = [o async for o in _iter_ops(model, [])]
|
||
assert [o["op"] for o in ops] == ["plan", "done"]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 画布改图：截断和空手而归都不能冒充「无需修改」
|
||
# ---------------------------------------------------------------------------
|
||
|
||
async def test_thinking_that_eats_the_whole_budget_is_an_error_not_unchanged(client, monkeypatch):
|
||
"""就是截图里那一轮：只有思考、没有正文，额度用完。"""
|
||
model = _Scripted([_thinking("Let me design the modification. "), _thinking("report_ans has 4 incoming edges."),
|
||
_openai_cut()], max_tokens=8192)
|
||
events = await _events(client, monkeypatch, model)
|
||
|
||
kinds = _kinds(events)
|
||
assert "final" not in kinds, "没说完的一轮不能把原图当成结果交回去"
|
||
assert "check" not in kinds, "自查对着原图报「通过」，是在替一个没发生的修改作证"
|
||
[err] = [e for e in events if e["op"] == "error"]
|
||
assert err["message"].startswith("助手本轮未完成：") and "长度上限" in err["message"]
|
||
assert "画布未修改" in err["hint"]
|
||
assert "length" in err["detail"] and "8192" in err["detail"]
|
||
|
||
|
||
async def test_a_cut_in_the_middle_of_the_ops_is_not_delivered_as_done(client, monkeypatch):
|
||
"""写到一半被截断：半张图不能当成品交付，更不能自动开跑。"""
|
||
model = _Scripted([
|
||
_line({"op": "plan", "summary": "加一支"}),
|
||
_line({"op": "add_node", "node": {"id": "yoy", "type": "llm", "label": "同比",
|
||
"config": {"prompt": "算同比", "assign_to": "yoy"}}}),
|
||
_anthropic_cut(),
|
||
])
|
||
events = await _events(client, monkeypatch, model)
|
||
kinds = _kinds(events)
|
||
assert "final" not in kinds
|
||
assert kinds[-1] == "error"
|
||
|
||
|
||
async def test_output_that_parses_to_nothing_is_an_error(client, monkeypatch):
|
||
"""格式跑偏（只写了说明文字、没有一行是操作），每一行都被跳过：一个操作都没有，也没有 done。当场带纠正重写一次还是这样，
|
||
才报错；诊断写明跳过了几行（两次的）。"""
|
||
model = _Scripted([AIMessageChunk(content="先看一下表结构，\n再决定怎么搭。\n"),
|
||
AIMessageChunk(content="", response_metadata={"finish_reason": "stop"})])
|
||
events = await _events(client, monkeypatch, model)
|
||
kinds = _kinds(events)
|
||
assert "final" not in kinds
|
||
[err] = [e for e in events if e["op"] == "error"]
|
||
assert err["message"].startswith("助手本轮未完成：") and "画布未修改" in err["hint"]
|
||
assert "跳过 4 行：4 行不是完整的 JSON" in err["detail"] and "先看一下表" not in err["detail"], err
|
||
assert err["message"] == "助手本轮未完成：模型的输出格式不对，没有一行能认出的操作" and model.calls == 2, err
|
||
|
||
|
||
async def test_a_pretty_printed_op_in_a_fence_is_read(client, monkeypatch):
|
||
"""整段放进 markdown 围栏、JSON 分了行：以前每一行都被跳过、报一个操作都没有；现在攒成一个操作照常落地。"""
|
||
pretty = "```json\n" + json.dumps({"op": "add_node", "node": {"id": "yoy", "type": "llm", "label": "同比",
|
||
"config": {"prompt": "算同比"}}},
|
||
indent=2, ensure_ascii=False) + "\n```\n"
|
||
model = _Scripted([AIMessageChunk(content=pretty),
|
||
AIMessageChunk(content="", response_metadata={"finish_reason": "stop"})])
|
||
events = await _events(client, monkeypatch, model)
|
||
assert "error" not in _kinds(events), events
|
||
assert any(e["op"] == "add_node" and e["node"]["id"] == "yoy" for e in events)
|
||
|
||
|
||
async def test_a_model_that_says_done_without_changes_is_still_unchanged(client, monkeypatch):
|
||
"""模型看过、明确收尾、一处不改：这才是真的「无需修改」，照旧交付。"""
|
||
model = _Scripted([_line({"op": "plan", "summary": "现有的图已经满足"}),
|
||
_line({"op": "done", "explanation": "不需要改"})])
|
||
events = await _events(client, monkeypatch, model)
|
||
final = events[-1]
|
||
assert final["op"] == "final" and final["explanation"] == "不需要改"
|
||
assert {n["id"] for n in final["graph"]["nodes"]} == {"start", "answer", "out"}
|
||
assert "error" not in _kinds(events)
|
||
|
||
|
||
async def test_a_cut_in_the_repair_round_is_reported(client, monkeypatch):
|
||
"""自查交回去修的那一轮被截断：要说修正没完成，不能说「自查通过」。"""
|
||
broken = [
|
||
_line({"op": "plan", "summary": "加个循环"}),
|
||
_line({"op": "add_node", "node": {"id": "lp", "type": "loop", "label": "循环",
|
||
"config": {"mode": "while", "condition": "foo(vars.x)",
|
||
"max_iterations": 2}}}),
|
||
_line({"op": "add_edge", "edge": {"source": "answer", "target": "lp"}}),
|
||
_line({"op": "add_edge", "edge": {"source": "lp", "target": "out", "sourceHandle": "done"}}),
|
||
_line({"op": "done", "explanation": "加好了"}),
|
||
]
|
||
model = _Scripted(broken, [_thinking("修一下条件"), _openai_cut()])
|
||
events = await _events(client, monkeypatch, model)
|
||
checks = [e for e in events if e["op"] == "check"]
|
||
assert checks[0]["status"] == "repairing"
|
||
assert any(c["status"] == "error" and "长度上限" in c["message"] for c in checks), checks
|
||
assert not any(c["status"] == "passed" for c in checks)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 额度
|
||
# ---------------------------------------------------------------------------
|
||
|
||
async def test_the_stream_asks_for_a_budget_that_leaves_room_after_thinking(client, monkeypatch):
|
||
asked: list[int | None] = []
|
||
|
||
async def _model(_session, spec):
|
||
asked.append(spec.max_tokens)
|
||
return _Scripted([_line({"op": "plan", "summary": "x"}), _line({"op": "done", "explanation": "y"})]), "m"
|
||
|
||
monkeypatch.setattr(cp, "get_chat_model", _model)
|
||
async with client.stream("POST", "/api/copilot/generate-stream",
|
||
json={"instruction": "改一下", "base_graph": BASE}) as r:
|
||
[line async for line in r.aiter_lines()]
|
||
assert asked and asked[0] >= 32768, asked
|
||
|
||
|
||
class _BudgetTooLarge(Exception):
|
||
"""仿 openai / anthropic SDK 的 400：一发请求就被拒，还没吐出任何东西。"""
|
||
|
||
status_code = 400
|
||
|
||
|
||
class _Rejects:
|
||
def __init__(self, message: str) -> None:
|
||
self.message = message
|
||
self.calls = 0
|
||
|
||
async def astream(self, messages):
|
||
self.calls += 1
|
||
raise _BudgetTooLarge(self.message)
|
||
yield # pragma: no cover - 让它成为异步生成器
|
||
|
||
|
||
@pytest.mark.parametrize("message", [
|
||
"Invalid max_tokens value, the valid range of max_tokens is [1, 8192]",
|
||
"max_tokens: 32768 > 8192, which is the maximum allowed number of output tokens",
|
||
"This model's maximum context length is 32768 tokens. However, you requested 40369 tokens "
|
||
"(7601 in the messages, 32768 in the completion).",
|
||
])
|
||
async def test_a_rejected_budget_falls_back_to_the_old_one(client, monkeypatch, message):
|
||
big = _Rejects(message)
|
||
small = _Scripted([_line({"op": "plan", "summary": "x"}),
|
||
_line({"op": "add_node", "node": {"id": "yoy", "type": "llm", "label": "同比",
|
||
"config": {"prompt": "算同比", "assign_to": "yoy"}}}),
|
||
_line({"op": "add_edge", "edge": {"source": "answer", "target": "yoy"}}),
|
||
_line({"op": "add_edge", "edge": {"source": "yoy", "target": "out"}}),
|
||
_line({"op": "done", "explanation": "加好了"})])
|
||
asked: list[int | None] = []
|
||
|
||
async def _model(_session, spec):
|
||
asked.append(spec.max_tokens)
|
||
return (big if spec.max_tokens and spec.max_tokens > 8192 else small), "m"
|
||
|
||
monkeypatch.setattr(cp, "get_chat_model", _model)
|
||
events: list[dict] = []
|
||
async with client.stream("POST", "/api/copilot/generate-stream",
|
||
json={"instruction": "加一支同比", "base_graph": BASE}) as r:
|
||
async for line in r.aiter_lines():
|
||
# 末尾那条 usage 是这一轮助手的用量（answer_ops），不是生成的操作
|
||
if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'):
|
||
events.append(json.loads(line[6:]))
|
||
assert 8192 in asked
|
||
assert big.calls == 1 and small.calls == 1
|
||
final = events[-1]
|
||
assert final["op"] == "final" and "yoy" in {n["id"] for n in final["graph"]["nodes"]}
|
||
|
||
|
||
async def test_other_rejections_are_not_retried(client, monkeypatch):
|
||
"""别的 400（模型名写错之类）照实报错，不拿小额度再撞一次。"""
|
||
big = _Rejects("model 'no-such-model' does not exist")
|
||
small = _Scripted([_line({"op": "done", "explanation": "不该到这里"})])
|
||
|
||
async def _model(_session, spec):
|
||
return (big if spec.max_tokens and spec.max_tokens > 8192 else small), "m"
|
||
|
||
monkeypatch.setattr(cp, "get_chat_model", _model)
|
||
events: list[dict] = []
|
||
async with client.stream("POST", "/api/copilot/generate-stream",
|
||
json={"instruction": "改一下", "base_graph": BASE}) as r:
|
||
async for line in r.aiter_lines():
|
||
# 末尾那条 usage 是这一轮助手的用量（answer_ops），不是生成的操作
|
||
if line.startswith("data: ") and not line.startswith('data: {"op": "usage"'):
|
||
events.append(json.loads(line[6:]))
|
||
assert small.calls == 0
|
||
assert events[-1]["op"] == "error"
|
||
|
||
|
||
def test_the_protocol_asks_not_to_draft_the_json_while_thinking():
|
||
"""思考里把整段 JSON 先写一遍，额度就去了一半：那一轮重放时思考的结尾正在逐字写口径卡的表达式。"""
|
||
assert "思考" in cp._STREAM_PROTOCOL and "不要在思考里" in cp._STREAM_PROTOCOL
|