mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-22 08:32:00 +00:00
助手把操作写成工具调用格式时照操作读;一个能落地的操作都没有时当场纠正一次;报错按诊断照实说
部署上画布助手连着几轮报「助手本轮未完成:模型没有给出任何修改,也没有说明原因」,又试了一次才成。本地用同一网关、同一模型、同一底图和需求重放复现:模型思考里写着「先输出 look」,正文却是一段 DeepSeek 的 DSML 工具调用(invoke name="look",参数 need、targets),意图和参数都对,只是不是约定的单行 JSON。读流只认 JSON 和原生工具调用,于是一个操作都没认出、整轮报错;画布不处理 next=retry,点重试原样再发,几率不变。提示词里「输出这一行就停下,系统把结果交回来」很像工具调用,却从没说这里没有工具调用接口。
- engine/toolcalls:take_leaked_calls 读写成文字的工具调用(DSML 竖线一到两个、带不带空格,<function_calls><invoke>,<tool_call>{name, arguments},DeepSeek 特殊 token 两种排法),按出现先后交出已写完的调用和没写完的那一段;final 时参数写完整、只差结尾的补上结尾再读,写到一半的不硬读。_MARKUP 补认 DSML 结尾标签和 function_calls / tool_calls 包装(agent 节点的识别跟着受益)。has_markup 只认标记本身
- copilot_gen._iter_ops:从有标记的那一行起攒,一次调用写完就按名字读成操作(名字是操作名的才认,规矩同原生工具调用 _call_op;op 写在最前,接着写时交回模型的是约定写法),读到 look 照样停下去查看;读不出的记「工具调用标记的名字不认识 / 参数解析不了 / 没写完」;raw 交出正文开头一段
- Generation:起草时一次调用读完、到这时还没有能落地的操作(plan、look 不算——查看完接着写什么都没认出的整轮照样是空的),把原话交回去、附 StreamDiagnosis.correction 那句纠正,当场重写一次(FORMAT_NUDGE_PROMPT,每轮一次);什么都没写的把纠正接在最后那条消息后面。诊断按次收(absorb),记 nudged。写成标记被读出、当场纠正都打 info 日志
- 报错说法:Generation.empty 默认用 StreamDiagnosis.headline(写成工具调用格式 / 只写了思考 / 什么都没有输出 / 格式不对 / 只说了打算),不再一律「也没有说明原因」;摘要注明当场纠正过
- 提示词:look 用法(四个入口共用)和画布、应用、蓝图的输出格式都写明这里没有工具调用接口
- 问数据失败原因 meta.cause(kind=copilot)带上 code、diagnostics
验证:本地真模型重放,修后 DSML 没再出现、都正常交付;把第一次调用换成读不出的 DSML 后交给真模型,都被当场纠正救回来(先 look 再出操作);原先抓到的真实 DSML 原文离线读成 look。测试:新增 tests/test_copilot_markup_ops.py(各写法、流式切碎、和 JSON 行混写、读不出的诊断、DSML look 驱动真查看、说明文字 / 只写思考 / 只写 plan / 查看后空续写的纠正、只纠正一次、画布流不报错、报错说法、提示词);空生成相关的旧测试按「一次生成先纠正一次」更新(基线夹具 no_ops 的说法)。后端全量通过。
This commit is contained in:
+1
-1
@@ -1006,7 +1006,7 @@
|
||||
},
|
||||
{
|
||||
"op": "error",
|
||||
"message": "助手本轮未完成:模型没有给出任何修改,也没有说明原因",
|
||||
"message": "助手本轮未完成:模型的输出格式不对,没有一行能认出的操作",
|
||||
"hint": "画布未修改。可以重试,或把需求描述得更具体",
|
||||
"detail": ""
|
||||
}
|
||||
|
||||
@@ -196,13 +196,16 @@ async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch)
|
||||
assert err["code"] == turn.EMPTY_GENERATION and err["next"] == "retry", err
|
||||
# 原因是重来时交给助手的那句纠正(这次连思考都没有),诊断写在 detail(不含原文)
|
||||
assert err["reason"].startswith("什么都没有输出") and "look" in err["reason"], err
|
||||
assert err["message"] == "助手本轮未完成:模型没有给出任何修改,也没有说明原因"
|
||||
# 说法按诊断照实写;起草时已经当场纠正过一次(第二次还是空的),整轮才交 error
|
||||
assert err["message"] == "助手本轮未完成:模型什么都没有输出"
|
||||
assert "正文 0 字,思考 0 字" in err["detail"] and err["diagnostics"]["ops"] == 0
|
||||
assert err["diagnostics"]["nudged"] is True and model.calls == 2, err
|
||||
|
||||
|
||||
async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
|
||||
"""评测的作答(app/eval/answer.Driver)走同一个决定:第一次一个操作都没给就带着原因重来一次,原因照实记在 retries;
|
||||
重来还不行才照实报建图失败(助手调用失败),不再试第三次。"""
|
||||
重来还不行才照实报建图失败(助手调用失败),不再试第三次。每一次生成里起草时已经当场纠正过一次(copilot_gen
|
||||
FORMAT_NUDGE_PROMPT),所以一次生成算两次调用。"""
|
||||
import json
|
||||
|
||||
import app.api.copilot as copilot
|
||||
@@ -210,7 +213,7 @@ async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
|
||||
from tests.test_usage_meter import Scripted
|
||||
|
||||
reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False)
|
||||
for replies, path in (([[""], [reply]], "reply"), ([[""]], "build_failed")):
|
||||
for replies, path, calls in (([[""], [""], [reply]], "reply", 3), ([[""]], "build_failed", 4)):
|
||||
model = Scripted(model="claude-opus-5", replies=replies)
|
||||
seen: list[str] = []
|
||||
|
||||
@@ -228,14 +231,14 @@ async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
|
||||
driver = answer.Driver("上周华中区销售额", datasource_ids=[], ctx=answer.TurnContext(
|
||||
suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None))
|
||||
t = await driver.drive()
|
||||
assert t.path == path and t.attempts == 1 and model.calls == 2, (t.path, t.attempts, model.calls, t.reason)
|
||||
assert t.path == path and t.attempts == 1 and model.calls == calls, (t.path, t.attempts, model.calls, t.reason)
|
||||
assert len(t.retries) == 1 and t.retries[0].startswith("什么都没有输出"), t.retries
|
||||
# 重来时问题后面附那句纠正,不原样重发
|
||||
assert seen[1] == turn.empty_retry_instruction("上周华中区销售额", t.retries[0]) != seen[0], seen
|
||||
assert "什么都没有输出" in seen[1]
|
||||
if path == "build_failed":
|
||||
assert t.copilot_failed and t.reason.startswith("助手本轮未完成"), t.reason
|
||||
assert t.usage["assistant"]["calls"] == 2, "两次生成都是这一题花的"
|
||||
assert t.usage["assistant"]["calls"] == calls, "两次生成(连同各自当场纠正的那一次)都是这一题花的"
|
||||
|
||||
|
||||
async def test_review_says_whether_and_how_to_retry(client, monkeypatch):
|
||||
|
||||
@@ -850,12 +850,13 @@ def test_attempt_records_say_what_the_page_used_to_say():
|
||||
|
||||
|
||||
async def test_an_empty_generation_is_retried_once_on_the_server(client, monkeypatch):
|
||||
"""助手只写了思考、一个操作都没给(error.next=retry):服务端带着原因重来一次(turn.retry kind=empty),不用页面在。"""
|
||||
"""助手只写了思考、一个操作都没给(error.next=retry):服务端带着原因重来一次(turn.retry kind=empty),不用页面在。
|
||||
一次建图里起草时已经当场纠正过一次(copilot_gen FORMAT_NUDGE_PROMPT),那一次也空着,整次建图才交 error。"""
|
||||
import app.api.copilot as copilot
|
||||
from tests.test_usage_meter import Scripted
|
||||
|
||||
reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False)
|
||||
model = Scripted(model="claude-opus-5", replies=[[""], [reply]])
|
||||
model = Scripted(model="claude-opus-5", replies=[[""], [""], [reply]])
|
||||
|
||||
async def _model(*_a, **_k):
|
||||
return model, "claude-opus-5"
|
||||
@@ -871,7 +872,7 @@ async def test_an_empty_generation_is_retried_once_on_the_server(client, monkeyp
|
||||
assert "正文 0 字" in retry["detail"], retry
|
||||
row = await _row(turn_id)
|
||||
assert row.status == "done" and row.answer == "上周没有华中区的数据" and row.meta["noQuery"] is True
|
||||
assert row.meta["usage"]["copilot"]["calls"] == 2, "两次建图都是这一轮花的"
|
||||
assert row.meta["usage"]["copilot"]["calls"] == 3, "两次建图(连同第一次当场纠正的那一次)都是这一轮花的"
|
||||
|
||||
|
||||
async def test_several_stranded_turns_of_one_conversation_can_be_resumed_together(client):
|
||||
|
||||
@@ -154,7 +154,10 @@ async def test_the_sse_output_is_unchanged_but_for_a_trailing_usage(client, monk
|
||||
reason = op.pop("reason")
|
||||
if name == "no_ops":
|
||||
# 一个操作都没给:原因是重来时交给助手的那句纠正,诊断(不含原文)写进 detail、diagnostics,别的照旧
|
||||
assert "每个操作单独一行" in reason and op.pop("diagnostics")["dropped"] == {"不是 JSON 对象": 1}, op
|
||||
# 起草时当场带纠正重写过一次(同一个剧本再吐一遍),跳过的行是两次的
|
||||
diagnostics = op.pop("diagnostics")
|
||||
assert "每个操作单独一行" in reason and diagnostics["dropped"] == {"不是 JSON 对象": 2}, op
|
||||
assert diagnostics["calls"] == 2 and diagnostics["nudged"] is True, diagnostics
|
||||
assert op["detail"].startswith("结束原因"), op
|
||||
op["detail"] = ""
|
||||
else:
|
||||
|
||||
@@ -93,10 +93,13 @@ async def test_a_native_tool_call_named_after_an_op_becomes_that_op():
|
||||
assert diag.dropped == {"工具调用的名字不认识": 1, "工具调用的参数解析不了": 1}, diag.dropped
|
||||
|
||||
|
||||
async def test_tool_call_markup_in_text_is_diagnosed():
|
||||
ops, diag = await _ops([AIMessageChunk(content='<tool_call>{"name": "look", "arguments": {}}</tool_call>\n')])
|
||||
async def test_tool_call_markup_that_is_not_an_op_is_diagnosed():
|
||||
# 名字是操作名的照操作读(tests/test_copilot_markup_ops.py);不是的照实记进诊断
|
||||
ops, diag = await _ops([AIMessageChunk(content='<tool_call>{"name": "run_sql", "arguments": {}}</tool_call>\n')])
|
||||
assert ops == [] and diag.markup == 1
|
||||
assert diag.dropped == {"工具调用标记的名字不认识": 1}, diag.dropped
|
||||
assert "不走工具调用" in diag.correction()
|
||||
assert diag.headline() == "模型把操作写成了工具调用的格式,没有按约定逐行输出,系统认不出"
|
||||
|
||||
|
||||
async def test_the_empty_error_carries_the_diagnosis_and_a_correction(client, monkeypatch): # noqa: F811
|
||||
@@ -107,9 +110,13 @@ async def test_the_empty_error_carries_the_diagnosis_and_a_correction(client, mo
|
||||
got = await _sse(client, BODY)
|
||||
[error] = [op for op in got if op["op"] == "error"]
|
||||
assert error["code"] == "copilot_empty" and error["next"] == "retry"
|
||||
assert error["message"] == "助手本轮未完成:模型没有给出任何修改,也没有说明原因"
|
||||
assert "正文 0 字" in error["detail"] and "思考 6 字" in error["detail"] and "结束原因 stop" in error["detail"]
|
||||
assert error["diagnostics"]["text_chars"] == 0 and error["diagnostics"]["thinking_chars"] == 6
|
||||
# 说法按诊断照实写,不再一律是「没有给出任何修改,也没有说明原因」
|
||||
assert error["message"] == "助手本轮未完成:模型只写了思考,没有给出任何操作"
|
||||
# 起草时当场纠正过一次(同一个模型又只写了思考):诊断是两次的
|
||||
assert "正文 0 字" in error["detail"] and "思考 12 字" in error["detail"] and "结束原因 stop" in error["detail"]
|
||||
assert "共调用模型 2 次" in error["detail"] and "当场带纠正重写过一次" in error["detail"], error["detail"]
|
||||
assert error["diagnostics"]["text_chars"] == 0 and error["diagnostics"]["thinking_chars"] == 12
|
||||
assert error["diagnostics"]["nudged"] is True
|
||||
assert "只写了思考" in error["reason"]
|
||||
# 重来时交给助手的话:问题后面附这句纠正,不是原样重发
|
||||
again = policy.empty_retry_instruction("上周各条线的产量", policy.error_reason(error))
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
"""模型把操作写成了工具调用的格式时,照操作读;一个操作都没认出时当场纠正一次;报错照实说。
|
||||
|
||||
部署上撞过:画布上让助手搭一张按天出数的图,连着几轮都是「助手本轮未完成:模型没有给出任何修改,也没有说明原因」,
|
||||
又试了一次才成。用同一个模型重放:模型思考里写着「先输出 look」,正文却是一段 DeepSeek 的 DSML 工具调用
|
||||
(<||DSML|| invoke name="look"> + need / targets 两个参数),意图和参数都对,只是不是约定的单行 JSON。读流只认 JSON 和
|
||||
原生工具调用,于是一个操作都没认出、整轮报错;画布上没有自动重来,点重试还是同样的几率。现在:
|
||||
|
||||
- 写成文字的工具调用(DSML、<invoke>、<tool_call>、DeepSeek 特殊 token)一次调用写完就按名字读成操作(名字是操作名的才认),
|
||||
流式一行行攒、读到 look 照样停下去查看;
|
||||
- 起草时一次调用读完、一个 plan 以外的操作都没认出:把原话交回去、附一句纠正,请它当场重写一次(每轮一次);
|
||||
- 还是没有,报错的说法按诊断照实写,不再一律是「没有给出任何修改,也没有说明原因」;
|
||||
- 提示词里写明这里没有工具调用接口。
|
||||
|
||||
需求、库名、表名都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessageChunk
|
||||
|
||||
from app.api.copilot_gen import FORMAT_NUDGE_PROMPT, Generation, GraphApplier, StreamDiagnosis, _iter_ops
|
||||
from app.engine.toolcalls import LeakedCall, has_markup, leaked_markup, take_leaked_calls
|
||||
from tests.test_copilot_look import START, Turns, _events, _graph_ops, _line, _ops, _tables, client, shop # noqa: F401
|
||||
|
||||
#: 部署上见到的那种写法:竖线两个、DSML 后面带空格,外层包的是 calls,参数带 string 属性
|
||||
DSML_LOOK = (
|
||||
"<||DSML|| calls>\n"
|
||||
'<||DSML|| invoke name="look">\n'
|
||||
'<||DSML|| parameter name="need" string="false">订单金额相关的表,要看清表名和字段</||DSML|| parameter>\n'
|
||||
'<||DSML|| parameter name="targets" string="false">[{"kind": "table", "name": "orders"}]</||DSML|| parameter>\n'
|
||||
"</||DSML|| invoke>\n"
|
||||
"</||DSML|| calls>\n"
|
||||
)
|
||||
LOOK = {"op": "look", "need": "订单金额相关的表,要看清表名和字段", "targets": [{"kind": "table", "name": "orders"}]}
|
||||
|
||||
|
||||
class _Model:
|
||||
def __init__(self, chunks: list[AIMessageChunk]) -> None:
|
||||
self.chunks = chunks
|
||||
|
||||
async def astream(self, messages):
|
||||
for chunk in self.chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
async def _read(text: str, *, size: int = 0, lines: list[str] | None = None) -> tuple[list[dict], StreamDiagnosis]:
|
||||
"""把 text 当作模型正文读一遍(size>0 时按这么多字切成小段流过来)。"""
|
||||
pieces = [text[i:i + size] for i in range(0, len(text), size)] if size else [text]
|
||||
diag = StreamDiagnosis()
|
||||
model = _Model([AIMessageChunk(content=p) for p in pieces]
|
||||
+ [AIMessageChunk(content="", response_metadata={"finish_reason": "stop"})])
|
||||
ops = [op async for op in _iter_ops(model, [], lines=lines, diag=diag)]
|
||||
return [op for op in ops if op["op"] != "thinking"], diag
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 认标记
|
||||
|
||||
@pytest.mark.parametrize("text, call", [
|
||||
# 部署上见到的 DSML
|
||||
(DSML_LOOK, LeakedCall("look", {k: v for k, v in LOOK.items() if k != "op"})),
|
||||
# 标准的 DSML:竖线一个、外层是 function_calls;string="true" 原样是字符串(哪怕看着像数字)
|
||||
('<|DSML|function_calls>\n<|DSML|invoke name="look">\n'
|
||||
'<|DSML|parameter name="need" string="true">2024 年的订单</|DSML|parameter>\n'
|
||||
'<|DSML|parameter name="limit" string="true">3</|DSML|parameter>\n'
|
||||
"</|DSML|invoke>\n</|DSML|function_calls>",
|
||||
LeakedCall("look", {"need": "2024 年的订单", "limit": "3"})),
|
||||
# 去掉 DSML 前缀的 <invoke>:没写 string 的,像 JSON 就按 JSON 读
|
||||
('<function_calls><invoke name="add_node"><parameter name="node">{"id": "q", "type": "llm"}</parameter>'
|
||||
'<parameter name="label">查订单</parameter></invoke></function_calls>',
|
||||
LeakedCall("add_node", {"node": {"id": "q", "type": "llm"}, "label": "查订单"})),
|
||||
# <tool_call>:arguments 可能是一段 JSON 字符串
|
||||
('<tool_call>{"name": "look", "arguments": "{\\"need\\": \\"订单表\\"}"}</tool_call>',
|
||||
LeakedCall("look", {"need": "订单表"})),
|
||||
('<tool_call>{"function": {"name": "done", "arguments": {"explanation": "好了"}}}</tool_call>',
|
||||
LeakedCall("done", {"explanation": "好了"})),
|
||||
# DeepSeek 特殊 token 的两种排法
|
||||
('<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>look\n```json\n{"need": "订单表"}\n```'
|
||||
"<|tool▁call▁end|><|tool▁calls▁end|>",
|
||||
LeakedCall("look", {"need": "订单表"})),
|
||||
('<|tool▁call▁begin|>look<|tool▁sep|>{"need": "订单表"}<|tool▁call▁end|>', LeakedCall("look", {"need": "订单表"})),
|
||||
])
|
||||
def test_each_markup_style_gives_its_call(text, call):
|
||||
assert has_markup(text)
|
||||
assert take_leaked_calls(text) == ([call], "")
|
||||
|
||||
|
||||
def test_an_unfinished_call_is_kept_until_its_end_arrives():
|
||||
calls, rest = take_leaked_calls('说明文字 <|DSML|function_calls>\n<|DSML|invoke name="look">\n')
|
||||
assert calls == [] and rest.startswith('<invoke name="look">')
|
||||
calls, rest = take_leaked_calls(rest + '<|DSML|parameter name="need" string="true">订单</|DSML|parameter>\n'
|
||||
"</|DSML|invoke>\n<|DSML|invoke name=\"done\">\n")
|
||||
assert calls == [LeakedCall("look", {"need": "订单"})] and rest.startswith('<invoke name="done">')
|
||||
# 流读完了还差结尾:参数都写完整了的补上结尾再读;参数写到一半、补上也读不出的照样交回(调用方记「没写完」)
|
||||
assert take_leaked_calls(rest, final=True) == ([LeakedCall("done", {})], "")
|
||||
half = '<invoke name="look">\n<parameter name="need" string="true">没写完'
|
||||
assert take_leaked_calls(half, final=True) == ([], half)
|
||||
assert take_leaked_calls("<tool_call>{坏的", final=True) == ([], "<tool_call>{坏的")
|
||||
|
||||
|
||||
def test_closing_dsml_tags_and_call_wrappers_count_as_leaked_markup():
|
||||
# 节点里的 agent 也用这个认:DSML 的结尾标签、<function_calls> 包装以前漏认
|
||||
assert leaked_markup("</|DSML|invoke>") and leaked_markup("<function_calls>") and leaked_markup("<tool_calls>")
|
||||
assert not has_markup('{"op": "look", "need": "订单"}') and not has_markup("先看一下 orders 表")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 读流
|
||||
|
||||
@pytest.mark.parametrize("size", [0, 7])
|
||||
async def test_a_dsml_look_is_read_as_a_look_and_the_stream_stops_there(size):
|
||||
lines: list[str] = []
|
||||
ops, diag = await _read(DSML_LOOK + _line({"op": "add_node", "node": {"id": "ghost"}}) + "\n", size=size,
|
||||
lines=lines)
|
||||
assert ops == [LOOK]
|
||||
assert diag.markup == 1 and diag.ops == 1 and diag.dropped == {}
|
||||
# 接着写时交回模型的是约定的写法,不是它写的那段标记
|
||||
assert lines == [_line(LOOK)]
|
||||
|
||||
|
||||
async def test_markup_and_json_lines_mix():
|
||||
text = (_line({"op": "plan", "summary": "先看表"}) + "\n" + DSML_LOOK.replace('name="look"', 'name="update_node"')
|
||||
.replace('name="need"', 'name="id"').replace('name="targets"', 'name="config"') + _line({"op": "done"}))
|
||||
ops, diag = await _read(text)
|
||||
assert [op["op"] for op in ops] == ["plan", "update_node", "done"]
|
||||
assert ops[1]["id"] == "订单金额相关的表,要看清表名和字段" and diag.markup == 1 and diag.dropped == {}
|
||||
|
||||
|
||||
async def test_markup_that_cannot_be_read_is_diagnosed():
|
||||
text = ('<|DSML|function_calls>\n<|DSML|invoke name="run_sql">\n'
|
||||
'<|DSML|parameter name="sql" string="true">SELECT 1</|DSML|parameter>\n</|DSML|invoke>\n'
|
||||
'<|DSML|invoke name="look">\n<|DSML|parameter name="need" string="true">没写完')
|
||||
ops, diag = await _read(text)
|
||||
assert ops == [] and diag.markup == 1
|
||||
assert diag.dropped == {"工具调用标记的名字不认识": 1, "工具调用标记没写完": 1}, diag.dropped
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 当场纠正
|
||||
|
||||
async def test_a_dsml_look_drives_a_real_look_and_the_graph_lands(shop):
|
||||
"""部署上那一幕:第一次调用就把 look 写成了 DSML。现在照样去查看、接着写、图落地,模型只调两次。"""
|
||||
model = Turns([AIMessageChunk(content=DSML_LOOK)], [START, *_graph_ops(shop.name)])
|
||||
nodes: dict = {}
|
||||
gen = Generation(model, system="SYS", applier=GraphApplier(nodes, []), looks=await _tables(shop))
|
||||
events = [op async for op in gen.draft("REQ")]
|
||||
assert len(model.calls) == 2 and not gen.diagnosis.nudged
|
||||
assert model.calls[1][2] == ("ai", _line(LOOK))
|
||||
look = next(e for e in events if e["op"] == "look")
|
||||
assert look["status"] == "ok" and look["objects"][0]["named"] == ["orders"]
|
||||
assert set(nodes) == {"start", "q", "out"}
|
||||
|
||||
|
||||
async def test_prose_gets_one_correction_with_its_own_words_handed_back(shop):
|
||||
model = Turns([AIMessageChunk(content="我先看一下订单表的结构,\n再决定怎么搭。\n")], [START, *_graph_ops(shop.name)])
|
||||
nodes: dict = {}
|
||||
gen = Generation(model, system="SYS", applier=GraphApplier(nodes, []), looks=await _tables(shop))
|
||||
[op async for op in gen.draft("REQ")]
|
||||
assert len(model.calls) == 2 and gen.diagnosis.nudged and set(nodes) == {"start", "q", "out"}
|
||||
system, human, ai, nudge = model.calls[1]
|
||||
assert human == ("human", "REQ") and ai == ("ai", "我先看一下订单表的结构,\n再决定怎么搭。")
|
||||
assert nudge[0] == "human" and nudge[1].startswith("上面这次输出里没有一行能认出的操作:")
|
||||
assert "每个操作单独一行" in nudge[1] and nudge[1].endswith("请按输出格式重新给出这一轮的操作。")
|
||||
|
||||
|
||||
async def test_an_empty_continuation_after_a_look_is_corrected_too(shop):
|
||||
"""真模型重放里撞过:先 look,查看完接着写的那一次交回一大段读不成的东西,整轮照样是空的。look 不算进展,照样纠正。"""
|
||||
look = {"op": "look", "need": "订单金额", "targets": [{"kind": "table", "name": "orders"}]}
|
||||
model = Turns([look], [AIMessageChunk(content='{"op": "add_node", "node": {"id": "q"}}{"op": "done"\n')],
|
||||
[START, *_graph_ops(shop.name)])
|
||||
nodes: dict = {}
|
||||
gen = Generation(model, system="SYS", applier=GraphApplier(nodes, []), looks=await _tables(shop))
|
||||
[op async for op in gen.draft("REQ")]
|
||||
assert len(model.calls) == 3 and gen.diagnosis.nudged and set(nodes) == {"start", "q", "out"}
|
||||
*_, result, said, nudge = model.calls[2]
|
||||
assert result[1].startswith("查看结果") and said == ("ai", '{"op": "add_node", "node": {"id": "q"}}{"op": "done"')
|
||||
assert nudge[1].startswith("上面这次输出里没有一行能认出的操作:")
|
||||
|
||||
|
||||
async def test_only_thinking_gets_the_correction_appended_to_the_request(shop):
|
||||
thinking = AIMessageChunk(content="", additional_kwargs={"reasoning_content": "先看表。"})
|
||||
model = Turns([thinking], [START, *_graph_ops(shop.name)])
|
||||
gen = Generation(model, system="SYS", applier=GraphApplier({}, []), looks=await _tables(shop))
|
||||
[op async for op in gen.draft("REQ")]
|
||||
assert len(model.calls) == 2
|
||||
system, human = model.calls[1]
|
||||
assert human[0] == "human" and human[1].startswith("REQ\n\n上面这次输出里没有一行能认出的操作:只写了思考")
|
||||
|
||||
|
||||
async def test_the_correction_is_given_once_and_never_after_real_ops(shop):
|
||||
# 只交了 plan、又什么都没给:纠正一次,还是这样就收场(调用方报 copilot_empty)
|
||||
plan_only = Turns([{"op": "plan", "summary": "打算先看表"}])
|
||||
gen = Generation(plan_only, system="SYS", applier=GraphApplier({}, []), looks=await _tables(shop))
|
||||
[op async for op in gen.draft("REQ")]
|
||||
assert len(plan_only.calls) == 2 and gen.diagnosis.nudged and gen.diagnosis.calls == 2
|
||||
assert "没有给出任何操作" in plan_only.calls[1][-1][1]
|
||||
assert gen.empty().args[0] == "模型只说了打算,没有给出任何修改"
|
||||
# 交过真操作(这里是「无需修改」的 done)不算跑偏,不纠正
|
||||
done = Turns([{"op": "done", "explanation": "不用改"}])
|
||||
gen = Generation(done, system="SYS", applier=GraphApplier({}, []), looks=await _tables(shop))
|
||||
[op async for op in gen.draft("REQ")]
|
||||
assert len(done.calls) == 1 and not gen.diagnosis.nudged
|
||||
|
||||
|
||||
async def test_the_canvas_stream_recovers_without_an_error(client, monkeypatch, shop): # noqa: F811
|
||||
model = Turns([AIMessageChunk(content="好的,我来设计。\n")], [START, *_graph_ops(shop.name)])
|
||||
events = await _events(client, monkeypatch, model, {"instruction": "订单总金额", "intent": "build",
|
||||
"datasource_ids": [shop.id]})
|
||||
assert "error" not in _ops(events) and events[-1]["op"] == "final", _ops(events)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 报错的说法、提示词
|
||||
|
||||
@pytest.mark.parametrize("diag, headline", [
|
||||
(StreamDiagnosis(markup=1, text_chars=200), "模型把操作写成了工具调用的格式,没有按约定逐行输出,系统认不出"),
|
||||
(StreamDiagnosis(tool_calls=["run_sql"]), "模型把操作写成了工具调用的格式,没有按约定逐行输出,系统认不出"),
|
||||
(StreamDiagnosis(thinking_chars=30), "模型只写了思考,没有给出任何操作"),
|
||||
(StreamDiagnosis(), "模型什么都没有输出"),
|
||||
(StreamDiagnosis(text_chars=40, dropped={"不是完整的 JSON": 2}), "模型的输出格式不对,没有一行能认出的操作"),
|
||||
(StreamDiagnosis(text_chars=40, ops=1), "模型只说了打算,没有给出任何修改"),
|
||||
])
|
||||
def test_the_headline_says_what_actually_came_back(diag, headline):
|
||||
assert diag.headline() == headline
|
||||
|
||||
|
||||
def test_the_prompts_say_there_is_no_tool_call_interface():
|
||||
from app.api import copilot, copilot_blueprint
|
||||
from app.look.service import GUIDE_HEAD_PROMPT
|
||||
|
||||
assert "这里没有工具调用接口" in GUIDE_HEAD_PROMPT and "DSML" in GUIDE_HEAD_PROMPT
|
||||
for protocol in (copilot._STREAM_PROTOCOL, copilot.APP_PROTOCOL_PROMPT, copilot_blueprint.BLUEPRINT_PROTOCOL_PROMPT):
|
||||
assert "这里没有工具调用接口" in protocol
|
||||
assert "{correction}" in FORMAT_NUDGE_PROMPT
|
||||
@@ -163,7 +163,8 @@ async def test_a_cut_in_the_middle_of_the_ops_is_not_delivered_as_done(client, m
|
||||
|
||||
|
||||
async def test_output_that_parses_to_nothing_is_an_error(client, monkeypatch):
|
||||
"""格式跑偏(只写了说明文字、没有一行是操作),每一行都被跳过:一个操作都没有,也没有 done。诊断写明跳过了几行。"""
|
||||
"""格式跑偏(只写了说明文字、没有一行是操作),每一行都被跳过:一个操作都没有,也没有 done。当场带纠正重写一次还是这样,
|
||||
才报错;诊断写明跳过了几行(两次的)。"""
|
||||
model = _Scripted([AIMessageChunk(content="先看一下表结构,\n再决定怎么搭。\n"),
|
||||
AIMessageChunk(content="", response_metadata={"finish_reason": "stop"})])
|
||||
events = await _events(client, monkeypatch, model)
|
||||
@@ -171,7 +172,8 @@ async def test_output_that_parses_to_nothing_is_an_error(client, monkeypatch):
|
||||
assert "final" not in kinds
|
||||
[err] = [e for e in events if e["op"] == "error"]
|
||||
assert err["message"].startswith("助手本轮未完成:") and "画布未修改" in err["hint"]
|
||||
assert "跳过 2 行:2 行不是完整的 JSON" in err["detail"] and "先看一下表" not in err["detail"], err
|
||||
assert "跳过 4 行:4 行不是完整的 JSON" in err["detail"] and "先看一下表" not in err["detail"], err
|
||||
assert err["message"] == "助手本轮未完成:模型的输出格式不对,没有一行能认出的操作" and model.calls == 2, err
|
||||
|
||||
|
||||
async def test_a_pretty_printed_op_in_a_fence_is_read(client, monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user