mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-20 05:02:00 +00:00
助手只写了思考、一个操作都没给时自动重来一次:问数据和评测走同一个决定
真实模型验收里 deepseek 偶发只出思考、不出操作,生成过程交出 error「助手本轮未完成:模型没有给出任何修改,也没有说明 原因」,问数据这一轮照实报了,但不会自动重来——而这种偶发情况重来一次多半就好,「重来一次」的规矩本来就有 (chat/turn 的 MAX_ATTEMPTS = 2,含第一次)。 - chat/turn:error 也由 next_step 判:机读码是 EMPTY_GENERATION(copilot_empty)的判 retry,别的仍判失败(被截断的不重来: 额度不变,重来还是截断);error_reason 取重来时交给助手的原因(error 带的 reason,没有就去掉「助手本轮未完成:」前缀); - api/copilot.answer_ops:一个操作都没给时 CopilotIncomplete 带上这个码;error 操作多带 reason 和 next(retry | fail),和 final 的 next 一样由服务端定; - 问数据页(store/chat.ts)照 error.next 走:retry 就带着原因重来一次(同 retry_instruction 的从零重建措辞,次数照 MAX_ATTEMPTS),试满了才照实报;进度写「助手上一次没有给出任何操作,正在重新生成工作流…」(lib/terms 的 CHAT_RETRY_TEXT); 第一次建图的用量照样记进这一轮; - 评测(eval/answer.Driver.drive)用同一个 next_step / error_reason / retry_instruction:先重来一次,还不行才记建图失败、 助手调用失败;每次自动重来的原因照实记在 turn.retries(没过检查、复核判不可用的重来也记),评测页逐题明细写出来。 测试:test_chat_turn_policy.py 的 test_next_step_branches(error 的两支和 error_reason)、 test_the_frontend_retries_an_error_the_server_says_to_retry、test_answer_ops_says_to_retry_an_empty_generation、 test_the_eval_driver_retries_an_empty_generation_once(重来后回答 / 重来还不行,两次的用量都算); test_copilot_answer_ops.py 的 SSE 对照多认 error 上新加的 next、reason、code。去掉修复回跑都失败。 浏览器:check-chat 新加 emptyretry 一节(重来一次后照常回答、交回去的话带原因、两次用量都记;两次都不行照实报、不试第三次), check-chat 840 项全过。
This commit is contained in:
@@ -48,6 +48,11 @@ def test_next_step_branches():
|
||||
assert turn.next_step(_final(issues=[ERROR], autorun=False)) == "fail", "没过检查比只建不跑优先"
|
||||
assert turn.next_step({"op": "reply", "text": "已经查过了"}) == "reply"
|
||||
assert turn.next_step({"op": "error", "message": "x"}) == "fail"
|
||||
empty = {"op": "error", "message": "助手本轮未完成:模型没有给出任何修改,也没有说明原因",
|
||||
"code": turn.EMPTY_GENERATION, "reason": "模型没有给出任何修改,也没有说明原因"}
|
||||
assert turn.next_step(empty) == "retry", "只写了思考、一个操作都没给:按重来一次的规矩自动重来"
|
||||
assert turn.error_reason(empty) == "模型没有给出任何修改,也没有说明原因"
|
||||
assert turn.error_reason({"op": "error", "message": "助手本轮未完成:接入返回了 502"}) == "接入返回了 502"
|
||||
assert turn.next_step(None) == "asked", "流结束时只有反问"
|
||||
assert set(turn.NEXT_STEPS) == {"run", "ready", "fail"}
|
||||
|
||||
@@ -167,6 +172,74 @@ async def _run(output: dict, graph: dict, code: str) -> str:
|
||||
return run.id
|
||||
|
||||
|
||||
def test_the_frontend_retries_an_error_the_server_says_to_retry():
|
||||
"""问数据页照服务端给的 next 走:error 操作带 next=retry(只写了思考、一个操作都没给)时带着原因重来一次,不再直接报
|
||||
「助手本轮未完成」。读 chat.ts 里 error 那一支(行为另由 scripts/check-chat.mjs 的 emptyretry 一节在浏览器里守)。"""
|
||||
branch = re.search(r"case 'error':(.*?)break\n", _ts(), re.S)
|
||||
assert branch, "chat.ts 里找不到 error 那一支"
|
||||
assert re.search(r"op\.next === 'retry'", branch.group(1)) and "retry(" in branch.group(1), branch.group(1)
|
||||
|
||||
|
||||
async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch):
|
||||
"""生成过程一个操作都没给(deepseek 偶发只出思考):error 操作带机读码 copilot_empty、原因、next=retry。"""
|
||||
import json
|
||||
|
||||
import app.api.copilot as copilot
|
||||
from tests.test_usage_meter import Scripted
|
||||
|
||||
model = Scripted(model="claude-opus-5", replies=[[""]])
|
||||
|
||||
async def _model(*_a, **_k):
|
||||
return model, "claude-opus-5"
|
||||
|
||||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||||
events = []
|
||||
async with client.stream("POST", "/api/copilot/generate-stream",
|
||||
json={"instruction": "上周各地区销售额", "intent": "answer"}) as r:
|
||||
async for line in r.aiter_lines():
|
||||
if line.startswith("data: "):
|
||||
events.append(json.loads(line[6:]))
|
||||
[err] = [e for e in events if e["op"] == "error"]
|
||||
assert err["code"] == turn.EMPTY_GENERATION and err["next"] == "retry", err
|
||||
assert err["reason"] == "模型没有给出任何修改,也没有说明原因" and err["message"].startswith("助手本轮未完成")
|
||||
|
||||
|
||||
async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
|
||||
"""评测的作答(app/eval/answer.Driver)走同一个决定:第一次一个操作都没给就带着原因重来一次,原因照实记在 retries;
|
||||
重来还不行才照实报建图失败(助手调用失败),不再试第三次。"""
|
||||
import json
|
||||
|
||||
import app.api.copilot as copilot
|
||||
from app.eval import answer
|
||||
from tests.test_usage_meter import Scripted
|
||||
|
||||
reply = json.dumps({"op": "reply", "text": "上周没有华中区的数据"}, ensure_ascii=False)
|
||||
for replies, path in (([[""], [reply]], "reply"), ([[""]], "build_failed")):
|
||||
model = Scripted(model="claude-opus-5", replies=replies)
|
||||
seen: list[str] = []
|
||||
|
||||
async def _model(*_a, _m=model, **_k):
|
||||
return _m, "claude-opus-5"
|
||||
|
||||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||||
real = answer.Driver._generate
|
||||
|
||||
async def spy(self, instruction, base_graph, _seen=seen, _real=real):
|
||||
_seen.append(instruction)
|
||||
return await _real(self, instruction, base_graph)
|
||||
|
||||
monkeypatch.setattr(answer.Driver, "_generate", spy)
|
||||
driver = answer.Driver("上周华中区销售额", datasource_ids=[], ctx=answer.TurnContext(
|
||||
suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None))
|
||||
t = await driver.drive()
|
||||
assert t.path == path and t.attempts == 1 and model.calls == 2, (t.path, t.attempts, model.calls, t.reason)
|
||||
assert t.retries == ["模型没有给出任何修改,也没有说明原因"], t.retries
|
||||
assert seen[1] == turn.retry_instruction("上周华中区销售额", t.retries[0], False), seen
|
||||
if path == "build_failed":
|
||||
assert t.copilot_failed and t.reason.startswith("助手本轮未完成"), t.reason
|
||||
assert t.usage["assistant"]["calls"] == 2, "两次生成都是这一题花的"
|
||||
|
||||
|
||||
async def test_review_says_whether_and_how_to_retry(client, monkeypatch):
|
||||
import app.api.copilot as copilot
|
||||
from app.providers.factory import ProviderNotConfigured
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
|
||||
- 几种结局(建图交付、直接回答、只问不改、自查改不好、模型出错、没给操作、输出被截断)的 SSE 输出,和抽出来之前
|
||||
逐条一样(剧本见 SCENARIOS,抽出来之前录下的输出在 fixtures/copilot_answer_ops_baseline.json),末尾只多一条 usage,
|
||||
final 只多一个 next(问数据一轮的决定,chat/turn.py);
|
||||
final 只多一个 next(问数据一轮的决定,chat/turn.py);error 多一个 next 和原因 reason(一个操作都没给的另带机读码 code,
|
||||
next 是 retry:问数据页和评测据此自动重来一次);
|
||||
- 直接迭代 answer_ops 和 SSE 收到的是同一串;
|
||||
- 开流前的拒绝(模型没配好、限定的数据源都不在)照旧是 HTTP 错误,不进流。
|
||||
|
||||
@@ -133,6 +134,8 @@ BODY = {"instruction": "数一数门店", "intent": "answer"}
|
||||
|
||||
#: final 带的 next(chat/turn.next_step):问数据页照它走
|
||||
NEXT = {"graph_ok": "run", "graph_fixed": "run", "check_failed": "fail", "ask_only": "ready"}
|
||||
#: error 带的 (next, code):只有一个操作都没给的重来一次
|
||||
ERROR_NEXT = {"error": ("fail", None), "no_ops": ("retry", "copilot_empty"), "truncated": ("fail", None)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", list(SCENARIOS))
|
||||
@@ -144,6 +147,11 @@ async def test_the_sse_output_is_unchanged_but_for_a_trailing_usage(client, monk
|
||||
# final 多了 next(问数据一轮的决定下沉到后端),别的一个字不变
|
||||
finals = [op for op in got if op["op"] == "final"]
|
||||
assert [op.pop("next") for op in finals] == ([NEXT[name]] if name in NEXT else [])
|
||||
errors = [op for op in got if op["op"] == "error"]
|
||||
assert [(op.pop("next"), op.pop("code", None)) for op in errors] == \
|
||||
([ERROR_NEXT[name]] if name in ERROR_NEXT else [])
|
||||
for op in errors:
|
||||
assert op["message"] == f"助手本轮未完成:{op.pop('reason')}", op
|
||||
assert got[:-1] == expected
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user