mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-17 07:31:57 +00:00
助手一个操作都没给(copilot_empty)时看得出模型交回了什么,自动重来带一句纠正,格式跑偏的操作不再被静默丢掉:copilot_gen 新增 StreamDiagnosis,起草阶段记结束原因、正文(不算空白)和思考的字数、认出几行操作、原生工具调用的名字、跳过的行按原因数(不是完整的 JSON、不是 JSON 对象、缺 op、工具调用名字不认识、参数解析不了)、正文里写成文字的工具调用标记,只记长度个数和原因、不记原文;copilot_empty 的 error 带 detail(一句话)和 diagnostics(机读),问数据、画布、应用润色、蓝图四处一致,非流式的 502 也带上。重来时交给助手的原因换成按诊断给的纠正(只写了思考 / 什么都没输出 → 直接输出操作、要看表先写一行 look;格式跑偏 → 每个操作单独一行完整的 JSON;用了工具调用 → 这里不走工具调用),chat/turn.empty_retry_instruction 附在问题后面,不原样重发;诊断随 turn.retry 事件记下。原来静默丢掉的写法照常读:分成多行写的 JSON(以「{」开头读不成就往后攒)、和围栏写在同一行的操作、原生工具调用里名字是操作名的(参数就是操作的字段)
This commit is contained in:
@@ -933,9 +933,11 @@ async def generate(
|
||||
async for _op in gen.draft(human):
|
||||
pass
|
||||
if applier.replied is None and gen.asked is None and all(o.get("op") == "plan" for o in applier.seen):
|
||||
raise CopilotIncomplete("模型没有给出任何修改,也没有说明原因")
|
||||
raise gen.empty()
|
||||
except Exception as e: # noqa: BLE001
|
||||
reason, _ = explain_error(e)
|
||||
if isinstance(e, CopilotIncomplete) and e.detail:
|
||||
reason = f"{reason};{e.detail}"
|
||||
raise HTTPException(
|
||||
502, f"模型未返回可用的工作流({reason})。请换一种描述重试;如反复出现,"
|
||||
"请在输入框的「助手使用的模型」中换用指令遵循能力更强的模型",
|
||||
@@ -1770,11 +1772,13 @@ async def answer_ops(payload: GenerateIn, *, catalog_view: Any = None):
|
||||
if gen.asked is None and all(o.get("op") == "plan" for o in applier.seen):
|
||||
# 只写了思考(deepseek 这类推理模型偶发)、或者格式跑偏:带机读码,问数据页和评测按 chat/turn 的规矩自动
|
||||
# 重来一次(error.next = retry),还不行再照实报
|
||||
raise CopilotIncomplete("模型没有给出任何修改,也没有说明原因",
|
||||
hint="画布未修改。可以重试,或把需求描述得更具体", code=turn.EMPTY_GENERATION)
|
||||
# 错误详情里记诊断(结束原因、正文和思考的长度、跳过的行和原因、工具调用……,不记原文),重来时交给助手的
|
||||
# 原因换成一句纠正(Generation.empty)
|
||||
raise gen.empty(hint="画布未修改。可以重试,或把需求描述得更具体")
|
||||
except CopilotIncomplete as e:
|
||||
failed = {"op": "error", "message": f"助手本轮未完成:{e}", "hint": e.hint, "detail": e.detail,
|
||||
"reason": str(e), **({"code": e.code} if e.code else {})}
|
||||
"reason": e.correction or str(e), **({"code": e.code} if e.code else {}),
|
||||
**({"diagnostics": e.diagnostics} if e.diagnostics else {})}
|
||||
yield {**failed, "next": turn.next_step(failed)}
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
@@ -2456,8 +2460,7 @@ async def app_stream(payload: AppStreamIn, session: AsyncSession = Depends(get_s
|
||||
return
|
||||
if gen.asked is None and gen.done is None and not applier.effective:
|
||||
# 流正常结束,却没有 done、也没有一个操作落地:不能按「无需修改」交付
|
||||
raise CopilotIncomplete("模型没有给出任何修改,也没有说明原因",
|
||||
hint="应用草稿未修改。可以重试,或把要求描述得更具体")
|
||||
raise gen.empty(hint="应用草稿未修改。可以重试,或把要求描述得更具体")
|
||||
async for op in _with_heartbeat(gen.repair(), phase="repairing"):
|
||||
yield _sse(op)
|
||||
remaining = await checker.issues()
|
||||
|
||||
@@ -321,8 +321,7 @@ async def blueprint_stream(payload: BlueprintStreamIn, session: AsyncSession = D
|
||||
meta={"looks": looks.records})
|
||||
return
|
||||
if gen.asked is None and gen.done is None and not applier.effective:
|
||||
raise copilot.CopilotIncomplete("模型没有给出任何修改,也没有说明原因",
|
||||
hint="蓝图未修改。可以重试,或把需求描述得更具体")
|
||||
raise gen.empty(hint="蓝图未修改。可以重试,或把需求描述得更具体")
|
||||
async for op in copilot._with_heartbeat(gen.repair(), phase="repairing", started=started):
|
||||
yield _sse(op)
|
||||
remaining = await checker.issues() if gen.asked is None else []
|
||||
|
||||
+209
-21
@@ -24,9 +24,11 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import aclosing
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.engine.schema import NodeType
|
||||
@@ -53,30 +55,120 @@ DRAIN_CHARS = 400
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def _parse_op_line(line: str) -> dict[str, Any] | None:
|
||||
"""解析一行操作。围栏行、空行、解析失败、缺 op 的一律返回 None——
|
||||
模型偶尔犯规不该毁掉整条流,跳过就是。"""
|
||||
#: 操作行前后的围栏:```json {"op":…} 这样和围栏写在同一行的,去掉围栏再读
|
||||
_FENCE_HEAD = re.compile(r"^`{3,}[A-Za-z0-9_-]*\s*")
|
||||
_FENCE_TAIL = re.compile(r"\s*`{3,}$")
|
||||
#: 一个操作分成多行写(缩进排版的 JSON)时最多攒多少行、多少字:攒不成一个完整的 JSON 就当跳过
|
||||
_PENDING_LINES = 200
|
||||
_PENDING_CHARS = 20000
|
||||
#: 操作的名字(各入口的都在里面):原生工具调用叫这些名字的,参数就是操作的字段,照操作读(_tool_ops)
|
||||
OP_NAMES = frozenset({
|
||||
"plan", "add_node", "update_node", "remove_node", "add_edge", "remove_edge", "done", "reply", "look", "ask",
|
||||
"catalog_patch", "use_caliber", "question",
|
||||
"app_add", "app_update", "app_remove", "app_move", "app_page", "app_set",
|
||||
"bp_ask", "bp_set", "bp_add", "bp_remove", "bp_template", "fill_joint",
|
||||
})
|
||||
|
||||
|
||||
def _op_text(line: str) -> str:
|
||||
"""一行里要按操作读的那段:去掉首尾空白和同一行上的围栏。只有围栏的行(```、```json)交回空串。"""
|
||||
text = line.strip()
|
||||
if not text or text.startswith("```"):
|
||||
return None
|
||||
if text.startswith("```"):
|
||||
text = _FENCE_HEAD.sub("", text, count=1)
|
||||
return _FENCE_TAIL.sub("", text).strip()
|
||||
|
||||
|
||||
def _classify(text: str) -> tuple[dict[str, Any] | None, str | None]:
|
||||
"""(操作, None) 或 (None, 为什么跳过)。空的交回 (None, None):不算跳过。"""
|
||||
if not text:
|
||||
return None, None
|
||||
try:
|
||||
obj = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(obj, dict) or "op" not in obj:
|
||||
return None
|
||||
return obj
|
||||
return None, "不是完整的 JSON"
|
||||
if not isinstance(obj, dict):
|
||||
return None, "不是 JSON 对象"
|
||||
if "op" not in obj:
|
||||
return None, "缺 op"
|
||||
return obj, None
|
||||
|
||||
|
||||
def _parse_op_line(line: str) -> dict[str, Any] | None:
|
||||
"""解析一行操作。围栏行、空行、解析失败、缺 op 的一律返回 None——
|
||||
模型偶尔犯规不该毁掉整条流,跳过就是(跳过的照实记进 StreamDiagnosis)。和围栏写在同一行的操作去掉围栏再读。"""
|
||||
return _classify(_op_text(line))[0]
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamDiagnosis:
|
||||
"""一轮生成里模型交回了什么:只记长度、个数和原因,不记原文。一个操作都没认出来(copilot_empty)时写进错误详情,看得出
|
||||
是只写了思考、格式跑偏、还是用了工具调用;自动重来时据此给一句纠正(correction)。
|
||||
|
||||
finish 是最后一次调用的结束原因(finish_reason / stop_reason);text_chars、thinking_chars 是正文(不算空白)和思考的
|
||||
字数;dropped 是跳过的行按原因数;tool_calls 是原生工具调用的
|
||||
名字(按操作读了的也记);markup 是正文里写成文字的工具调用标记(<tool_call>、DeepSeek 的 DSML……)出现了几次调用。"""
|
||||
|
||||
calls: int = 0
|
||||
finish: str = ""
|
||||
text_chars: int = 0
|
||||
thinking_chars: int = 0
|
||||
ops: int = 0
|
||||
tool_calls: list[str] = field(default_factory=list)
|
||||
dropped: dict[str, int] = field(default_factory=dict)
|
||||
markup: int = 0
|
||||
|
||||
def drop(self, reason: str) -> None:
|
||||
self.dropped[reason] = self.dropped.get(reason, 0) + 1
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {"calls": self.calls, "finish": self.finish or None, "text_chars": self.text_chars,
|
||||
"thinking_chars": self.thinking_chars, "ops": self.ops, "tool_calls": list(self.tool_calls),
|
||||
"dropped": dict(self.dropped), "markup": self.markup}
|
||||
|
||||
def summary(self) -> str:
|
||||
"""给人看的一句:「结束原因 stop;正文 0 字,思考 11 字;认出 0 行操作;……」。"""
|
||||
parts = [f"结束原因 {self.finish or '未报'}", f"正文 {self.text_chars} 字,思考 {self.thinking_chars} 字",
|
||||
f"认出 {self.ops} 行操作"]
|
||||
if self.tool_calls:
|
||||
parts.append(f"原生工具调用 {len(self.tool_calls)} 个({'、'.join(self.tool_calls[:5])})")
|
||||
if self.dropped:
|
||||
parts.append(f"跳过 {sum(self.dropped.values())} 行:" + "、".join(f"{n} 行{why}" for why, n in self.dropped.items()))
|
||||
if self.markup:
|
||||
parts.append("正文里有写成文字的工具调用标记")
|
||||
if self.calls > 1:
|
||||
parts.append(f"共调用模型 {self.calls} 次")
|
||||
return ";".join(parts)
|
||||
|
||||
def correction(self) -> str:
|
||||
"""自动重来时交给助手的那句纠正(chat/turn.empty_retry_instruction 接在问题后面)。"""
|
||||
if self.tool_calls:
|
||||
return (f"用了工具调用接口({'、'.join(self.tool_calls[:3])}),这里不走工具调用;每个操作写成正文里单独一行的 "
|
||||
"JSON,要看表就写一行 {\"op\":\"look\",…}")
|
||||
if self.markup:
|
||||
return "把操作写成了工具调用的标记,这里不走工具调用;每个操作写成单独一行的 JSON,要看表就写一行 {\"op\":\"look\",…}"
|
||||
if not self.text_chars and self.thinking_chars:
|
||||
return "只写了思考、没有给出任何操作;这次请直接输出操作(每行一个 JSON),要看表就先输出一行 look"
|
||||
if not self.text_chars:
|
||||
return "什么都没有输出;这次请直接输出操作(每行一个 JSON),要看表就先输出一行 look"
|
||||
if self.dropped:
|
||||
return ("输出里没有一行能认出的操作;每个操作单独一行、写成一行完整的 JSON(带 op),不要夹说明文字,"
|
||||
"要看表就先输出一行 look")
|
||||
return "没有给出任何操作;请先 look 看表,或者直接给出操作"
|
||||
|
||||
|
||||
class CopilotIncomplete(Exception):
|
||||
"""模型这一轮没说完:输出被截断,或者一个操作都没给。画布不能按「无需修改」交付。code 是机读码:一个操作都没给的是
|
||||
chat/turn.EMPTY_GENERATION(问数据一轮按它自动重来一次),截断的不给。"""
|
||||
chat/turn.EMPTY_GENERATION(问数据一轮按它自动重来一次),截断的不给。correction 是重来时交给助手的那句纠正(error 的
|
||||
reason),diagnostics 是机读的诊断(StreamDiagnosis.as_dict),都只在一个操作都没给时有。"""
|
||||
|
||||
def __init__(self, reason: str, *, hint: str = "", detail: str = "", code: str = "") -> None:
|
||||
def __init__(self, reason: str, *, hint: str = "", detail: str = "", code: str = "", correction: str = "",
|
||||
diagnostics: dict[str, Any] | None = None) -> None:
|
||||
super().__init__(reason)
|
||||
self.hint = hint or "画布未修改。可以重试,或把需求拆成几步分别提出"
|
||||
self.detail = detail
|
||||
self.code = code
|
||||
self.correction = correction
|
||||
self.diagnostics = diagnostics
|
||||
|
||||
|
||||
async def _drain(stream: Any, already: int) -> None:
|
||||
@@ -92,7 +184,29 @@ async def _drain(stream: Any, already: int) -> None:
|
||||
return
|
||||
|
||||
|
||||
async def _iter_ops(model: Any, messages: list[Any], *, lines: list[str] | None = None):
|
||||
def _tool_ops(calls: dict[int, dict[str, str]], diag: StreamDiagnosis) -> list[dict[str, Any]]:
|
||||
"""原生工具调用(流里的 tool_call_chunks,按 index 拼好的 {name, args})→ 操作:名字是操作名的,参数就是操作的字段。
|
||||
名字不认识、参数解析不了的照实记进诊断——以前整个工具调用一声不响就丢了。"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for index in sorted(calls):
|
||||
name, args = calls[index].get("name") or "", calls[index].get("args") or ""
|
||||
diag.tool_calls.append(name or "(没有名字)")
|
||||
if name not in OP_NAMES:
|
||||
diag.drop("工具调用的名字不认识")
|
||||
continue
|
||||
try:
|
||||
fields = json.loads(args) if args.strip() else {}
|
||||
except json.JSONDecodeError:
|
||||
fields = None
|
||||
if not isinstance(fields, dict):
|
||||
diag.drop("工具调用的参数解析不了")
|
||||
continue
|
||||
out.append({**{k: v for k, v in fields.items() if k != "op"}, "op": name})
|
||||
return out
|
||||
|
||||
|
||||
async def _iter_ops(model: Any, messages: list[Any], *, lines: list[str] | None = None,
|
||||
diag: StreamDiagnosis | None = None):
|
||||
"""流式读模型输出,按行切出操作。done、look、ask 之后即停——后面就算有闲话也不要了;look 之后停下也就是
|
||||
「读到这一行就停止本次输出」。停下之后不马上关流:交出那一行、调用方接着要下一条时先把流读完(_drain,内容丢掉,最多
|
||||
DRAIN_CHARS 字),流末尾那一段用量才收得到——以前读到这一行就关流,输出 token 一个都拿不到,问数据这一轮的花费多半是下限
|
||||
@@ -101,35 +215,99 @@ async def _iter_ops(model: Any, messages: list[Any], *, lines: list[str] | None
|
||||
除了操作,还把模型的思考原样转出去(thinking_text:Claude 的 thinking 块、OpenAI 兼容系的 reasoning_content),
|
||||
不把非 JSON 行当成思考。lines 给了的话,认出来的操作行原样记进去:接着写时作为 assistant 消息交回模型。
|
||||
|
||||
格式跑偏的几种写法照样认(以前一声不响就丢了):一个操作分成多行写的 JSON(以「{」开头的一行读不成,就接着往后攒,
|
||||
攒到读得成为止;中途来了一行自己就读得成的,攒着的那段算跳过)、和围栏写在同一行的操作、原生工具调用里名字是操作名
|
||||
的(_tool_ops,流读完再按先后交出)。diag 给了的话,交回了什么(长度、个数、跳过的原因)记进去,不记原文。
|
||||
|
||||
没等到 done 就因为额度用完停下的,抛 CopilotIncomplete:截断之前的操作照常转出去,由调用方决定整轮作废。
|
||||
"""
|
||||
from app.engine.state import message_text as _text
|
||||
from app.engine.state import thinking_text
|
||||
from app.engine.toolcalls import leaked_markup
|
||||
|
||||
diag = diag if diag is not None else StreamDiagnosis()
|
||||
diag.calls += 1
|
||||
buffer = ""
|
||||
cut = ""
|
||||
pending: list[str] = []
|
||||
calls: dict[int, dict[str, str]] = {}
|
||||
head = "" # 正文的开头一段:只用来认写成文字的工具调用标记,不进诊断
|
||||
|
||||
def read(line: str) -> dict[str, Any] | None:
|
||||
"""一行 → 操作(认出来了)或 None;多行的 JSON 攒在 pending 里。"""
|
||||
text = _op_text(line)
|
||||
if pending:
|
||||
op, _ = _classify(text)
|
||||
if op is not None:
|
||||
diag.drop("不是完整的 JSON") # 自己就是一个操作的一行来了:攒着的那段读不成了
|
||||
pending.clear()
|
||||
return _single(op, None)
|
||||
pending.append(text)
|
||||
joined = "\n".join(pending)
|
||||
op, why = _classify(joined)
|
||||
if op is not None:
|
||||
pending.clear()
|
||||
return _single(op, None)
|
||||
if len(pending) > _PENDING_LINES or len(joined) > _PENDING_CHARS:
|
||||
diag.drop("不是完整的 JSON")
|
||||
pending.clear()
|
||||
return None
|
||||
op, why = _classify(text)
|
||||
if why == "不是完整的 JSON" and text.startswith("{"):
|
||||
pending.append(text) # 可能是分成多行写的一个操作:接着往后攒
|
||||
return None
|
||||
return _single(op, why)
|
||||
|
||||
def _single(op: dict[str, Any] | None, why: str | None) -> dict[str, Any] | None:
|
||||
if why:
|
||||
diag.drop(why)
|
||||
if op is not None:
|
||||
diag.ops += 1
|
||||
return op
|
||||
|
||||
def record(op: dict[str, Any]) -> None:
|
||||
if lines is not None:
|
||||
lines.append(json.dumps(op, ensure_ascii=False))
|
||||
|
||||
async with aclosing(model.astream(messages)) as stream:
|
||||
async for chunk in stream:
|
||||
reasoning = thinking_text(chunk)
|
||||
if reasoning:
|
||||
diag.thinking_chars += len(reasoning)
|
||||
yield {"op": "thinking", "delta": reasoning}
|
||||
for part in getattr(chunk, "tool_call_chunks", None) or []:
|
||||
slot = calls.setdefault(int(part.get("index") or 0), {"name": "", "args": ""})
|
||||
slot["name"] = slot["name"] or str(part.get("name") or "")
|
||||
slot["args"] += str(part.get("args") or "")
|
||||
meta = getattr(chunk, "response_metadata", None) or {}
|
||||
if meta.get("finish_reason") or meta.get("stop_reason"):
|
||||
diag.finish = str(meta.get("finish_reason") or meta.get("stop_reason"))
|
||||
|
||||
buffer += _text(chunk)
|
||||
text = _text(chunk)
|
||||
diag.text_chars += sum(1 for c in text if not c.isspace())
|
||||
if len(head) < 4000:
|
||||
head += text[:4000 - len(head)]
|
||||
buffer += text
|
||||
while "\n" in buffer:
|
||||
line, buffer = buffer.split("\n", 1)
|
||||
op = _parse_op_line(line)
|
||||
op = read(line)
|
||||
if op:
|
||||
if lines is not None:
|
||||
lines.append(line.strip())
|
||||
record(op)
|
||||
yield op
|
||||
if op.get("op") in _STOP_OPS:
|
||||
await _drain(stream, len(buffer))
|
||||
return
|
||||
cut = _cut_off(chunk) or cut
|
||||
op = _parse_op_line(buffer)
|
||||
if op:
|
||||
if lines is not None:
|
||||
lines.append(buffer.strip())
|
||||
if leaked_markup(head):
|
||||
diag.markup += 1
|
||||
tail = [op for op in [read(buffer)] if op]
|
||||
if pending:
|
||||
diag.drop("不是完整的 JSON")
|
||||
pending.clear()
|
||||
tools = _tool_ops(calls, diag)
|
||||
diag.ops += len(tools)
|
||||
for op in tail + tools:
|
||||
record(op)
|
||||
yield op
|
||||
if op.get("op") in _STOP_OPS:
|
||||
return
|
||||
@@ -709,6 +887,15 @@ class Generation:
|
||||
self.repair_error: Exception | None = None
|
||||
#: 这一轮调了几次主模型(含接着写、修正),验收时统计
|
||||
self.calls = 0
|
||||
#: 起草阶段模型交回了什么(只记长度、个数和原因):一个操作都没认出来时写进错误详情
|
||||
self.diagnosis = StreamDiagnosis()
|
||||
|
||||
def empty(self, message: str = "模型没有给出任何修改,也没有说明原因", *, hint: str = "") -> CopilotIncomplete:
|
||||
"""一个操作都没给(copilot_empty)的报错:带诊断(detail、diagnostics)和重来时交给助手的纠正(correction)。"""
|
||||
from app.chat.turn import EMPTY_GENERATION
|
||||
|
||||
return CopilotIncomplete(message, hint=hint, code=EMPTY_GENERATION, detail=self.diagnosis.summary(),
|
||||
correction=self.diagnosis.correction(), diagnostics=self.diagnosis.as_dict())
|
||||
|
||||
async def draft(self, request: str) -> AsyncIterator[dict[str, Any]]:
|
||||
async for op in self._converse(request, LookBudget(self.look_limit), repair=False):
|
||||
@@ -752,7 +939,8 @@ class Generation:
|
||||
lines: list[str] = []
|
||||
look: dict[str, Any] | None = None
|
||||
self.calls += 1
|
||||
async with aclosing(_iter_ops(self.model, messages, lines=lines)) as ops:
|
||||
async with aclosing(_iter_ops(self.model, messages, lines=lines,
|
||||
diag=None if repair else self.diagnosis)) as ops:
|
||||
async for op in ops:
|
||||
kind = op.get("op")
|
||||
if kind == "thinking":
|
||||
|
||||
@@ -163,7 +163,8 @@ class TurnDriver:
|
||||
async def on_op(self, op: dict[str, Any]) -> None:
|
||||
"""生成过程的每一条操作(含思考、心跳),收下之前先交给这里。"""
|
||||
|
||||
async def on_retry(self, reason: str, *, kind: str, base_graph: dict[str, Any] | None) -> None:
|
||||
async def on_retry(self, reason: str, *, kind: str, base_graph: dict[str, Any] | None,
|
||||
detail: str | None = None) -> None:
|
||||
"""要自动重来一次了(kind:empty 生成过程一个操作都没给、blocked 没过检查、review 复核判不可用)。"""
|
||||
|
||||
def prepare_graph(self, graph: dict[str, Any]) -> dict[str, Any]:
|
||||
@@ -338,11 +339,14 @@ class TurnDriver:
|
||||
# 生成过程一个操作都没给(只写了思考):按重来一次的规矩带着原因重来(chat/turn.next_step)
|
||||
step = terminal.get("next") or policy.next_step(terminal)
|
||||
if step == policy.NEXT_RETRY and t.attempts + 1 < policy.MAX_ATTEMPTS:
|
||||
# 原因是生成过程给的那句纠正(只写了思考、格式跑偏、用了工具调用……),附在问题后面重来,
|
||||
# 不原样重发;诊断(error 的 detail)随重来的事件记下
|
||||
reason = policy.error_reason(terminal)
|
||||
t.attempts += 1
|
||||
t.retries.append(reason)
|
||||
instruction = policy.retry_instruction(self.question, reason, policy.has_nodes(base_graph))
|
||||
await self.on_retry(reason, kind="empty", base_graph=base_graph)
|
||||
instruction = policy.empty_retry_instruction(self.question, reason)
|
||||
await self.on_retry(reason, kind="empty", base_graph=base_graph,
|
||||
detail=str(terminal.get("detail") or "") or None)
|
||||
continue
|
||||
t.path, t.reason, t.copilot_failed = "build_failed", str(terminal.get("message") or "助手本轮未完成"), True
|
||||
return t
|
||||
|
||||
@@ -191,13 +191,14 @@ class ChatDriver(TurnDriver):
|
||||
finally:
|
||||
await self._flush_thinking()
|
||||
|
||||
async def on_retry(self, reason: str, *, kind: str, base_graph: dict[str, Any] | None) -> None:
|
||||
async def on_retry(self, reason: str, *, kind: str, base_graph: dict[str, Any] | None,
|
||||
detail: str | None = None) -> None:
|
||||
from app.chat import turn as policy
|
||||
|
||||
# 换下来的那次运行交出去并进 earlier;这一次从「还没有运行」起,收尾时不再把它当这一次的运行记一遍
|
||||
run, self.last_run = (self.last_run if kind == "review" else None), None
|
||||
await self.execution.retrying(reason, kind=kind, base=policy.has_nodes(base_graph), attempt=self.turn.attempts,
|
||||
run=run)
|
||||
run=run, detail=detail)
|
||||
|
||||
async def run_input(self, session: AsyncSession, graph: dict[str, Any]) -> dict[str, Any]:
|
||||
"""入口填问题和这一轮之前的几轮问答(conversations.run_history,同以前页面上的 historyOf):重试、重新运行较早的
|
||||
@@ -364,13 +365,16 @@ class ChatTurnExecution(runtime.JobRun):
|
||||
meta={"runId": run.id, "runClass": run.run_class, "runStatus": run.status})
|
||||
await self.emit("run", {"run_id": run.id, "status": run.status, "run_class": run.run_class})
|
||||
|
||||
async def retrying(self, reason: str, *, kind: str, base: bool, attempt: int, run: Any = None) -> None:
|
||||
"""自动重来一次:换下来的那次运行的用量并进 earlier;这一行回到「还没有运行」。"""
|
||||
async def retrying(self, reason: str, *, kind: str, base: bool, attempt: int, run: Any = None,
|
||||
detail: str | None = None) -> None:
|
||||
"""自动重来一次:换下来的那次运行的用量并进 earlier;这一行回到「还没有运行」。detail:生成过程一个操作都没给时的
|
||||
诊断(结束原因、正文和思考的长度、跳过的行……,不含原文),随 turn.retry 事件记下。"""
|
||||
ops: list[tuple[Any, ...]] = [("set", "run", record.usage_total(run.usage) or {})] if run is not None else []
|
||||
self.run_id = None
|
||||
await self._job(attempt=attempt, run_id=None)
|
||||
usage = await self.save(run_id=None, meta={"runId": None, "runStatus": None}, usage=[*ops, ("retire",)])
|
||||
await self.emit("turn.retry", {"kind": kind, "reason": reason, "base": base, "attempt": attempt, "usage": usage})
|
||||
await self.emit("turn.retry", {"kind": kind, "reason": reason, "base": base, "attempt": attempt, "usage": usage,
|
||||
**({"detail": detail} if detail else {})})
|
||||
|
||||
# ---------------- 执行 ----------------
|
||||
|
||||
|
||||
@@ -51,6 +51,12 @@ RETRY_WITH_BASE = ("这个工作流上一次运行没有成功:{reason}。\n\n
|
||||
"改完继续回答原来的问题:{question}")
|
||||
#: 没有图可交回(上一次连图都没建出来)时:原问题后面缀上原因,从零再建
|
||||
RETRY_FROM_SCRATCH = "{question}\n\n(上一次没有成功:{reason}。请针对这个原因调整,重新给出完整的工作流。)"
|
||||
#: 生成过程一个操作都没给(EMPTY_GENERATION)时自动重来交给助手的话:原问题后面缀上那句纠正(error 的 reason,见
|
||||
#: copilot_gen.StreamDiagnosis.correction:只写了思考、格式跑偏、用了工具调用各有一句)。不原样重发——原样重发多半还是
|
||||
#: 只写思考(部署上撞过两次都是空的)
|
||||
RETRY_EMPTY = "{question}\n\n(上一次没有给出可用的操作:{reason}。)"
|
||||
#: 没有纠正可用(老服务端只给了「模型没有给出任何修改」)时补的那句
|
||||
EMPTY_FALLBACK = "这次请直接按输出格式给出操作(每行一个 JSON),要看表就先输出一行 look"
|
||||
#: 用户点「重试本轮」时交给助手的话(以前在问数据页 retryTurn 里拼,一轮搬到服务端之后在这里):有图时说的是「没有给出可用的
|
||||
#: 结果」——上一次未必是运行失败(停止了、被驳回、结论不可用都算),原因是那一次的归档说法(chat/record.attempt_reason);
|
||||
#: 没有图的同 RETRY_FROM_SCRATCH
|
||||
@@ -149,6 +155,14 @@ def retry_instruction(question: str, reason: str, has_base: bool) -> str:
|
||||
return template.format(question=question, reason=reason)
|
||||
|
||||
|
||||
def empty_retry_instruction(question: str, reason: str) -> str:
|
||||
"""生成过程一个操作都没给、自动重来时交给助手的话:问题后面附那句纠正(reason);纠正里没说怎么做的补 EMPTY_FALLBACK。"""
|
||||
reason = reason.strip().rstrip("。")
|
||||
if "look" not in reason:
|
||||
reason = f"{reason};{EMPTY_FALLBACK}" if reason else EMPTY_FALLBACK
|
||||
return RETRY_EMPTY.format(question=question, reason=reason)
|
||||
|
||||
|
||||
def user_retry_instruction(question: str, reason: str, has_base: bool) -> str:
|
||||
"""用户点「重试本轮」时交给助手的话。has_base:有上一次建出的图可以交回去改。"""
|
||||
template = RETRY_USER_WITH_BASE if has_base else RETRY_FROM_SCRATCH
|
||||
@@ -160,7 +174,8 @@ def has_nodes(graph: Any) -> bool:
|
||||
return isinstance(graph, Mapping) and bool(graph.get("nodes"))
|
||||
|
||||
|
||||
__all__ = ["EMPTY_GENERATION", "MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY", "NEXT_REPLY", "NEXT_RETRY",
|
||||
"NEXT_RUN", "NEXT_STEPS", "QUESTION_FIELD", "RETRY_ERRORS", "RETRY_FROM_SCRATCH", "RETRY_USER_WITH_BASE",
|
||||
"RETRY_WITH_BASE", "blocked_reason", "error_reason", "fill_input", "has_nodes", "next_step", "retry_after_review",
|
||||
__all__ = ["EMPTY_FALLBACK", "EMPTY_GENERATION", "MAX_ATTEMPTS", "NEXT_ASKED", "NEXT_FAIL", "NEXT_READY",
|
||||
"NEXT_REPLY", "NEXT_RETRY", "NEXT_RUN", "NEXT_STEPS", "QUESTION_FIELD", "RETRY_EMPTY", "RETRY_ERRORS",
|
||||
"RETRY_FROM_SCRATCH", "RETRY_USER_WITH_BASE", "RETRY_WITH_BASE", "blocked_reason",
|
||||
"empty_retry_instruction", "error_reason", "fill_input", "has_nodes", "next_step", "retry_after_review",
|
||||
"retry_instruction", "review_reason", "user_retry_instruction"]
|
||||
|
||||
@@ -194,7 +194,10 @@ async def test_answer_ops_says_to_retry_an_empty_generation(client, monkeypatch)
|
||||
events.append(json.loads(line[6:]))
|
||||
[err] = [e for e in events if e["op"] == "error"]
|
||||
assert err["code"] == turn.EMPTY_GENERATION and err["next"] == "retry", err
|
||||
assert err["reason"] == "模型没有给出任何修改,也没有说明原因" and err["message"].startswith("助手本轮未完成")
|
||||
# 原因是重来时交给助手的那句纠正(这次连思考都没有),诊断写在 detail(不含原文)
|
||||
assert err["reason"].startswith("什么都没有输出") and "look" in err["reason"], err
|
||||
assert err["message"] == "助手本轮未完成:模型没有给出任何修改,也没有说明原因"
|
||||
assert "正文 0 字,思考 0 字" in err["detail"] and err["diagnostics"]["ops"] == 0
|
||||
|
||||
|
||||
async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
|
||||
@@ -226,8 +229,10 @@ async def test_the_eval_driver_retries_an_empty_generation_once(monkeypatch):
|
||||
suite_id="s", suite_name="虚构题集", eval_run_id="e", case_id=None, case_seq=1, actor=None))
|
||||
t = await driver.drive()
|
||||
assert t.path == path and t.attempts == 1 and model.calls == 2, (t.path, t.attempts, model.calls, t.reason)
|
||||
assert t.retries == ["模型没有给出任何修改,也没有说明原因"], t.retries
|
||||
assert seen[1] == turn.retry_instruction("上周华中区销售额", t.retries[0], False), seen
|
||||
assert len(t.retries) == 1 and t.retries[0].startswith("什么都没有输出"), t.retries
|
||||
# 重来时问题后面附那句纠正,不原样重发
|
||||
assert seen[1] == turn.empty_retry_instruction("上周华中区销售额", t.retries[0]) != seen[0], seen
|
||||
assert "什么都没有输出" in seen[1]
|
||||
if path == "build_failed":
|
||||
assert t.copilot_failed and t.reason.startswith("助手本轮未完成"), t.reason
|
||||
assert t.usage["assistant"]["calls"] == 2, "两次生成都是这一题花的"
|
||||
|
||||
@@ -866,7 +866,9 @@ async def test_an_empty_generation_is_retried_once_on_the_server(client, monkeyp
|
||||
await _done(turn_id)
|
||||
events = await _events(turn_id)
|
||||
[retry] = [e["data"] for e in events if e["type"] == "turn.retry"]
|
||||
assert retry["kind"] == "empty" and retry["reason"] == "模型没有给出任何修改,也没有说明原因"
|
||||
# 原因是重来时交给助手的那句纠正(这次连思考都没有),诊断(不含原文)随事件记下
|
||||
assert retry["kind"] == "empty" and retry["reason"].startswith("什么都没有输出"), retry
|
||||
assert "正文 0 字" in retry["detail"], retry
|
||||
row = await _row(turn_id)
|
||||
assert row.status == "done" and row.answer == "上周没有华中区的数据" and row.meta["noQuery"] is True
|
||||
assert row.meta["usage"]["copilot"]["calls"] == 2, "两次建图都是这一轮花的"
|
||||
|
||||
@@ -151,7 +151,14 @@ async def test_the_sse_output_is_unchanged_but_for_a_trailing_usage(client, monk
|
||||
assert [(op.pop("next"), op.pop("code", None)) for op in errors] == \
|
||||
([ERROR_NEXT[name]] if name in ERROR_NEXT else [])
|
||||
for op in errors:
|
||||
assert op["message"] == f"助手本轮未完成:{op.pop('reason')}", op
|
||||
reason = op.pop("reason")
|
||||
if name == "no_ops":
|
||||
# 一个操作都没给:原因是重来时交给助手的那句纠正,诊断(不含原文)写进 detail、diagnostics,别的照旧
|
||||
assert "每个操作单独一行" in reason and op.pop("diagnostics")["dropped"] == {"不是 JSON 对象": 1}, op
|
||||
assert op["detail"].startswith("结束原因"), op
|
||||
op["detail"] = ""
|
||||
else:
|
||||
assert op["message"] == f"助手本轮未完成:{reason}", op
|
||||
assert got[:-1] == expected
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""助手一个操作都没给(copilot_empty)时看得出模型交回了什么,自动重来那一次带一句纠正;格式跑偏的操作不再被静默丢掉。
|
||||
|
||||
部署上撞过:问数据建图两次都是空的——模型在思考里说「先看一下表」,却一个操作都没给,自动重来一次还是这样,错误详情为空,
|
||||
看不出模型实际返回了什么。现在:
|
||||
|
||||
- 诊断(只记长度、个数和原因,不记原文):结束原因、正文和思考各多少字、认出几行操作、原生工具调用几个(叫什么)、跳过了
|
||||
哪些行(不是完整的 JSON、缺 op……)、正文里有没有写成文字的工具调用标记。写进 error 的 detail(给人看的一句)和
|
||||
diagnostics(机读);
|
||||
- 重来时交给助手的原因换成一句纠正(只写了思考 → 直接输出操作、要看表先写一行 look;格式跑偏 → 每个操作单独一行完整的
|
||||
JSON;用了工具调用 → 这里不走工具调用),不原样重发;
|
||||
- 原来会被静默丢掉的几种写法认出来:一个操作分成多行写的 JSON、和围栏写在同一行的操作、原生工具调用里名字是操作名的
|
||||
(look、add_node……,参数就是操作的字段)。名字不认识、参数解析不了的照实记进诊断。
|
||||
|
||||
需求、表名都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessageChunk
|
||||
|
||||
from app.api.copilot_gen import StreamDiagnosis, _iter_ops
|
||||
from app.chat import turn as policy
|
||||
from tests.test_copilot_answer_ops import _sse, _use, client # noqa: F401
|
||||
|
||||
|
||||
class _Model:
|
||||
def __init__(self, chunks: list[AIMessageChunk]) -> None:
|
||||
self.chunks = chunks
|
||||
|
||||
async def astream(self, messages):
|
||||
for chunk in self.chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
def _think(text: str) -> AIMessageChunk:
|
||||
return AIMessageChunk(content="", additional_kwargs={"reasoning_content": text})
|
||||
|
||||
|
||||
async def _ops(chunks: list[AIMessageChunk]) -> tuple[list[dict], StreamDiagnosis]:
|
||||
diag = StreamDiagnosis()
|
||||
ops = [op async for op in _iter_ops(_Model(chunks), [], diag=diag)]
|
||||
return [op for op in ops if op["op"] != "thinking"], diag
|
||||
|
||||
|
||||
async def test_only_thinking_is_diagnosed():
|
||||
ops, diag = await _ops([_think("先看一下表结构再决定。"),
|
||||
AIMessageChunk(content="", response_metadata={"finish_reason": "stop"})])
|
||||
assert ops == []
|
||||
assert diag.text_chars == 0 and diag.thinking_chars == len("先看一下表结构再决定。") and diag.ops == 0
|
||||
assert diag.finish == "stop"
|
||||
assert "只写了思考" in diag.correction() and "look" in diag.correction()
|
||||
summary = diag.summary()
|
||||
assert "结束原因 stop" in summary and "正文 0 字" in summary and "思考 11 字" in summary
|
||||
assert "先看一下表" not in summary and "先看一下表" not in json.dumps(diag.as_dict(), ensure_ascii=False)
|
||||
|
||||
|
||||
async def test_skipped_lines_are_counted_with_reasons():
|
||||
# 最后一行以「{」开头、读不成:可能是分成多行写的操作,往后攒,流读完还是读不成的算跳过
|
||||
text = '这是说明文字\n[1, 2]\n{"note": "缺 op"}\n{"op": "look", "need": \n'
|
||||
ops, diag = await _ops([AIMessageChunk(content=text)])
|
||||
assert ops == []
|
||||
assert diag.dropped == {"不是完整的 JSON": 2, "不是 JSON 对象": 1, "缺 op": 1}, diag.dropped
|
||||
assert "跳过 4 行:2 行不是完整的 JSON、1 行不是 JSON 对象、1 行缺 op" in diag.summary()
|
||||
assert "每个操作单独一行" in diag.correction()
|
||||
|
||||
|
||||
async def test_a_pretty_printed_op_is_read_not_dropped():
|
||||
text = '{"op": "plan", "summary": "先看表"}\n{\n "op": "look",\n "need": "产量表的字段"\n}\n'
|
||||
ops, diag = await _ops([AIMessageChunk(content=text)])
|
||||
assert [op["op"] for op in ops] == ["plan", "look"] and ops[1]["need"] == "产量表的字段"
|
||||
assert diag.dropped == {} and diag.ops == 2
|
||||
|
||||
|
||||
async def test_an_op_on_the_fence_line_is_read():
|
||||
text = '```json {"op": "plan", "summary": "先看表"}\n{"op": "look", "need": "产量表"}```\n'
|
||||
ops, _ = await _ops([AIMessageChunk(content=text)])
|
||||
assert [op["op"] for op in ops] == ["plan", "look"]
|
||||
|
||||
|
||||
async def test_a_native_tool_call_named_after_an_op_becomes_that_op():
|
||||
chunks = [AIMessageChunk(content="", tool_call_chunks=[
|
||||
{"name": "look", "args": '{"need": "产量', "id": "c1", "index": 0}]),
|
||||
AIMessageChunk(content="", tool_call_chunks=[{"name": None, "args": '表的字段"}', "id": None, "index": 0}]),
|
||||
AIMessageChunk(content="", tool_call_chunks=[{"name": "run_sql", "args": '{"sql": "SELECT 1"}', "id": "c2",
|
||||
"index": 1}]),
|
||||
AIMessageChunk(content="", tool_call_chunks=[{"name": "add_node", "args": '{"node": ', "id": "c3",
|
||||
"index": 2}])]
|
||||
ops, diag = await _ops(chunks)
|
||||
assert ops == [{"op": "look", "need": "产量表的字段"}]
|
||||
assert diag.tool_calls == ["look", "run_sql", "add_node"]
|
||||
assert diag.dropped == {"工具调用的名字不认识": 1, "工具调用的参数解析不了": 1}, diag.dropped
|
||||
|
||||
|
||||
async def test_tool_call_markup_in_text_is_diagnosed():
|
||||
ops, diag = await _ops([AIMessageChunk(content='<tool_call>{"name": "look", "arguments": {}}</tool_call>\n')])
|
||||
assert ops == [] and diag.markup == 1
|
||||
assert "不走工具调用" in diag.correction()
|
||||
|
||||
|
||||
async def test_the_empty_error_carries_the_diagnosis_and_a_correction(client, monkeypatch): # noqa: F811
|
||||
from tests.test_copilot_answer_ops import BODY
|
||||
|
||||
_use(monkeypatch, _Model([_think("先看一下表。"), AIMessageChunk(content="", response_metadata={
|
||||
"finish_reason": "stop"})]))
|
||||
got = await _sse(client, BODY)
|
||||
[error] = [op for op in got if op["op"] == "error"]
|
||||
assert error["code"] == "copilot_empty" and error["next"] == "retry"
|
||||
assert error["message"] == "助手本轮未完成:模型没有给出任何修改,也没有说明原因"
|
||||
assert "正文 0 字" in error["detail"] and "思考 6 字" in error["detail"] and "结束原因 stop" in error["detail"]
|
||||
assert error["diagnostics"]["text_chars"] == 0 and error["diagnostics"]["thinking_chars"] == 6
|
||||
assert "只写了思考" in error["reason"]
|
||||
# 重来时交给助手的话:问题后面附这句纠正,不是原样重发
|
||||
again = policy.empty_retry_instruction("上周各条线的产量", policy.error_reason(error))
|
||||
assert again.startswith("上周各条线的产量") and "只写了思考" in again and again != "上周各条线的产量"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reason", ["", "模型没有给出任何修改,也没有说明原因"])
|
||||
def test_without_a_correction_the_retry_still_asks_for_ops(reason):
|
||||
again = policy.empty_retry_instruction("产量是多少", reason)
|
||||
assert "look" in again and "产量是多少" in again
|
||||
@@ -163,15 +163,27 @@ async def test_a_cut_in_the_middle_of_the_ops_is_not_delivered_as_done(client, m
|
||||
|
||||
|
||||
async def test_output_that_parses_to_nothing_is_an_error(client, monkeypatch):
|
||||
"""格式跑偏(整段放进 markdown 围栏、JSON 分了行),每一行都被跳过:一个操作都没有,也没有 done。"""
|
||||
pretty = "```json\n" + json.dumps({"op": "add_node", "node": {"id": "yoy", "type": "llm"}}, indent=2) + "\n```\n"
|
||||
model = _Scripted([AIMessageChunk(content=pretty),
|
||||
"""格式跑偏(只写了说明文字、没有一行是操作),每一行都被跳过:一个操作都没有,也没有 done。诊断写明跳过了几行。"""
|
||||
model = _Scripted([AIMessageChunk(content="先看一下表结构,\n再决定怎么搭。\n"),
|
||||
AIMessageChunk(content="", response_metadata={"finish_reason": "stop"})])
|
||||
events = await _events(client, monkeypatch, model)
|
||||
kinds = _kinds(events)
|
||||
assert "final" not in kinds
|
||||
[err] = [e for e in events if e["op"] == "error"]
|
||||
assert err["message"].startswith("助手本轮未完成:") and "画布未修改" in err["hint"]
|
||||
assert "跳过 2 行:2 行不是完整的 JSON" in err["detail"] and "先看一下表" not in err["detail"], err
|
||||
|
||||
|
||||
async def test_a_pretty_printed_op_in_a_fence_is_read(client, monkeypatch):
|
||||
"""整段放进 markdown 围栏、JSON 分了行:以前每一行都被跳过、报一个操作都没有;现在攒成一个操作照常落地。"""
|
||||
pretty = "```json\n" + json.dumps({"op": "add_node", "node": {"id": "yoy", "type": "llm", "label": "同比",
|
||||
"config": {"prompt": "算同比"}}},
|
||||
indent=2, ensure_ascii=False) + "\n```\n"
|
||||
model = _Scripted([AIMessageChunk(content=pretty),
|
||||
AIMessageChunk(content="", response_metadata={"finish_reason": "stop"})])
|
||||
events = await _events(client, monkeypatch, model)
|
||||
assert "error" not in _kinds(events), events
|
||||
assert any(e["op"] == "add_node" and e["node"]["id"] == "yoy" for e in events)
|
||||
|
||||
|
||||
async def test_a_model_that_says_done_without_changes_is_still_unchanged(client, monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user