mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:31:58 +00:00
起草引擎按失败原因处理的三处修正:结构化输出回了一段说明文字(不是 JSON、也没停在额度上,例如网关接受了 response_format 却没照做)不再因为带着思考就当成额度用完去拆,也不直接判失败,这次起草后面改走纯文本、这一段换纯文本问一次(纯文本另带格式说明);只有停在额度上、或者正文为空而有思考才算额度用完。补问之后整列没列出来的列记没写出来(「模型的回复里漏了这些列,补问一次仍没有给出」),表是 partial、covered 不含 llm,以前起草过的待确认模型项不再被删(以前补问只给空壳也记 ok,还把这一轮没给的旧项删掉);列出了、字段留空的照旧算拿不准;补问时整张表还一项都没有的,照只给空壳的规矩拆小、换纯文本,不直接记失败。几处没写出来的原因按基础原因去重,拆小重试过的在最后补一句,不再出现同一句说两遍。假模型照提示词把请写的列都列出来(字段留空是拿不准),要模拟漏列的传 list_all=False
This commit is contained in:
@@ -18,8 +18,12 @@
|
||||
两半再问(多表的段按表拆,单表的按列拆),最小 MODEL_SPLIT_MIN_COLUMNS 列,最多拆 MODEL_SPLIT_DEPTH 层。
|
||||
拆不开的空壳(结构化输出给的)换纯文本问一次:有的网关的结构化输出只给空壳、纯文本却能写——写出来了,这次起草
|
||||
后面都走纯文本。
|
||||
- 结构化输出回了一段说明文字(不是 JSON、也没停在额度上):网关接受了结构化输出的参数却没照做(例如开了丢弃不认识
|
||||
的参数),提示词里又没有格式说明,模型就写成了说明文字。这次起草后面改走纯文本,这一段换纯文本问一次。有思考
|
||||
不等于额度用完:只有停在额度上、或者正文为空而有思考才算。
|
||||
- 被截断的 JSON:先把已经完整的表、列对象捡回来,只补剩下的。
|
||||
- 一段里缺了的表、列只补缺的,补一次。补过仍然没给的列算模型拿不准、留空,不算没写出来;整张表一项都没有才算。
|
||||
- 一段里缺了的表、列只补缺的,补一次。补问之后:模型列出了、字段留空的列是拿不准,不算没写出来;整列没列出来的
|
||||
算没写出来(这张表是 partial,旧的模型起草项不删);整张表还一项都没有的,照只给空壳处理(拆小、换纯文本)。
|
||||
- 结构化输出这个接口不支持(400 / 422 提到 response_format、json_schema、tool……,或者模型类不支持):这次起草
|
||||
后面改走纯文本、从正文取 JSON(纯文本那条路另外说清 JSON 的格式)。
|
||||
- 服务方嫌额度太大:退回 MODEL_FALLBACK_MAX_TOKENS 重来,这次起草后面都用它(catalog.DraftModel.shrink)。
|
||||
@@ -44,7 +48,7 @@ import json
|
||||
import logging
|
||||
import re
|
||||
import weakref
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
@@ -72,6 +76,7 @@ REASON_HOLLOW = "模型只给了空壳,没有写出可用的内容"
|
||||
REASON_TRUNCATED = "模型的输出被截断,这一段没写完"
|
||||
REASON_FORMAT = "模型的回复不是约定的格式"
|
||||
REASON_MISSING = "模型的回复里没有这张表"
|
||||
REASON_UNANSWERED = "模型的回复里漏了这些列,补问一次仍没有给出"
|
||||
REASON_TIMEOUT = "模型 {seconds} 秒内没有写完这一段"
|
||||
REASON_CALL = "调用模型出错:{line}"
|
||||
REASON_RETRIED = "(已重试 {n} 次)"
|
||||
@@ -222,6 +227,28 @@ class _Reply:
|
||||
exhausted: bool = False
|
||||
#: 这一段最终没写出来的原因(重试用完、别的错误、不是约定的格式)
|
||||
error: str | None = None
|
||||
#: 结构化输出回了一段说明文字(error 是「不是约定的格式」):换纯文本还可能写出来
|
||||
prose: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Why:
|
||||
"""一部分没写出来的原因。base 给人看的原因;split 拆小重试过。"""
|
||||
|
||||
base: str
|
||||
split: bool = False
|
||||
|
||||
|
||||
def _join(whys: Iterable[_Why]) -> str:
|
||||
"""几处没写出来的原因拼成一句:同样的原因只说一次(最多说两种),拆小重试过的在最后补一次(只看说出来的那几种)。
|
||||
不然宽表的表级那段拆不开、列分段拆过时,同一句原因会说两遍。"""
|
||||
whys = list(whys)
|
||||
shown = list(dict.fromkeys(w.base for w in whys))[:2]
|
||||
told = [w for w in whys if w.base in shown]
|
||||
text = ";".join(shown)
|
||||
if any(w.split for w in told):
|
||||
text += REASON_SPLIT
|
||||
return text
|
||||
|
||||
|
||||
class _Cut:
|
||||
@@ -480,11 +507,13 @@ def _structured_reply(raw: Any, parsed: Any) -> _Reply:
|
||||
reply = _parse_text(text, cut)
|
||||
if reply is not None:
|
||||
return reply
|
||||
if cut or _thought(raw):
|
||||
return _Reply(exhausted=True)
|
||||
if cut or (not texts and _thought(raw)):
|
||||
return _Reply(exhausted=True) # 停在额度上,或者正文为空而有思考(同 _text_reply)
|
||||
if not texts:
|
||||
return _Reply() # 什么都没写:按空壳处理
|
||||
return _Reply(error=REASON_FORMAT)
|
||||
# 写了字却不是约定的格式、也没停在额度上:网关接受了结构化输出的参数却没照做,模型写成了说明文字。有没有思考都
|
||||
# 一样,不是额度的问题;换纯文本(另带格式说明)还能写出来
|
||||
return _Reply(error=REASON_FORMAT, prose=True)
|
||||
|
||||
|
||||
def _text_reply(raw: Any) -> _Reply:
|
||||
@@ -546,13 +575,17 @@ class _Table:
|
||||
got_table: bool = False
|
||||
got_columns: set[str] = field(default_factory=set)
|
||||
#: 没写出来的列(表级的项记在 _TABLE)→ 原因
|
||||
failed: dict[str, str] = field(default_factory=dict)
|
||||
failed: dict[str, _Why] = field(default_factory=dict)
|
||||
parts_done: int = 0
|
||||
finished: bool = False
|
||||
|
||||
def whole(self, piece: _Piece) -> bool:
|
||||
return piece.table_level and len(piece.columns) == len(self.columns)
|
||||
|
||||
def has_any(self) -> bool:
|
||||
"""这张表已经拿到了模型起草的内容(表级的项或任何一列)。"""
|
||||
return self.got_table or bool(self.got_columns)
|
||||
|
||||
|
||||
def _restrict(notes: Mapping[str, Any], piece: _Piece) -> dict[str, Any]:
|
||||
"""只留这一段请模型写的部分:没请写表级的项就不要表级的项,列只要请写的那几列。"""
|
||||
@@ -665,10 +698,13 @@ class _Run:
|
||||
last: bool = False) -> None:
|
||||
"""问一段,拿到的并进各表;按失败原因拆开、补缺、记下没写出来的。
|
||||
|
||||
depth:已经拆了几层(截断后只补剩下的也算一层)。followup:这一段是在补缺(补过就不再补,空着的算模型拿不准)。
|
||||
mode / last:拆不开的空壳换纯文本问的那一次(再空就记没写出来)。"""
|
||||
depth:已经拆了几层(截断后只补剩下的也算一层)。followup:这一段是在补缺(补过就不再补,见 _after_followup)。
|
||||
mode / last:换纯文本问的那一次——拆不开的空壳、结构化输出回了说明文字(再不行就记没写出来)。"""
|
||||
reply = await self._ask(pieces, mode)
|
||||
if reply.error is not None:
|
||||
if reply.prose and not last:
|
||||
await self._prose_to_text(pieces, depth, followup)
|
||||
return
|
||||
self._fail(pieces, reply.error)
|
||||
return
|
||||
gains = self._take(pieces, reply.entries)
|
||||
@@ -689,19 +725,45 @@ class _Run:
|
||||
else:
|
||||
self._fail(rest, self._reason(kind, split=True))
|
||||
return
|
||||
if not anything:
|
||||
if followup:
|
||||
return
|
||||
if not anything and not followup:
|
||||
if last:
|
||||
self._fail(pieces, lambda p: self._reason("hollow", split=depth > 0, piece=p))
|
||||
return
|
||||
await self._split(pieces, depth, "hollow", followup)
|
||||
return
|
||||
if followup:
|
||||
return
|
||||
rest = self._remainder(pieces, gains, truncated=False)
|
||||
if rest:
|
||||
await self._segment(rest, depth, True)
|
||||
if not rest:
|
||||
return
|
||||
if followup:
|
||||
await self._after_followup(rest, depth, last=last)
|
||||
return
|
||||
await self._segment(rest, depth, True)
|
||||
|
||||
async def _after_followup(self, rest: list[_Piece], depth: int, *, last: bool) -> None:
|
||||
"""补问过仍然没给的部分(补过就不再补)。
|
||||
|
||||
- 这张表已经拿到了内容:没列出来的列记没写出来(这张表是 partial,旧的模型起草项不删)。模型列出了、字段留空
|
||||
的列是拿不准,不在这里(_remainder 只把没列出来的列交过来);
|
||||
- 这张表到现在还一项都没有(回复里没有这张表、只给了空壳):照第一次只给空壳的规矩,拆小再问,拆不开换纯文本
|
||||
问一次。拆出来的半段是新的一段(还能补缺一次),层数照算。"""
|
||||
given = [p for p in rest if self.tables[p.table].has_any()]
|
||||
empty = [p for p in rest if not self.tables[p.table].has_any()]
|
||||
if given:
|
||||
self._fail(given, _Why(REASON_UNANSWERED))
|
||||
if not empty:
|
||||
return
|
||||
if last:
|
||||
self._fail(empty, lambda p: self._reason("hollow", split=depth > 0, piece=p))
|
||||
return
|
||||
await self._split(empty, depth, "hollow", False)
|
||||
|
||||
async def _prose_to_text(self, pieces: list[_Piece], depth: int, followup: bool) -> None:
|
||||
"""结构化输出回了一段说明文字:网关接受了结构化输出的参数却没照做。这次起草后面都走纯文本(另带格式说明),
|
||||
这一段换纯文本问一次。纯文本额度用完了还能照常拆小(拆出来的半段也走纯文本,不再先问一遍结构化输出)。"""
|
||||
if self.mode != "text":
|
||||
self.mode = "text"
|
||||
logger.info("起草数据目录:结构化输出回了说明文字,后面改走纯文本")
|
||||
await self._segment(pieces, depth, followup, mode="text", last=True)
|
||||
|
||||
async def _split(self, pieces: list[_Piece], depth: int, kind: str, followup: bool) -> None:
|
||||
halves = _halves(pieces) if depth < int(C.MODEL_SPLIT_DEPTH) else None
|
||||
@@ -724,8 +786,9 @@ class _Run:
|
||||
self._fail(pieces, lambda p: self._reason(kind, split=depth > 0, piece=p))
|
||||
|
||||
def _remainder(self, pieces: list[_Piece], gains: list[_Gain], *, truncated: bool) -> list[_Piece]:
|
||||
"""还没写出来的部分:回复里没有这张表、或者这张表一项可用内容都没有的整段重问;有内容的只问没给的列。
|
||||
截断的回复里表级的项也没拿到的,表级的项一起重问(没截断时表级空着算模型拿不准)。"""
|
||||
"""还没写出来的部分:回复里没有这张表、或者这张表一项可用内容都没有的整段重问;有内容的只问没列出来的列
|
||||
(列出了、字段留空的算模型拿不准,不再问)。截断的回复里表级的项也没拿到的,表级的项一起重问(没截断时表级
|
||||
空着算模型拿不准:这张表在回复里,表级的字段留空和列的字段留空一样)。"""
|
||||
out: list[_Piece] = []
|
||||
for piece, gain in zip(pieces, gains):
|
||||
if not gain.named or not gain.items:
|
||||
@@ -874,18 +937,20 @@ class _Run:
|
||||
st.notes.setdefault(key, value)
|
||||
st.got_table = True
|
||||
|
||||
def _fail(self, pieces: list[_Piece], reason: str | Callable[[_Piece], str]) -> None:
|
||||
def _fail(self, pieces: list[_Piece], reason: _Why | str | Callable[[_Piece], _Why]) -> None:
|
||||
"""这几段没写出来:还没拿到内容的列(和表级的项)记下原因。"""
|
||||
for piece in pieces:
|
||||
st = self.tables[piece.table]
|
||||
why = reason(piece) if callable(reason) else reason
|
||||
if isinstance(why, str):
|
||||
why = _Why(why)
|
||||
for col in piece.columns:
|
||||
if col not in st.got_columns:
|
||||
st.failed.setdefault(col, why)
|
||||
if piece.table_level and not st.got_table:
|
||||
st.failed.setdefault(_TABLE, why)
|
||||
|
||||
def _reason(self, kind: str, *, split: bool, piece: _Piece | None = None) -> str:
|
||||
def _reason(self, kind: str, *, split: bool, piece: _Piece | None = None) -> _Why:
|
||||
if kind == "exhausted":
|
||||
base = REASON_EXHAUSTED.format(budget=self._budget())
|
||||
elif kind == "truncated":
|
||||
@@ -896,7 +961,7 @@ class _Run:
|
||||
base = C.MODEL_EMPTY_REASON if st.named else REASON_MISSING
|
||||
else:
|
||||
base = REASON_HOLLOW
|
||||
return base + (REASON_SPLIT if split else "")
|
||||
return _Why(base, split=split)
|
||||
|
||||
# ---------------------------------------------------------------- 收尾
|
||||
|
||||
@@ -906,8 +971,8 @@ class _Run:
|
||||
failed_table = _TABLE in st.failed and not st.got_table
|
||||
total = len(st.parts) or 1
|
||||
if not items:
|
||||
reasons = list(dict.fromkeys(st.failed.values()))
|
||||
reason = ";".join(reasons[:2]) if reasons else (C.MODEL_EMPTY_REASON if st.named else REASON_MISSING)
|
||||
whys = list(st.failed.values())
|
||||
reason = _join(whys) if whys else (C.MODEL_EMPTY_REASON if st.named else REASON_MISSING)
|
||||
return ModelOutcome("failed", {}, 0, reason, total, st.parts_done)
|
||||
notes = copy.deepcopy(st.notes)
|
||||
if failed_cols or failed_table:
|
||||
@@ -919,7 +984,7 @@ class _Run:
|
||||
def _partial_text(st: _Table, failed_cols: list[str], failed_table: bool) -> str:
|
||||
"""只写出一部分时说清哪部分没写出来、为什么、怎么办。"""
|
||||
keys = ([_TABLE] if failed_table else []) + failed_cols
|
||||
reason = ";".join(list(dict.fromkeys(st.failed[k] for k in keys))[:2])
|
||||
reason = _join(st.failed[k] for k in keys)
|
||||
n = len(st.columns)
|
||||
if len(st.parts) > 1:
|
||||
part_of = {c: p.part for p in st.parts for c in p.columns}
|
||||
@@ -1056,5 +1121,5 @@ class _Run:
|
||||
|
||||
|
||||
__all__ = ["BUDGET_WORDS", "Background", "Hooks", "ModelOutcome", "REASON_CALL", "REASON_EXHAUSTED", "REASON_FORMAT",
|
||||
"REASON_HOLLOW", "REASON_MISSING", "REASON_RETRIED", "REASON_SPLIT", "REASON_TIMEOUT", "REASON_TRUNCATED",
|
||||
"classify_failure", "model_gate", "parts_for", "run_model_draft"]
|
||||
"REASON_HOLLOW", "REASON_MISSING", "REASON_RETRIED", "REASON_SPLIT", "REASON_TIMEOUT",
|
||||
"REASON_TRUNCATED", "REASON_UNANSWERED", "classify_failure", "model_gate", "parts_for", "run_model_draft"]
|
||||
|
||||
+38
-1
@@ -6,6 +6,8 @@
|
||||
- 结构化输出(with_structured_output(...).ainvoke)从提示词里认出这一段请它写哪些表、每张表写哪部分(requested:
|
||||
整张表、只写表级的项、只写列出的几列、表级的项加几列),返回 {"tables": [{"table": 表名, …}, …]},只给请它写的
|
||||
那部分(整张表给 replies[表名] 的全部;只写几列时只给 replies 里这几列);
|
||||
- 照提示词的要求,请它写的每一列都列出来:replies 里没写的列只给列名({"name": 列名},字段留空,是拿不准),
|
||||
不整列漏掉。list_all=False 时只给 replies 里有的列(模拟漏列的模型:漏掉的列会被补问一次,补问后仍没给的算没写出来);
|
||||
- 直接 ainvoke 是纯文本那条路:同样的内容写成一段正文(AIMessage),前面带一句话;
|
||||
- fail 里的表所在的那一段抛异常,用来验证「一段失败只影响这一段」;
|
||||
- calls 记下每次收到的提示词全文,用来核对交给模型的内容(只有结构,没有数据);modes 记下每次走的是
|
||||
@@ -69,11 +71,44 @@ def answer(reply: dict, name: str, part: str, columns: list[str] | None) -> dict
|
||||
return out
|
||||
|
||||
|
||||
def listed(text: str) -> dict[str, list[str]]:
|
||||
"""提示词里每张表的结构描述列出了哪些列(「- 列名 类型」那几行)。"""
|
||||
out: dict[str, list[str]] = {}
|
||||
for chunk in text.split("\n\n"):
|
||||
lines = chunk.split("\n")
|
||||
if not lines[0].startswith("表 "):
|
||||
continue
|
||||
name = lines[0][2:]
|
||||
if name.endswith("(视图)"):
|
||||
name = name[:-len("(视图)")]
|
||||
out[name] = [line[2:].split(" ")[0] for line in lines if line.startswith("- ")]
|
||||
return out
|
||||
|
||||
|
||||
def wanted_columns(text: str, name: str, part: str, columns: list[str] | None) -> list[str]:
|
||||
"""这一段请它写这张表的哪几列:整张表是结构描述里的全部列;只写表级的项时没有。"""
|
||||
if part == "full":
|
||||
return listed(text).get(name, [])
|
||||
if part in ("columns", "table+columns"):
|
||||
return list(columns or [])
|
||||
return []
|
||||
|
||||
|
||||
def with_every_column(entry: dict, wanted: list[str]) -> dict:
|
||||
"""请写的列都列出来:没写的只给列名(字段留空)。"""
|
||||
have = {c.get("name") for c in entry.get("columns") or [] if isinstance(c, dict)}
|
||||
missing = [{"name": c} for c in wanted if c not in have]
|
||||
if not missing:
|
||||
return entry
|
||||
return {**entry, "columns": list(entry.get("columns") or []) + missing}
|
||||
|
||||
|
||||
class FakeModel:
|
||||
def __init__(self, replies: dict[str, dict], fail: set[str] = frozenset(), *,
|
||||
structured: str = "tables", text: str = "tables") -> None:
|
||||
structured: str = "tables", text: str = "tables", list_all: bool = True) -> None:
|
||||
self.replies, self.fail, self.calls = replies, set(fail), []
|
||||
self.structured, self.text = structured, text
|
||||
self.list_all = list_all
|
||||
self.modes: list[str] = []
|
||||
|
||||
def with_structured_output(self, schema: Any, **_: Any) -> "_Structured":
|
||||
@@ -100,6 +135,8 @@ class FakeModel:
|
||||
if shape == "hollow":
|
||||
return {"tables": [{"table": t} for t in names]}
|
||||
answers = [answer(self.replies[t], t, *asked[t]) for t in names]
|
||||
if self.list_all:
|
||||
answers = [with_every_column(a, wanted_columns(text, t, *asked[t])) for a, t in zip(answers, names)]
|
||||
if shape == "single":
|
||||
return answers[0] if answers else {}
|
||||
return {"tables": answers}
|
||||
|
||||
@@ -275,11 +275,14 @@ async def test_model_single_table_object_without_wrapper_is_recognized():
|
||||
|
||||
async def test_hollow_structured_output_retries_with_text():
|
||||
"""结构化输出只给了 {"tables":[{"table":"visits"}]} 这样的空壳、这一段又拆不开(不到 10 列):改走纯文本再问一次,
|
||||
用纯文本给的内容。纯文本只给了表级的项、没给列:再补问一次没给的列(只问缺的,data/catalog_draft.py)。"""
|
||||
model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, structured="hollow", text="tables")
|
||||
用纯文本给的内容。纯文本只给了表级的项、一列都没列出来:再补问一次没给的列(只问缺的,data/catalog_draft.py);
|
||||
补问后仍没给,这几列算没写出来,表是只写出一部分(旧的模型起草项不删)。"""
|
||||
model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, structured="hollow", text="tables",
|
||||
list_all=False)
|
||||
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
|
||||
assert model.modes == ["structured", "text", "text"]
|
||||
assert got["visits"].notes == {"label": make_item("入园记录", "llm"), "kind": make_item("fact", "llm")}
|
||||
assert got["visits"].covered == frozenset() and "有 5 列没写出来" in got["visits"].error
|
||||
|
||||
|
||||
async def test_hollow_structured_then_single_text_object():
|
||||
|
||||
@@ -304,6 +304,7 @@ async def test_truncated_tool_call_arguments_are_salvaged_too():
|
||||
|
||||
|
||||
async def test_missing_tables_are_asked_again_once():
|
||||
"""缺了的表只补缺的、补一次;补问时整张表还是一项都没有,按只给空壳处理(拆不开就换纯文本问一次)。"""
|
||||
tables = {"a_tab": _table(4), "b_tab": _table(4), "c_tab": _table(4)}
|
||||
ok = answering(tables)
|
||||
|
||||
@@ -315,13 +316,34 @@ async def test_missing_tables_are_asked_again_once():
|
||||
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source(tables), ["a_tab", "b_tab", "c_tab"])
|
||||
assert len(model.calls) == 2
|
||||
assert len(model.calls) == 3
|
||||
assert model.calls[1].asked == {"b_tab": ("full", None), "c_tab": ("full", None)}
|
||||
assert model.calls[2].asked == {"c_tab": ("full", None)} and model.calls[2].mode == "text"
|
||||
assert _status(got) == {"a_tab": "ok", "b_tab": "ok", "c_tab": "failed"}
|
||||
assert got["c_tab"].error == "模型的回复里没有这张表"
|
||||
|
||||
|
||||
async def test_columns_the_model_skips_are_asked_once_and_then_left_empty():
|
||||
async def test_a_table_still_missing_after_the_followup_is_split_like_a_hollow_shell():
|
||||
"""补问时整张表还是一项都没有:照第一次的规则拆小(这里按列拆成两半),不直接记失败。"""
|
||||
tables = {"a_tab": _table(4), "b_tab": _table(12)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
reply = ok(call)
|
||||
if call.n < 2:
|
||||
reply.obj["tables"] = [e for e in reply.obj["tables"] if e["table"] == "a_tab"]
|
||||
return reply
|
||||
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source(tables), ["a_tab", "b_tab"])
|
||||
assert _status(got) == {"a_tab": "ok", "b_tab": "ok"}
|
||||
cols = [f"c{i:03d}" for i in range(1, 13)]
|
||||
assert [c.asked for c in model.calls[2:]] == [{"b_tab": ("table+columns", cols[:6])},
|
||||
{"b_tab": ("columns", cols[6:])}]
|
||||
|
||||
|
||||
async def test_columns_still_missing_after_the_followup_are_not_written():
|
||||
"""补问过仍然没列出来的列算没写出来:表是 partial,说清哪部分没写出来(不算拿不准,旧的模型起草项不删)。"""
|
||||
tables = {"t8": _table(8)}
|
||||
ok = answering(tables)
|
||||
|
||||
@@ -335,8 +357,51 @@ async def test_columns_the_model_skips_are_asked_once_and_then_left_empty():
|
||||
got = await D.run_model_draft(model, _source(tables), ["t8"])
|
||||
assert len(model.calls) == 2 and model.calls[1].asked == {"t8": ("columns", ["c007", "c008"])}
|
||||
o = got["t8"]
|
||||
assert o.status == "ok" and o.error is None, "补问过仍然没给的列算拿不准、留空"
|
||||
assert o.status == "partial"
|
||||
assert o.error == "这张表 8 列,有 2 列没写出来:模型的回复里漏了这些列,补问一次仍没有给出。可以只重试这张表"
|
||||
assert set(o.notes["columns"]) == {f"c{i:03d}" for i in range(1, 7)}
|
||||
compat = (await catalog.draft_with_model(Scripted(handle), _source(tables), ["t8"]))["t8"]
|
||||
assert compat.covered == frozenset(), "没写全的不算完整起草过"
|
||||
|
||||
|
||||
async def test_a_followup_that_comes_back_as_an_empty_shell_leaves_the_table_partial():
|
||||
"""40 列的表第一次只回了 20 列,补问只给一个空壳:没写出来的 20 列照实记下,表是 partial。"""
|
||||
tables = {"t40": _table(40)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
if call.n == 0:
|
||||
reply = ok(call)
|
||||
reply.obj["tables"][0]["columns"] = reply.obj["tables"][0]["columns"][:20]
|
||||
return reply
|
||||
return Reply(obj={"tables": [{"table": "t40"}]})
|
||||
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source(tables), ["t40"])
|
||||
o = got["t40"]
|
||||
assert len(model.calls) == 2
|
||||
assert model.calls[1].asked == {"t40": ("columns", [f"c{i:03d}" for i in range(21, 41)])}
|
||||
assert o.status == "partial" and len(o.notes["columns"]) == 20
|
||||
assert o.error == "这张表 40 列,有 20 列没写出来:模型的回复里漏了这些列,补问一次仍没有给出。可以只重试这张表"
|
||||
|
||||
|
||||
async def test_columns_listed_with_empty_fields_are_left_to_the_model():
|
||||
"""列出了、字段留空的列是模型拿不准:不补问,也不算没写出来。"""
|
||||
tables = {"t8": _table(8)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
reply = ok(call)
|
||||
for entry in reply.obj["tables"]:
|
||||
entry["columns"] = [c if c["name"] not in ("c007", "c008") else {"name": c["name"]}
|
||||
for c in entry.get("columns", [])]
|
||||
return reply
|
||||
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source(tables), ["t8"])
|
||||
assert len(model.calls) == 1
|
||||
assert got["t8"].status == "ok" and got["t8"].error is None
|
||||
assert set(got["t8"].notes["columns"]) == {f"c{i:03d}" for i in range(1, 7)}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -382,6 +447,34 @@ async def test_structured_output_unsupported_switches_to_text_for_the_rest():
|
||||
assert [c.mode for c in model.calls] == ["structured", "text", "text", "text"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", ["", "先想想每一列是什么……"], ids=["plain", "with-thinking"])
|
||||
async def test_structured_output_that_comes_back_as_prose_falls_back_to_text(thinking):
|
||||
"""网关接受了结构化输出的参数却没照做(模型写成了一段说明文字):换纯文本问,不当成额度用完去拆,也不直接判失败。"""
|
||||
tables = {"t40": _table(40)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
if call.mode == "structured":
|
||||
return Reply(text="好的。这张表记录的是一笔笔业务,各列的含义如下:第一列是编号……", thinking=thinking)
|
||||
return ok(call)
|
||||
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source(tables), ["t40"])
|
||||
assert got["t40"].status == "ok" and len(got["t40"].notes["columns"]) == 40
|
||||
assert [c.mode for c in model.calls] == ["structured", "text"]
|
||||
assert catalog.CATALOG_DRAFT_TEXT_PROMPT in model.calls[1].text
|
||||
|
||||
|
||||
async def test_prose_on_both_paths_is_a_format_failure_not_a_budget_one():
|
||||
def handle(call: Call) -> Reply:
|
||||
return Reply(text="这张表看起来是业务明细,具体含义需要结合业务确认。", thinking="……")
|
||||
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source({"t40": _table(40)}), ["t40"])
|
||||
assert [c.mode for c in model.calls] == ["structured", "text"], "格式不对不拆小重试"
|
||||
assert got["t40"].status == "failed" and got["t40"].error == "模型的回复不是约定的格式"
|
||||
|
||||
|
||||
async def test_a_model_without_structured_output_goes_to_text():
|
||||
class TextOnly(Scripted):
|
||||
def with_structured_output(self, schema, **_):
|
||||
@@ -500,6 +593,33 @@ async def test_tables_are_packed_up_to_40_columns_and_4_tables():
|
||||
])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 报错去重
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _hollow(call: Call) -> Reply:
|
||||
return Reply(obj={"tables": [{"table": t} for t in call.asked]})
|
||||
|
||||
|
||||
async def test_the_same_reason_is_said_once_even_when_some_parts_were_split():
|
||||
"""宽表的表级那段拆不开、列分段拆过:原因只说一次,拆过的在最后补一句。"""
|
||||
model = Scripted(_hollow)
|
||||
got = await D.run_model_draft(model, _source({"wide45": _table(45)}), ["wide45"])
|
||||
o = got["wide45"]
|
||||
assert o.status == "failed"
|
||||
assert o.error == "模型只给了空壳,没有写出可用的内容;已拆小重试仍没写出"
|
||||
|
||||
|
||||
def test_reasons_are_joined_by_their_base():
|
||||
W = D._Why
|
||||
exhausted = D.REASON_EXHAUSTED.format(budget=32768)
|
||||
assert D._join([W(D.REASON_HOLLOW), W(D.REASON_HOLLOW, split=True)]) == D.REASON_HOLLOW + D.REASON_SPLIT
|
||||
assert D._join([W(exhausted, split=True), W(D.REASON_HOLLOW)]) == \
|
||||
f"{exhausted};{D.REASON_HOLLOW}{D.REASON_SPLIT}"
|
||||
assert D._join([W("甲"), W("乙"), W("丙", split=True)]) == "甲;乙", "没说出来的原因不补后缀"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 起草要求、用量
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@@ -215,6 +215,57 @@ async def test_a_table_whose_model_part_failed_is_failed_but_structure_is_writte
|
||||
assert by["channels"]["status"] == "ok"
|
||||
|
||||
|
||||
class HalfThenHollow:
|
||||
"""假模型:第一次只给前一半列,补问时只给一个空壳(表名对上、一项都没有)。"""
|
||||
|
||||
model_name = "half-model"
|
||||
|
||||
def __init__(self, table: str, reply: dict) -> None:
|
||||
self.table, self.reply, self.calls = table, reply, 0
|
||||
|
||||
def with_structured_output(self, schema, **_):
|
||||
outer = self
|
||||
|
||||
class _S:
|
||||
async def ainvoke(self, messages):
|
||||
outer.calls += 1
|
||||
if outer.calls == 1:
|
||||
cols = outer.reply["columns"]
|
||||
return {"tables": [{"table": outer.table, **outer.reply, "columns": cols[:len(cols) // 2]}]}
|
||||
return {"tables": [{"table": outer.table}]}
|
||||
|
||||
return _S()
|
||||
|
||||
async def ainvoke(self, messages):
|
||||
raise AssertionError("这里不该走纯文本")
|
||||
|
||||
|
||||
async def test_a_followup_that_comes_back_empty_keeps_the_earlier_model_items(client, monkeypatch):
|
||||
"""补问只给空壳:表是 partial、任务是 partial,以前起草过的列(待确认的模型起草项)不因这次没写全被删掉。"""
|
||||
cols = [_col(f"c{i:02d}") for i in range(1, 9)]
|
||||
sid = await _source({"tickets": {"qualified": "tickets", "columns": cols, "primary_key": ["c01"], "comment": None,
|
||||
"is_view": False, "foreign_keys": []}})
|
||||
full = {"label": "门票", "kind": "fact",
|
||||
"columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 9)]}
|
||||
use(monkeypatch, FakeModel({"tickets": full}))
|
||||
first = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"], "use_model": True})).json()
|
||||
await finished(first["job"]["id"])
|
||||
before = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"]
|
||||
assert {c for c, items in before.items() if "label" in items} == {f"c{i:02d}" for i in range(1, 9)}
|
||||
|
||||
model = HalfThenHollow("tickets", full)
|
||||
use(monkeypatch, model)
|
||||
second = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"], "use_model": True})).json()
|
||||
await finished(second["job"]["id"])
|
||||
job = await get_job(client, sid, second["job"]["id"])
|
||||
task = job["tasks"][0]
|
||||
assert model.calls == 2
|
||||
assert job["status"] == "partial" and task["status"] == "partial" and task["removed"] == 0
|
||||
assert "有 4 列没写出来" in task["model_error"] and "可以只重试这张表" in task["model_error"]
|
||||
after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"]
|
||||
assert {c for c, items in after.items() if "label" in items} == {f"c{i:02d}" for i in range(1, 9)}
|
||||
|
||||
|
||||
async def test_a_job_that_cannot_start_is_failed_with_the_reason(client, monkeypatch):
|
||||
"""建任务时起草模型还能用,开始起草时已经用不了(接入刚被停用):任务整体 failed,error 写原因,没开始的表也记 failed。"""
|
||||
sid = await _source()
|
||||
|
||||
Reference in New Issue
Block a user