mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 22:21:59 +00:00
ThearchyHelios
801 lines
36 KiB
Python
801 lines
36 KiB
Python
"""数据目录模型起草的引擎（data/catalog_draft.py）：分段、按失败原因处理、捡回被截断的部分、起草要求、用量。
|
||
|
||
用假模型复刻实测查明的几种失败（不调用真实模型）：
|
||
- 思考加输出用完额度：finish_reason=length 且正文为空（只有思考）、LengthFinishReasonError → 拆成两半再问；
|
||
- 只给空壳 → 拆开再问；拆不开的换纯文本问一次，写出来了这次起草后面都走纯文本；
|
||
- 被截断的 JSON：已经完整的表、列对象捡回来，只补剩下的；
|
||
- 只回一部分表：只补缺的，补一次；
|
||
- 服务方嫌额度太大 → 退回 8192，后面都用它；结构化输出不支持 → 后面改走纯文本；
|
||
- 网络错误、5xx、超时 → 退避重试，最多 2 次，报错写明重试过几次。
|
||
|
||
表名、列名、说明文字都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import inspect
|
||
import json
|
||
from dataclasses import dataclass, field
|
||
from types import SimpleNamespace
|
||
from typing import Any, Callable
|
||
|
||
import pytest
|
||
from langchain_core.messages import AIMessage
|
||
|
||
from app.data import catalog
|
||
from app.data import catalog_draft as D
|
||
from tests.fixtures.catalog.fake_model import answer, requested
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 夹具
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _col(name: str, type_: str = "INTEGER") -> dict:
|
||
return {"name": name, "type": type_, "nullable": True, "comment": None}
|
||
|
||
|
||
def _table(n: int, prefix: str = "c", comment: str | None = None) -> dict:
|
||
return {"qualified": "", "schema": None, "columns": [_col(f"{prefix}{i:03d}") for i in range(1, n + 1)],
|
||
"primary_key": [f"{prefix}001"], "comment": comment, "is_view": False, "foreign_keys": []}
|
||
|
||
|
||
def _source(tables: dict, **kw) -> SimpleNamespace:
|
||
for name, meta in tables.items():
|
||
meta["qualified"] = name
|
||
return SimpleNamespace(id="src", name="shop", kind="sqlite", origin="manual",
|
||
description=kw.pop("description", ""), schema_cache={"tables": tables})
|
||
|
||
|
||
def _reply(table: str, meta: dict) -> dict:
|
||
"""一张表的完整草稿：表级三项 + 每列中文名和度量类型。"""
|
||
return {"label": f"{table}的中文名", "grain": "每条记录一行", "kind": "fact",
|
||
"columns": [{"name": c["name"], "label": f"{c['name']}的中文名", "measure": "attribute"}
|
||
for c in meta["columns"]]}
|
||
|
||
|
||
@dataclass
|
||
class Reply:
|
||
"""一次回复。obj 是正常的 JSON 对象；text 给了就原样作正文（截断的 JSON 之类）。"""
|
||
|
||
obj: Any = None
|
||
text: str | None = None
|
||
finish: str = "stop"
|
||
thinking: str = ""
|
||
usage: tuple[int, int] = (0, 0)
|
||
|
||
def render(self, mode: str, include_raw: bool) -> Any:
|
||
if self.text is not None:
|
||
content = self.text
|
||
else:
|
||
content = json.dumps(self.obj, ensure_ascii=False) if self.obj is not None else ""
|
||
msg = AIMessage(
|
||
content=content,
|
||
additional_kwargs={"reasoning_content": self.thinking} if self.thinking else {},
|
||
response_metadata={"finish_reason": self.finish},
|
||
usage_metadata={"input_tokens": self.usage[0], "output_tokens": self.usage[1],
|
||
"total_tokens": sum(self.usage)},
|
||
)
|
||
if mode == "text":
|
||
return msg
|
||
parsed = self.obj if self.text is None else None
|
||
if not include_raw:
|
||
return parsed
|
||
return {"raw": msg, "parsed": parsed, "parsing_error": None if parsed is not None else ValueError("坏的")}
|
||
|
||
|
||
@dataclass
|
||
class Call:
|
||
mode: str
|
||
text: str
|
||
asked: dict
|
||
n: int
|
||
|
||
|
||
class Scripted:
|
||
"""每次调用交给 handler(call) 决定怎么回：返回 Reply，或者抛异常；handler 可以是协程。"""
|
||
|
||
def __init__(self, handler: Callable[[Call], Any], name: str = "scripted-model") -> None:
|
||
self.handler, self.model_name = handler, name
|
||
self.calls: list[Call] = []
|
||
|
||
def with_structured_output(self, schema: Any, include_raw: bool = False, **_: Any) -> Any:
|
||
owner = self
|
||
|
||
class _S:
|
||
async def ainvoke(self, messages):
|
||
return await owner._run(messages, "structured", include_raw)
|
||
|
||
return _S()
|
||
|
||
async def ainvoke(self, messages: list[Any]) -> Any:
|
||
return await self._run(messages, "text", False)
|
||
|
||
async def _run(self, messages: list[Any], mode: str, include_raw: bool) -> Any:
|
||
text = "\n".join(str(getattr(m, "content", m)) for m in messages)
|
||
call = Call(mode=mode, text=text, asked=requested(text), n=len(self.calls))
|
||
self.calls.append(call)
|
||
out = self.handler(call)
|
||
if inspect.isawaitable(out):
|
||
out = await out
|
||
return out.render(mode, include_raw)
|
||
|
||
|
||
def answering(tables: dict, **kw) -> Callable[[Call], Reply]:
|
||
"""照请求写：每张表按请它写的那部分给出 _reply 的内容。"""
|
||
def handle(call: Call) -> Reply:
|
||
return Reply(obj={"tables": [answer(_reply(t, tables[t]), t, *call.asked[t]) for t in call.asked]}, **kw)
|
||
return handle
|
||
|
||
|
||
class _Status(Exception):
|
||
def __init__(self, status: int, text: str) -> None:
|
||
super().__init__(text)
|
||
self.status_code = status
|
||
|
||
|
||
class LengthFinishReasonError(Exception):
|
||
"""和 openai 的同名异常一样：解析结构化输出时发现 finish_reason=length。"""
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__("Could not parse response content as the length limit was reached")
|
||
self.completion = None
|
||
|
||
|
||
class ConnectError(Exception):
|
||
pass
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _no_wait(monkeypatch):
|
||
"""退避不真等：记下要等多久。"""
|
||
waited: list[float] = []
|
||
|
||
async def fake_sleep(seconds: float) -> None:
|
||
waited.append(seconds)
|
||
|
||
monkeypatch.setattr(D, "_sleep", fake_sleep)
|
||
return waited
|
||
|
||
|
||
def _status(outcomes: dict) -> dict:
|
||
return {t: o.status for t, o in outcomes.items()}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 思考用完额度、空壳：拆开再问
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_thinking_used_up_the_budget_splits_the_segment():
|
||
tables = {"orders_x": _table(24)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.n == 0: # 只想不写：正文为空、只有思考，额度用完
|
||
return Reply(text="", finish="length", thinking="先逐列想一遍……")
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["orders_x"])
|
||
o = got["orders_x"]
|
||
assert o.status == "ok" and o.error is None and o.items == 3 + 24 * 2
|
||
assert len(model.calls) == 3
|
||
cols = [f"c{i:03d}" for i in range(1, 25)]
|
||
assert model.calls[1].asked == {"orders_x": ("table+columns", cols[:12])}
|
||
assert model.calls[2].asked == {"orders_x": ("columns", cols[12:])}
|
||
# 后半段在前半段之后问，带着刚起草的表级内容做背景
|
||
assert catalog.CATALOG_DRAFT_PROMPT_LABELS["drafted"] in model.calls[2].text
|
||
assert "orders_x的中文名" in model.calls[2].text
|
||
|
||
|
||
async def test_length_finish_reason_error_also_splits():
|
||
tables = {"orders_x": _table(24)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.n == 0:
|
||
raise LengthFinishReasonError()
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["orders_x"])
|
||
assert got["orders_x"].status == "ok" and len(model.calls) == 3
|
||
|
||
|
||
async def test_hollow_shell_splits_the_pack_by_table():
|
||
tables = {"a_tab": _table(6), "b_tab": _table(6)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.n == 0:
|
||
return Reply(obj={"tables": [{"table": "a_tab"}, {"table": "b_tab"}]})
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["a_tab", "b_tab"])
|
||
assert _status(got) == {"a_tab": "ok", "b_tab": "ok"}
|
||
assert [set(c.asked) for c in model.calls] == [{"a_tab", "b_tab"}, {"a_tab"}, {"b_tab"}]
|
||
assert all(c.asked[t][0] == "full" for c in model.calls[1:] for t in c.asked)
|
||
|
||
|
||
async def test_splitting_stops_at_three_levels_and_five_columns():
|
||
tables = {"wide40": _table(40)}
|
||
model = Scripted(lambda call: Reply(text="", finish="length", thinking="……"))
|
||
got = await D.run_model_draft(model, _source(tables), ["wide40"])
|
||
o = got["wide40"]
|
||
assert len(model.calls) == 1 + 2 + 4 + 8, "40 → 20 → 10 → 5 列，最多拆三层"
|
||
assert min(len(c.asked["wide40"][1] or []) for c in model.calls[1:]) == 5
|
||
assert o.status == "failed" and o.items == 0
|
||
assert o.error == "模型的思考和输出用完了额度（32768 个 token），这一段没写出来；已拆小重试仍没写出"
|
||
|
||
small = Scripted(lambda call: Reply(text="", finish="length"))
|
||
got = await D.run_model_draft(small, _source({"t9": _table(9)}), ["t9"])
|
||
assert len(small.calls) == 1, "不到 10 列拆不出两段 5 列"
|
||
assert got["t9"].error == "模型的思考和输出用完了额度（32768 个 token），这一段没写出来"
|
||
|
||
|
||
async def test_an_unsplittable_hollow_shell_tries_text_once_then_stays_on_text():
|
||
"""有的网关的结构化输出只给空壳：拆不开时换纯文本问一次，写出来了，这次起草后面直接走纯文本。
|
||
|
||
用宽表看先后：表级那段（拆不开）写完之后才问列分段。"""
|
||
tables = {"wide45": _table(45)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.mode == "structured":
|
||
return Reply(obj={"tables": [{"table": t} for t in call.asked]})
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["wide45"])
|
||
assert got["wide45"].status == "ok" and len(got["wide45"].notes["columns"]) == 45
|
||
assert [c.mode for c in model.calls] == ["structured", "text", "text", "text"]
|
||
assert catalog.CATALOG_DRAFT_TEXT_PROMPT in model.calls[1].text, "纯文本那条路要说清 JSON 的格式"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 截断、缺表、缺列
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_truncated_json_keeps_complete_columns_and_asks_only_the_rest():
|
||
tables = {"t12": _table(12)}
|
||
full = json.dumps({"tables": [{"table": "t12", **_reply("t12", tables["t12"])}]}, ensure_ascii=False)
|
||
cut = full[:full.index('"c006"') + 8] # 第 6 列写到一半
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.n == 0:
|
||
return Reply(text=cut, finish="length")
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["t12"])
|
||
o = got["t12"]
|
||
assert o.status == "ok" and set(o.notes["columns"]) == {f"c{i:03d}" for i in range(1, 13)}
|
||
assert o.notes["label"]["value"] == "t12的中文名"
|
||
assert len(model.calls) == 2
|
||
assert model.calls[1].asked == {"t12": ("columns", [f"c{i:03d}" for i in range(6, 13)])}
|
||
|
||
|
||
async def test_truncated_tool_call_arguments_are_salvaged_too():
|
||
"""结构化输出走工具调用时，截断的参数在 invalid_tool_calls 里（一段字符串）。"""
|
||
tables = {"t12": _table(12)}
|
||
full = json.dumps({"tables": [{"table": "t12", **_reply("t12", tables["t12"])}]}, ensure_ascii=False)
|
||
cut = full[:full.index('"c004"') + 3] # 第 4 列的列名写到一半
|
||
ok = answering(tables)
|
||
|
||
class ToolModel(Scripted):
|
||
async def _run(self, messages, mode, include_raw):
|
||
if not self.calls:
|
||
self.calls.append(Call(mode, "", {}, 0))
|
||
raw = AIMessage(content="", invalid_tool_calls=[{"name": "catalog_draft", "args": cut, "id": "x",
|
||
"error": None, "type": "invalid_tool_call"}],
|
||
response_metadata={"finish_reason": "length"})
|
||
return {"raw": raw, "parsed": None, "parsing_error": ValueError("截断")}
|
||
return await super()._run(messages, mode, include_raw)
|
||
|
||
model = ToolModel(ok)
|
||
got = await D.run_model_draft(model, _source(tables), ["t12"])
|
||
assert got["t12"].status == "ok"
|
||
assert model.calls[1].asked == {"t12": ("columns", [f"c{i:03d}" for i in range(4, 13)])}
|
||
|
||
|
||
async def test_missing_tables_are_asked_again_once():
|
||
"""缺了的表只补缺的、补一次；补问时整张表还是一项都没有，按只给空壳处理（拆不开就换纯文本问一次）。"""
|
||
tables = {"a_tab": _table(4), "b_tab": _table(4), "c_tab": _table(4)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
reply = ok(call)
|
||
keep = {0: {"a_tab"}, 1: {"b_tab"}}.get(call.n, set())
|
||
reply.obj["tables"] = [e for e in reply.obj["tables"] if e["table"] in keep]
|
||
return reply
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["a_tab", "b_tab", "c_tab"])
|
||
assert len(model.calls) == 3
|
||
assert model.calls[1].asked == {"b_tab": ("full", None), "c_tab": ("full", None)}
|
||
assert model.calls[2].asked == {"c_tab": ("full", None)} and model.calls[2].mode == "text"
|
||
assert _status(got) == {"a_tab": "ok", "b_tab": "ok", "c_tab": "failed"}
|
||
assert got["c_tab"].error == "模型的回复里没有这张表"
|
||
|
||
|
||
async def test_a_table_still_missing_after_the_followup_is_split_like_a_hollow_shell():
|
||
"""补问时整张表还是一项都没有：照第一次的规则拆小（这里按列拆成两半），不直接记失败。"""
|
||
tables = {"a_tab": _table(4), "b_tab": _table(12)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
reply = ok(call)
|
||
if call.n < 2:
|
||
reply.obj["tables"] = [e for e in reply.obj["tables"] if e["table"] == "a_tab"]
|
||
return reply
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["a_tab", "b_tab"])
|
||
assert _status(got) == {"a_tab": "ok", "b_tab": "ok"}
|
||
cols = [f"c{i:03d}" for i in range(1, 13)]
|
||
assert [c.asked for c in model.calls[2:]] == [{"b_tab": ("table+columns", cols[:6])},
|
||
{"b_tab": ("columns", cols[6:])}]
|
||
|
||
|
||
async def test_columns_still_missing_after_the_followup_are_not_written():
|
||
"""补问过仍然没列出来的列算没写出来：表是 partial，说清哪部分没写出来（不算拿不准，旧的模型起草项不删）。"""
|
||
tables = {"t8": _table(8)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
reply = ok(call)
|
||
for entry in reply.obj["tables"]:
|
||
entry["columns"] = [c for c in entry.get("columns", []) if c["name"] not in ("c007", "c008")]
|
||
return reply
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["t8"])
|
||
assert len(model.calls) == 2 and model.calls[1].asked == {"t8": ("columns", ["c007", "c008"])}
|
||
o = got["t8"]
|
||
assert o.status == "partial"
|
||
assert o.error == "这张表 8 列，有 2 列没写出来：模型的回复里漏了这些列，补问一次仍没有给出。可以只重试这张表"
|
||
assert set(o.notes["columns"]) == {f"c{i:03d}" for i in range(1, 7)}
|
||
compat = (await catalog.draft_with_model(Scripted(handle), _source(tables), ["t8"]))["t8"]
|
||
assert compat.covered == frozenset(), "没写全的不算完整起草过"
|
||
|
||
|
||
async def test_a_followup_that_comes_back_as_an_empty_shell_leaves_the_table_partial():
|
||
"""40 列的表第一次只回了 20 列，补问只给一个空壳：没写出来的 20 列照实记下，表是 partial。"""
|
||
tables = {"t40": _table(40)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.n == 0:
|
||
reply = ok(call)
|
||
reply.obj["tables"][0]["columns"] = reply.obj["tables"][0]["columns"][:20]
|
||
return reply
|
||
return Reply(obj={"tables": [{"table": "t40"}]})
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["t40"])
|
||
o = got["t40"]
|
||
assert len(model.calls) == 2
|
||
assert model.calls[1].asked == {"t40": ("columns", [f"c{i:03d}" for i in range(21, 41)])}
|
||
assert o.status == "partial" and len(o.notes["columns"]) == 20
|
||
assert o.error == "这张表 40 列，有 20 列没写出来：模型的回复里漏了这些列，补问一次仍没有给出。可以只重试这张表"
|
||
|
||
|
||
async def test_columns_listed_with_empty_fields_are_left_to_the_model():
|
||
"""列出了、字段留空的列是模型拿不准：不补问，也不算没写出来。"""
|
||
tables = {"t8": _table(8)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
reply = ok(call)
|
||
for entry in reply.obj["tables"]:
|
||
entry["columns"] = [c if c["name"] not in ("c007", "c008") else {"name": c["name"]}
|
||
for c in entry.get("columns", [])]
|
||
return reply
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["t8"])
|
||
assert len(model.calls) == 1
|
||
assert got["t8"].status == "ok" and got["t8"].error is None
|
||
assert set(got["t8"].notes["columns"]) == {f"c{i:03d}" for i in range(1, 7)}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 额度被拒、结构化输出不支持、网络错误、超时
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_budget_rejection_falls_back_to_8192_for_the_rest():
|
||
tables = {"wide45": _table(45)}
|
||
|
||
def refuse(call: Call) -> Reply:
|
||
raise _Status(400, "max_tokens: 32768 > 8192, which is the maximum allowed number of output tokens")
|
||
|
||
big, small = Scripted(refuse), Scripted(answering(tables))
|
||
model = catalog.DraftModel(big, small, "m")
|
||
got = await D.run_model_draft(model, _source(tables), ["wide45"])
|
||
assert got["wide45"].status == "ok"
|
||
assert len(big.calls) == 1 and len(small.calls) == 3, "退回之后，表级那段重来、两个列分段都用小额度"
|
||
assert model.budget == 8192
|
||
|
||
|
||
async def test_exhausted_message_names_the_budget_actually_used():
|
||
def refuse(call: Call) -> Reply:
|
||
raise _Status(400, "max_tokens is too large")
|
||
|
||
model = catalog.DraftModel(Scripted(refuse), Scripted(lambda c: Reply(text="", finish="length")), "m")
|
||
got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"])
|
||
assert got["t4"].error == "模型的思考和输出用完了额度（8192 个 token），这一段没写出来"
|
||
|
||
|
||
async def test_structured_output_unsupported_switches_to_text_for_the_rest():
|
||
tables = {"wide45": _table(45)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.mode == "structured":
|
||
raise _Status(400, "This response_format type is unavailable now")
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["wide45"])
|
||
assert got["wide45"].status == "ok"
|
||
assert [c.mode for c in model.calls] == ["structured", "text", "text", "text"]
|
||
|
||
|
||
@pytest.mark.parametrize("thinking", ["", "先想想每一列是什么……"], ids=["plain", "with-thinking"])
|
||
async def test_structured_output_that_comes_back_as_prose_falls_back_to_text(thinking):
|
||
"""网关接受了结构化输出的参数却没照做（模型写成了一段说明文字）：换纯文本问，不当成额度用完去拆，也不直接判失败。"""
|
||
tables = {"t40": _table(40)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.mode == "structured":
|
||
return Reply(text="好的。这张表记录的是一笔笔业务，各列的含义如下：第一列是编号……", thinking=thinking)
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["t40"])
|
||
assert got["t40"].status == "ok" and len(got["t40"].notes["columns"]) == 40
|
||
assert [c.mode for c in model.calls] == ["structured", "text"]
|
||
assert catalog.CATALOG_DRAFT_TEXT_PROMPT in model.calls[1].text
|
||
|
||
|
||
async def test_prose_on_both_paths_is_a_format_failure_not_a_budget_one():
|
||
def handle(call: Call) -> Reply:
|
||
return Reply(text="这张表看起来是业务明细，具体含义需要结合业务确认。", thinking="……")
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source({"t40": _table(40)}), ["t40"])
|
||
assert [c.mode for c in model.calls] == ["structured", "text"], "格式不对不拆小重试"
|
||
assert got["t40"].status == "failed" and got["t40"].error == "模型的回复不是约定的格式"
|
||
|
||
|
||
async def test_a_model_without_structured_output_goes_to_text():
|
||
class TextOnly(Scripted):
|
||
def with_structured_output(self, schema, **_):
|
||
raise NotImplementedError
|
||
|
||
tables = {"t4": _table(4)}
|
||
model = TextOnly(answering(tables))
|
||
got = await D.run_model_draft(model, _source(tables), ["t4"])
|
||
assert got["t4"].status == "ok" and [c.mode for c in model.calls] == ["text"]
|
||
|
||
|
||
async def test_network_errors_are_retried_with_backoff(_no_wait):
|
||
tables = {"t4": _table(4)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.n < 2:
|
||
raise ConnectError("All connection attempts failed")
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["t4"])
|
||
assert got["t4"].status == "ok" and len(model.calls) == 3
|
||
assert _no_wait == list(catalog.MODEL_RETRY_DELAYS_S)
|
||
|
||
|
||
async def test_server_errors_give_up_after_two_retries():
|
||
def handle(call: Call) -> Reply:
|
||
raise _Status(503, "Service Unavailable")
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"])
|
||
assert len(model.calls) == 3
|
||
assert got["t4"].status == "failed"
|
||
assert got["t4"].error == "调用模型出错：Service Unavailable（已重试 2 次）"
|
||
|
||
|
||
async def test_other_errors_are_not_retried():
|
||
def handle(call: Call) -> Reply:
|
||
raise RuntimeError("网关拒绝了这次请求")
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"])
|
||
assert len(model.calls) == 1 and got["t4"].error == "调用模型出错：网关拒绝了这次请求"
|
||
|
||
|
||
async def test_a_call_past_the_time_limit_is_retried_then_reported(monkeypatch):
|
||
monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05)
|
||
|
||
async def slow(call: Call) -> Reply:
|
||
await asyncio.sleep(1)
|
||
return Reply(obj={"tables": []})
|
||
|
||
model = Scripted(slow)
|
||
got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"])
|
||
assert len(model.calls) == 3
|
||
assert got["t4"].error == "模型 0.05 秒内没有写完这一段（已重试 2 次）"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 分段：宽表拆成表级一段 + 列分段，窄表拼段
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_a_wide_table_is_split_into_a_table_part_and_column_parts():
|
||
tables = {"wide95": _table(95, comment="虚构的宽表")}
|
||
model = Scripted(answering(tables))
|
||
done: list[str] = []
|
||
|
||
async def part_done(table: str) -> None:
|
||
done.append(table)
|
||
|
||
got = await D.run_model_draft(model, _source(tables), ["wide95"], hooks=D.Hooks(part_done=part_done))
|
||
o = got["wide95"]
|
||
assert o.status == "ok" and o.parts_total == 4 and o.parts_done == 4 and done == ["wide95"] * 4
|
||
assert len(o.notes["columns"]) == 95 and o.notes["label"]["value"] == "wide95的中文名"
|
||
first, *rest = model.calls
|
||
assert first.asked == {"wide95": ("table", None)}
|
||
assert catalog.CATALOG_DRAFT_PROMPT_LABELS["brief_columns"].format(n=95) in first.text
|
||
assert sorted(len(c.asked["wide95"][1]) for c in rest) == [15, 40, 40]
|
||
for call in rest:
|
||
assert call.asked["wide95"][0] == "columns"
|
||
assert "wide95的中文名" in call.text and "虚构的宽表" in call.text, "列分段带着表注释和表级起草结果"
|
||
assert {c for call in rest for c in call.asked["wide95"][1]} == {f"c{i:03d}" for i in range(1, 96)}
|
||
|
||
|
||
async def test_a_wide_table_with_one_failed_part_is_partial_and_says_which():
|
||
tables = {"wide95": _table(95)}
|
||
ok = answering(tables)
|
||
|
||
def handle(call: Call) -> Reply:
|
||
part, cols = call.asked["wide95"]
|
||
if part == "columns" and "c041" in cols:
|
||
raise RuntimeError("网关拒绝")
|
||
return ok(call)
|
||
|
||
model = Scripted(handle)
|
||
got = await D.run_model_draft(model, _source(tables), ["wide95"])
|
||
o = got["wide95"]
|
||
assert o.status == "partial" and len(o.notes["columns"]) == 55
|
||
assert o.error == "这张表 95 列，分 4 段起草，第 3 段没写出来（共 40 列）：调用模型出错：网关拒绝。可以只重试这张表"
|
||
compat = (await catalog.draft_with_model(Scripted(handle), _source(tables), ["wide95"]))["wide95"]
|
||
assert compat.covered == frozenset() and compat.error == o.error, "没写全的不算完整起草过，旧的模型项不删"
|
||
|
||
|
||
async def test_tables_are_packed_up_to_40_columns_and_4_tables():
|
||
widths = {"ta": 15, "tb": 15, "tc": 15, "td": 5, "te": 5, "tf": 5, "tg": 5, "th": 41}
|
||
tables = {name: _table(n) for name, n in widths.items()}
|
||
model = Scripted(answering(tables))
|
||
got = await D.run_model_draft(model, _source(tables), list(widths))
|
||
assert set(_status(got).values()) == {"ok"}
|
||
asked = sorted((tuple(sorted(c.asked)), c.asked[next(iter(c.asked))][0]) for c in model.calls)
|
||
assert asked == sorted([
|
||
(("ta", "tb"), "full"), (("tc", "td", "te", "tf"), "full"), (("tg",), "full"),
|
||
(("th",), "table"), (("th",), "columns"), (("th",), "columns"),
|
||
])
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 报错去重、止损、几次起草轮流用模型
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _hollow(call: Call) -> Reply:
|
||
return Reply(obj={"tables": [{"table": t} for t in call.asked]})
|
||
|
||
|
||
async def test_the_same_reason_is_said_once_even_when_some_parts_were_split(monkeypatch):
|
||
"""宽表的表级那段拆不开、列分段拆过：原因只说一次，拆过的在最后补一句。"""
|
||
monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99)
|
||
model = Scripted(_hollow)
|
||
got = await D.run_model_draft(model, _source({"wide45": _table(45)}), ["wide45"])
|
||
o = got["wide45"]
|
||
assert o.status == "failed"
|
||
assert o.error == "模型只给了空壳，没有写出可用的内容；已拆小重试仍没写出"
|
||
|
||
|
||
def test_reasons_are_joined_by_their_base():
|
||
W = D._Why
|
||
exhausted = D.REASON_EXHAUSTED.format(budget=32768)
|
||
assert D._join([W(D.REASON_HOLLOW), W(D.REASON_HOLLOW, split=True)]) == D.REASON_HOLLOW + D.REASON_SPLIT
|
||
assert D._join([W(exhausted, split=True), W(D.REASON_HOLLOW)]) == \
|
||
f"{exhausted}；{D.REASON_HOLLOW}{D.REASON_SPLIT}"
|
||
assert D._join([W("甲"), W("乙"), W("丙", split=True)]) == "甲；乙", "没说出来的原因不补后缀"
|
||
assert D._join([W(D.REASON_HOLLOW, halted=True), W(D.REASON_HOLLOW)]) == \
|
||
D.REASON_HOLLOW + D.REASON_HALTED.format(n=catalog.MODEL_GIVE_UP_AFTER)
|
||
|
||
|
||
def _by_table(model: Scripted) -> dict[str, int]:
|
||
out: dict[str, int] = {}
|
||
for call in model.calls:
|
||
for name in call.asked:
|
||
out[name] = out.get(name, 0) + 1
|
||
return out
|
||
|
||
|
||
async def test_after_two_segments_in_a_row_come_to_nothing_the_rest_are_not_split(monkeypatch):
|
||
"""模型明显用不了（思考总是吃满额度）：连续 2 段拆到底仍一项没写出，后面的段只问一次、不再拆，报错说明已停止拆小重试。
|
||
有一段写出来了就恢复拆小。一次只起草一段（并发 1），先后是确定的。"""
|
||
monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1)
|
||
monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1)
|
||
assert catalog.MODEL_GIVE_UP_AFTER == 2
|
||
tables = {f"t{i}": _table(10) for i in range(1, 5)}
|
||
exhausted = Reply(text="", finish="length", thinking="……")
|
||
|
||
model = Scripted(lambda call: exhausted)
|
||
got = await D.run_model_draft(model, _source(tables), list(tables))
|
||
assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1}
|
||
budget = D.REASON_EXHAUSTED.format(budget=32768)
|
||
assert got["t1"].error == budget + D.REASON_SPLIT
|
||
assert got["t3"].error == budget + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型"
|
||
|
||
ok = answering(tables)
|
||
model = Scripted(lambda call: ok(call) if "t3" in call.asked else exhausted)
|
||
got = await D.run_model_draft(model, _source(tables), list(tables))
|
||
assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 3}, "t3 写出来了，t4 照常拆小"
|
||
assert got["t3"].status == "ok" and got["t4"].error == budget + D.REASON_SPLIT
|
||
|
||
|
||
async def test_the_column_parts_of_a_wide_table_stop_splitting_too(monkeypatch):
|
||
"""宽表的列分段一段一段排队起草（一段拆到底再起草下一段），止损在宽表里同样起作用。
|
||
|
||
95 列：表级那段拆不开（结构化、纯文本各一次）；第 2 段 40 列拆到底（15 次结构化 + 8 次纯文本）；这时已经连续 2 段
|
||
没写出来，第 3、4 段只问一次、换纯文本一次。"""
|
||
monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1)
|
||
model = Scripted(_hollow)
|
||
got = await D.run_model_draft(model, _source({"wide95": _table(95)}), ["wide95"])
|
||
assert len(model.calls) == 2 + 23 + 2 + 2
|
||
assert got["wide95"].error == (D.REASON_HOLLOW + D.REASON_SPLIT
|
||
+ "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型")
|
||
|
||
|
||
async def test_a_wide_table_that_never_comes_back_is_bounded_by_the_first_wave():
|
||
"""默认并发（3 段同时起草）：294 列一直只给空壳，最多是表级那段 2 次、头 3 个列分段拆到底（各 23 次）、其余 5 段
|
||
各 2 次。以前所有列分段一下子都排上去、一层一层地拆，要 168 次。"""
|
||
model = Scripted(_hollow)
|
||
got = await D.run_model_draft(model, _source({"wide294": _table(294)}), ["wide294"])
|
||
assert got["wide294"].status == "failed"
|
||
assert len(model.calls) <= 2 + 3 * 23 + 5 * 2
|
||
|
||
|
||
async def test_a_big_draft_does_not_hold_the_model_ahead_of_a_later_one(monkeypatch):
|
||
"""全进程的调用名额先来先放：一次起草不把它的所有段一下子都排上去，后开始的那次起草（另一个数据源）能轮上。"""
|
||
monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1)
|
||
monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1)
|
||
big = {f"a{i}": _table(4) for i in range(1, 5)}
|
||
small = {"b1": _table(4)}
|
||
order: list[str] = []
|
||
release = asyncio.Event()
|
||
|
||
def handler(tables: dict) -> Callable[[Call], Any]:
|
||
ok = answering(tables)
|
||
|
||
async def handle(call: Call) -> Reply:
|
||
order.append(next(iter(call.asked)))
|
||
if len(order) == 1:
|
||
await release.wait()
|
||
return ok(call)
|
||
return handle
|
||
|
||
first = asyncio.create_task(D.run_model_draft(Scripted(handler(big)), _source(big), list(big)))
|
||
while not order:
|
||
await asyncio.sleep(0)
|
||
second = asyncio.create_task(D.run_model_draft(Scripted(handler(small)), _source(small), ["b1"]))
|
||
for _ in range(20):
|
||
await asyncio.sleep(0)
|
||
release.set()
|
||
got_big, got_small = await asyncio.gather(first, second)
|
||
assert set(_status(got_big).values()) == {"ok"} and _status(got_small) == {"b1": "ok"}
|
||
assert order == ["a1", "b1", "a2", "a3", "a4"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 起草要求、用量
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_guidance_goes_into_the_prompt_after_the_rules():
|
||
tables = {"t4": _table(4)}
|
||
model = Scripted(answering(tables))
|
||
src = _source(tables, description="虚构的景区票务库")
|
||
await D.run_model_draft(model, src, ["t4"],
|
||
background=D.Background(guidance="金额一律按元写单位", note="这次重点看会员相关的列"))
|
||
text = model.calls[0].text
|
||
block = ("使用者补充的背景和要求：\n数据源说明：虚构的景区票务库\n起草要求：金额一律按元写单位\n"
|
||
"本次说明：这次重点看会员相关的列")
|
||
assert block in text
|
||
rule = "它和这里的规矩冲突时，以这里的规矩为准"
|
||
assert rule in catalog.CATALOG_DRAFT_SYSTEM
|
||
assert text.index(rule) < text.index(block) < text.index("请为下面") < text.index("表 t4")
|
||
|
||
plain = Scripted(answering(tables))
|
||
await D.run_model_draft(plain, _source(tables), ["t4"])
|
||
assert "使用者补充的背景和要求：" not in plain.calls[0].text, "都没有就不出现这一段（系统提示词里那条规矩照旧在）"
|
||
only_note = Scripted(answering(tables))
|
||
await D.run_model_draft(only_note, _source(tables), ["t4"], background=D.Background(note="只看金额"))
|
||
assert "使用者补充的背景和要求：\n本次说明：只看金额\n\n请为下面" in only_note.calls[0].text
|
||
|
||
|
||
async def test_the_prompt_asks_not_to_draft_every_column_while_thinking():
|
||
assert "思考时不必把要输出的内容逐列草拟一遍" in catalog.CATALOG_DRAFT_SYSTEM
|
||
|
||
|
||
async def test_usage_is_added_up_through_the_hooks():
|
||
tables = {"orders_x": _table(24)}
|
||
ok = answering(tables, usage=(120, 60))
|
||
|
||
def handle(call: Call) -> Reply:
|
||
if call.n == 0:
|
||
return Reply(text="", finish="length", thinking="……", usage=(100, 32768))
|
||
return ok(call)
|
||
|
||
seen: list[tuple[int, int]] = []
|
||
|
||
async def usage(i: int, o: int) -> None:
|
||
seen.append((i, o))
|
||
|
||
await D.run_model_draft(Scripted(handle), _source(tables), ["orders_x"], hooks=D.Hooks(usage=usage))
|
||
assert seen == [(100, 32768), (120, 60), (120, 60)]
|
||
|
||
|
||
def test_budget_words_match_the_other_drafts():
|
||
from app.data import recipe_ai
|
||
|
||
assert D.BUDGET_WORDS == recipe_ai._BUDGET_WORDS
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 从正文里取 JSON
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize("text, tables, truncated", [
|
||
# 前面有一句话、带括号的说明、围栏
|
||
('按 {table} 的格式写好了：\n```json\n{"tables": [{"table": "a", "label": "甲"}]}\n```', ["a"], False),
|
||
('[说明] 草稿：{"tables": [{"table": "a"}]} 以上', ["a"], False),
|
||
# 外面多包了一层
|
||
('{"result": {"tables": [{"table": "a"}, {"table": "b"}]}}', ["a", "b"], False),
|
||
# 截断在字符串、数字、字面量、键名中间：只收写完了的
|
||
('{"tables": [{"table": "a", "label": "甲"}, {"table": "b", "label": "乙', ["a", "b"], True),
|
||
('{"tables": [{"table": "a", "columns": [{"name": "x", "label": "某"}, {"name": "y", "len": 12', ["a"], True),
|
||
('{"tables": [{"table": "a", "keys": ["x", "y"', ["a"], True),
|
||
('{"tables": [{"table": "a"}, {"tab', ["a"], True),
|
||
('{"tables": [{"table": "a", "flag": tr', ["a"], True),
|
||
])
|
||
def test_json_is_taken_from_the_text(text, tables, truncated):
|
||
reply = D._parse_text(text, cut=False)
|
||
assert reply is not None and [e["table"] for e in reply.entries] == tables
|
||
assert reply.truncated is truncated
|
||
|
||
|
||
def test_cut_values_are_dropped_and_complete_ones_kept():
|
||
text = '{"tables": [{"table": "a", "label": "甲", "keys": ["x", "y"], "columns": [{"name": "x", "label": "某"}, ' \
|
||
'{"name": "y", "label": "半'
|
||
entry = D._parse_text(text, cut=True).entries[0]
|
||
assert entry == {"table": "a", "label": "甲", "keys": ["x", "y"], "columns": [{"name": "x", "label": "某"}]}
|
||
cut_keys = D._parse_text('{"tables": [{"table": "a", "label": "甲", "keys": ["x", "y"', cut=True).entries[0]
|
||
assert cut_keys == {"table": "a", "label": "甲"}, "没写完的业务主键整项不要，不能只留前半"
|
||
|
||
|
||
@pytest.mark.parametrize("text", ["好的，我来写", "{坏的 json}", ""])
|
||
def test_not_json_is_not_a_reply(text):
|
||
assert D._parse_text(text, cut=False) is None
|