mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 05:42:01 +00:00
起草引擎按验收修正:列分段均分、超时拆小、段大小随模型减半、每段 8 次封顶、提醒、超时与没写出来的计数、模型留空的列、不插队、原因写清哪几段哪几列
- 均分:超过 40 列的表,列分段数是列数除以 40 向上取整,各段列数平均(相差不超过 1),不再留下 2 到 8 列的尾段(实测 会思考的模型常对这种小段只给空壳);表级那段、窄表拼段、parts_total 不变。 - 拆不开的段(不到 10 列)只给空壳:再问一次,结构化输出换纯文本,已经走纯文本的原样再问,仍是空壳才记没写出来。 - 超时当作这一段太大:不再原样重试同样大小的段,按额度用完的办法拆小再问;网络错误、429、5xx 仍原样退避重试。 - 段大小随模型调整:一次起草里额度用完或超时累计 2 次,之后新开始的段列数上限减半(40 → 20 → 10,最低 10),新开始 的段超过上限就先按上限均分开(多表的段按表重新拼);任务记当前的上限 part_columns。 - 封顶:一段(连同它拆出来的、补问的、重试的)最多调用 8 次,到了次数还没写完的记没写出来、原因写清。止损照旧。 - 提醒:最近 6 次调用有 4 次以上没写出来(额度用完、超时、空壳、出错),任务的 notice 写一句提醒(次数、原因,建议在 「设置 → 偏好设置」的「起草模型」换一个模型、减少表或停止);之后最近 6 次里少于 2 次就清掉。 - 用量:进度回调 usage 换成 call_done(CallReport:用量、没写出来的原因种类、当前的提醒和列数上限),任务的用量加 timeouts、failed_calls;超时的调用 token 照实记 0。 - 留空的列:请模型写了、模型也列出了,中文名和含义却都没给(或者给的都没通过校验)的列记在 ModelOutcome.blank,逐表 记 blank_columns 和最多 5 个列名 blank_sample,任务上是合计,目录清单的 last_draft 也带列数;表的状态不因此变。已确认 的、库注释已经说清的列不算。补问时把漏的列列出来、字段却留空的,算留空,不再记成「漏了这些列」。 - 公平排队:宽表的表级那段写完,它的列分段排到已经排着的段后面(先来先服务),不再插到队首让窄表一直等。 - 原因:几段因为同一个原因没写出来时说「这几段」,同一段拆出来的几块仍是「这一段」;只有真拆小过才补「已拆小重试仍 没写出」(截断后只补剩下的不算);每段写上列的范围,例如「第 5 段(列名 到 列名,共 33 列)」,窄表写出是哪几列。 - 库:catalog_draft_jobs 加 timeouts、failed_calls、part_columns、notice,catalog_draft_tasks 加 blank_columns、 blank_sample,旧库启动时补列(存量回填 0 和空)。假模型加 hang(头几次调用一直不回,用来模拟超时)。
This commit is contained in:
+14
-3
@@ -10,6 +10,7 @@
|
||||
不整列漏掉。list_all=False 时只给 replies 里有的列(模拟漏列的模型:漏掉的列会被补问一次,补问后仍没给的算没写出来);
|
||||
- 直接 ainvoke 是纯文本那条路:同样的内容写成一段正文(AIMessage),前面带一句话;
|
||||
- fail 里的表所在的那一段抛异常,用来验证「一段失败只影响这一段」;
|
||||
- hang=N:头 N 次调用一直不回(等起草引擎按时限掐掉,当作超时);
|
||||
- calls 记下每次收到的提示词全文,用来核对交给模型的内容(只有结构,没有数据);modes 记下每次走的是
|
||||
structured 还是 text。
|
||||
|
||||
@@ -22,6 +23,7 @@ structured、text 模拟实测见过的几种回复(两条路各自设):
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
@@ -105,10 +107,11 @@ def with_every_column(entry: dict, wanted: list[str]) -> dict:
|
||||
|
||||
class FakeModel:
|
||||
def __init__(self, replies: dict[str, dict], fail: set[str] = frozenset(), *,
|
||||
structured: str = "tables", text: str = "tables", list_all: bool = True) -> None:
|
||||
structured: str = "tables", text: str = "tables", list_all: bool = True, hang: int = 0) -> None:
|
||||
self.replies, self.fail, self.calls = replies, set(fail), []
|
||||
self.structured, self.text = structured, text
|
||||
self.list_all = list_all
|
||||
self.hang = hang
|
||||
self.modes: list[str] = []
|
||||
|
||||
def with_structured_output(self, schema: Any, **_: Any) -> "_Structured":
|
||||
@@ -118,9 +121,17 @@ class FakeModel:
|
||||
async def ainvoke(self, messages: list[Any]) -> Any:
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
reply = self._reply(messages, "text")
|
||||
reply = await self._answer(messages, "text")
|
||||
return AIMessage(content="草稿如下:" + json.dumps(reply, ensure_ascii=False))
|
||||
|
||||
async def _answer(self, messages: list[Any], mode: str) -> dict:
|
||||
if self.hang > 0:
|
||||
self.hang -= 1
|
||||
self.calls.append("\n".join(str(getattr(m, "content", m)) for m in messages))
|
||||
self.modes.append(mode)
|
||||
await asyncio.sleep(3600)
|
||||
return self._reply(messages, mode)
|
||||
|
||||
def _reply(self, messages: list[Any], mode: str) -> dict:
|
||||
text = "\n".join(str(getattr(m, "content", m)) for m in messages)
|
||||
self.calls.append(text)
|
||||
@@ -147,4 +158,4 @@ class _Structured:
|
||||
self.owner = owner
|
||||
|
||||
async def ainvoke(self, messages: list[Any]) -> dict:
|
||||
return self.owner._reply(messages, "structured")
|
||||
return await self.owner._answer(messages, "structured")
|
||||
|
||||
@@ -324,7 +324,7 @@ async def test_hollow_structured_output_retries_with_text():
|
||||
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
|
||||
assert model.modes == ["structured", "text", "text"]
|
||||
assert got["visits"].notes == {"label": make_item("入园记录", "llm"), "kind": make_item("fact", "llm")}
|
||||
assert got["visits"].covered == frozenset() and "有 5 列没写出来" in got["visits"].error
|
||||
assert got["visits"].covered == frozenset() and "有 5 列(id 到 status)没写出来" in got["visits"].error
|
||||
|
||||
|
||||
async def test_hollow_structured_then_single_text_object():
|
||||
|
||||
@@ -6,7 +6,11 @@
|
||||
- 被截断的 JSON:已经完整的表、列对象捡回来,只补剩下的;
|
||||
- 只回一部分表:只补缺的,补一次;
|
||||
- 服务方嫌额度太大 → 退回 8192,后面都用它;结构化输出不支持 → 后面改走纯文本;
|
||||
- 网络错误、5xx、超时 → 退避重试,最多 2 次,报错写明重试过几次。
|
||||
- 网络错误、5xx → 退避重试,最多 2 次,报错写明重试过几次;超时当作这一段太大,拆小再问,不原样重试。
|
||||
|
||||
验收后修正的几条(用假模型复现):宽表的列分段均分、没有很小的尾段;拆不开的空壳纯文本也再问一次;额度用完或超时
|
||||
累计 2 次后新开始的段列数减半;一段最多调用 8 次;最近 6 次调用 4 次没写出来时提醒、少于 2 次时清掉;记超时和没写
|
||||
出来的调用;模型留空的列计数;窄表不被宽表的列分段插队;原因写清是哪几段、哪几列。
|
||||
|
||||
表名、列名、说明文字都是虚构的。
|
||||
"""
|
||||
@@ -163,6 +167,34 @@ def _status(outcomes: dict) -> dict:
|
||||
return {t: o.status for t, o in outcomes.items()}
|
||||
|
||||
|
||||
def _hollow(call: Call) -> Reply:
|
||||
return Reply(obj={"tables": [{"table": t} for t in call.asked]})
|
||||
|
||||
|
||||
def _exhausted(seg: str = "这一段", budget: int = 32768) -> str:
|
||||
return D.REASON_EXHAUSTED.format(budget=budget, seg=seg)
|
||||
|
||||
|
||||
EXHAUSTED = Reply(text="", finish="length", thinking="……")
|
||||
|
||||
|
||||
class Reports:
|
||||
"""进度回调 call_done 收到的每次调用的结果。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.items: list[D.CallReport] = []
|
||||
|
||||
async def __call__(self, report: D.CallReport) -> None:
|
||||
self.items.append(report)
|
||||
|
||||
def hooks(self) -> D.Hooks:
|
||||
return D.Hooks(call_done=self)
|
||||
|
||||
@property
|
||||
def failures(self) -> list[str | None]:
|
||||
return [r.failure for r in self.items]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 思考用完额度、空壳:拆开再问
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -220,9 +252,10 @@ async def test_hollow_shell_splits_the_pack_by_table():
|
||||
assert all(c.asked[t][0] == "full" for c in model.calls[1:] for t in c.asked)
|
||||
|
||||
|
||||
async def test_splitting_stops_at_three_levels_and_five_columns():
|
||||
async def test_splitting_stops_at_three_levels_and_five_columns(monkeypatch):
|
||||
monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99)
|
||||
tables = {"wide40": _table(40)}
|
||||
model = Scripted(lambda call: Reply(text="", finish="length", thinking="……"))
|
||||
model = Scripted(lambda call: EXHAUSTED)
|
||||
got = await D.run_model_draft(model, _source(tables), ["wide40"])
|
||||
o = got["wide40"]
|
||||
assert len(model.calls) == 1 + 2 + 4 + 8, "40 → 20 → 10 → 5 列,最多拆三层"
|
||||
@@ -236,6 +269,30 @@ async def test_splitting_stops_at_three_levels_and_five_columns():
|
||||
assert got["t9"].error == "模型的思考和输出用完了额度(32768 个 token),这一段没写出来"
|
||||
|
||||
|
||||
async def test_one_part_makes_at_most_eight_calls(monkeypatch):
|
||||
"""一段(连同它拆出来的、补问的、重试的)最多调用 8 次:40 列一直用完额度,拆到第 8 次就停,剩下没问的记没写出来、
|
||||
原因写清。以前一段 40 列拆到底要 15 次。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99)
|
||||
assert catalog.MODEL_PART_MAX_CALLS == 8
|
||||
model = Scripted(lambda call: EXHAUSTED)
|
||||
got = await D.run_model_draft(model, _source({"t40": _table(40)}), ["t40"])
|
||||
assert len(model.calls) == 8
|
||||
o = got["t40"]
|
||||
assert o.status == "failed"
|
||||
assert o.error == (_exhausted() + ";每段最多调用 8 次模型,这一段到了次数仍没写完,已停止重试"
|
||||
+ D.REASON_SPLIT)
|
||||
|
||||
# 网络错误的重试也算在这 8 次里
|
||||
def flaky(call: Call) -> Reply:
|
||||
raise ConnectError("All connection attempts failed")
|
||||
|
||||
monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 2)
|
||||
model = Scripted(flaky)
|
||||
got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"])
|
||||
assert len(model.calls) == 2
|
||||
assert got["t4"].error == "每段最多调用 2 次模型,这一段到了次数仍没写完,已停止重试"
|
||||
|
||||
|
||||
async def test_an_unsplittable_hollow_shell_tries_text_once_then_stays_on_text():
|
||||
"""有的网关的结构化输出只给空壳:拆不开时换纯文本问一次,写出来了,这次起草后面直接走纯文本。
|
||||
|
||||
@@ -255,6 +312,31 @@ async def test_an_unsplittable_hollow_shell_tries_text_once_then_stays_on_text()
|
||||
assert catalog.CATALOG_DRAFT_TEXT_PROMPT in model.calls[1].text, "纯文本那条路要说清 JSON 的格式"
|
||||
|
||||
|
||||
def _text_only(handle_text: Callable[[Call], Reply]) -> Callable[[Call], Reply]:
|
||||
"""结构化输出不支持(这次起草一开始就改走纯文本),纯文本交给 handle_text。"""
|
||||
def handle(call: Call) -> Reply:
|
||||
if call.mode == "structured":
|
||||
raise _Status(400, "This response_format type is unavailable now")
|
||||
return handle_text(call)
|
||||
return handle
|
||||
|
||||
|
||||
async def test_an_unsplittable_hollow_shell_is_asked_again_in_text_mode_too():
|
||||
"""已经走纯文本时,拆不开的段(不到 10 列)只给空壳:原样再问一次,写出来就用;仍是空壳才记没写出来。
|
||||
以前纯文本模式下不补问,整张表直接记没写出来。"""
|
||||
tables = {"t8": _table(8)}
|
||||
ok = answering(tables)
|
||||
model = Scripted(_text_only(lambda call: _hollow(call) if call.n == 1 else ok(call)))
|
||||
got = await D.run_model_draft(model, _source(tables), ["t8"])
|
||||
assert [c.mode for c in model.calls] == ["structured", "text", "text"]
|
||||
assert got["t8"].status == "ok" and len(got["t8"].notes["columns"]) == 8
|
||||
|
||||
model = Scripted(_text_only(_hollow))
|
||||
got = await D.run_model_draft(model, _source(tables), ["t8"])
|
||||
assert [c.mode for c in model.calls] == ["structured", "text", "text"], "再问一次,不多问"
|
||||
assert got["t8"].status == "failed" and got["t8"].error == catalog.MODEL_EMPTY_REASON
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 截断、缺表、缺列
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -358,12 +440,35 @@ async def test_columns_still_missing_after_the_followup_are_not_written():
|
||||
assert len(model.calls) == 2 and model.calls[1].asked == {"t8": ("columns", ["c007", "c008"])}
|
||||
o = got["t8"]
|
||||
assert o.status == "partial"
|
||||
assert o.error == "这张表 8 列,有 2 列没写出来:模型的回复里漏了这些列,补问一次仍没有给出。可以只重试这张表"
|
||||
assert o.error == ("这张表 8 列,有 2 列(c007、c008)没写出来:模型的回复里漏了这些列,补问一次仍没有给出。"
|
||||
"可以只重试这张表")
|
||||
assert set(o.notes["columns"]) == {f"c{i:03d}" for i in range(1, 7)}
|
||||
assert o.blank == [], "没写出来的列不算留空"
|
||||
compat = (await catalog.draft_with_model(Scripted(handle), _source(tables), ["t8"]))["t8"]
|
||||
assert compat.covered == frozenset(), "没写全的不算完整起草过"
|
||||
|
||||
|
||||
async def test_columns_listed_blank_in_the_followup_are_left_blank_not_failed():
|
||||
"""补问时模型把漏的列列出来了、字段却留空:是拿不准(记留空),不算「漏了这些列」。"""
|
||||
tables = {"t8": _table(8)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
reply = ok(call)
|
||||
entry = reply.obj["tables"][0]
|
||||
if call.n == 0:
|
||||
entry["columns"] = [c for c in entry["columns"] if c["name"] not in ("c003", "c007")]
|
||||
else:
|
||||
entry["columns"] = [{"name": c["name"]} for c in entry["columns"]]
|
||||
return reply
|
||||
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source(tables), ["t8"])
|
||||
assert model.calls[1].asked == {"t8": ("columns", ["c003", "c007"])}
|
||||
o = got["t8"]
|
||||
assert o.status == "ok" and o.error is None and o.blank == ["c003", "c007"]
|
||||
|
||||
|
||||
async def test_a_followup_that_comes_back_as_an_empty_shell_leaves_the_table_partial():
|
||||
"""40 列的表第一次只回了 20 列,补问只给一个空壳:没写出来的 20 列照实记下,表是 partial。"""
|
||||
tables = {"t40": _table(40)}
|
||||
@@ -382,11 +487,12 @@ async def test_a_followup_that_comes_back_as_an_empty_shell_leaves_the_table_par
|
||||
assert len(model.calls) == 2
|
||||
assert model.calls[1].asked == {"t40": ("columns", [f"c{i:03d}" for i in range(21, 41)])}
|
||||
assert o.status == "partial" and len(o.notes["columns"]) == 20
|
||||
assert o.error == "这张表 40 列,有 20 列没写出来:模型的回复里漏了这些列,补问一次仍没有给出。可以只重试这张表"
|
||||
assert o.error == ("这张表 40 列,有 20 列(c021 到 c040)没写出来:模型的回复里漏了这些列,补问一次仍没有给出。"
|
||||
"可以只重试这张表")
|
||||
|
||||
|
||||
async def test_columns_listed_with_empty_fields_are_left_to_the_model():
|
||||
"""列出了、字段留空的列是模型拿不准:不补问,也不算没写出来。"""
|
||||
"""列出了、字段留空的列是模型拿不准:不补问,也不算没写出来;记成留空(表的状态不因此变)。"""
|
||||
tables = {"t8": _table(8)}
|
||||
ok = answering(tables)
|
||||
|
||||
@@ -402,6 +508,30 @@ async def test_columns_listed_with_empty_fields_are_left_to_the_model():
|
||||
assert len(model.calls) == 1
|
||||
assert got["t8"].status == "ok" and got["t8"].error is None
|
||||
assert set(got["t8"].notes["columns"]) == {f"c{i:03d}" for i in range(1, 7)}
|
||||
assert got["t8"].blank == ["c007", "c008"]
|
||||
|
||||
|
||||
async def test_blank_columns_are_the_ones_without_a_label_or_meaning():
|
||||
"""留空的列:请模型写了、中文名和含义却都没给(只给了度量类型、单位,或者给的都没通过校验)。已确认的、库注释已经说清
|
||||
的不算:给模型的提示词里它们以此为准,模型不写也不缺。"""
|
||||
meta = _table(9)
|
||||
meta["columns"][5]["comment"] = "会员等级" # c006:库注释就是中文名
|
||||
meta["columns"][8]["comment"] = "C009" # c009:注释就是列名,说明不了什么
|
||||
tables = {"t9": meta}
|
||||
existing = {"t9": {"columns": {"c005": {"label": catalog.make_item("已确认的名字", "human")}}}}
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
return Reply(obj={"tables": [{"table": "t9", "label": "某表", "columns": [
|
||||
{"name": "c001", "label": "第一列"},
|
||||
{"name": "c002", "meaning": "第二列的含义"},
|
||||
{"name": "c003", "measure": "attribute", "unit": "元"}, # 中文名、含义都没给
|
||||
{"name": "c004", "label": "=c001+c002"}, # 像公式:校验丢掉
|
||||
{"name": "c005"}, {"name": "c006"}, {"name": "c007"}, {"name": "c008"}, {"name": "c009"}]}]})
|
||||
|
||||
got = await D.run_model_draft(Scripted(handle), _source(tables), ["t9"], existing=existing)
|
||||
o = got["t9"]
|
||||
assert o.status == "ok" and o.error is None
|
||||
assert o.blank == ["c003", "c004", "c007", "c008", "c009"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -521,7 +651,29 @@ async def test_other_errors_are_not_retried():
|
||||
assert len(model.calls) == 1 and got["t4"].error == "调用模型出错:网关拒绝了这次请求"
|
||||
|
||||
|
||||
async def test_a_call_past_the_time_limit_is_retried_then_reported(monkeypatch):
|
||||
async def test_a_call_past_the_time_limit_is_split_not_retried(monkeypatch, _no_wait):
|
||||
"""超时当作这一段太大:不原样重试同样大小的段,拆成两半再问(同额度用完)。超时的调用拿不到用量,记 0。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05)
|
||||
tables = {"t24": _table(24)}
|
||||
ok = answering(tables, usage=(100, 50))
|
||||
|
||||
async def handle(call: Call) -> Reply:
|
||||
if call.n == 0:
|
||||
await asyncio.sleep(1)
|
||||
return ok(call)
|
||||
|
||||
model, reports = Scripted(handle), Reports()
|
||||
got = await D.run_model_draft(model, _source(tables), ["t24"], hooks=reports.hooks())
|
||||
cols = [f"c{i:03d}" for i in range(1, 25)]
|
||||
assert got["t24"].status == "ok"
|
||||
assert [c.asked for c in model.calls] == [{"t24": ("full", None)}, {"t24": ("table+columns", cols[:12])},
|
||||
{"t24": ("columns", cols[12:])}]
|
||||
assert reports.failures == ["timeout", None, None]
|
||||
assert [(r.input_tokens, r.output_tokens) for r in reports.items] == [(0, 0), (100, 50), (100, 50)]
|
||||
assert _no_wait == [], "不退避重试"
|
||||
|
||||
|
||||
async def test_a_small_part_past_the_time_limit_is_not_retried(monkeypatch):
|
||||
monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05)
|
||||
|
||||
async def slow(call: Call) -> Reply:
|
||||
@@ -530,8 +682,8 @@ async def test_a_call_past_the_time_limit_is_retried_then_reported(monkeypatch):
|
||||
|
||||
model = Scripted(slow)
|
||||
got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"])
|
||||
assert len(model.calls) == 3
|
||||
assert got["t4"].error == "模型 0.05 秒内没有写完这一段(已重试 2 次)"
|
||||
assert len(model.calls) == 1, "拆不开、也不原样重试"
|
||||
assert got["t4"].status == "failed" and got["t4"].error == "模型 0.05 秒内没有写完这一段"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -554,13 +706,30 @@ async def test_a_wide_table_is_split_into_a_table_part_and_column_parts():
|
||||
first, *rest = model.calls
|
||||
assert first.asked == {"wide95": ("table", None)}
|
||||
assert catalog.CATALOG_DRAFT_PROMPT_LABELS["brief_columns"].format(n=95) in first.text
|
||||
assert sorted(len(c.asked["wide95"][1]) for c in rest) == [15, 40, 40]
|
||||
assert sorted(len(c.asked["wide95"][1]) for c in rest) == [31, 32, 32]
|
||||
for call in rest:
|
||||
assert call.asked["wide95"][0] == "columns"
|
||||
assert "wide95的中文名" in call.text and "虚构的宽表" in call.text, "列分段带着表注释和表级起草结果"
|
||||
assert {c for call in rest for c in call.asked["wide95"][1]} == {f"c{i:03d}" for i in range(1, 96)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n, sizes", [(164, [33, 33, 33, 33, 32]), (95, [32, 32, 31]), (41, [21, 20]),
|
||||
(80, [40, 40]), (81, [27, 27, 27])])
|
||||
async def test_column_parts_are_even_without_a_small_tail(monkeypatch, n, sizes):
|
||||
"""列分段数 = 列数 / 40 向上取整,各段列数平均(相差不超过 1):不再留下几列的尾段(实测会思考的模型常对这种小段只给
|
||||
空壳)。表级那段照旧在最前面。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1)
|
||||
tables = {"wide": _table(n)}
|
||||
model = Scripted(answering(tables))
|
||||
got = await D.run_model_draft(model, _source(tables), ["wide"])
|
||||
o = got["wide"]
|
||||
assert o.status == "ok" and o.parts_total == 1 + len(sizes) == D.parts_for(n)
|
||||
assert model.calls[0].asked == {"wide": ("table", None)}
|
||||
parts = [c.asked["wide"][1] for c in model.calls[1:]]
|
||||
assert [len(p) for p in parts] == sizes
|
||||
assert [c for p in parts for c in p] == [f"c{i:03d}" for i in range(1, n + 1)], "按表结构的顺序连续切开"
|
||||
|
||||
|
||||
async def test_a_wide_table_with_one_failed_part_is_partial_and_says_which():
|
||||
tables = {"wide95": _table(95)}
|
||||
ok = answering(tables)
|
||||
@@ -574,12 +743,69 @@ async def test_a_wide_table_with_one_failed_part_is_partial_and_says_which():
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source(tables), ["wide95"])
|
||||
o = got["wide95"]
|
||||
assert o.status == "partial" and len(o.notes["columns"]) == 55
|
||||
assert o.error == "这张表 95 列,分 4 段起草,第 3 段没写出来(共 40 列):调用模型出错:网关拒绝。可以只重试这张表"
|
||||
assert o.status == "partial" and len(o.notes["columns"]) == 63
|
||||
assert o.error == ("这张表 95 列,分 4 段起草,第 3 段(c033 到 c064,共 32 列)没写出来:调用模型出错:网关拒绝。"
|
||||
"可以只重试这张表")
|
||||
compat = (await catalog.draft_with_model(Scripted(handle), _source(tables), ["wide95"]))["wide95"]
|
||||
assert compat.covered == frozenset() and compat.error == o.error, "没写全的不算完整起草过,旧的模型项不删"
|
||||
|
||||
|
||||
async def test_reasons_name_each_part_and_say_these_parts(monkeypatch):
|
||||
"""几段因为同一个原因没写出来:原因里写「这几段」;每段写上列的范围(列名照表结构原样);拆过的才补「已拆小重试」。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 99)
|
||||
tables = {"wide95": _table(95)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
part, cols = call.asked["wide95"]
|
||||
if part == "columns" and cols[0] >= "c033": # 第 3、4 段(c033 起)一直用完额度
|
||||
return EXHAUSTED
|
||||
return ok(call)
|
||||
|
||||
got = await D.run_model_draft(Scripted(handle), _source(tables), ["wide95"])
|
||||
o = got["wide95"]
|
||||
assert o.status == "partial" and len(o.notes["columns"]) == 32
|
||||
assert o.error == ("这张表 95 列,分 4 段起草,第 3 段(c033 到 c064,共 32 列)、第 4 段(c065 到 c095,共 31 列)没写出来:"
|
||||
+ _exhausted("这几段") + D.REASON_SPLIT + "。可以只重试这张表")
|
||||
|
||||
|
||||
async def test_a_part_that_was_never_halved_does_not_say_it_was():
|
||||
"""被截断后只补剩下的(不是拆小):补的那次也用完了额度、又拆不开,原因里不写「已拆小重试」。"""
|
||||
tables = {"t9": _table(9)}
|
||||
full = json.dumps({"tables": [{"table": "t9", **_reply("t9", tables["t9"])}]}, ensure_ascii=False)
|
||||
cut = full[:full.index('"c005"') + 8]
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
return Reply(text=cut, finish="length") if call.n == 0 else EXHAUSTED
|
||||
|
||||
model = Scripted(handle)
|
||||
got = await D.run_model_draft(model, _source(tables), ["t9"])
|
||||
o = got["t9"]
|
||||
assert len(model.calls) == 2 and model.calls[1].asked == {"t9": ("columns", [f"c{i:03d}" for i in range(5, 10)])}
|
||||
assert o.status == "partial"
|
||||
assert o.error == "这张表 9 列,有 5 列(c005 到 c009)没写出来:" + _exhausted() + "。可以只重试这张表"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing, said", [
|
||||
(["c004"], "有 1 列(c004)"),
|
||||
(["c003", "c007"], "有 2 列(c003、c007)"),
|
||||
(["c002", "c004", "c006", "c008"], "有 4 列(c002、c004、c006 等)"),
|
||||
(["c003", "c004", "c005", "c006"], "有 4 列(c003 到 c006)"),
|
||||
])
|
||||
async def test_columns_not_written_are_named(missing, said):
|
||||
tables = {"t8": _table(8)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
reply = ok(call)
|
||||
for entry in reply.obj["tables"]:
|
||||
entry["columns"] = [c for c in entry.get("columns", []) if c["name"] not in missing]
|
||||
return reply
|
||||
|
||||
got = await D.run_model_draft(Scripted(handle), _source(tables), ["t8"])
|
||||
assert got["t8"].error == f"这张表 8 列,{said}没写出来:{D.REASON_UNANSWERED}。可以只重试这张表"
|
||||
|
||||
|
||||
async def test_tables_are_packed_up_to_40_columns_and_4_tables():
|
||||
widths = {"ta": 15, "tb": 15, "tc": 15, "td": 5, "te": 5, "tf": 5, "tg": 5, "th": 41}
|
||||
tables = {name: _table(n) for name, n in widths.items()}
|
||||
@@ -598,13 +824,10 @@ async def test_tables_are_packed_up_to_40_columns_and_4_tables():
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _hollow(call: Call) -> Reply:
|
||||
return Reply(obj={"tables": [{"table": t} for t in call.asked]})
|
||||
|
||||
|
||||
async def test_the_same_reason_is_said_once_even_when_some_parts_were_split(monkeypatch):
|
||||
"""宽表的表级那段拆不开、列分段拆过:原因只说一次,拆过的在最后补一句。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99)
|
||||
monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99)
|
||||
model = Scripted(_hollow)
|
||||
got = await D.run_model_draft(model, _source({"wide45": _table(45)}), ["wide45"])
|
||||
o = got["wide45"]
|
||||
@@ -613,14 +836,19 @@ async def test_the_same_reason_is_said_once_even_when_some_parts_were_split(monk
|
||||
|
||||
|
||||
def test_reasons_are_joined_by_their_base():
|
||||
"""(原因, 第几段):同样的原因只说一次;同一个原因落在几段上写「这几段」,同一段拆出来的几块仍是「这一段」。"""
|
||||
W = D._Why
|
||||
exhausted = D.REASON_EXHAUSTED.format(budget=32768)
|
||||
assert D._join([W(D.REASON_HOLLOW), W(D.REASON_HOLLOW, split=True)]) == D.REASON_HOLLOW + D.REASON_SPLIT
|
||||
assert D._join([W(exhausted, split=True), W(D.REASON_HOLLOW)]) == \
|
||||
f"{exhausted};{D.REASON_HOLLOW}{D.REASON_SPLIT}"
|
||||
assert D._join([W("甲"), W("乙"), W("丙", split=True)]) == "甲;乙", "没说出来的原因不补后缀"
|
||||
assert D._join([W(D.REASON_HOLLOW, halted=True), W(D.REASON_HOLLOW)]) == \
|
||||
ex = D.REASON_EXHAUSTED.format(budget=32768, seg=D.SEG)
|
||||
assert D._join([(W(D.REASON_HOLLOW), 1), (W(D.REASON_HOLLOW, split=True), 2)]) == \
|
||||
D.REASON_HOLLOW + D.REASON_SPLIT
|
||||
assert D._join([(W(ex, split=True), 2), (W(D.REASON_HOLLOW), 1)]) == \
|
||||
f"{_exhausted()};{D.REASON_HOLLOW}{D.REASON_SPLIT}"
|
||||
assert D._join([(W(ex), 2), (W(ex), 3)]) == _exhausted("这几段")
|
||||
assert D._join([(W(ex), 2), (W(ex, split=True), 2)]) == _exhausted() + D.REASON_SPLIT
|
||||
assert D._join([(W("甲"), 1), (W("乙"), 1), (W("丙", split=True), 1)]) == "甲;乙", "没说出来的原因不补后缀"
|
||||
assert D._join([(W(D.REASON_HOLLOW, halted=True), 1), (W(D.REASON_HOLLOW), 2)]) == \
|
||||
D.REASON_HOLLOW + D.REASON_HALTED.format(n=catalog.MODEL_GIVE_UP_AFTER)
|
||||
assert D._join([(W("调用模型出错:{坏的}"), 1)]) == "调用模型出错:{坏的}", "报错原文里的花括号原样留着"
|
||||
|
||||
|
||||
def _by_table(model: Scripted) -> dict[str, int]:
|
||||
@@ -643,7 +871,7 @@ async def test_after_two_segments_in_a_row_come_to_nothing_the_rest_are_not_spli
|
||||
model = Scripted(lambda call: exhausted)
|
||||
got = await D.run_model_draft(model, _source(tables), list(tables))
|
||||
assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1}
|
||||
budget = D.REASON_EXHAUSTED.format(budget=32768)
|
||||
budget = _exhausted()
|
||||
assert got["t1"].error == budget + D.REASON_SPLIT
|
||||
assert got["t3"].error == budget + ";模型连续 2 段都没写出来,已停止拆小重试,建议更换起草模型"
|
||||
|
||||
@@ -657,23 +885,23 @@ async def test_after_two_segments_in_a_row_come_to_nothing_the_rest_are_not_spli
|
||||
async def test_the_column_parts_of_a_wide_table_stop_splitting_too(monkeypatch):
|
||||
"""宽表的列分段一段一段排队起草(一段拆到底再起草下一段),止损在宽表里同样起作用。
|
||||
|
||||
95 列:表级那段拆不开(结构化、纯文本各一次);第 2 段 40 列拆到底(15 次结构化 + 8 次纯文本);这时已经连续 2 段
|
||||
没写出来,第 3、4 段只问一次、换纯文本一次。"""
|
||||
95 列:表级那段拆不开(结构化、纯文本各一次);第 2 段 32 列拆小、到 8 次封顶;这时已经连续 2 段没写出来,第 3、4 段
|
||||
只问一次、换纯文本一次。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1)
|
||||
model = Scripted(_hollow)
|
||||
got = await D.run_model_draft(model, _source({"wide95": _table(95)}), ["wide95"])
|
||||
assert len(model.calls) == 2 + 23 + 2 + 2
|
||||
assert got["wide95"].error == (D.REASON_HOLLOW + D.REASON_SPLIT
|
||||
+ ";模型连续 2 段都没写出来,已停止拆小重试,建议更换起草模型")
|
||||
assert len(model.calls) == 2 + 8 + 2 + 2
|
||||
assert got["wide95"].error == (D.REASON_HOLLOW + ";每段最多调用 8 次模型,这一段到了次数仍没写完,已停止重试"
|
||||
+ D.REASON_SPLIT + ";模型连续 2 段都没写出来,已停止拆小重试,建议更换起草模型")
|
||||
|
||||
|
||||
async def test_a_wide_table_that_never_comes_back_is_bounded_by_the_first_wave():
|
||||
"""默认并发(3 段同时起草):294 列一直只给空壳,最多是表级那段 2 次、头 3 个列分段拆到底(各 23 次)、其余 5 段
|
||||
各 2 次。以前所有列分段一下子都排上去、一层一层地拆,要 168 次。"""
|
||||
"""默认并发(3 段同时起草):294 列(8 个列分段)一直只给空壳,最多是表级那段 2 次、头 3 个列分段各 8 次(封顶)、
|
||||
其余 5 段各 2 次。以前所有列分段一下子都排上去、一层一层地拆,要 168 次。"""
|
||||
model = Scripted(_hollow)
|
||||
got = await D.run_model_draft(model, _source({"wide294": _table(294)}), ["wide294"])
|
||||
assert got["wide294"].status == "failed"
|
||||
assert len(model.calls) <= 2 + 3 * 23 + 5 * 2
|
||||
assert got["wide294"].status == "failed" and got["wide294"].parts_total == 9
|
||||
assert len(model.calls) <= 2 + 3 * 8 + 5 * 2
|
||||
|
||||
|
||||
async def test_a_big_draft_does_not_hold_the_model_ahead_of_a_later_one(monkeypatch):
|
||||
@@ -707,6 +935,85 @@ async def test_a_big_draft_does_not_hold_the_model_ahead_of_a_later_one(monkeypa
|
||||
assert order == ["a1", "b1", "a2", "a3", "a4"]
|
||||
|
||||
|
||||
def _order(model: Scripted) -> list[tuple[str, str]]:
|
||||
return [(name, part) for call in model.calls for name, (part, _) in call.asked.items()]
|
||||
|
||||
|
||||
async def test_narrow_tables_are_not_cut_in_by_the_column_parts_of_a_wide_one(monkeypatch):
|
||||
"""一个任务里按表的先后排队:宽表的表级那段写完,它的列分段排到已经排着的窄表段后面(先来先服务),不插队。
|
||||
以前插到队首,窄表要等宽表的列分段全部派完。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1)
|
||||
monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1)
|
||||
tables = {"wide": _table(45), "na": _table(4), "nb": _table(4)}
|
||||
model = Scripted(answering(tables))
|
||||
got = await D.run_model_draft(model, _source(tables), ["wide", "na", "nb"])
|
||||
assert set(_status(got).values()) == {"ok"}
|
||||
assert _order(model) == [("wide", "table"), ("na", "full"), ("nb", "full"), ("wide", "columns"),
|
||||
("wide", "columns")]
|
||||
|
||||
|
||||
async def test_after_two_exhausted_or_timed_out_calls_new_parts_are_halved(monkeypatch):
|
||||
"""额度用完或超时累计 2 次,之后新开始的段列数上限减半(40 → 20 → 10,最低 10);已经开始的段照旧(按原因拆小)。
|
||||
每次调用的结果里带着当前的上限(part_columns)。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1)
|
||||
monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1)
|
||||
tables = {f"t{i}": _table(40) for i in range(1, 4)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
(_, (part, cols)), = call.asked.items()
|
||||
width = 40 if part == "full" else len(cols)
|
||||
return EXHAUSTED if width > 10 else ok(call) # 这个模型一次只写得出 10 列
|
||||
|
||||
model, reports = Scripted(handle), Reports()
|
||||
got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks())
|
||||
assert set(_status(got).values()) == {"ok"}
|
||||
asked = {name: [(part, len(cols or [])) for c in model.calls for n, (part, cols) in c.asked.items() if n == name]
|
||||
for name in tables}
|
||||
# t1:整张 40 列开始,用完额度拆成 20、20,又各拆成 10、10
|
||||
assert asked["t1"][0] == ("full", 0) and len(asked["t1"]) == 7
|
||||
# 第 2 次用完额度后上限是 20:t2 一开始就按 20 列切成两段(各自再拆一次)
|
||||
assert asked["t2"][0] == ("table+columns", 20) and ("columns", 20) in asked["t2"]
|
||||
# 第 4 次之后上限是 10:t3 一开始就切成 4 段 10 列,一次都没用完额度
|
||||
assert asked["t3"] == [("table+columns", 10), ("columns", 10), ("columns", 10), ("columns", 10)]
|
||||
caps = [r.part_columns for r in reports.items]
|
||||
assert list(dict.fromkeys(caps)) == [40, 20, 10]
|
||||
assert caps[:2] == [40, 20], "第 2 次用完额度的那次调用之后就是 20"
|
||||
assert min(caps) == 10 == catalog.MODEL_MIN_PART_COLUMNS
|
||||
|
||||
|
||||
async def test_the_notice_comes_when_four_of_six_calls_fail_and_goes_when_fewer_than_two(monkeypatch):
|
||||
"""最近 6 次调用有 4 次以上没写出来:提醒(写明次数和原因,建议换起草模型、减少表或停止);之后最近 6 次里没写出来
|
||||
的少于 2 次就清掉。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1)
|
||||
monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1)
|
||||
tables = {f"t{i}": _table(4) for i in range(1, 10)}
|
||||
ok = answering(tables)
|
||||
|
||||
def handle(call: Call) -> Reply:
|
||||
name = next(iter(call.asked))
|
||||
if name in ("t1", "t2"):
|
||||
raise RuntimeError("网关拒绝了这次请求")
|
||||
if name == "t3":
|
||||
return EXHAUSTED
|
||||
if name == "t4":
|
||||
return _hollow(call)
|
||||
return ok(call)
|
||||
|
||||
reports = Reports()
|
||||
got = await D.run_model_draft(Scripted(handle), _source(tables), list(tables), hooks=reports.hooks())
|
||||
assert reports.failures == ["error", "error", "exhausted", "hollow", "hollow"] + [None] * 5
|
||||
notices = [r.notice for r in reports.items]
|
||||
assert notices[:3] == [None] * 3
|
||||
assert notices[3] == ("最近 4 次调用有 4 次没写出来(额度用完 1 次、只给空壳 1 次、出错 2 次)。可以在「设置 → 偏好设置」"
|
||||
"的「起草模型」换一个模型、减少这次起草的表,或者停止这次起草")
|
||||
assert notices[4].startswith("最近 5 次调用有 5 次没写出来")
|
||||
assert notices[7].startswith("最近 6 次调用有 3 次没写出来"), "还有 2 次以上没写出来:留着,次数照实写"
|
||||
assert notices[8].startswith("最近 6 次调用有 2 次没写出来")
|
||||
assert notices[9] is None, "最近 6 次只有 1 次没写出来:清掉"
|
||||
assert got["t9"].status == "ok"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 起草要求、用量
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -747,13 +1054,11 @@ async def test_usage_is_added_up_through_the_hooks():
|
||||
return Reply(text="", finish="length", thinking="……", usage=(100, 32768))
|
||||
return ok(call)
|
||||
|
||||
seen: list[tuple[int, int]] = []
|
||||
|
||||
async def usage(i: int, o: int) -> None:
|
||||
seen.append((i, o))
|
||||
|
||||
await D.run_model_draft(Scripted(handle), _source(tables), ["orders_x"], hooks=D.Hooks(usage=usage))
|
||||
assert seen == [(100, 32768), (120, 60), (120, 60)]
|
||||
reports = Reports()
|
||||
await D.run_model_draft(Scripted(handle), _source(tables), ["orders_x"], hooks=reports.hooks())
|
||||
assert [(r.input_tokens, r.output_tokens, r.failure) for r in reports.items] == \
|
||||
[(100, 32768, "exhausted"), (120, 60, None), (120, 60, None)]
|
||||
assert {r.notice for r in reports.items} == {None} and {r.part_columns for r in reports.items} == {40}
|
||||
|
||||
|
||||
def test_budget_words_match_the_other_drafts():
|
||||
|
||||
@@ -6,7 +6,9 @@
|
||||
- 停止:排队的直接 cancelled;进行中的停掉正在进行的模型调用,没写完的表记 cancelled,已经写进目录的不回滚;已结束的 409
|
||||
- 服务启动时把没结束的任务标成 interrupted(没结束的表记 cancelled、写明服务重启),不自动续跑;关停时同样
|
||||
- 重试:只含原任务里 failed、partial、cancelled 的表,本次说明沿用、起草要求取数据源当前的;没有可重试的 422
|
||||
- 目录清单:每行 last_draft,顶层 guidance、active_job
|
||||
- 目录清单:每行 last_draft(带模型留空的列数),顶层 guidance、active_job
|
||||
- 重新起草时模型这次留空、给的值没通过校验:以前起草好的模型项留着,删除只在列已经不在表结构里时才有
|
||||
- 用量记超时次数、没写出来的调用次数;任务记当前的列数上限、提醒(最近 6 次调用 4 次没写出来);逐表记模型留空的列
|
||||
- 起草要求存在数据源 options 里(上传表格的源存在它自己那一行),数据源接口不显示、整份回写 options 时留着,拼连接串时摘掉
|
||||
|
||||
表名、列名、说明文字都是虚构的。
|
||||
@@ -261,11 +263,136 @@ async def test_a_followup_that_comes_back_empty_keeps_the_earlier_model_items(cl
|
||||
task = job["tasks"][0]
|
||||
assert model.calls == 2
|
||||
assert job["status"] == "partial" and task["status"] == "partial" and task["removed"] == 0
|
||||
assert "有 4 列没写出来" in task["model_error"] and "可以只重试这张表" in task["model_error"]
|
||||
assert "有 4 列(c05 到 c08)没写出来" in task["model_error"] and "可以只重试这张表" in task["model_error"]
|
||||
after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"]
|
||||
assert {c for c, items in after.items() if "label" in items} == {f"c{i:02d}" for i in range(1, 9)}
|
||||
|
||||
|
||||
def _tickets(n: int = 6) -> dict:
|
||||
return {"tickets": {"qualified": "tickets", "columns": [_col(f"c{i:02d}") for i in range(1, n + 1)],
|
||||
"primary_key": ["c01"], "comment": None, "is_view": False, "foreign_keys": []}}
|
||||
|
||||
|
||||
async def _draft(client, sid: str, monkeypatch, model, tables: list[str]) -> dict:
|
||||
use(monkeypatch, model)
|
||||
r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": tables, "use_model": True})
|
||||
assert r.status_code == 202, r.text
|
||||
await finished(r.json()["job"]["id"])
|
||||
return await get_job(client, sid, r.json()["job"]["id"])
|
||||
|
||||
|
||||
async def test_a_redraft_keeps_what_the_model_left_blank_this_time(client, monkeypatch):
|
||||
"""重新起草(比如补了起草要求再起草一次)时模型这次留空、或者给的值没通过校验:以前起草好的模型项留着,不算删除;
|
||||
给了合格新值的才替换。以前整库重跑一次删掉几百项,表仍记「已完成」。"""
|
||||
sid = await _source(_tickets())
|
||||
full = {"label": "门票", "kind": "fact",
|
||||
"columns": [{"name": f"c{i:02d}", "label": f"第{i}列", "meaning": f"第{i}列的含义"} for i in range(1, 7)]}
|
||||
first = await _draft(client, sid, monkeypatch, FakeModel({"tickets": full}), ["tickets"])
|
||||
assert first["tasks"][0]["status"] == "ok" and first["blank_columns"] == 0
|
||||
before = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]
|
||||
|
||||
# 这一次:表的中文名换了;c03 的中文名像公式(校验丢掉)、含义换了;c04 只给了中文名;其余列列出了但都留空
|
||||
again = {"label": "门票明细", "columns": [{"name": "c03", "label": "=c01+c02", "meaning": "新的含义"},
|
||||
{"name": "c04", "label": "第4列"}]}
|
||||
second = await _draft(client, sid, monkeypatch, FakeModel({"tickets": again}), ["tickets"])
|
||||
task = second["tasks"][0]
|
||||
assert task["status"] == "ok" and task["removed"] == 0 and second["removed"] == 0
|
||||
assert (task["added"], task["updated"]) == (0, 2)
|
||||
after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]
|
||||
assert after["label"]["value"] == "门票明细" and after["kind"] == before["kind"]
|
||||
assert after["columns"]["c03"]["label"] == before["columns"]["c03"]["label"]
|
||||
assert after["columns"]["c03"]["meaning"]["value"] == "新的含义"
|
||||
for col in ("c01", "c02", "c05", "c06"):
|
||||
assert after["columns"][col] == before["columns"][col], col
|
||||
# 模型这次留空的列照实记下(表的状态不因此变)
|
||||
assert task["blank_columns"] == 4 and task["blank_sample"] == ["c01", "c02", "c05", "c06"]
|
||||
|
||||
|
||||
async def test_model_items_of_a_column_gone_from_the_schema_are_removed(client, monkeypatch):
|
||||
sid = await _source(_tickets())
|
||||
full = {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 7)]}
|
||||
await _draft(client, sid, monkeypatch, FakeModel({"tickets": full}), ["tickets"])
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
row.schema_cache = {"tables": _tickets(5)} # 重新探查过:c06 已经没有了
|
||||
await session.commit()
|
||||
job = await _draft(client, sid, monkeypatch, FakeModel({"tickets": {"label": "门票"}}), ["tickets"])
|
||||
assert job["tasks"][0]["removed"] == 1
|
||||
after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]
|
||||
assert set(after["columns"]) == {"c01", "c02", "c03", "c04", "c05"}
|
||||
|
||||
|
||||
async def test_usage_counts_timeouts_and_failed_calls(client, monkeypatch):
|
||||
"""超时的调用拆小再问,用量里记超时几次、没写出来几次(超时的调用拿不到用量,token 记 0);额度用完或超时累计 2 次,
|
||||
新开始的段列数上限减半,任务上记着当前的上限。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05)
|
||||
sid = await _source(_tickets(24))
|
||||
replies = {"tickets": {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"}
|
||||
for i in range(1, 25)]}}
|
||||
model = FakeModel(replies, hang=2)
|
||||
job = await _draft(client, sid, monkeypatch, model, ["tickets"])
|
||||
assert job["status"] == "succeeded" and job["tasks"][0]["status"] == "ok"
|
||||
assert job["usage"] == {"calls": 5, "input_tokens": 0, "output_tokens": 0, "timeouts": 2, "failed_calls": 2}
|
||||
assert job["part_columns"] == 20 and job["notice"] is None
|
||||
# 24 列整张问超时 → 前 12 列(带表级的项)超时 → 拆成 6、6 → 后 12 列
|
||||
assert [len(requested(c)["tickets"][1] or []) for c in model.calls] == [0, 12, 6, 6, 12]
|
||||
|
||||
plain = await _draft(client, sid, monkeypatch, FakeModel(replies), ["tickets"])
|
||||
assert plain["part_columns"] == 40 and plain["usage"]["timeouts"] == plain["usage"]["failed_calls"] == 0
|
||||
|
||||
|
||||
async def test_the_notice_is_kept_on_the_job(client, monkeypatch):
|
||||
"""最近 6 次调用有 4 次以上没写出来:任务上写一句提醒(次数、原因、建议换起草模型);不用模型的任务没有。"""
|
||||
monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1)
|
||||
tables = {f"t{i}": {"qualified": f"t{i}", "columns": [_col("id"), _col("v")], "primary_key": ["id"],
|
||||
"comment": None, "is_view": False, "foreign_keys": []} for i in range(1, 6)}
|
||||
sid = await _source(tables)
|
||||
replies = {name: {"label": f"表{name}"} for name in tables}
|
||||
job = await _draft(client, sid, monkeypatch, FakeModel(replies, fail={"t1", "t2", "t3", "t4"}), list(tables))
|
||||
assert job["usage"]["calls"] == 5 and job["usage"]["failed_calls"] == 4
|
||||
assert job["notice"] == ("最近 5 次调用有 4 次没写出来(出错 4 次)。可以在「设置 → 偏好设置」的「起草模型」换一个模型、"
|
||||
"减少这次起草的表,或者停止这次起草")
|
||||
plain = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["t1"]})).json()["job"]
|
||||
await finished(plain["id"])
|
||||
plain = await get_job(client, sid, plain["id"])
|
||||
assert plain["notice"] is None and plain["part_columns"] is None and plain["blank_columns"] == 0
|
||||
|
||||
|
||||
async def test_blank_columns_reach_the_task_the_job_and_the_list(client, monkeypatch):
|
||||
"""模型列出了、中文名和含义都没给的列:逐表记列数和最多 5 个列名,任务上是合计,目录清单的 last_draft 也带列数。"""
|
||||
tables = {**_tickets(8), "gates": _tables()["gates"]}
|
||||
sid = await _source(tables)
|
||||
replies = {"tickets": {"label": "门票", "columns": [{"name": "c01", "label": "编号"}]},
|
||||
"gates": {"label": "闸口", "columns": [{"name": "id", "label": "编号"}]}}
|
||||
job = await _draft(client, sid, monkeypatch, FakeModel(replies), ["tickets", "gates"])
|
||||
by = {t["table_name"]: t for t in job["tasks"]}
|
||||
assert by["tickets"]["status"] == "ok" and by["tickets"]["blank_columns"] == 7
|
||||
assert by["tickets"]["blank_sample"] == ["c02", "c03", "c04", "c05", "c06"]
|
||||
assert by["gates"]["blank_columns"] == 1 and by["gates"]["blank_sample"] == ["gate_code"]
|
||||
assert job["blank_columns"] == 8
|
||||
rows = {r["table_name"]: r for r in (await client.get(_base(sid))).json()["tables"]}
|
||||
assert rows["tickets"]["last_draft"]["blank_columns"] == 7 and rows["gates"]["last_draft"]["blank_columns"] == 1
|
||||
|
||||
|
||||
def test_old_job_tables_get_the_new_columns():
|
||||
"""升级前建的起草任务表:新列补上,存量行的计数是 0、提醒和列数上限为空。"""
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from app.db.base import _migrate
|
||||
|
||||
db = create_engine("sqlite://")
|
||||
with db.begin() as conn:
|
||||
conn.execute(text("CREATE TABLE catalog_draft_jobs (id VARCHAR(32) PRIMARY KEY, calls INTEGER)"))
|
||||
conn.execute(text("CREATE TABLE catalog_draft_tasks (id VARCHAR(32) PRIMARY KEY, removed INTEGER)"))
|
||||
conn.execute(text("INSERT INTO catalog_draft_jobs (id, calls) VALUES ('j1', 3)"))
|
||||
conn.execute(text("INSERT INTO catalog_draft_tasks (id, removed) VALUES ('t1', 0)"))
|
||||
_migrate(conn)
|
||||
_migrate(conn)
|
||||
assert conn.execute(text("SELECT timeouts, failed_calls, part_columns, notice FROM catalog_draft_jobs")) \
|
||||
.one() == (0, 0, None, None)
|
||||
assert conn.execute(text("SELECT blank_columns, blank_sample FROM catalog_draft_tasks")).one() == (0, None)
|
||||
|
||||
|
||||
async def test_a_job_that_cannot_start_is_failed_with_the_reason(client, monkeypatch):
|
||||
"""建任务时起草模型还能用,开始起草时已经用不了(接入刚被停用):任务整体 failed,error 写原因,没开始的表也记 failed。"""
|
||||
sid = await _source()
|
||||
|
||||
Reference in New Issue
Block a user