A2 模型起草一项都没写进去时不再算成功:认出没有 tables 外壳的单个表对象;结构化输出是空壳(只有表名、没有可用字段)时改走纯文本再问一次;某张表最终没有可用内容记 model_error「模型没有给出这张表的可用内容」;起草结果每张表多一个 model_items(模型给了几项可用内容),界面据此区分「和现有目录一致」与「模型没给内容」;「没有变化」改为只说起草出的内容与现有目录一致,另加「目录没有更新」一句;假模型能模拟空壳、单个对象、某条路不可用三种实测回复

This commit is contained in:
ThearchyHelios
2026-10-04 01:48:07 +08:00
parent 15d5caf19f
commit 0df5e7d4af
7 changed files with 177 additions and 29 deletions
+7 -2
View File
@@ -227,7 +227,11 @@ async def put_catalog_table(source_id: str, table: str, payload: CatalogPutIn,
async def draft_catalog(source_id: str, payload: CatalogDraftIn,
x_actor: str | None = Header(default=None),
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""同步起草。返回每张表新增、更新、删除了几项;模型用不了或某张表的模型起草失败,照实写在结果里。"""
"""同步起草。返回每张表新增、更新、删除了几项;模型用不了或某张表的模型起草失败,照实写在结果里。
model_items:模型给了这张表几项可用内容(没用模型、模型这部分失败时为 null)。界面据此区分「模型给了内容、
只是和现有目录一致」(model_items > 0、三个数都是 0)和「模型没给内容」(model_error 有值)。
"""
_, source = await _resolved(session, source_id)
cache = source.schema_cache or {}
if not cache.get("tables"):
@@ -235,7 +239,8 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
report = await catalog.draft_catalog(session, source, tables=payload.tables or None,
use_model=payload.use_model, actor=actor_of(x_actor))
rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed,
"version": r.version, "error": r.error, "model_error": r.model_error} for r in report.tables]
"version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items}
for r in report.tables]
return {
"tables": rows,
"total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")},
+57 -19
View File
@@ -40,7 +40,7 @@ import hashlib
import json
import re
from collections import Counter
from collections.abc import Iterable, Mapping
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
@@ -1576,8 +1576,16 @@ def _batches(names: list[str], tables: Mapping[str, Any]) -> list[list[str]]:
return out
#: 模型给了这张表、但一项可用内容都没有(只有表名的空壳,或者给的值全不合规):这张表记模型失败,不算起草成功。
#: 不然界面只看到「新增 0 项」,会说成「没有变化」
MODEL_EMPTY_REASON = "模型没有给出这张表的可用内容"
def _model_tables(raw: Any) -> list[dict[str, Any]] | None:
"""模型的回复 → 每张表一项的列表。结构化输出给的对象、正文里的 JSON 都认;对不上格式返回 None。"""
"""模型的回复 → 每张表一项的列表。结构化输出给的对象、正文里的 JSON 都认;对不上格式返回 None。
实测模型有时不带 tables 外壳、只回单个表对象({"table": "x", "label": …}),也认成一张表。
"""
if hasattr(raw, "model_dump"):
raw = raw.model_dump()
if isinstance(raw, str):
@@ -1591,31 +1599,43 @@ def _model_tables(raw: Any) -> list[dict[str, Any]] | None:
except ValueError:
return None
if isinstance(raw, dict):
raw = raw.get("tables")
raw = [raw] if "tables" not in raw and raw.get("table") else raw.get("tables")
if not isinstance(raw, list):
return None
return [t for t in raw if isinstance(t, dict)]
async def _ask_model(model: Any, messages: list[Any]) -> list[dict[str, Any]]:
"""问一批。先走结构化输出;模型不支持、调用报错、或给回来的不是约定的结构,再退回纯文本、从正文里取 JSON。
async def _ask_model(model: Any, messages: list[Any],
usable: Callable[[list[dict[str, Any]]], bool] | None = None) -> list[dict[str, Any]]:
"""问一批。先走结构化输出;模型不支持、调用报错、给回来的不是约定的结构,或者 usable 说它不够用(实测见过
结构化输出只给 {"tables":[{"table":"x"}]} 这样的空壳),再退回纯文本、从正文里取 JSON。
结构化输出报错也退回(同 judge._ask):有的网关不支持工具调用,一调就报错,纯文本却能用。代价是真的
网络故障会多试一次,两次加起来仍受 MODEL_TIMEOUT_S 约束。
两条路都拿到了东西时,结构化的在前、纯文本的在后一起交回(同一张表各有一份,调用方挑能用的那份);纯文本那次
失败了就只交回结构化的那份(调用方据此给空壳的表记失败)。
"""
try:
parsed = _model_tables(await model.with_structured_output(CATALOG_DRAFT_PROMPT_SCHEMA).ainvoke(messages))
except Exception: # noqa: BLE001 - 退回纯文本
parsed = None
if parsed is not None:
if parsed is not None and (usable is None or usable(parsed)):
return parsed
from app.engine.state import message_text
reply = await model.ainvoke(messages)
parsed = _model_tables(message_text(reply))
if parsed is None:
try:
reply = await model.ainvoke(messages)
except Exception: # noqa: BLE001 - 结构化那份还在:交回它,空壳的表由调用方记失败
if parsed is not None:
return parsed
raise
text = _model_tables(message_text(reply))
if text is None:
if parsed is not None:
return parsed
raise ValueError("模型的回复不是约定的格式")
return parsed
return [*(parsed or []), *text]
def _clean_text(value: Any, limit: int = _TEXT_MAX) -> str | None:
@@ -1695,9 +1715,22 @@ async def draft_with_model(model: Any, source: Any, tables: list[str], *,
existing=(existing or {}).get(n)) for n in batch)
messages = [SystemMessage(content=CATALOG_DRAFT_SYSTEM),
HumanMessage(content=CATALOG_DRAFT_PROMPT.format(n=len(batch), tables=described))]
def drafted(entries: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""这批里每张表能用的那份:同一张表给了几份(结构化、纯文本各一份)时取第一份有可用内容的。"""
out: dict[str, dict[str, Any]] = {}
for entry in entries:
name = str(entry.get("table"))
if name in batch and name not in out:
notes = _from_model(entry, all_tables[name], system=system)
if notes:
out[name] = notes
return out
try:
async with gate:
got = await asyncio.wait_for(_ask_model(model, messages), timeout=MODEL_TIMEOUT_S)
got = await asyncio.wait_for(
_ask_model(model, messages, usable=lambda entries: len(drafted(entries)) == len(batch)),
timeout=MODEL_TIMEOUT_S)
except asyncio.TimeoutError:
for n in batch:
results[n] = f"模型起草超时:{MODEL_TIMEOUT_S} 秒内没有收到回复"
@@ -1708,14 +1741,13 @@ async def draft_with_model(model: Any, source: Any, tables: list[str], *,
for n in batch:
results[n] = f"模型起草失败:{first_line(e) or describe_exception(e)}"
return
by_name = {str(t.get("table")): t for t in got}
named = {str(t.get("table")) for t in got}
usable = drafted(got)
for n in batch:
entry = by_name.get(n)
if entry is None:
results[n] = "模型的回复里没有这张表"
continue
results[n] = TableDraft(notes=_from_model(entry, all_tables[n], system=system),
covered=frozenset({"llm"}))
if n in usable:
results[n] = TableDraft(notes=usable[n], covered=frozenset({"llm"}))
else:
results[n] = MODEL_EMPTY_REASON if n in named else "模型的回复里没有这张表"
await asyncio.gather(*(run(b) for b in _batches(names, all_tables)))
return {n: results[n] for n in names if n in results}
@@ -1737,6 +1769,9 @@ class TableDraftResult:
version: int = 0
error: str | None = None
model_error: str | None = None
#: 模型给了这张表几项可用内容(并入目录之前,按项数)。没用模型、模型这部分失败时为 None。
#: 大于 0 而新增、更新、删除都是 0,才是「模型给了内容、只是和现有目录一致」
model_items: int | None = None
@dataclass
@@ -1800,6 +1835,8 @@ async def draft_catalog(session: AsyncSession, source: Any, *, tables: list[str]
modeled = llm.get(table)
if isinstance(modeled, str):
result.model_error = modeled
elif isinstance(modeled, TableDraft):
result.model_items = len(_slots(modeled.notes))
for _attempt in range(3):
entry = await read_entry(session, source.id, table)
version = entry.version if entry else 0
@@ -2188,7 +2225,8 @@ __all__ = [
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX", "PROFILE_VERIFY_COVERAGE",
"MODEL_EMPTY_REASON", "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX",
"PROFILE_VERIFY_COVERAGE",
"PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL",
"TABLE_KINDS", "TableDraft", "TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit",
"apply_patch", "codes_complete", "describe_slot", "draft_catalog", "draft_structure", "draft_with_model",
+41 -6
View File
@@ -3,28 +3,63 @@
model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, fail={"orders"})
await catalog.draft_with_model(model, source, ["visits"])
- 结构化输出(with_structured_output)返回自己;ainvoke 从提示词里认出这一批有哪些表(「表 <表名>」),
- 结构化输出(with_structured_output(...).ainvoke)从提示词里认出这一批有哪些表(「表 <表名>」),
返回 {"tables": [{"table": 表名, **replies[表名]}, …]};
- 直接 ainvoke 是纯文本那条路:同样的内容写成一段正文(AIMessage),前面带一句话;
- fail 里的表所在的那一批抛异常,用来验证「一批失败只影响这一批」;
- calls 记下每次收到的提示词全文,用来核对交给模型的内容(只有结构,没有数据)。
- calls 记下每次收到的提示词全文,用来核对交给模型的内容(只有结构,没有数据);modes 记下每次走的是
structured 还是 text。
structured、text 模拟实测见过的几种回复(两条路各自设):
- "tables":正常,{"tables": [...]};
- "hollow":退化的空壳,表名对上了、一个字段都没有:{"tables": [{"table": 表名}, …]};
- "single":只给第一张表的对象,没有 tables 外壳:{"table": 表名, …};
- "error":这条路用不了(网关不支持工具调用之类),一调就抛异常。
"""
from __future__ import annotations
import json
from typing import Any
class FakeModel:
def __init__(self, replies: dict[str, dict], fail: set[str] = frozenset()) -> None:
def __init__(self, replies: dict[str, dict], fail: set[str] = frozenset(), *,
structured: str = "tables", text: str = "tables") -> None:
self.replies, self.fail, self.calls = replies, set(fail), []
self.structured, self.text = structured, text
self.modes: list[str] = []
def with_structured_output(self, schema: Any, **_: Any) -> "FakeModel":
def with_structured_output(self, schema: Any, **_: Any) -> "_Structured":
assert schema["title"]
return self
return _Structured(self)
async def ainvoke(self, messages: list[Any]) -> dict:
async def ainvoke(self, messages: list[Any]) -> Any:
from langchain_core.messages import AIMessage
reply = self._reply(messages, "text")
return AIMessage(content="草稿如下:" + json.dumps(reply, ensure_ascii=False))
def _reply(self, messages: list[Any], mode: str) -> dict:
text = "\n".join(str(getattr(m, "content", m)) for m in messages)
self.calls.append(text)
self.modes.append(mode)
names = [t for t in self.replies if f"表 {t}\n" in text or f"表 {t}(" in text]
if any(t in self.fail for t in names):
raise RuntimeError("网关超时")
shape = self.structured if mode == "structured" else self.text
if shape == "error":
raise NotImplementedError("这条路用不了")
if shape == "hollow":
return {"tables": [{"table": t} for t in names]}
if shape == "single":
return {"table": names[0], **self.replies[names[0]]} if names else {}
return {"tables": [{"table": t, **self.replies[t]} for t in names]}
class _Structured:
def __init__(self, owner: FakeModel) -> None:
self.owner = owner
async def ainvoke(self, messages: list[Any]) -> dict:
return self.owner._reply(messages, "structured")
+3
View File
@@ -162,6 +162,9 @@ async def test_draft_defaults_to_top_tables_and_model_via_settings(client, sourc
body = r.json()
assert len(body["tables"]) == 20 and body["tables"][0]["table_name"] == "visits"
assert body["model_used"] is True and body["model"] == "fake-model"
# 每张表模型给了几项可用内容:模型给了内容的表是正数;回复里没有的表记 model_error,没有这个数
assert body["tables"][0]["model_items"] == 2 and body["tables"][0]["model_error"] is None
assert all(t["model_items"] is None and t["model_error"] for t in body["tables"][1:])
detail = (await client.get(f"{_base(source_id)}/visits")).json()
assert detail["notes"]["label"]["source"] == "llm" and detail["usage"] == 7
+59
View File
@@ -263,6 +263,41 @@ async def test_model_without_structured_output_falls_back_to_text():
assert got["visits"].notes == {"label": make_item("入园记录", "llm")}
# ---- 实测见过的几种退化回复(A2):一项都没写进去时不能说「没有变化」
async def test_model_single_table_object_without_wrapper_is_recognized():
"""纯文本兜底时只回了单个表对象,没有 tables 外壳:照样认出来。"""
model = FakeModel({"visits": {"label": "入园记录"}}, structured="error", text="single")
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
assert got["visits"].notes == {"label": make_item("入园记录", "llm")}
async def test_hollow_structured_output_retries_with_text():
"""结构化输出只给了 {"tables":[{"table":"visits"}]} 这样的空壳:改走纯文本再问一次,用纯文本给的内容。"""
model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, structured="hollow", text="tables")
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
assert model.modes == ["structured", "text"]
assert got["visits"].notes == {"label": make_item("入园记录", "llm"), "kind": make_item("fact", "llm")}
async def test_hollow_structured_then_single_text_object():
model = FakeModel({"visits": {"label": "入园记录"}}, structured="hollow", text="single")
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
assert got["visits"].notes == {"label": make_item("入园记录", "llm")}
async def test_table_with_no_usable_content_is_a_model_error():
"""表名对上了、一个字段都没有(两条路都是空壳):这张表记模型失败,不算起草成功。"""
model = FakeModel({"visits": {"label": "入园记录"}}, structured="hollow", text="hollow")
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
assert got["visits"] == catalog.MODEL_EMPTY_REASON
# 给了字段、但没有一项合规(列名对不上、取值不在范围里)同样算没有可用内容
junk = FakeModel({"visits": {"kind": "planet", "columns": [{"name": "ghost", "label": "不存在的列"}]}})
got = await catalog.draft_with_model(junk, _source(_scenic_tables()), ["visits"])
assert got["visits"] == catalog.MODEL_EMPTY_REASON
async def test_model_draft_respects_system_notes_of_imported_tables():
"""导入表格:表说明已经写了粒度、列说明写了单位,模型起草的粒度和单位不进目录,免得重复或矛盾。"""
src = _source({"日报": _table(_col("日期", "TEXT", comment="格式 YYYY-MM-DD"), _col("客流"), pk=(),
@@ -383,6 +418,30 @@ async def test_draft_catalog_model_failure_on_one_table(monkeypatch):
assert by["channels"].model_error is None and by["channels"].added == 1
async def test_draft_catalog_empty_model_reply_is_not_no_change():
"""模型一项可用内容都没给:这张表记 model_error,没有 model_items,界面据此不说「没有变化」。
模型给了内容、只是和现有目录一致:没有 model_error,model_items 是模型给的项数。"""
row = await _db_source(_scenic_tables())
hollow = FakeModel({"visits": {"label": "入园记录"}}, structured="hollow", text="hollow")
async with SessionLocal() as session:
report = await catalog.draft_catalog(session, row, tables=["visits"], use_model=True, model=hollow)
result = report.tables[0]
assert result.model_error == catalog.MODEL_EMPTY_REASON and result.model_items is None
assert (result.added, result.updated, result.removed) == (0, 0, 0)
model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}})
async with SessionLocal() as session:
first = await catalog.draft_catalog(session, row, tables=["visits"], use_model=True, model=model)
again = await catalog.draft_catalog(session, row, tables=["visits"], use_model=True, model=model)
assert first.tables[0].model_items == 2 and first.tables[0].added == 2
assert again.tables[0].model_error is None and again.tables[0].model_items == 2
assert (again.tables[0].added, again.tables[0].updated, again.tables[0].removed) == (0, 0, 0)
# 不用模型时没有这个数
async with SessionLocal() as session:
plain = await catalog.draft_catalog(session, row, tables=["visits"], use_model=False)
assert plain.tables[0].model_items is None
async def test_draft_catalog_defaults_to_most_used_tables(monkeypatch):
tables = {f"t{i:02d}": _table(_col("id"), fks=[]) for i in range(25)}
row = await _db_source(tables)
+4 -1
View File
@@ -2050,7 +2050,10 @@ export const CATALOG_TEXT = {
draftFailed: '起草中断',
draftSummary: (added: number, updated: number, removed: number) =>
`新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`,
draftNoChange: '没有变化:起草结果与现有目录一致',
// 只在起草确实给出了内容、只是和现有目录一致时用:有表的模型起草失败或没给出可用内容(model_error)时
// 不能说「没有变化」,改用 draftNothingWritten
draftNoChange: '没有变化:起草出的内容与现有目录一致',
draftNothingWritten: '目录没有更新:模型没有给出可用的内容,原因见下方',
draftModelSkipped: (reason: string) => `模型未参与起草:${reason.replace(/[。.]$/, '')}。已按注释、外键和命名起草。`,
draftModelUsed: (model: string) => `模型:${model}`,
draftTableErrors: (n: number) => `${formatNumber(n)} 张表未能起草`,
+6 -1
View File
@@ -2746,8 +2746,13 @@ export interface CatalogDraftRow {
version: number
/** 这张表没有起草(表结构里没有、写入一直冲突) */
error: string | null
/** 只是模型那部分失败,其余来源照常写入 */
/** 只是模型那部分失败,其余来源照常写入。模型给了这张表、却一项可用内容都没有,也记在这里 */
model_error: string | null
/**
* 模型给了这张表几项可用内容(并入目录之前)。没用模型、模型这部分失败时为 null(老后端没有这个键)。
* 大于 0 而新增、更新、删除都是 0,才是「模型给了内容、只是和现有目录一致」
*/
model_items?: number | null
}
/** POST /datasources/{id}/catalog/draft */