mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 03:01:58 +00:00
A2 模型起草一项都没写进去时不再算成功:认出没有 tables 外壳的单个表对象;结构化输出是空壳(只有表名、没有可用字段)时改走纯文本再问一次;某张表最终没有可用内容记 model_error「模型没有给出这张表的可用内容」;起草结果每张表多一个 model_items(模型给了几项可用内容),界面据此区分「和现有目录一致」与「模型没给内容」;「没有变化」改为只说起草出的内容与现有目录一致,另加「目录没有更新」一句;假模型能模拟空壳、单个对象、某条路不可用三种实测回复
This commit is contained in:
@@ -227,7 +227,11 @@ async def put_catalog_table(source_id: str, table: str, payload: CatalogPutIn,
|
||||
async def draft_catalog(source_id: str, payload: CatalogDraftIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""同步起草。返回每张表新增、更新、删除了几项;模型用不了或某张表的模型起草失败,照实写在结果里。"""
|
||||
"""同步起草。返回每张表新增、更新、删除了几项;模型用不了或某张表的模型起草失败,照实写在结果里。
|
||||
|
||||
model_items:模型给了这张表几项可用内容(没用模型、模型这部分失败时为 null)。界面据此区分「模型给了内容、
|
||||
只是和现有目录一致」(model_items > 0、三个数都是 0)和「模型没给内容」(model_error 有值)。
|
||||
"""
|
||||
_, source = await _resolved(session, source_id)
|
||||
cache = source.schema_cache or {}
|
||||
if not cache.get("tables"):
|
||||
@@ -235,7 +239,8 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
|
||||
report = await catalog.draft_catalog(session, source, tables=payload.tables or None,
|
||||
use_model=payload.use_model, actor=actor_of(x_actor))
|
||||
rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed,
|
||||
"version": r.version, "error": r.error, "model_error": r.model_error} for r in report.tables]
|
||||
"version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items}
|
||||
for r in report.tables]
|
||||
return {
|
||||
"tables": rows,
|
||||
"total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")},
|
||||
|
||||
+57
-19
@@ -40,7 +40,7 @@ import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Mapping
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
@@ -1576,8 +1576,16 @@ def _batches(names: list[str], tables: Mapping[str, Any]) -> list[list[str]]:
|
||||
return out
|
||||
|
||||
|
||||
#: 模型给了这张表、但一项可用内容都没有(只有表名的空壳,或者给的值全不合规):这张表记模型失败,不算起草成功。
|
||||
#: 不然界面只看到「新增 0 项」,会说成「没有变化」
|
||||
MODEL_EMPTY_REASON = "模型没有给出这张表的可用内容"
|
||||
|
||||
|
||||
def _model_tables(raw: Any) -> list[dict[str, Any]] | None:
|
||||
"""模型的回复 → 每张表一项的列表。结构化输出给的对象、正文里的 JSON 都认;对不上格式返回 None。"""
|
||||
"""模型的回复 → 每张表一项的列表。结构化输出给的对象、正文里的 JSON 都认;对不上格式返回 None。
|
||||
|
||||
实测模型有时不带 tables 外壳、只回单个表对象({"table": "x", "label": …}),也认成一张表。
|
||||
"""
|
||||
if hasattr(raw, "model_dump"):
|
||||
raw = raw.model_dump()
|
||||
if isinstance(raw, str):
|
||||
@@ -1591,31 +1599,43 @@ def _model_tables(raw: Any) -> list[dict[str, Any]] | None:
|
||||
except ValueError:
|
||||
return None
|
||||
if isinstance(raw, dict):
|
||||
raw = raw.get("tables")
|
||||
raw = [raw] if "tables" not in raw and raw.get("table") else raw.get("tables")
|
||||
if not isinstance(raw, list):
|
||||
return None
|
||||
return [t for t in raw if isinstance(t, dict)]
|
||||
|
||||
|
||||
async def _ask_model(model: Any, messages: list[Any]) -> list[dict[str, Any]]:
|
||||
"""问一批。先走结构化输出;模型不支持、调用报错、或给回来的不是约定的结构,再退回纯文本、从正文里取 JSON。
|
||||
async def _ask_model(model: Any, messages: list[Any],
|
||||
usable: Callable[[list[dict[str, Any]]], bool] | None = None) -> list[dict[str, Any]]:
|
||||
"""问一批。先走结构化输出;模型不支持、调用报错、给回来的不是约定的结构,或者 usable 说它不够用(实测见过
|
||||
结构化输出只给 {"tables":[{"table":"x"}]} 这样的空壳),再退回纯文本、从正文里取 JSON。
|
||||
|
||||
结构化输出报错也退回(同 judge._ask):有的网关不支持工具调用,一调就报错,纯文本却能用。代价是真的
|
||||
网络故障会多试一次,两次加起来仍受 MODEL_TIMEOUT_S 约束。
|
||||
|
||||
两条路都拿到了东西时,结构化的在前、纯文本的在后一起交回(同一张表各有一份,调用方挑能用的那份);纯文本那次
|
||||
失败了就只交回结构化的那份(调用方据此给空壳的表记失败)。
|
||||
"""
|
||||
try:
|
||||
parsed = _model_tables(await model.with_structured_output(CATALOG_DRAFT_PROMPT_SCHEMA).ainvoke(messages))
|
||||
except Exception: # noqa: BLE001 - 退回纯文本
|
||||
parsed = None
|
||||
if parsed is not None:
|
||||
if parsed is not None and (usable is None or usable(parsed)):
|
||||
return parsed
|
||||
from app.engine.state import message_text
|
||||
|
||||
reply = await model.ainvoke(messages)
|
||||
parsed = _model_tables(message_text(reply))
|
||||
if parsed is None:
|
||||
try:
|
||||
reply = await model.ainvoke(messages)
|
||||
except Exception: # noqa: BLE001 - 结构化那份还在:交回它,空壳的表由调用方记失败
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
raise
|
||||
text = _model_tables(message_text(reply))
|
||||
if text is None:
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
raise ValueError("模型的回复不是约定的格式")
|
||||
return parsed
|
||||
return [*(parsed or []), *text]
|
||||
|
||||
|
||||
def _clean_text(value: Any, limit: int = _TEXT_MAX) -> str | None:
|
||||
@@ -1695,9 +1715,22 @@ async def draft_with_model(model: Any, source: Any, tables: list[str], *,
|
||||
existing=(existing or {}).get(n)) for n in batch)
|
||||
messages = [SystemMessage(content=CATALOG_DRAFT_SYSTEM),
|
||||
HumanMessage(content=CATALOG_DRAFT_PROMPT.format(n=len(batch), tables=described))]
|
||||
def drafted(entries: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
"""这批里每张表能用的那份:同一张表给了几份(结构化、纯文本各一份)时取第一份有可用内容的。"""
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for entry in entries:
|
||||
name = str(entry.get("table"))
|
||||
if name in batch and name not in out:
|
||||
notes = _from_model(entry, all_tables[name], system=system)
|
||||
if notes:
|
||||
out[name] = notes
|
||||
return out
|
||||
|
||||
try:
|
||||
async with gate:
|
||||
got = await asyncio.wait_for(_ask_model(model, messages), timeout=MODEL_TIMEOUT_S)
|
||||
got = await asyncio.wait_for(
|
||||
_ask_model(model, messages, usable=lambda entries: len(drafted(entries)) == len(batch)),
|
||||
timeout=MODEL_TIMEOUT_S)
|
||||
except asyncio.TimeoutError:
|
||||
for n in batch:
|
||||
results[n] = f"模型起草超时:{MODEL_TIMEOUT_S} 秒内没有收到回复"
|
||||
@@ -1708,14 +1741,13 @@ async def draft_with_model(model: Any, source: Any, tables: list[str], *,
|
||||
for n in batch:
|
||||
results[n] = f"模型起草失败:{first_line(e) or describe_exception(e)}"
|
||||
return
|
||||
by_name = {str(t.get("table")): t for t in got}
|
||||
named = {str(t.get("table")) for t in got}
|
||||
usable = drafted(got)
|
||||
for n in batch:
|
||||
entry = by_name.get(n)
|
||||
if entry is None:
|
||||
results[n] = "模型的回复里没有这张表"
|
||||
continue
|
||||
results[n] = TableDraft(notes=_from_model(entry, all_tables[n], system=system),
|
||||
covered=frozenset({"llm"}))
|
||||
if n in usable:
|
||||
results[n] = TableDraft(notes=usable[n], covered=frozenset({"llm"}))
|
||||
else:
|
||||
results[n] = MODEL_EMPTY_REASON if n in named else "模型的回复里没有这张表"
|
||||
|
||||
await asyncio.gather(*(run(b) for b in _batches(names, all_tables)))
|
||||
return {n: results[n] for n in names if n in results}
|
||||
@@ -1737,6 +1769,9 @@ class TableDraftResult:
|
||||
version: int = 0
|
||||
error: str | None = None
|
||||
model_error: str | None = None
|
||||
#: 模型给了这张表几项可用内容(并入目录之前,按项数)。没用模型、模型这部分失败时为 None。
|
||||
#: 大于 0 而新增、更新、删除都是 0,才是「模型给了内容、只是和现有目录一致」
|
||||
model_items: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1800,6 +1835,8 @@ async def draft_catalog(session: AsyncSession, source: Any, *, tables: list[str]
|
||||
modeled = llm.get(table)
|
||||
if isinstance(modeled, str):
|
||||
result.model_error = modeled
|
||||
elif isinstance(modeled, TableDraft):
|
||||
result.model_items = len(_slots(modeled.notes))
|
||||
for _attempt in range(3):
|
||||
entry = await read_entry(session, source.id, table)
|
||||
version = entry.version if entry else 0
|
||||
@@ -2188,7 +2225,8 @@ __all__ = [
|
||||
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
|
||||
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
|
||||
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
|
||||
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX", "PROFILE_VERIFY_COVERAGE",
|
||||
"MODEL_EMPTY_REASON", "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX",
|
||||
"PROFILE_VERIFY_COVERAGE",
|
||||
"PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL",
|
||||
"TABLE_KINDS", "TableDraft", "TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit",
|
||||
"apply_patch", "codes_complete", "describe_slot", "draft_catalog", "draft_structure", "draft_with_model",
|
||||
|
||||
+41
-6
@@ -3,28 +3,63 @@
|
||||
model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, fail={"orders"})
|
||||
await catalog.draft_with_model(model, source, ["visits"])
|
||||
|
||||
- 结构化输出(with_structured_output)返回自己;ainvoke 从提示词里认出这一批有哪些表(「表 <表名>」),
|
||||
- 结构化输出(with_structured_output(...).ainvoke)从提示词里认出这一批有哪些表(「表 <表名>」),
|
||||
返回 {"tables": [{"table": 表名, **replies[表名]}, …]};
|
||||
- 直接 ainvoke 是纯文本那条路:同样的内容写成一段正文(AIMessage),前面带一句话;
|
||||
- fail 里的表所在的那一批抛异常,用来验证「一批失败只影响这一批」;
|
||||
- calls 记下每次收到的提示词全文,用来核对交给模型的内容(只有结构,没有数据)。
|
||||
- calls 记下每次收到的提示词全文,用来核对交给模型的内容(只有结构,没有数据);modes 记下每次走的是
|
||||
structured 还是 text。
|
||||
|
||||
structured、text 模拟实测见过的几种回复(两条路各自设):
|
||||
|
||||
- "tables":正常,{"tables": [...]};
|
||||
- "hollow":退化的空壳,表名对上了、一个字段都没有:{"tables": [{"table": 表名}, …]};
|
||||
- "single":只给第一张表的对象,没有 tables 外壳:{"table": 表名, …};
|
||||
- "error":这条路用不了(网关不支持工具调用之类),一调就抛异常。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
class FakeModel:
|
||||
def __init__(self, replies: dict[str, dict], fail: set[str] = frozenset()) -> None:
|
||||
def __init__(self, replies: dict[str, dict], fail: set[str] = frozenset(), *,
|
||||
structured: str = "tables", text: str = "tables") -> None:
|
||||
self.replies, self.fail, self.calls = replies, set(fail), []
|
||||
self.structured, self.text = structured, text
|
||||
self.modes: list[str] = []
|
||||
|
||||
def with_structured_output(self, schema: Any, **_: Any) -> "FakeModel":
|
||||
def with_structured_output(self, schema: Any, **_: Any) -> "_Structured":
|
||||
assert schema["title"]
|
||||
return self
|
||||
return _Structured(self)
|
||||
|
||||
async def ainvoke(self, messages: list[Any]) -> dict:
|
||||
async def ainvoke(self, messages: list[Any]) -> Any:
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
reply = self._reply(messages, "text")
|
||||
return AIMessage(content="草稿如下:" + json.dumps(reply, ensure_ascii=False))
|
||||
|
||||
def _reply(self, messages: list[Any], mode: str) -> dict:
|
||||
text = "\n".join(str(getattr(m, "content", m)) for m in messages)
|
||||
self.calls.append(text)
|
||||
self.modes.append(mode)
|
||||
names = [t for t in self.replies if f"表 {t}\n" in text or f"表 {t}(" in text]
|
||||
if any(t in self.fail for t in names):
|
||||
raise RuntimeError("网关超时")
|
||||
shape = self.structured if mode == "structured" else self.text
|
||||
if shape == "error":
|
||||
raise NotImplementedError("这条路用不了")
|
||||
if shape == "hollow":
|
||||
return {"tables": [{"table": t} for t in names]}
|
||||
if shape == "single":
|
||||
return {"table": names[0], **self.replies[names[0]]} if names else {}
|
||||
return {"tables": [{"table": t, **self.replies[t]} for t in names]}
|
||||
|
||||
|
||||
class _Structured:
|
||||
def __init__(self, owner: FakeModel) -> None:
|
||||
self.owner = owner
|
||||
|
||||
async def ainvoke(self, messages: list[Any]) -> dict:
|
||||
return self.owner._reply(messages, "structured")
|
||||
|
||||
@@ -162,6 +162,9 @@ async def test_draft_defaults_to_top_tables_and_model_via_settings(client, sourc
|
||||
body = r.json()
|
||||
assert len(body["tables"]) == 20 and body["tables"][0]["table_name"] == "visits"
|
||||
assert body["model_used"] is True and body["model"] == "fake-model"
|
||||
# 每张表模型给了几项可用内容:模型给了内容的表是正数;回复里没有的表记 model_error,没有这个数
|
||||
assert body["tables"][0]["model_items"] == 2 and body["tables"][0]["model_error"] is None
|
||||
assert all(t["model_items"] is None and t["model_error"] for t in body["tables"][1:])
|
||||
detail = (await client.get(f"{_base(source_id)}/visits")).json()
|
||||
assert detail["notes"]["label"]["source"] == "llm" and detail["usage"] == 7
|
||||
|
||||
|
||||
@@ -263,6 +263,41 @@ async def test_model_without_structured_output_falls_back_to_text():
|
||||
assert got["visits"].notes == {"label": make_item("入园记录", "llm")}
|
||||
|
||||
|
||||
# ---- 实测见过的几种退化回复(A2):一项都没写进去时不能说「没有变化」
|
||||
|
||||
|
||||
async def test_model_single_table_object_without_wrapper_is_recognized():
|
||||
"""纯文本兜底时只回了单个表对象,没有 tables 外壳:照样认出来。"""
|
||||
model = FakeModel({"visits": {"label": "入园记录"}}, structured="error", text="single")
|
||||
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
|
||||
assert got["visits"].notes == {"label": make_item("入园记录", "llm")}
|
||||
|
||||
|
||||
async def test_hollow_structured_output_retries_with_text():
|
||||
"""结构化输出只给了 {"tables":[{"table":"visits"}]} 这样的空壳:改走纯文本再问一次,用纯文本给的内容。"""
|
||||
model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, structured="hollow", text="tables")
|
||||
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
|
||||
assert model.modes == ["structured", "text"]
|
||||
assert got["visits"].notes == {"label": make_item("入园记录", "llm"), "kind": make_item("fact", "llm")}
|
||||
|
||||
|
||||
async def test_hollow_structured_then_single_text_object():
|
||||
model = FakeModel({"visits": {"label": "入园记录"}}, structured="hollow", text="single")
|
||||
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
|
||||
assert got["visits"].notes == {"label": make_item("入园记录", "llm")}
|
||||
|
||||
|
||||
async def test_table_with_no_usable_content_is_a_model_error():
|
||||
"""表名对上了、一个字段都没有(两条路都是空壳):这张表记模型失败,不算起草成功。"""
|
||||
model = FakeModel({"visits": {"label": "入园记录"}}, structured="hollow", text="hollow")
|
||||
got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"])
|
||||
assert got["visits"] == catalog.MODEL_EMPTY_REASON
|
||||
# 给了字段、但没有一项合规(列名对不上、取值不在范围里)同样算没有可用内容
|
||||
junk = FakeModel({"visits": {"kind": "planet", "columns": [{"name": "ghost", "label": "不存在的列"}]}})
|
||||
got = await catalog.draft_with_model(junk, _source(_scenic_tables()), ["visits"])
|
||||
assert got["visits"] == catalog.MODEL_EMPTY_REASON
|
||||
|
||||
|
||||
async def test_model_draft_respects_system_notes_of_imported_tables():
|
||||
"""导入表格:表说明已经写了粒度、列说明写了单位,模型起草的粒度和单位不进目录,免得重复或矛盾。"""
|
||||
src = _source({"日报": _table(_col("日期", "TEXT", comment="格式 YYYY-MM-DD"), _col("客流"), pk=(),
|
||||
@@ -383,6 +418,30 @@ async def test_draft_catalog_model_failure_on_one_table(monkeypatch):
|
||||
assert by["channels"].model_error is None and by["channels"].added == 1
|
||||
|
||||
|
||||
async def test_draft_catalog_empty_model_reply_is_not_no_change():
|
||||
"""模型一项可用内容都没给:这张表记 model_error,没有 model_items,界面据此不说「没有变化」。
|
||||
模型给了内容、只是和现有目录一致:没有 model_error,model_items 是模型给的项数。"""
|
||||
row = await _db_source(_scenic_tables())
|
||||
hollow = FakeModel({"visits": {"label": "入园记录"}}, structured="hollow", text="hollow")
|
||||
async with SessionLocal() as session:
|
||||
report = await catalog.draft_catalog(session, row, tables=["visits"], use_model=True, model=hollow)
|
||||
result = report.tables[0]
|
||||
assert result.model_error == catalog.MODEL_EMPTY_REASON and result.model_items is None
|
||||
assert (result.added, result.updated, result.removed) == (0, 0, 0)
|
||||
|
||||
model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}})
|
||||
async with SessionLocal() as session:
|
||||
first = await catalog.draft_catalog(session, row, tables=["visits"], use_model=True, model=model)
|
||||
again = await catalog.draft_catalog(session, row, tables=["visits"], use_model=True, model=model)
|
||||
assert first.tables[0].model_items == 2 and first.tables[0].added == 2
|
||||
assert again.tables[0].model_error is None and again.tables[0].model_items == 2
|
||||
assert (again.tables[0].added, again.tables[0].updated, again.tables[0].removed) == (0, 0, 0)
|
||||
# 不用模型时没有这个数
|
||||
async with SessionLocal() as session:
|
||||
plain = await catalog.draft_catalog(session, row, tables=["visits"], use_model=False)
|
||||
assert plain.tables[0].model_items is None
|
||||
|
||||
|
||||
async def test_draft_catalog_defaults_to_most_used_tables(monkeypatch):
|
||||
tables = {f"t{i:02d}": _table(_col("id"), fks=[]) for i in range(25)}
|
||||
row = await _db_source(tables)
|
||||
|
||||
@@ -2050,7 +2050,10 @@ export const CATALOG_TEXT = {
|
||||
draftFailed: '起草中断',
|
||||
draftSummary: (added: number, updated: number, removed: number) =>
|
||||
`新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`,
|
||||
draftNoChange: '没有变化:起草结果与现有目录一致',
|
||||
// 只在起草确实给出了内容、只是和现有目录一致时用:有表的模型起草失败或没给出可用内容(model_error)时
|
||||
// 不能说「没有变化」,改用 draftNothingWritten
|
||||
draftNoChange: '没有变化:起草出的内容与现有目录一致',
|
||||
draftNothingWritten: '目录没有更新:模型没有给出可用的内容,原因见下方',
|
||||
draftModelSkipped: (reason: string) => `模型未参与起草:${reason.replace(/[。.]$/, '')}。已按注释、外键和命名起草。`,
|
||||
draftModelUsed: (model: string) => `模型:${model}`,
|
||||
draftTableErrors: (n: number) => `${formatNumber(n)} 张表未能起草`,
|
||||
|
||||
@@ -2746,8 +2746,13 @@ export interface CatalogDraftRow {
|
||||
version: number
|
||||
/** 这张表没有起草(表结构里没有、写入一直冲突) */
|
||||
error: string | null
|
||||
/** 只是模型那部分失败,其余来源照常写入 */
|
||||
/** 只是模型那部分失败,其余来源照常写入。模型给了这张表、却一项可用内容都没有,也记在这里 */
|
||||
model_error: string | null
|
||||
/**
|
||||
* 模型给了这张表几项可用内容(并入目录之前)。没用模型、模型这部分失败时为 null(老后端没有这个键)。
|
||||
* 大于 0 而新增、更新、删除都是 0,才是「模型给了内容、只是和现有目录一致」
|
||||
*/
|
||||
model_items?: number | null
|
||||
}
|
||||
|
||||
/** POST /datasources/{id}/catalog/draft */
|
||||
|
||||
Reference in New Issue
Block a user