数据目录阶段 4A:助手协议新增目录修改提案 catalog_patch,服务端核对数据源、表、路径和取值后附版本与改前改后转出、绝不自动写入,新增提案预览与保存接口

This commit is contained in:
ThearchyHelios
2026-10-03 11:19:44 +08:00
parent 5df10ac260
commit 1a760726f7
4 changed files with 714 additions and 12 deletions
+56
View File
@@ -4,6 +4,8 @@
- GET /catalog/{table} 单表目录 + 表结构
- PUT /catalog/{table} 整份提交(乐观锁),改动过的项记为人工确认
- POST /catalog/{table}/review 单项审阅:确认、驳回、撤销审阅
- POST /catalog/{table}/patch/preview 目录修改提案的预览(只读):对着当前目录算改前、改后
- POST /catalog/{table}/patch 保存目录修改提案(乐观锁),记为人工确认
- POST /catalog/draft 同步起草(注释、外键、命名推断,可选模型)
规则都在 data/catalog.py,这里只做取源、转换形状和把异常翻成状态码:版本不符 409,结构不合规、审阅路径
@@ -40,6 +42,21 @@ class CatalogReviewIn(BaseModel):
if_version: int = Field(ge=0)
class CatalogPatchChange(BaseModel):
#: 审阅接口的路径写法;新增关联关系写 relations.new
path: str = Field(min_length=1, max_length=600)
value: Any
reason: str | None = Field(default=None, max_length=2000)
class CatalogPatchPreviewIn(BaseModel):
changes: list[CatalogPatchChange] = Field(min_length=1, max_length=catalog.PATCH_MAX_CHANGES)
class CatalogPatchIn(CatalogPatchPreviewIn):
if_version: int = Field(ge=0)
class CatalogDraftIn(BaseModel):
#: 要起草的表;不给或为空时按使用次数取前 20 张
tables: list[str] | None = Field(default=None, max_length=200)
@@ -224,3 +241,42 @@ async def review_catalog_item(source_id: str, table: str, payload: CatalogReview
except catalog.CatalogPathError as e:
raise HTTPException(422, str(e)) from e
return await _detail(session, source, key, entry)
def _tables_of(source: Any) -> dict[str, Any] | None:
"""核对提案用的表结构;没有表结构(尚未探查)时不核对列名。"""
return ((source.schema_cache or {}).get("tables") or None) if source is not None else None
@router.post("/{source_id}/catalog/{table}/patch/preview")
async def preview_catalog_patch(source_id: str, table: str, payload: CatalogPatchPreviewIn,
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""目录修改提案的预览:对着当前目录逐项算改前、改后,不写库。
助手转出的提案卡片在保存遇到 409(别人刚改过这张表)时用它重新载入:改前的值、码值补充后的样子都按
最新的目录重算,人看过之后再带新版本保存。
"""
_, source = await _resolved(session, source_id)
key, entry = await _table_key(session, source, table)
plan = catalog.plan_patch(entry.notes if entry else {}, [c.model_dump() for c in payload.changes], table=key,
tables=_tables_of(source))
return {"table_name": key, "version": entry.version if entry else 0,
"changes": [c.as_dict() for c in plan.changes], "problems": plan.problems}
@router.post("/{source_id}/catalog/{table}/patch")
async def save_catalog_patch(source_id: str, table: str, payload: CatalogPatchIn,
x_actor: str | None = Header(default=None),
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""保存一条目录修改提案:改动过的项记为人工填写、已确认。版本不符 409;有一项不合法 422,整条不写。"""
_, source = await _resolved(session, source_id)
key, _ = await _table_key(session, source, table)
try:
entry = await catalog.save_patch(session, source.id, key, [c.model_dump() for c in payload.changes],
if_version=payload.if_version, actor=actor_of(x_actor),
tables=_tables_of(source))
except catalog.CatalogConflict as e:
raise _conflict(e) from e
except catalog.CatalogInvalid as e:
raise _invalid(e) from e
return await _detail(session, source, key, entry)
+95 -6
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import copy
import json
import logging
import math
import re
import time
@@ -30,6 +31,7 @@ from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_mod
from app.tools.registry import all_specs
router = APIRouter(prefix="/api/copilot", tags=["copilot"])
logger = logging.getLogger(__name__)
async def _sources(session: AsyncSession, scope: list[str] | None) -> list[Any]:
@@ -795,7 +797,22 @@ _STREAM_PROTOCOL = """\
("设计一个每天跑的工作流")时给 false —— 他要的是这张图,不是这一次的结果
- 如果这句话根本不需要工作流(见下面的路径约定),**第一行也是最后一行**就输出:
{"op":"reply","text":"直接回答的内容"}
这时不要输出任何别的操作"""
这时除了下面的 catalog_patch,不要输出任何别的操作
- 数据目录修改提案(可选):用户在这句话里**明确陈述了一条数据事实**时,另输出一行
{"op":"catalog_patch","source":"数据源名","table":"表名","changes":[{"path":"…","value":…,"reason":"一句话依据"}]}
它只是提案,用户确认后才写进数据目录。可以和图操作一起出现(写在 done 之前),也可以只配一行 reply(写在 reply 之前)
· path:表级项直接写字段名(label 中文名、description 说明、grain 粒度、keys 业务主键列表、
kind 表类型 fact/dimension/snapshot/log/config、business_date {"column","rule","timezone"}、
valid_filter 有效记录的 SQL 条件、dedup 去重规则);列级项写 columns.<列名>.<字段>(label、meaning 含义、
unit 单位、measure 度量类型 flow/stock/ratio/identifier/status/attribute、codes 码值 {"码":"含义"});
已有的关联关系写 relations.<编号>;新增关联关系写 relations.new,value 写
{"columns":[本表列],"to_table":"目标表","to_columns":[目标列],"cardinality":"many_to_one/one_to_one/one_to_many"}
· codes 只写要补充或更正的码,原有的码会保留;valid_filter 要写成完整的条件(原有条件也要带上)
· 例:「在园人数是存量,不能跨天相加」→ columns.<那一列>.measure 写 stock;「status=9 表示作废,统计时要排除」→
columns.status.codes 写 {"9":"作废"},再把 valid_filter 补上排除条件;「channel_visits 是渠道和入园记录的关联表」→
description 和两条 relations.new
· 只有用户明确说出了数据事实才提,不要从自己的推测、查询结果或提问里提;表名、列名必须是上面数据源里真实存在的
· 计算公式不进数据目录:「转化率=下单数/访问数」这类指标怎么算的说法属于口径卡,不要提"""
def _parse_op_line(line: str) -> dict[str, Any] | None:
@@ -813,6 +830,56 @@ def _parse_op_line(line: str) -> dict[str, Any] | None:
return obj
#: 模型只发了目录修改提案、既没回答也没改图时,补的那句回答
PATCH_ONLY_REPLY = "已按你的说明整理出数据目录的修改建议,确认无误后可保存到数据目录。"
async def catalog_patch_op(op: dict[str, Any], sources: list[Any]) -> dict[str, Any] | None:
"""模型发的 catalog_patch → 转给前端的提案;整条不成立时返回 None。**只读,不写目录。**
核对数据源(只认这一轮给助手看的源,按名字)、表(按表结构认)、每一项的路径和取值格式(catalog.plan_patch,
和保存时同一套),附上当前版本、改前和改后的值。不合法的项丢掉、记一条日志,其余照转;一项都不剩就整条不转。
写入只发生在人点「保存到数据目录」之后(/catalog/{table}/patch,带着这里给的版本)。
开流之后请求的会话不能再用,这里另开一个短会话读目录:版本取提案到达的这一刻,比开流前读的新。
"""
from app.data import catalog, table_versions
from app.db.base import SessionLocal
name = op.get("source")
table = op.get("table")
row = next((s for s in sources if isinstance(name, str) and s.name == name), None)
if row is None:
logger.warning("目录修改提案被丢弃:数据源「%s」不在这一轮的范围内", name)
return None
try:
async with SessionLocal() as session:
source = await table_versions.resolve_source(session, row)
tables = (source.schema_cache or {}).get("tables") or {}
key = catalog.resolve_table_name(source.schema_cache, table) if isinstance(table, str) else None
if key is None:
logger.warning("目录修改提案被丢弃:数据源「%s」中没有表「%s」", name, table)
return None
entry = await catalog.read_entry(session, row.id, key)
except Exception: # noqa: BLE001 - 提案是附加的,读不到目录就不提,不能让这一轮生成失败
logger.exception("目录修改提案被丢弃:读取「%s」的数据目录失败", name)
return None
plan = catalog.plan_patch(entry.notes if entry else {}, op.get("changes"), table=key, tables=tables)
for problem in plan.problems:
logger.warning("目录修改提案(%s.%s)有一项被丢弃:%s", name, key, problem)
# 已经是这个值、也确认过的项没什么可提的
changes = [c.as_dict() for c in plan.changes if c.state != "same"]
if not changes:
logger.warning("目录修改提案被丢弃:%s.%s 没有需要修改的项", name, key)
return None
label = (entry.notes.get("label") if entry else None) or {}
return {
"op": "catalog_patch", "source": row.name, "source_id": row.id, "table": key,
"table_label": label.get("value") if label.get("status") != "rejected" else None,
"version": entry.version if entry else 0, "changes": changes,
}
_NODE_TYPES = frozenset(t.value for t in NodeType)
#: 协作成员条目里写 "remove": true 就把这个成员拿掉。不复用 null:null 在合并里
@@ -1760,6 +1827,10 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g
messages = [("system", system), ("human", human)]
if (shown := context.op()) is not None:
yield _sse(shown)
# reply 先压着:模型可能把目录修改提案写在 reply 后面,而前端收到 reply 就收尾(画布那边之后的操作一律不收),
# 提案得排在它前面转出。reply 之后只再认 catalog_patch,其余一概不理
replied: dict[str, Any] | None = None
patches = 0
try:
async for op in _with_heartbeat(_iter_ops(model, messages), started=started):
kind = op.get("op")
@@ -1767,11 +1838,19 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g
if kind in ("thinking", "heartbeat"):
yield f"data: {json.dumps(op, ensure_ascii=False)}\n\n"
continue
if kind == "catalog_patch":
# 目录修改提案:核对后转给前端做成卡片,人点「保存」才写。不进图,也不算这一轮的图操作
if (checked := await catalog_patch_op(op, sources)) is not None:
patches += 1
yield _sse(checked)
continue
if replied is not None:
continue
if kind == "reply":
# 这句话不需要工作流。原样转给前端,然后收流——后面那套
# 这句话不需要工作流:等模型说完(只收它后面的提案)再转给前端、收流——后面那套
# 排版校验对着一张空图跑,只会得到"图是空的,先拖一个节点进来"
yield f"data: {json.dumps(op, ensure_ascii=False)}\n\n"
return
replied = op
continue
if kind == "done":
explanation = str(op.get("explanation", ""))
# 用户要的是流程本身时("设计一个每天跑的工作流"),
@@ -1785,6 +1864,14 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g
skipped_types.append(str(bad_type))
if changed or kind in ("plan", "done"):
yield f"data: {json.dumps(op, ensure_ascii=False)}\n\n"
if replied is not None:
yield _sse(replied)
return
# 只发了目录修改提案,既没回答也没改图(用户只是在说一条数据事实):补一句回答收尾,
# 不能按下面的「模型没有给出任何修改」报错——提案已经摆在用户面前了
if patches and all(o.get("op") == "plan" for o in seen_ops):
yield _sse({"op": "reply", "text": PATCH_ONLY_REPLY})
return
# 流正常结束,却没有 done、也没有一个操作能认(格式跑偏,每一行都被跳过):
# 往下走就是拿原图去自查、交付,界面上写「无需修改」——而模型什么都没说
if all(o.get("op") == "plan" for o in seen_ops):
@@ -2012,7 +2099,8 @@ async def assist_publish_fix(
try:
async for op in _iter_ops(chat, [("system", system), ("human", _publish_fix_request(base, errors, level))]):
kind = op.get("op")
if kind in ("thinking", "heartbeat"):
# 目录修改提案只在用户亲口陈述数据事实时才有;这里的「用户消息」是系统拼的修复请求,提了也不转
if kind in ("thinking", "heartbeat", "catalog_patch"):
continue
if kind == "plan":
plan = str(op.get("summary") or "")
@@ -2197,7 +2285,8 @@ async def assist_upgrade(
nonlocal plan
async for op in _iter_ops(chat, [("system", system), ("human", request)]):
kind = op.get("op")
if kind in ("thinking", "heartbeat"):
# 同发布修复:这里没有用户陈述的数据事实,目录修改提案不收
if kind in ("thinking", "heartbeat", "catalog_patch"):
continue
if kind in ("plan", "done"):
# 修补轮的开场和收尾不是新方案,不覆盖第一轮的说明
+236 -6
View File
@@ -674,6 +674,235 @@ async def save_human_edit(session: AsyncSession, source_id: str, table: str, sub
return await write_entry(session, source_id, table, notes, if_version=current, actor=actor)
# ==========================================================================
# 目录修改提案:对话里的纠正(阶段 4A)
#
# 用户在助手里说出一条数据事实(「status=9 表示作废」),模型发一条 catalog_patch;这里逐项核对、算出改前和
# 改后,转给前端做成卡片。**提案本身从不落库**:要人点「保存到数据目录」,带着读到的版本走 save_patch,
# 改动记为人工填写、已确认——和人工编辑同一条规矩,只是改动由模型起草。
# ==========================================================================
#: 新增关联关系的路径:relations.new,值写两端;编号按两端算(relation_id),同一条关系不论谁提都是同一个编号
NEW_RELATION_PATH = "relations.new"
#: 一条提案最多几项:用户说的是一两条事实,几十项的「提案」多半是模型在重写整张表
PATCH_MAX_CHANGES = 20
_REASON_MAX = 500
#: 看起来像计算公式的文字:「转化率=下单数/访问数」「= SUM(amount)」。公式进口径卡,目录只记数据事实。
#: 不认减号:「上线日期=2024-01-01」不是公式。有效记录条件本来就是 SQL 条件,不查它
_FORMULA = re.compile(r"[==][^=<>!]*?[+*/×÷]|\b(?:sum|count|avg|average)\s*\(", re.IGNORECASE)
_FORMULA_CHECKED = frozenset({("t", "label"), ("t", "description"), ("t", "grain"), ("c", "label"),
("c", "meaning"), ("c", "unit"), ("c", "codes")})
@dataclass(frozen=True)
class PatchChange:
"""提案里的一项,核对过的。
- path:规范化后的路径(新增关系写成 relations.<编号>);
- before / before_status:当前目录里这一项的值和状态(被驳回的和没有的都算没有,为 None);
- after:保存后的值(码值是补充后的完整对照);value:要交回保存的原样取值(码值只有补充的那几个,
409 之后在最新的目录上重新补);
- state:change 值有变化;confirm 值相同、但还不是已确认(保存即确认);same 已经是这个值且已确认。
"""
path: str
before: Any
before_status: str | None
after: Any
value: Any
reason: str
state: str
def as_dict(self) -> dict[str, Any]:
return {"path": self.path, "before": self.before, "before_status": self.before_status, "after": self.after,
"value": self.value, "reason": self.reason, "state": self.state}
@dataclass(frozen=True)
class PatchPlan:
"""一条提案核对的结果:合法的项(changes)和不合法的项的中文说明(problems)。"""
changes: list[PatchChange]
problems: list[str]
def _column_names(meta: Mapping[str, Any] | None) -> set[str] | None:
"""表结构里的列名;不知道(表结构里没有这张表)时为 None,不核对。"""
if not isinstance(meta, Mapping):
return None
return {str(c.get("name")) for c in meta.get("columns") or [] if isinstance(c, Mapping) and c.get("name")}
def _formula_like(key: Slot, value: Any) -> bool:
if key[:1] + key[-1:] not in _FORMULA_CHECKED:
return False
texts = list(value.values()) if isinstance(value, dict) else [value]
return any(isinstance(t, str) and _FORMULA.search(t) for t in texts)
def _patch_relation(table: str, raw: Any, path_id: str | None, tables: Mapping[str, Any] | None,
columns: set[str] | None) -> tuple[dict[str, Any] | None, str | None]:
"""关联关系的取值 → (关系的值部分, 问题)。值写两端(columns、to_table、to_columns),基数可选。"""
if not isinstance(raw, Mapping):
return None, "关联关系的值应写明本表字段、目标表和目标字段"
to_table = raw.get("to_table")
if isinstance(to_table, str) and tables is not None:
to_table = resolve_table_name({"tables": tables}, to_table)
if to_table is None:
return None, f"关联关系的目标表「{raw.get('to_table')}」不在数据源的表结构里"
value = {"columns": raw.get("columns"), "to_table": to_table, "to_columns": raw.get("to_columns"),
"cardinality": raw.get("cardinality"), "coverage": None}
probe = {"id": "probe", **value, "source": "human", "status": "confirmed"}
if problems := _relation_problems(probe, set()):
return None, problems[0].replace("关联关系 probe ", "关联关系")
if columns is not None and (missing := [c for c in value["columns"] if c not in columns]):
return None, f"关联关系的本表字段「{'、'.join(missing)}」不在表结构里"
target = _column_names((tables or {}).get(to_table)) if tables is not None else None
if target is not None and (missing := [c for c in value["to_columns"] if c not in target]):
return None, f"关联关系的目标字段「{'、'.join(missing)}」不在 {to_table} 的表结构里"
rid = relation_id(table, value["columns"], to_table, value["to_columns"])
if path_id is not None and path_id != rid:
return None, f"关联关系 {path_id} 的两端与取值不一致:要改指向,请驳回原关系后再新增"
return value, None
def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
tables: Mapping[str, Any] | None = None) -> PatchPlan:
"""逐项核对一条目录修改提案,算出改前、改后。不改入参、不抛异常:不合法的项写进 problems。
changes 是 [{path, value, reason?}]。path 用审阅接口的写法:表级项写字段名,列级项 columns.<列名>.<字段>,
关系 relations.<编号>;新增关系写 relations.new。tables 是数据源 schema_cache 的 tables:给了就核对列名、
业务主键、业务日期列、关系两端在不在表结构里(模型会照着名字编列名);不给不核对。
规则:
- 取值格式和 validate_notes 同一套(_TABLE_VALUE_RULES / _COLUMN_VALUE_RULES / _relation_problems)。
- 码值是补充:在现有(没被驳回的)码值上加或改给出的码,不整份替换——「status=9 表示作废」不该把已确认的
其余码值删掉。
- 看起来像计算公式的文字不收(_FORMULA):公式进口径卡。
- 同一个路径出现两次只认第一次。
"""
meta = (tables or {}).get(table) if tables is not None else None
columns = _column_names(meta)
slots = _slots(notes)
out: list[PatchChange] = []
problems: list[str] = []
seen: set[str] = set()
if not isinstance(changes, list) or not changes:
return PatchPlan([], ["提案里没有要修改的项"])
if len(changes) > PATCH_MAX_CHANGES:
problems.append(f"提案里有 {len(changes)} 项,只核对前 {PATCH_MAX_CHANGES} 项")
changes = changes[:PATCH_MAX_CHANGES]
for raw in changes:
if not isinstance(raw, Mapping) or not isinstance(raw.get("path"), str) or "value" not in raw:
problems.append("提案里有一项不是「路径 + 取值」的形式")
continue
path = raw["path"].strip()
value = copy.deepcopy(raw["value"])
reason = raw.get("reason")
reason = reason.strip()[:_REASON_MAX] if isinstance(reason, str) else ""
if path == NEW_RELATION_PATH:
rel, problem = _patch_relation(table, value, None, tables, columns)
if problem:
problems.append(problem)
continue
key: Slot = ("r", relation_id(table, rel["columns"], rel["to_table"], rel["to_columns"]))
value = rel
else:
try:
key = parse_path(path)
except CatalogPathError:
problems.append(f"无法识别要修改的项({path[:60]})")
continue
if key[0] == "r":
rel, problem = _patch_relation(table, value, key[1], tables, columns)
if problem:
problems.append(problem)
continue
value = rel
where = describe_slot(key) if key[0] != "r" else "关联关系"
if key[0] == "t":
check, hint = _TABLE_VALUE_RULES[key[1]]
if not check(value):
problems.append(f"{where}{hint}")
continue
if columns is not None:
cited = value if key[1] == "keys" else [value["column"]] if key[1] == "business_date" else []
if missing := [c for c in cited if c not in columns]:
problems.append(f"{where}里的「{'、'.join(missing)}」不在表结构里")
continue
elif key[0] == "c":
if columns is not None and key[1] not in columns:
problems.append(f"列 {key[1]} 不在表结构里")
continue
check, hint = _COLUMN_VALUE_RULES[key[2]]
if not check(value):
problems.append(f"{where}{hint}")
continue
if _formula_like(key, value):
problems.append(f"{where}看起来是计算公式:公式请写进口径卡,数据目录只记数据事实")
continue
norm = f"relations.{key[1]}" if key[0] == "r" else path
if norm in seen:
problems.append(f"{where}在提案里出现了两次,只采用第一次")
continue
seen.add(norm)
prev = slots.get(key)
live = prev if prev is not None and prev.get("status") != "rejected" else None
before = _value_of(key, live) if live is not None else None
after = value
if key[0] == "c" and key[2] == "codes" and isinstance(before, dict):
after = {**before, **value}
if before is not None and before == after:
state = "same" if live.get("status") == "confirmed" else "confirm"
else:
state = "change"
out.append(PatchChange(path=norm, before=copy.deepcopy(before),
before_status=live.get("status") if live is not None else None,
after=after, value=value, reason=reason, state=state))
return PatchPlan(out, problems)
def apply_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
tables: Mapping[str, Any] | None = None, at: str | None = None) -> dict[str, Any]:
"""把一条提案并进目录,返回新目录(不改入参)。有一项不合法就整条不收,抛 CatalogInvalid。
值有变化的项记为人工填写、已确认(和人工编辑同一条规矩:改动出自人的确认,不是模型起草);值没变的只把
状态改成已确认、来源不变(等同单项确认);已经确认过的原样不动。
"""
plan = plan_patch(notes, changes, table=table, tables=tables)
if plan.problems:
raise CatalogInvalid(plan.problems)
at = at or now_iso()
slots = _slots(notes)
for change in plan.changes:
key = parse_path(change.path)
if change.state == "same":
continue
prev = slots.get(key)
if change.state == "confirm" and prev is not None:
slots[key] = {**prev, "status": "confirmed", "updated_at": at}
continue
if key[0] == "r":
slots[key] = {"id": key[1], **change.after, "source": "human", "status": "confirmed", "updated_at": at}
else:
slots[key] = {"value": change.after, "source": "human", "status": "confirmed", "updated_at": at}
out = _assemble(slots)
if problems := validate_notes(out):
raise CatalogInvalid(problems)
return out
async def save_patch(session: AsyncSession, source_id: str, table: str, changes: Any, *, if_version: int,
actor: str | None, tables: Mapping[str, Any] | None = None) -> CatalogEntry:
"""保存一条目录修改提案(规则见 apply_patch,乐观锁同 write_entry)。版本不符抛 CatalogConflict。"""
entry = await read_entry(session, source_id, table)
current = entry.version if entry else 0
if if_version != current:
raise CatalogConflict(table, current)
notes = apply_patch(entry.notes if entry else {}, changes, table=table, tables=tables)
return await write_entry(session, source_id, table, notes, if_version=current, actor=actor)
# ==========================================================================
# 起草:数据库注释、外键约束、命名推断(只看探查缓存,不发数据库查询)
# ==========================================================================
@@ -1690,11 +1919,12 @@ __all__ = [
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
"MergeStats", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS",
"TableDraft", "TableDraftResult", "apply_human_edit", "describe_slot", "draft_catalog", "draft_structure",
"draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths",
"make_item", "merge_notes", "now_iso", "parse_path", "query_snapshot_meta", "read_catalog", "read_entry",
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND",
"REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult",
"apply_human_edit", "apply_patch", "describe_slot", "draft_catalog", "draft_structure", "draft_with_model",
"fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths", "make_item",
"merge_notes", "now_iso", "parse_path", "plan_patch", "query_snapshot_meta", "read_catalog", "read_entry",
"relation_graph", "relation_id", "render_table_index", "render_table_notes", "resolve_draft_model",
"resolve_table_name", "review_entry", "review_item", "same_notes", "save_human_edit", "status_counts",
"system_notes_source", "table_usage", "validate_notes", "visible_notes", "write_entry",
"resolve_table_name", "review_entry", "review_item", "same_notes", "save_human_edit", "save_patch",
"status_counts", "system_notes_source", "table_usage", "validate_notes", "visible_notes", "write_entry",
]
+327
View File
@@ -0,0 +1,327 @@
"""对话里的纠正变成目录修改提案(阶段 4A)。
用户在助手(画布)或问数据页说出一条数据事实(「status=9 表示作废」),模型发一条 catalog_patch 操作;服务端
核对数据源、表、路径和取值格式,附上当前版本、改前和改后的值转给前端——**服务端绝不自动写入**,要人在卡片上点
「保存到数据目录」,带着读到的版本走 /catalog/{table}/patch(记为人工填写、已确认)。
- 纯函数:plan_patch 逐项核对并算出改前、改后;码值是补充,不整份替换;新增关联关系写 relations.new,编号按两端算;
计算公式不进目录。
- 流:合法的转出、带版本和改前值;不合法的丢弃并记日志;写在 reply 后面的提案也排到 reply 前面转出(前端收到
reply 就收尾);只有提案、没有回答时补一句回答;全程不写库。
- 接口:预览只读;保存带 if_version,版本不符 409,X-Actor 记成署名。
"""
from __future__ import annotations
import json
import logging
import uuid
from types import SimpleNamespace
from urllib.parse import quote
import pytest
from httpx import ASGITransport, AsyncClient
from app.data import catalog
from app.data.engine import engines
from app.data.introspect import introspect
from tests.fixtures.catalog import scenic_notes
_CACHE: dict[str, dict] = {}
async def _schema(path: str) -> dict:
if path not in _CACHE:
probe = SimpleNamespace(id=f"probe-{uuid.uuid4().hex[:6]}", name="scenic", kind="sqlite", database=path,
host=None, port=None, username=None, password=None, options={}, readonly=True,
description="", schema_cache={}, origin="manual")
try:
_CACHE[path] = await introspect(probe)
finally:
await engines.invalidate(probe.id)
return _CACHE[path]
def _visits() -> dict:
return scenic_notes.notes()["visits"]
# --------------------------------------------------------------------------
# plan_patch / apply_patch:纯函数
# --------------------------------------------------------------------------
async def test_plan_patch_reports_before_and_after(scenic_db):
tables = (await _schema(scenic_db))["tables"]
plan = catalog.plan_patch(_visits(), [
{"path": "columns.visitor_count.measure", "value": "stock", "reason": "在园人数是存量,不能跨天相加"},
{"path": "kind", "value": "fact"},
], table="visits", tables=tables)
assert plan.problems == []
measure, kind = plan.changes
assert (measure.path, measure.before, measure.before_status, measure.after) == (
"columns.visitor_count.measure", "flow", "confirmed", "stock")
assert measure.state == "change" and measure.reason == "在园人数是存量,不能跨天相加"
assert (kind.path, kind.before, kind.before_status, kind.after, kind.state) == ("kind", None, None, "fact", "change")
async def test_codes_are_supplemented_not_replaced(scenic_db):
tables = (await _schema(scenic_db))["tables"]
plan = catalog.plan_patch(_visits(), [{"path": "columns.status.codes", "value": {"9": "作废"}}],
table="visits", tables=tables)
[change] = plan.changes
assert change.before == {"1": "有效", "0": "作废"}
assert change.after == {"1": "有效", "0": "作废", "9": "作废"}
# 交回保存的是原样的补充值:409 之后在最新的目录上重新补
assert change.value == {"9": "作废"}
async def test_new_relation_gets_its_id_from_both_ends(scenic_db):
tables = (await _schema(scenic_db))["tables"]
value = {"columns": ["visit_id"], "to_table": "visits", "to_columns": ["id"], "cardinality": "many_to_one"}
plan = catalog.plan_patch({}, [{"path": "relations.new", "value": value}], table="channel_visits", tables=tables)
[change] = plan.changes
rid = catalog.relation_id("channel_visits", ["visit_id"], "visits", ["id"])
assert change.path == f"relations.{rid}" and change.before is None
assert change.after == {**value, "coverage": None}
# 已有编号、两端对得上:改基数
notes = catalog.apply_patch({}, [{"path": "relations.new", "value": value}], table="channel_visits", tables=tables)
again = catalog.plan_patch(notes, [{"path": f"relations.{rid}", "value": {**value, "cardinality": "one_to_one"}}],
table="channel_visits", tables=tables)
assert again.changes[0].before["cardinality"] == "many_to_one" and again.problems == []
# 编号对不上两端:不认(改指向是另一条关系)
moved = catalog.plan_patch(notes, [{"path": f"relations.{rid}",
"value": {**value, "to_table": "channels", "to_columns": ["id"]}}],
table="channel_visits", tables=tables)
assert moved.changes == [] and len(moved.problems) == 1
async def test_invalid_changes_are_reported(scenic_db):
tables = (await _schema(scenic_db))["tables"]
plan = catalog.plan_patch(_visits(), [
{"path": "formula", "value": "下单数 / 访问数"}, # 不是目录的项
{"path": "columns.visitor_count.measure", "value": "累计值"}, # 不在可选值内
{"path": "columns.ghost.meaning", "value": "不存在的列"}, # 表结构里没有这一列
{"path": "columns.visitor_count.meaning", "value": "转化率=下单数/访问数"}, # 计算公式不进目录
{"path": "keys", "value": ["ticket_no", "nope"]}, # 业务主键里有不存在的列
{"path": "relations.new", "value": {"columns": ["park_id"], "to_table": "ghost_table", "to_columns": ["id"]}},
"不是对象",
], table="visits", tables=tables)
assert plan.changes == []
assert len(plan.problems) == 7
assert any("口径卡" in p for p in plan.problems)
# 报错写界面上的叫法,不裸写键名
assert not any("measure" in p or "meaning" in p for p in plan.problems)
async def test_same_value_states(scenic_db):
tables = (await _schema(scenic_db))["tables"]
proposed = scenic_notes.notes("proposed")["visits"]
plan = catalog.plan_patch(proposed, [{"path": "columns.visitor_count.measure", "value": "flow"}],
table="visits", tables=tables)
# 值没变、只是推断:保存即确认
assert plan.changes[0].state == "confirm"
plan = catalog.plan_patch(_visits(), [{"path": "columns.visitor_count.measure", "value": "flow"}],
table="visits", tables=tables)
assert plan.changes[0].state == "same"
async def test_apply_patch_records_human_confirmed(scenic_db):
tables = (await _schema(scenic_db))["tables"]
proposed = scenic_notes.notes("proposed")["visits"]
out = catalog.apply_patch(proposed, [
{"path": "columns.visitor_count.measure", "value": "stock"},
{"path": "valid_filter", "value": "status = 1"}, # 值没变:只确认,来源不变
], table="visits", tables=tables, at="2026-10-03T00:00:00+00:00")
measure = out["columns"]["visitor_count"]["measure"]
assert (measure["value"], measure["source"], measure["status"]) == ("stock", "human", "confirmed")
assert (out["valid_filter"]["source"], out["valid_filter"]["status"]) == ("llm", "confirmed")
# 其余项原样
assert out["columns"]["status"] == proposed["columns"]["status"]
with pytest.raises(catalog.CatalogInvalid):
catalog.apply_patch(proposed, [{"path": "kind", "value": "nope"}], table="visits", tables=tables)
# --------------------------------------------------------------------------
# 协议
# --------------------------------------------------------------------------
def test_protocol_describes_catalog_patch():
from app.api.copilot import _STREAM_PROTOCOL
assert '"op":"catalog_patch"' in _STREAM_PROTOCOL
assert "relations.new" in _STREAM_PROTOCOL and "columns.<列名>.<字段>" in _STREAM_PROTOCOL
# 两条边界:只在用户明确陈述数据事实时提;计算公式进口径卡
assert "明确" in _STREAM_PROTOCOL and "口径卡" in _STREAM_PROTOCOL
# --------------------------------------------------------------------------
# 流:校验后转出,不写库
# --------------------------------------------------------------------------
class _Scripted:
def __init__(self, ops):
self.ops = ops
self.calls: list[str] = []
async def astream(self, messages):
from langchain_core.messages import AIMessageChunk
self.calls.append(next(text for role, text in messages if role == "system"))
for op in self.ops:
yield AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n")
@pytest.fixture
async def scenic_source(scenic_db):
"""库里一个带目录的景区数据源;用完删掉(级联删掉目录),不留在测试库里。"""
from app.db.base import SessionLocal
from app.db.models import DataSource
name = f"scenic_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
row = DataSource(name=name, kind="sqlite", database=scenic_db, readonly=True)
row.schema_cache = await _schema(scenic_db)
session.add(row)
await session.commit()
for table, notes in scenic_notes.notes().items():
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
source_id = row.id
yield SimpleNamespace(id=source_id, name=name)
async with SessionLocal() as session:
row = await session.get(DataSource, source_id)
if row is not None:
await session.delete(row)
await session.commit()
async def _stream(monkeypatch, model, source, base_graph=None):
import app.api.copilot as copilot
from app.main import app
async def _model(*_a, **_k):
return model, "mock-fast"
monkeypatch.setattr(copilot, "get_chat_model", _model)
events: list[dict] = []
body = {"instruction": "status=9 表示作废,统计时要排除", "intent": "answer", "datasource_ids": [source.id]}
if base_graph:
body["base_graph"] = base_graph
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r:
async for line in r.aiter_lines():
if line.startswith("data: "):
events.append(json.loads(line[6:]))
return events
async def _entry(source_id: str, table: str):
from app.db.base import SessionLocal
async with SessionLocal() as session:
return await catalog.read_entry(session, source_id, table)
async def test_stream_forwards_checked_patch_and_writes_nothing(scenic_source, monkeypatch, caplog):
before = await _entry(scenic_source.id, "visits")
patch = {"op": "catalog_patch", "source": scenic_source.name, "table": "visits", "changes": [
{"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说明 status=9 表示作废"},
{"path": "valid_filter", "value": "status = 1 AND status <> 9", "reason": "统计时排除作废"},
{"path": "columns.status.colour", "value": "红"}, # 不合法:丢弃,其余照转
]}
model = _Scripted([
{"op": "catalog_patch", "source": "不存在的库", "table": "visits",
"changes": [{"path": "kind", "value": "fact"}]}, # 数据源不在范围内:整条丢弃
{"op": "catalog_patch", "source": scenic_source.name, "table": "ghost",
"changes": [{"path": "kind", "value": "fact"}]}, # 表不存在:整条丢弃
{"op": "reply", "text": "好的,已整理成数据目录的修改建议。"},
patch, # 写在 reply 后面:照样转出,排在前面
])
with caplog.at_level(logging.WARNING, logger="app.api.copilot"):
events = await _stream(monkeypatch, model, scenic_source)
ops = [e["op"] for e in events]
assert ops.count("catalog_patch") == 1 and ops[-1] == "reply"
assert ops.index("catalog_patch") < ops.index("reply")
[out] = [e for e in events if e["op"] == "catalog_patch"]
assert out["source"] == scenic_source.name and out["source_id"] == scenic_source.id
assert out["table"] == "visits" and out["table_label"] == "入园记录" and out["version"] == before.version
codes, valid = out["changes"]
assert codes["path"] == "columns.status.codes" and codes["before"] == {"1": "有效", "0": "作废"}
assert codes["after"] == {"1": "有效", "0": "作废", "9": "作废"} and codes["value"] == {"9": "作废"}
assert codes["before_status"] == "confirmed" and codes["reason"] == "用户说明 status=9 表示作废"
assert valid["before"] == "status = 1" and valid["after"] == "status = 1 AND status <> 9"
# 丢掉的记了日志
assert sum("目录修改提案" in r.getMessage() for r in caplog.records) >= 3
# 服务端没写库
after = await _entry(scenic_source.id, "visits")
assert after.version == before.version and after.notes == before.notes
# 协议进了 system
assert '"op":"catalog_patch"' in model.calls[0]
async def test_patch_only_gets_a_reply(scenic_source, monkeypatch):
"""只发了提案、没回答也没改图:补一句回答收尾,不当成「模型没给出任何修改」。"""
model = _Scripted([{"op": "catalog_patch", "source": scenic_source.name, "table": "visits",
"changes": [{"path": "columns.visitor_count.measure", "value": "stock"}]}])
events = await _stream(monkeypatch, model, scenic_source)
assert [e["op"] for e in events if e["op"] not in ("model", "heartbeat", "context")] == ["catalog_patch", "reply"]
assert "数据目录" in events[-1]["text"]
async def test_patch_alongside_graph_ops(scenic_source, monkeypatch):
"""和图操作一起出现:提案照转,图照常收尾。"""
model = _Scripted([
{"op": "plan", "summary": "查入园人数"},
{"op": "catalog_patch", "source": scenic_source.name, "table": "visits",
"changes": [{"path": "columns.visitor_count.measure", "value": "stock"}]},
{"op": "add_node", "node": {"id": "start", "type": "input", "label": "输入", "config": {}}},
{"op": "add_node", "node": {"id": "out", "type": "output", "label": "成果", "config": {"fields": []}}},
{"op": "add_edge", "edge": {"source": "start", "target": "out"}},
{"op": "done", "explanation": "", "run": False},
])
events = await _stream(monkeypatch, model, scenic_source)
ops = [e["op"] for e in events]
assert "catalog_patch" in ops and ops[-1] == "final"
assert ops.index("catalog_patch") < ops.index("final")
# --------------------------------------------------------------------------
# 接口:预览(只读)与保存
# --------------------------------------------------------------------------
async def test_preview_and_save(scenic_source):
from app.main import app
base = f"/api/datasources/{scenic_source.id}/catalog/visits"
changes = [{"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "作废单"}]
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
r = await client.post(f"{base}/patch/preview", json={"changes": changes})
assert r.status_code == 200, r.text
preview = r.json()
version = preview["version"]
assert preview["changes"][0]["after"] == {"1": "有效", "0": "作废", "9": "作废"}
assert preview["problems"] == []
# 预览不写库
assert (await _entry(scenic_source.id, "visits")).version == version
r = await client.post(f"{base}/patch", json={"changes": changes, "if_version": version},
headers={"X-Actor": quote("王敏")})
assert r.status_code == 200, r.text
detail = r.json()
assert detail["version"] == version + 1 and detail["updated_by"] == "王敏"
codes = detail["notes"]["columns"]["status"]["codes"]
assert codes["value"]["9"] == "作废" and (codes["source"], codes["status"]) == ("human", "confirmed")
# 拿旧版本再存:409,什么都不写
r = await client.post(f"{base}/patch", json={"changes": changes, "if_version": version})
assert r.status_code == 409 and "刚被修改过" in r.json()["detail"]
# 重新预览:已经是这个值了
again = (await client.post(f"{base}/patch/preview", json={"changes": changes})).json()
assert again["version"] == version + 1 and again["changes"][0]["state"] == "same"
# 不合法的取值:422,原话说清
r = await client.post(f"{base}/patch", json={"changes": [{"path": "kind", "value": "nope"}],
"if_version": version + 1})
assert r.status_code == 422 and "表类型" in r.json()["detail"]