From 1a760726f7bb04f53c89e7816185bb45a7ddb429 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sat, 3 Oct 2026 11:19:44 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95=E9=98=B6?= =?UTF-8?q?=E6=AE=B5=204A=EF=BC=9A=E5=8A=A9=E6=89=8B=E5=8D=8F=E8=AE=AE?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=9B=AE=E5=BD=95=E4=BF=AE=E6=94=B9=E6=8F=90?= =?UTF-8?q?=E6=A1=88=20catalog=5Fpatch=EF=BC=8C=E6=9C=8D=E5=8A=A1=E7=AB=AF?= =?UTF-8?q?=E6=A0=B8=E5=AF=B9=E6=95=B0=E6=8D=AE=E6=BA=90=E3=80=81=E8=A1=A8?= =?UTF-8?q?=E3=80=81=E8=B7=AF=E5=BE=84=E5=92=8C=E5=8F=96=E5=80=BC=E5=90=8E?= =?UTF-8?q?=E9=99=84=E7=89=88=E6=9C=AC=E4=B8=8E=E6=94=B9=E5=89=8D=E6=94=B9?= =?UTF-8?q?=E5=90=8E=E8=BD=AC=E5=87=BA=E3=80=81=E7=BB=9D=E4=B8=8D=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E5=86=99=E5=85=A5=EF=BC=8C=E6=96=B0=E5=A2=9E=E6=8F=90?= =?UTF-8?q?=E6=A1=88=E9=A2=84=E8=A7=88=E4=B8=8E=E4=BF=9D=E5=AD=98=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/catalog.py | 56 +++++ backend/app/api/copilot.py | 101 ++++++++- backend/app/data/catalog.py | 242 +++++++++++++++++++- backend/tests/test_catalog_patch.py | 327 ++++++++++++++++++++++++++++ 4 files changed, 714 insertions(+), 12 deletions(-) create mode 100644 backend/tests/test_catalog_patch.py diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 3bde3acd..e1c96af0 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -4,6 +4,8 @@ - GET /catalog/{table} 单表目录 + 表结构 - PUT /catalog/{table} 整份提交(乐观锁),改动过的项记为人工确认 - POST /catalog/{table}/review 单项审阅:确认、驳回、撤销审阅 +- POST /catalog/{table}/patch/preview 目录修改提案的预览(只读):对着当前目录算改前、改后 +- POST /catalog/{table}/patch 保存目录修改提案(乐观锁),记为人工确认 - POST /catalog/draft 同步起草(注释、外键、命名推断,可选模型) 规则都在 data/catalog.py,这里只做取源、转换形状和把异常翻成状态码:版本不符 409,结构不合规、审阅路径 @@ -40,6 +42,21 @@ class CatalogReviewIn(BaseModel): if_version: int = Field(ge=0) +class CatalogPatchChange(BaseModel): + #: 审阅接口的路径写法;新增关联关系写 relations.new + path: str = Field(min_length=1, max_length=600) + value: Any + reason: str | None = Field(default=None, max_length=2000) + + +class CatalogPatchPreviewIn(BaseModel): + changes: list[CatalogPatchChange] = Field(min_length=1, max_length=catalog.PATCH_MAX_CHANGES) + + +class CatalogPatchIn(CatalogPatchPreviewIn): + if_version: int = Field(ge=0) + + class CatalogDraftIn(BaseModel): #: 要起草的表;不给或为空时按使用次数取前 20 张 tables: list[str] | None = Field(default=None, max_length=200) @@ -224,3 +241,42 @@ async def review_catalog_item(source_id: str, table: str, payload: CatalogReview except catalog.CatalogPathError as e: raise HTTPException(422, str(e)) from e return await _detail(session, source, key, entry) + + +def _tables_of(source: Any) -> dict[str, Any] | None: + """核对提案用的表结构;没有表结构(尚未探查)时不核对列名。""" + return ((source.schema_cache or {}).get("tables") or None) if source is not None else None + + +@router.post("/{source_id}/catalog/{table}/patch/preview") +async def preview_catalog_patch(source_id: str, table: str, payload: CatalogPatchPreviewIn, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """目录修改提案的预览:对着当前目录逐项算改前、改后,不写库。 + + 助手转出的提案卡片在保存遇到 409(别人刚改过这张表)时用它重新载入:改前的值、码值补充后的样子都按 + 最新的目录重算,人看过之后再带新版本保存。 + """ + _, source = await _resolved(session, source_id) + key, entry = await _table_key(session, source, table) + plan = catalog.plan_patch(entry.notes if entry else {}, [c.model_dump() for c in payload.changes], table=key, + tables=_tables_of(source)) + return {"table_name": key, "version": entry.version if entry else 0, + "changes": [c.as_dict() for c in plan.changes], "problems": plan.problems} + + +@router.post("/{source_id}/catalog/{table}/patch") +async def save_catalog_patch(source_id: str, table: str, payload: CatalogPatchIn, + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """保存一条目录修改提案:改动过的项记为人工填写、已确认。版本不符 409;有一项不合法 422,整条不写。""" + _, source = await _resolved(session, source_id) + key, _ = await _table_key(session, source, table) + try: + entry = await catalog.save_patch(session, source.id, key, [c.model_dump() for c in payload.changes], + if_version=payload.if_version, actor=actor_of(x_actor), + tables=_tables_of(source)) + except catalog.CatalogConflict as e: + raise _conflict(e) from e + except catalog.CatalogInvalid as e: + raise _invalid(e) from e + return await _detail(session, source, key, entry) diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 87782cea..7ef959d0 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import copy import json +import logging import math import re import time @@ -30,6 +31,7 @@ from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_mod from app.tools.registry import all_specs router = APIRouter(prefix="/api/copilot", tags=["copilot"]) +logger = logging.getLogger(__name__) async def _sources(session: AsyncSession, scope: list[str] | None) -> list[Any]: @@ -795,7 +797,22 @@ _STREAM_PROTOCOL = """\ ("设计一个每天跑的工作流")时给 false —— 他要的是这张图,不是这一次的结果 - 如果这句话根本不需要工作流(见下面的路径约定),**第一行也是最后一行**就输出: {"op":"reply","text":"直接回答的内容"} - 这时不要输出任何别的操作""" + 这时除了下面的 catalog_patch,不要输出任何别的操作 +- 数据目录修改提案(可选):用户在这句话里**明确陈述了一条数据事实**时,另输出一行 + {"op":"catalog_patch","source":"数据源名","table":"表名","changes":[{"path":"…","value":…,"reason":"一句话依据"}]} + 它只是提案,用户确认后才写进数据目录。可以和图操作一起出现(写在 done 之前),也可以只配一行 reply(写在 reply 之前) + · path:表级项直接写字段名(label 中文名、description 说明、grain 粒度、keys 业务主键列表、 + kind 表类型 fact/dimension/snapshot/log/config、business_date {"column","rule","timezone"}、 + valid_filter 有效记录的 SQL 条件、dedup 去重规则);列级项写 columns.<列名>.<字段>(label、meaning 含义、 + unit 单位、measure 度量类型 flow/stock/ratio/identifier/status/attribute、codes 码值 {"码":"含义"}); + 已有的关联关系写 relations.<编号>;新增关联关系写 relations.new,value 写 + {"columns":[本表列],"to_table":"目标表","to_columns":[目标列],"cardinality":"many_to_one/one_to_one/one_to_many"} + · codes 只写要补充或更正的码,原有的码会保留;valid_filter 要写成完整的条件(原有条件也要带上) + · 例:「在园人数是存量,不能跨天相加」→ columns.<那一列>.measure 写 stock;「status=9 表示作废,统计时要排除」→ + columns.status.codes 写 {"9":"作废"},再把 valid_filter 补上排除条件;「channel_visits 是渠道和入园记录的关联表」→ + description 和两条 relations.new + · 只有用户明确说出了数据事实才提,不要从自己的推测、查询结果或提问里提;表名、列名必须是上面数据源里真实存在的 + · 计算公式不进数据目录:「转化率=下单数/访问数」这类指标怎么算的说法属于口径卡,不要提""" def _parse_op_line(line: str) -> dict[str, Any] | None: @@ -813,6 +830,56 @@ def _parse_op_line(line: str) -> dict[str, Any] | None: return obj +#: 模型只发了目录修改提案、既没回答也没改图时,补的那句回答 +PATCH_ONLY_REPLY = "已按你的说明整理出数据目录的修改建议,确认无误后可保存到数据目录。" + + +async def catalog_patch_op(op: dict[str, Any], sources: list[Any]) -> dict[str, Any] | None: + """模型发的 catalog_patch → 转给前端的提案;整条不成立时返回 None。**只读,不写目录。** + + 核对数据源(只认这一轮给助手看的源,按名字)、表(按表结构认)、每一项的路径和取值格式(catalog.plan_patch, + 和保存时同一套),附上当前版本、改前和改后的值。不合法的项丢掉、记一条日志,其余照转;一项都不剩就整条不转。 + 写入只发生在人点「保存到数据目录」之后(/catalog/{table}/patch,带着这里给的版本)。 + + 开流之后请求的会话不能再用,这里另开一个短会话读目录:版本取提案到达的这一刻,比开流前读的新。 + """ + from app.data import catalog, table_versions + from app.db.base import SessionLocal + + name = op.get("source") + table = op.get("table") + row = next((s for s in sources if isinstance(name, str) and s.name == name), None) + if row is None: + logger.warning("目录修改提案被丢弃:数据源「%s」不在这一轮的范围内", name) + return None + try: + async with SessionLocal() as session: + source = await table_versions.resolve_source(session, row) + tables = (source.schema_cache or {}).get("tables") or {} + key = catalog.resolve_table_name(source.schema_cache, table) if isinstance(table, str) else None + if key is None: + logger.warning("目录修改提案被丢弃:数据源「%s」中没有表「%s」", name, table) + return None + entry = await catalog.read_entry(session, row.id, key) + except Exception: # noqa: BLE001 - 提案是附加的,读不到目录就不提,不能让这一轮生成失败 + logger.exception("目录修改提案被丢弃:读取「%s」的数据目录失败", name) + return None + plan = catalog.plan_patch(entry.notes if entry else {}, op.get("changes"), table=key, tables=tables) + for problem in plan.problems: + logger.warning("目录修改提案(%s.%s)有一项被丢弃:%s", name, key, problem) + # 已经是这个值、也确认过的项没什么可提的 + changes = [c.as_dict() for c in plan.changes if c.state != "same"] + if not changes: + logger.warning("目录修改提案被丢弃:%s.%s 没有需要修改的项", name, key) + return None + label = (entry.notes.get("label") if entry else None) or {} + return { + "op": "catalog_patch", "source": row.name, "source_id": row.id, "table": key, + "table_label": label.get("value") if label.get("status") != "rejected" else None, + "version": entry.version if entry else 0, "changes": changes, + } + + _NODE_TYPES = frozenset(t.value for t in NodeType) #: 协作成员条目里写 "remove": true 就把这个成员拿掉。不复用 null:null 在合并里 @@ -1760,6 +1827,10 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g messages = [("system", system), ("human", human)] if (shown := context.op()) is not None: yield _sse(shown) + # reply 先压着:模型可能把目录修改提案写在 reply 后面,而前端收到 reply 就收尾(画布那边之后的操作一律不收), + # 提案得排在它前面转出。reply 之后只再认 catalog_patch,其余一概不理 + replied: dict[str, Any] | None = None + patches = 0 try: async for op in _with_heartbeat(_iter_ops(model, messages), started=started): kind = op.get("op") @@ -1767,11 +1838,19 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g if kind in ("thinking", "heartbeat"): yield f"data: {json.dumps(op, ensure_ascii=False)}\n\n" continue + if kind == "catalog_patch": + # 目录修改提案:核对后转给前端做成卡片,人点「保存」才写。不进图,也不算这一轮的图操作 + if (checked := await catalog_patch_op(op, sources)) is not None: + patches += 1 + yield _sse(checked) + continue + if replied is not None: + continue if kind == "reply": - # 这句话不需要工作流。原样转给前端,然后收流——后面那套 + # 这句话不需要工作流:等模型说完(只收它后面的提案)再转给前端、收流——后面那套 # 排版校验对着一张空图跑,只会得到"图是空的,先拖一个节点进来" - yield f"data: {json.dumps(op, ensure_ascii=False)}\n\n" - return + replied = op + continue if kind == "done": explanation = str(op.get("explanation", "")) # 用户要的是流程本身时("设计一个每天跑的工作流"), @@ -1785,6 +1864,14 @@ async def generate_stream(payload: GenerateIn, session: AsyncSession = Depends(g skipped_types.append(str(bad_type)) if changed or kind in ("plan", "done"): yield f"data: {json.dumps(op, ensure_ascii=False)}\n\n" + if replied is not None: + yield _sse(replied) + return + # 只发了目录修改提案,既没回答也没改图(用户只是在说一条数据事实):补一句回答收尾, + # 不能按下面的「模型没有给出任何修改」报错——提案已经摆在用户面前了 + if patches and all(o.get("op") == "plan" for o in seen_ops): + yield _sse({"op": "reply", "text": PATCH_ONLY_REPLY}) + return # 流正常结束,却没有 done、也没有一个操作能认(格式跑偏,每一行都被跳过): # 往下走就是拿原图去自查、交付,界面上写「无需修改」——而模型什么都没说 if all(o.get("op") == "plan" for o in seen_ops): @@ -2012,7 +2099,8 @@ async def assist_publish_fix( try: async for op in _iter_ops(chat, [("system", system), ("human", _publish_fix_request(base, errors, level))]): kind = op.get("op") - if kind in ("thinking", "heartbeat"): + # 目录修改提案只在用户亲口陈述数据事实时才有;这里的「用户消息」是系统拼的修复请求,提了也不转 + if kind in ("thinking", "heartbeat", "catalog_patch"): continue if kind == "plan": plan = str(op.get("summary") or "") @@ -2197,7 +2285,8 @@ async def assist_upgrade( nonlocal plan async for op in _iter_ops(chat, [("system", system), ("human", request)]): kind = op.get("op") - if kind in ("thinking", "heartbeat"): + # 同发布修复:这里没有用户陈述的数据事实,目录修改提案不收 + if kind in ("thinking", "heartbeat", "catalog_patch"): continue if kind in ("plan", "done"): # 修补轮的开场和收尾不是新方案,不覆盖第一轮的说明 diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 25fae279..5af60d7a 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -674,6 +674,235 @@ async def save_human_edit(session: AsyncSession, source_id: str, table: str, sub return await write_entry(session, source_id, table, notes, if_version=current, actor=actor) +# ========================================================================== +# 目录修改提案:对话里的纠正(阶段 4A) +# +# 用户在助手里说出一条数据事实(「status=9 表示作废」),模型发一条 catalog_patch;这里逐项核对、算出改前和 +# 改后,转给前端做成卡片。**提案本身从不落库**:要人点「保存到数据目录」,带着读到的版本走 save_patch, +# 改动记为人工填写、已确认——和人工编辑同一条规矩,只是改动由模型起草。 +# ========================================================================== + +#: 新增关联关系的路径:relations.new,值写两端;编号按两端算(relation_id),同一条关系不论谁提都是同一个编号 +NEW_RELATION_PATH = "relations.new" +#: 一条提案最多几项:用户说的是一两条事实,几十项的「提案」多半是模型在重写整张表 +PATCH_MAX_CHANGES = 20 +_REASON_MAX = 500 +#: 看起来像计算公式的文字:「转化率=下单数/访问数」「= SUM(amount)」。公式进口径卡,目录只记数据事实。 +#: 不认减号:「上线日期=2024-01-01」不是公式。有效记录条件本来就是 SQL 条件,不查它 +_FORMULA = re.compile(r"[==][^=<>!]*?[+*/×÷]|\b(?:sum|count|avg|average)\s*\(", re.IGNORECASE) +_FORMULA_CHECKED = frozenset({("t", "label"), ("t", "description"), ("t", "grain"), ("c", "label"), + ("c", "meaning"), ("c", "unit"), ("c", "codes")}) + + +@dataclass(frozen=True) +class PatchChange: + """提案里的一项,核对过的。 + + - path:规范化后的路径(新增关系写成 relations.<编号>); + - before / before_status:当前目录里这一项的值和状态(被驳回的和没有的都算没有,为 None); + - after:保存后的值(码值是补充后的完整对照);value:要交回保存的原样取值(码值只有补充的那几个, + 409 之后在最新的目录上重新补); + - state:change 值有变化;confirm 值相同、但还不是已确认(保存即确认);same 已经是这个值且已确认。 + """ + + path: str + before: Any + before_status: str | None + after: Any + value: Any + reason: str + state: str + + def as_dict(self) -> dict[str, Any]: + return {"path": self.path, "before": self.before, "before_status": self.before_status, "after": self.after, + "value": self.value, "reason": self.reason, "state": self.state} + + +@dataclass(frozen=True) +class PatchPlan: + """一条提案核对的结果:合法的项(changes)和不合法的项的中文说明(problems)。""" + + changes: list[PatchChange] + problems: list[str] + + +def _column_names(meta: Mapping[str, Any] | None) -> set[str] | None: + """表结构里的列名;不知道(表结构里没有这张表)时为 None,不核对。""" + if not isinstance(meta, Mapping): + return None + return {str(c.get("name")) for c in meta.get("columns") or [] if isinstance(c, Mapping) and c.get("name")} + + +def _formula_like(key: Slot, value: Any) -> bool: + if key[:1] + key[-1:] not in _FORMULA_CHECKED: + return False + texts = list(value.values()) if isinstance(value, dict) else [value] + return any(isinstance(t, str) and _FORMULA.search(t) for t in texts) + + +def _patch_relation(table: str, raw: Any, path_id: str | None, tables: Mapping[str, Any] | None, + columns: set[str] | None) -> tuple[dict[str, Any] | None, str | None]: + """关联关系的取值 → (关系的值部分, 问题)。值写两端(columns、to_table、to_columns),基数可选。""" + if not isinstance(raw, Mapping): + return None, "关联关系的值应写明本表字段、目标表和目标字段" + to_table = raw.get("to_table") + if isinstance(to_table, str) and tables is not None: + to_table = resolve_table_name({"tables": tables}, to_table) + if to_table is None: + return None, f"关联关系的目标表「{raw.get('to_table')}」不在数据源的表结构里" + value = {"columns": raw.get("columns"), "to_table": to_table, "to_columns": raw.get("to_columns"), + "cardinality": raw.get("cardinality"), "coverage": None} + probe = {"id": "probe", **value, "source": "human", "status": "confirmed"} + if problems := _relation_problems(probe, set()): + return None, problems[0].replace("关联关系 probe ", "关联关系") + if columns is not None and (missing := [c for c in value["columns"] if c not in columns]): + return None, f"关联关系的本表字段「{'、'.join(missing)}」不在表结构里" + target = _column_names((tables or {}).get(to_table)) if tables is not None else None + if target is not None and (missing := [c for c in value["to_columns"] if c not in target]): + return None, f"关联关系的目标字段「{'、'.join(missing)}」不在 {to_table} 的表结构里" + rid = relation_id(table, value["columns"], to_table, value["to_columns"]) + if path_id is not None and path_id != rid: + return None, f"关联关系 {path_id} 的两端与取值不一致:要改指向,请驳回原关系后再新增" + return value, None + + +def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, + tables: Mapping[str, Any] | None = None) -> PatchPlan: + """逐项核对一条目录修改提案,算出改前、改后。不改入参、不抛异常:不合法的项写进 problems。 + + changes 是 [{path, value, reason?}]。path 用审阅接口的写法:表级项写字段名,列级项 columns.<列名>.<字段>, + 关系 relations.<编号>;新增关系写 relations.new。tables 是数据源 schema_cache 的 tables:给了就核对列名、 + 业务主键、业务日期列、关系两端在不在表结构里(模型会照着名字编列名);不给不核对。 + + 规则: + - 取值格式和 validate_notes 同一套(_TABLE_VALUE_RULES / _COLUMN_VALUE_RULES / _relation_problems)。 + - 码值是补充:在现有(没被驳回的)码值上加或改给出的码,不整份替换——「status=9 表示作废」不该把已确认的 + 其余码值删掉。 + - 看起来像计算公式的文字不收(_FORMULA):公式进口径卡。 + - 同一个路径出现两次只认第一次。 + """ + meta = (tables or {}).get(table) if tables is not None else None + columns = _column_names(meta) + slots = _slots(notes) + out: list[PatchChange] = [] + problems: list[str] = [] + seen: set[str] = set() + if not isinstance(changes, list) or not changes: + return PatchPlan([], ["提案里没有要修改的项"]) + if len(changes) > PATCH_MAX_CHANGES: + problems.append(f"提案里有 {len(changes)} 项,只核对前 {PATCH_MAX_CHANGES} 项") + changes = changes[:PATCH_MAX_CHANGES] + for raw in changes: + if not isinstance(raw, Mapping) or not isinstance(raw.get("path"), str) or "value" not in raw: + problems.append("提案里有一项不是「路径 + 取值」的形式") + continue + path = raw["path"].strip() + value = copy.deepcopy(raw["value"]) + reason = raw.get("reason") + reason = reason.strip()[:_REASON_MAX] if isinstance(reason, str) else "" + if path == NEW_RELATION_PATH: + rel, problem = _patch_relation(table, value, None, tables, columns) + if problem: + problems.append(problem) + continue + key: Slot = ("r", relation_id(table, rel["columns"], rel["to_table"], rel["to_columns"])) + value = rel + else: + try: + key = parse_path(path) + except CatalogPathError: + problems.append(f"无法识别要修改的项({path[:60]})") + continue + if key[0] == "r": + rel, problem = _patch_relation(table, value, key[1], tables, columns) + if problem: + problems.append(problem) + continue + value = rel + where = describe_slot(key) if key[0] != "r" else "关联关系" + if key[0] == "t": + check, hint = _TABLE_VALUE_RULES[key[1]] + if not check(value): + problems.append(f"{where}{hint}") + continue + if columns is not None: + cited = value if key[1] == "keys" else [value["column"]] if key[1] == "business_date" else [] + if missing := [c for c in cited if c not in columns]: + problems.append(f"{where}里的「{'、'.join(missing)}」不在表结构里") + continue + elif key[0] == "c": + if columns is not None and key[1] not in columns: + problems.append(f"列 {key[1]} 不在表结构里") + continue + check, hint = _COLUMN_VALUE_RULES[key[2]] + if not check(value): + problems.append(f"{where}{hint}") + continue + if _formula_like(key, value): + problems.append(f"{where}看起来是计算公式:公式请写进口径卡,数据目录只记数据事实") + continue + norm = f"relations.{key[1]}" if key[0] == "r" else path + if norm in seen: + problems.append(f"{where}在提案里出现了两次,只采用第一次") + continue + seen.add(norm) + prev = slots.get(key) + live = prev if prev is not None and prev.get("status") != "rejected" else None + before = _value_of(key, live) if live is not None else None + after = value + if key[0] == "c" and key[2] == "codes" and isinstance(before, dict): + after = {**before, **value} + if before is not None and before == after: + state = "same" if live.get("status") == "confirmed" else "confirm" + else: + state = "change" + out.append(PatchChange(path=norm, before=copy.deepcopy(before), + before_status=live.get("status") if live is not None else None, + after=after, value=value, reason=reason, state=state)) + return PatchPlan(out, problems) + + +def apply_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, + tables: Mapping[str, Any] | None = None, at: str | None = None) -> dict[str, Any]: + """把一条提案并进目录,返回新目录(不改入参)。有一项不合法就整条不收,抛 CatalogInvalid。 + + 值有变化的项记为人工填写、已确认(和人工编辑同一条规矩:改动出自人的确认,不是模型起草);值没变的只把 + 状态改成已确认、来源不变(等同单项确认);已经确认过的原样不动。 + """ + plan = plan_patch(notes, changes, table=table, tables=tables) + if plan.problems: + raise CatalogInvalid(plan.problems) + at = at or now_iso() + slots = _slots(notes) + for change in plan.changes: + key = parse_path(change.path) + if change.state == "same": + continue + prev = slots.get(key) + if change.state == "confirm" and prev is not None: + slots[key] = {**prev, "status": "confirmed", "updated_at": at} + continue + if key[0] == "r": + slots[key] = {"id": key[1], **change.after, "source": "human", "status": "confirmed", "updated_at": at} + else: + slots[key] = {"value": change.after, "source": "human", "status": "confirmed", "updated_at": at} + out = _assemble(slots) + if problems := validate_notes(out): + raise CatalogInvalid(problems) + return out + + +async def save_patch(session: AsyncSession, source_id: str, table: str, changes: Any, *, if_version: int, + actor: str | None, tables: Mapping[str, Any] | None = None) -> CatalogEntry: + """保存一条目录修改提案(规则见 apply_patch,乐观锁同 write_entry)。版本不符抛 CatalogConflict。""" + entry = await read_entry(session, source_id, table) + current = entry.version if entry else 0 + if if_version != current: + raise CatalogConflict(table, current) + notes = apply_patch(entry.notes if entry else {}, changes, table=table, tables=tables) + return await write_entry(session, source_id, table, notes, if_version=current, actor=actor) + + # ========================================================================== # 起草:数据库注释、外键约束、命名推断(只看探查缓存,不发数据库查询) # ========================================================================== @@ -1690,11 +1919,12 @@ __all__ = [ "CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict", "CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES", "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", - "MergeStats", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", - "TableDraft", "TableDraftResult", "apply_human_edit", "describe_slot", "draft_catalog", "draft_structure", - "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths", - "make_item", "merge_notes", "now_iso", "parse_path", "query_snapshot_meta", "read_catalog", "read_entry", + "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", + "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult", + "apply_human_edit", "apply_patch", "describe_slot", "draft_catalog", "draft_structure", "draft_with_model", + "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths", "make_item", + "merge_notes", "now_iso", "parse_path", "plan_patch", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", "render_table_notes", "resolve_draft_model", - "resolve_table_name", "review_entry", "review_item", "same_notes", "save_human_edit", "status_counts", - "system_notes_source", "table_usage", "validate_notes", "visible_notes", "write_entry", + "resolve_table_name", "review_entry", "review_item", "same_notes", "save_human_edit", "save_patch", + "status_counts", "system_notes_source", "table_usage", "validate_notes", "visible_notes", "write_entry", ] diff --git a/backend/tests/test_catalog_patch.py b/backend/tests/test_catalog_patch.py new file mode 100644 index 00000000..37de087f --- /dev/null +++ b/backend/tests/test_catalog_patch.py @@ -0,0 +1,327 @@ +"""对话里的纠正变成目录修改提案(阶段 4A)。 + +用户在助手(画布)或问数据页说出一条数据事实(「status=9 表示作废」),模型发一条 catalog_patch 操作;服务端 +核对数据源、表、路径和取值格式,附上当前版本、改前和改后的值转给前端——**服务端绝不自动写入**,要人在卡片上点 +「保存到数据目录」,带着读到的版本走 /catalog/{table}/patch(记为人工填写、已确认)。 + +- 纯函数:plan_patch 逐项核对并算出改前、改后;码值是补充,不整份替换;新增关联关系写 relations.new,编号按两端算; + 计算公式不进目录。 +- 流:合法的转出、带版本和改前值;不合法的丢弃并记日志;写在 reply 后面的提案也排到 reply 前面转出(前端收到 + reply 就收尾);只有提案、没有回答时补一句回答;全程不写库。 +- 接口:预览只读;保存带 if_version,版本不符 409,X-Actor 记成署名。 +""" +from __future__ import annotations + +import json +import logging +import uuid +from types import SimpleNamespace +from urllib.parse import quote + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data import catalog +from app.data.engine import engines +from app.data.introspect import introspect +from tests.fixtures.catalog import scenic_notes + +_CACHE: dict[str, dict] = {} + + +async def _schema(path: str) -> dict: + if path not in _CACHE: + probe = SimpleNamespace(id=f"probe-{uuid.uuid4().hex[:6]}", name="scenic", kind="sqlite", database=path, + host=None, port=None, username=None, password=None, options={}, readonly=True, + description="", schema_cache={}, origin="manual") + try: + _CACHE[path] = await introspect(probe) + finally: + await engines.invalidate(probe.id) + return _CACHE[path] + + +def _visits() -> dict: + return scenic_notes.notes()["visits"] + + +# -------------------------------------------------------------------------- +# plan_patch / apply_patch:纯函数 +# -------------------------------------------------------------------------- + + +async def test_plan_patch_reports_before_and_after(scenic_db): + tables = (await _schema(scenic_db))["tables"] + plan = catalog.plan_patch(_visits(), [ + {"path": "columns.visitor_count.measure", "value": "stock", "reason": "在园人数是存量,不能跨天相加"}, + {"path": "kind", "value": "fact"}, + ], table="visits", tables=tables) + assert plan.problems == [] + measure, kind = plan.changes + assert (measure.path, measure.before, measure.before_status, measure.after) == ( + "columns.visitor_count.measure", "flow", "confirmed", "stock") + assert measure.state == "change" and measure.reason == "在园人数是存量,不能跨天相加" + assert (kind.path, kind.before, kind.before_status, kind.after, kind.state) == ("kind", None, None, "fact", "change") + + +async def test_codes_are_supplemented_not_replaced(scenic_db): + tables = (await _schema(scenic_db))["tables"] + plan = catalog.plan_patch(_visits(), [{"path": "columns.status.codes", "value": {"9": "作废"}}], + table="visits", tables=tables) + [change] = plan.changes + assert change.before == {"1": "有效", "0": "作废"} + assert change.after == {"1": "有效", "0": "作废", "9": "作废"} + # 交回保存的是原样的补充值:409 之后在最新的目录上重新补 + assert change.value == {"9": "作废"} + + +async def test_new_relation_gets_its_id_from_both_ends(scenic_db): + tables = (await _schema(scenic_db))["tables"] + value = {"columns": ["visit_id"], "to_table": "visits", "to_columns": ["id"], "cardinality": "many_to_one"} + plan = catalog.plan_patch({}, [{"path": "relations.new", "value": value}], table="channel_visits", tables=tables) + [change] = plan.changes + rid = catalog.relation_id("channel_visits", ["visit_id"], "visits", ["id"]) + assert change.path == f"relations.{rid}" and change.before is None + assert change.after == {**value, "coverage": None} + # 已有编号、两端对得上:改基数 + notes = catalog.apply_patch({}, [{"path": "relations.new", "value": value}], table="channel_visits", tables=tables) + again = catalog.plan_patch(notes, [{"path": f"relations.{rid}", "value": {**value, "cardinality": "one_to_one"}}], + table="channel_visits", tables=tables) + assert again.changes[0].before["cardinality"] == "many_to_one" and again.problems == [] + # 编号对不上两端:不认(改指向是另一条关系) + moved = catalog.plan_patch(notes, [{"path": f"relations.{rid}", + "value": {**value, "to_table": "channels", "to_columns": ["id"]}}], + table="channel_visits", tables=tables) + assert moved.changes == [] and len(moved.problems) == 1 + + +async def test_invalid_changes_are_reported(scenic_db): + tables = (await _schema(scenic_db))["tables"] + plan = catalog.plan_patch(_visits(), [ + {"path": "formula", "value": "下单数 / 访问数"}, # 不是目录的项 + {"path": "columns.visitor_count.measure", "value": "累计值"}, # 不在可选值内 + {"path": "columns.ghost.meaning", "value": "不存在的列"}, # 表结构里没有这一列 + {"path": "columns.visitor_count.meaning", "value": "转化率=下单数/访问数"}, # 计算公式不进目录 + {"path": "keys", "value": ["ticket_no", "nope"]}, # 业务主键里有不存在的列 + {"path": "relations.new", "value": {"columns": ["park_id"], "to_table": "ghost_table", "to_columns": ["id"]}}, + "不是对象", + ], table="visits", tables=tables) + assert plan.changes == [] + assert len(plan.problems) == 7 + assert any("口径卡" in p for p in plan.problems) + # 报错写界面上的叫法,不裸写键名 + assert not any("measure" in p or "meaning" in p for p in plan.problems) + + +async def test_same_value_states(scenic_db): + tables = (await _schema(scenic_db))["tables"] + proposed = scenic_notes.notes("proposed")["visits"] + plan = catalog.plan_patch(proposed, [{"path": "columns.visitor_count.measure", "value": "flow"}], + table="visits", tables=tables) + # 值没变、只是推断:保存即确认 + assert plan.changes[0].state == "confirm" + plan = catalog.plan_patch(_visits(), [{"path": "columns.visitor_count.measure", "value": "flow"}], + table="visits", tables=tables) + assert plan.changes[0].state == "same" + + +async def test_apply_patch_records_human_confirmed(scenic_db): + tables = (await _schema(scenic_db))["tables"] + proposed = scenic_notes.notes("proposed")["visits"] + out = catalog.apply_patch(proposed, [ + {"path": "columns.visitor_count.measure", "value": "stock"}, + {"path": "valid_filter", "value": "status = 1"}, # 值没变:只确认,来源不变 + ], table="visits", tables=tables, at="2026-10-03T00:00:00+00:00") + measure = out["columns"]["visitor_count"]["measure"] + assert (measure["value"], measure["source"], measure["status"]) == ("stock", "human", "confirmed") + assert (out["valid_filter"]["source"], out["valid_filter"]["status"]) == ("llm", "confirmed") + # 其余项原样 + assert out["columns"]["status"] == proposed["columns"]["status"] + with pytest.raises(catalog.CatalogInvalid): + catalog.apply_patch(proposed, [{"path": "kind", "value": "nope"}], table="visits", tables=tables) + + +# -------------------------------------------------------------------------- +# 协议 +# -------------------------------------------------------------------------- + + +def test_protocol_describes_catalog_patch(): + from app.api.copilot import _STREAM_PROTOCOL + + assert '"op":"catalog_patch"' in _STREAM_PROTOCOL + assert "relations.new" in _STREAM_PROTOCOL and "columns.<列名>.<字段>" in _STREAM_PROTOCOL + # 两条边界:只在用户明确陈述数据事实时提;计算公式进口径卡 + assert "明确" in _STREAM_PROTOCOL and "口径卡" in _STREAM_PROTOCOL + + +# -------------------------------------------------------------------------- +# 流:校验后转出,不写库 +# -------------------------------------------------------------------------- + + +class _Scripted: + def __init__(self, ops): + self.ops = ops + self.calls: list[str] = [] + + async def astream(self, messages): + from langchain_core.messages import AIMessageChunk + + self.calls.append(next(text for role, text in messages if role == "system")) + for op in self.ops: + yield AIMessageChunk(content=json.dumps(op, ensure_ascii=False) + "\n") + + +@pytest.fixture +async def scenic_source(scenic_db): + """库里一个带目录的景区数据源;用完删掉(级联删掉目录),不留在测试库里。""" + from app.db.base import SessionLocal + from app.db.models import DataSource + + name = f"scenic_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=scenic_db, readonly=True) + row.schema_cache = await _schema(scenic_db) + session.add(row) + await session.commit() + for table, notes in scenic_notes.notes().items(): + await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏") + source_id = row.id + yield SimpleNamespace(id=source_id, name=name) + async with SessionLocal() as session: + row = await session.get(DataSource, source_id) + if row is not None: + await session.delete(row) + await session.commit() + + +async def _stream(monkeypatch, model, source, base_graph=None): + import app.api.copilot as copilot + from app.main import app + + async def _model(*_a, **_k): + return model, "mock-fast" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + events: list[dict] = [] + body = {"instruction": "status=9 表示作废,统计时要排除", "intent": "answer", "datasource_ids": [source.id]} + if base_graph: + body["base_graph"] = base_graph + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + async with client.stream("POST", "/api/copilot/generate-stream", json=body) as r: + async for line in r.aiter_lines(): + if line.startswith("data: "): + events.append(json.loads(line[6:])) + return events + + +async def _entry(source_id: str, table: str): + from app.db.base import SessionLocal + + async with SessionLocal() as session: + return await catalog.read_entry(session, source_id, table) + + +async def test_stream_forwards_checked_patch_and_writes_nothing(scenic_source, monkeypatch, caplog): + before = await _entry(scenic_source.id, "visits") + patch = {"op": "catalog_patch", "source": scenic_source.name, "table": "visits", "changes": [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说明 status=9 表示作废"}, + {"path": "valid_filter", "value": "status = 1 AND status <> 9", "reason": "统计时排除作废"}, + {"path": "columns.status.colour", "value": "红"}, # 不合法:丢弃,其余照转 + ]} + model = _Scripted([ + {"op": "catalog_patch", "source": "不存在的库", "table": "visits", + "changes": [{"path": "kind", "value": "fact"}]}, # 数据源不在范围内:整条丢弃 + {"op": "catalog_patch", "source": scenic_source.name, "table": "ghost", + "changes": [{"path": "kind", "value": "fact"}]}, # 表不存在:整条丢弃 + {"op": "reply", "text": "好的,已整理成数据目录的修改建议。"}, + patch, # 写在 reply 后面:照样转出,排在前面 + ]) + with caplog.at_level(logging.WARNING, logger="app.api.copilot"): + events = await _stream(monkeypatch, model, scenic_source) + ops = [e["op"] for e in events] + assert ops.count("catalog_patch") == 1 and ops[-1] == "reply" + assert ops.index("catalog_patch") < ops.index("reply") + [out] = [e for e in events if e["op"] == "catalog_patch"] + assert out["source"] == scenic_source.name and out["source_id"] == scenic_source.id + assert out["table"] == "visits" and out["table_label"] == "入园记录" and out["version"] == before.version + codes, valid = out["changes"] + assert codes["path"] == "columns.status.codes" and codes["before"] == {"1": "有效", "0": "作废"} + assert codes["after"] == {"1": "有效", "0": "作废", "9": "作废"} and codes["value"] == {"9": "作废"} + assert codes["before_status"] == "confirmed" and codes["reason"] == "用户说明 status=9 表示作废" + assert valid["before"] == "status = 1" and valid["after"] == "status = 1 AND status <> 9" + # 丢掉的记了日志 + assert sum("目录修改提案" in r.getMessage() for r in caplog.records) >= 3 + # 服务端没写库 + after = await _entry(scenic_source.id, "visits") + assert after.version == before.version and after.notes == before.notes + # 协议进了 system + assert '"op":"catalog_patch"' in model.calls[0] + + +async def test_patch_only_gets_a_reply(scenic_source, monkeypatch): + """只发了提案、没回答也没改图:补一句回答收尾,不当成「模型没给出任何修改」。""" + model = _Scripted([{"op": "catalog_patch", "source": scenic_source.name, "table": "visits", + "changes": [{"path": "columns.visitor_count.measure", "value": "stock"}]}]) + events = await _stream(monkeypatch, model, scenic_source) + assert [e["op"] for e in events if e["op"] not in ("model", "heartbeat", "context")] == ["catalog_patch", "reply"] + assert "数据目录" in events[-1]["text"] + + +async def test_patch_alongside_graph_ops(scenic_source, monkeypatch): + """和图操作一起出现:提案照转,图照常收尾。""" + model = _Scripted([ + {"op": "plan", "summary": "查入园人数"}, + {"op": "catalog_patch", "source": scenic_source.name, "table": "visits", + "changes": [{"path": "columns.visitor_count.measure", "value": "stock"}]}, + {"op": "add_node", "node": {"id": "start", "type": "input", "label": "输入", "config": {}}}, + {"op": "add_node", "node": {"id": "out", "type": "output", "label": "成果", "config": {"fields": []}}}, + {"op": "add_edge", "edge": {"source": "start", "target": "out"}}, + {"op": "done", "explanation": "", "run": False}, + ]) + events = await _stream(monkeypatch, model, scenic_source) + ops = [e["op"] for e in events] + assert "catalog_patch" in ops and ops[-1] == "final" + assert ops.index("catalog_patch") < ops.index("final") + + +# -------------------------------------------------------------------------- +# 接口:预览(只读)与保存 +# -------------------------------------------------------------------------- + + +async def test_preview_and_save(scenic_source): + from app.main import app + + base = f"/api/datasources/{scenic_source.id}/catalog/visits" + changes = [{"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "作废单"}] + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.post(f"{base}/patch/preview", json={"changes": changes}) + assert r.status_code == 200, r.text + preview = r.json() + version = preview["version"] + assert preview["changes"][0]["after"] == {"1": "有效", "0": "作废", "9": "作废"} + assert preview["problems"] == [] + # 预览不写库 + assert (await _entry(scenic_source.id, "visits")).version == version + + r = await client.post(f"{base}/patch", json={"changes": changes, "if_version": version}, + headers={"X-Actor": quote("王敏")}) + assert r.status_code == 200, r.text + detail = r.json() + assert detail["version"] == version + 1 and detail["updated_by"] == "王敏" + codes = detail["notes"]["columns"]["status"]["codes"] + assert codes["value"]["9"] == "作废" and (codes["source"], codes["status"]) == ("human", "confirmed") + + # 拿旧版本再存:409,什么都不写 + r = await client.post(f"{base}/patch", json={"changes": changes, "if_version": version}) + assert r.status_code == 409 and "刚被修改过" in r.json()["detail"] + # 重新预览:已经是这个值了 + again = (await client.post(f"{base}/patch/preview", json={"changes": changes})).json() + assert again["version"] == version + 1 and again["changes"][0]["state"] == "same" + + # 不合法的取值:422,原话说清 + r = await client.post(f"{base}/patch", json={"changes": [{"path": "kind", "value": "nope"}], + "if_version": version + 1}) + assert r.status_code == 422 and "表类型" in r.json()["detail"]