From adc56da6cffff274a553f1224e694fa655930c72 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 07:59:51 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E5=AE=A1=E9=98=85=E9=98=9F=E5=88=97=EF=BC=9A=E6=8E=A8=E6=96=AD?= =?UTF-8?q?=E7=9A=84=E9=A1=B9=E6=8C=89=E3=80=8C=E4=BD=BF=E7=94=A8=E6=AC=A1?= =?UTF-8?q?=E6=95=B0=E6=9D=83=E9=87=8D=20=C3=97=20=E9=A3=8E=E9=99=A9?= =?UTF-8?q?=E6=9D=83=E9=87=8D=20=C3=97=20=E6=9D=A5=E6=BA=90=E6=9D=83?= =?UTF-8?q?=E9=87=8D=E3=80=8D=E6=8E=92=E5=A5=BD=EF=BC=88=E6=9C=89=E6=95=88?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=9D=A1=E4=BB=B6=E3=80=81=E5=8E=BB=E9=87=8D?= =?UTF-8?q?=E8=A7=84=E5=88=99=E3=80=81=E4=B8=9A=E5=8A=A1=E6=97=A5=E6=9C=9F?= =?UTF-8?q?=E3=80=81=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB=E6=9C=80=E9=AB=98?= =?UTF-8?q?=EF=BC=8C=E6=A8=A1=E5=9E=8B=E8=B5=B7=E8=8D=89=E3=80=81=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E6=8E=A8=E6=96=AD=E6=AF=94=E5=BA=93=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=E4=BC=98=E5=85=88=EF=BC=89=EF=BC=8C=E5=B8=A6=E6=8E=92=E5=9C=A8?= =?UTF-8?q?=E8=BF=99=E9=87=8C=E7=9A=84=E7=90=86=E7=94=B1=EF=BC=8C=E5=8F=AF?= =?UTF-8?q?=E6=8C=89=E5=BD=B1=E5=93=8D=E6=95=B0=E5=AD=97=E7=9A=84=E9=A1=B9?= =?UTF-8?q?=20/=20=E5=90=8D=E7=A7=B0=E5=92=8C=E8=AF=B4=E6=98=8E=E3=80=81?= =?UTF-8?q?=E6=8C=89=E8=A1=A8=E7=AD=9B=E9=80=89=E5=92=8C=E5=88=86=E9=A1=B5?= =?UTF-8?q?=EF=BC=9B=E6=89=B9=E9=87=8F=E7=A1=AE=E8=AE=A4=E3=80=81=E9=A9=B3?= =?UTF-8?q?=E5=9B=9E=E3=80=81=E6=92=A4=E9=94=80=E5=AE=A1=E9=98=85=E9=80=90?= =?UTF-8?q?=E8=A1=A8=E6=8C=89=E7=89=88=E6=9C=AC=E4=B9=90=E8=A7=82=E9=94=81?= =?UTF-8?q?=EF=BC=8C=E5=90=8C=E4=B8=80=E5=BC=A0=E8=A1=A8=E7=9A=84=E5=87=A0?= =?UTF-8?q?=E6=9D=A1=E4=B8=80=E6=AC=A1=E5=86=99=E5=85=A5=EF=BC=8C=E4=B8=80?= =?UTF-8?q?=E5=BC=A0=E8=A1=A8=E5=86=B2=E7=AA=81=E4=B8=8D=E5=BD=B1=E5=93=8D?= =?UTF-8?q?=E5=88=AB=E7=9A=84=E8=A1=A8=EF=BC=9B=E8=A1=A8=E8=A2=AB=E5=93=AA?= =?UTF-8?q?=E4=BA=9B=E8=BF=90=E8=A1=8C=E7=94=A8=E8=BF=87=EF=BC=88=E6=96=B0?= =?UTF-8?q?=E7=9A=84=E5=9C=A8=E5=89=8D=EF=BC=8C=E5=8F=96=E6=B3=95=E5=90=8C?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0=EF=BC=8C=E6=8A=BD=E5=87=BA?= =?UTF-8?q?=20query=5Fsnapshot=5Ftables=20=E5=85=B1=E7=94=A8=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E5=AE=A1=E9=98=85=E8=BF=9B=E5=BA=A6=EF=BC=88=E5=B8=B8?= =?UTF-8?q?=E7=94=A8=2020=20=E5=BC=A0=E8=A1=A8=E3=80=81=E5=85=A8=E9=83=A8?= =?UTF-8?q?=E3=80=81=E5=BD=B1=E5=93=8D=E6=95=B0=E5=AD=97=E7=9A=84=E9=A1=B9?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/catalog.py | 62 +++- backend/app/data/catalog.py | 42 ++- backend/app/data/catalog_review.py | 404 +++++++++++++++++++++ backend/tests/test_catalog_review_queue.py | 357 ++++++++++++++++++ 4 files changed, 850 insertions(+), 15 deletions(-) create mode 100644 backend/app/data/catalog_review.py create mode 100644 backend/tests/test_catalog_review_queue.py diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 0405a0f7..7680eb54 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -13,6 +13,9 @@ - POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py） - POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py） +- GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅， + GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行 + （data/catalog_review.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 @@ -24,7 +27,7 @@ from __future__ import annotations from typing import Any, Literal -from fastapi import APIRouter, Body, Depends, Header, HTTPException +from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query from fastapi.exceptions import RequestValidationError from pydantic import BaseModel, Field, ValidationError from sqlalchemy.ext.asyncio import AsyncSession @@ -35,6 +38,7 @@ from app.api.coded import ( ) from app.api.runs import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions +from app.data import catalog_review from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -234,6 +238,30 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi } +# ---- 审阅队列、审阅进度（要登记在 GET /catalog/{table} 前面：不然 review-queue、review-progress 会被当成表名） + + +@router.get("/{source_id}/catalog/review-queue") +async def catalog_review_queue(source_id: str, + limit: int = Query(50, ge=1, le=catalog_review.QUEUE_LIMIT_MAX), + offset: int = Query(0, ge=0), + kind: Literal["all", "facts", "names"] = Query("all"), + table: str | None = Query(None, max_length=255), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """审阅队列：只列推断的项，按使用次数、风险、来源排好（data/catalog_review.py）。kind 筛影响数字的项（facts）或名称 + 和说明（names），table 只看一张表。""" + _, source = await _resolved(session, source_id) + return await catalog_review.review_queue(session, source, kind=kind, table=table or None, limit=limit, + offset=offset) + + +@router.get("/{source_id}/catalog/review-progress") +async def catalog_review_progress(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """审阅进度：常用 20 张表、全部、影响数字的项各自确认了多少。""" + _, source = await _resolved(session, source_id) + return await catalog_review.review_progress(session, source) + + @router.get("/{source_id}/catalog/{table}") async def get_catalog_table(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: @@ -527,6 +555,38 @@ async def review_catalog_item(source_id: str, table: str, payload: CatalogReview return await _detail(session, source, key, entry) +class CatalogBatchReviewItem(BaseModel): + table_name: str = Field(min_length=1, max_length=255) + path: str = Field(min_length=1, max_length=600) + action: Literal["confirm", "reject", "reset"] + if_version: int = Field(ge=0) + + +class CatalogBatchReviewIn(BaseModel): + items: list[CatalogBatchReviewItem] = Field(min_length=1, max_length=catalog_review.BATCH_MAX) + + +@router.post("/{source_id}/catalog/review/batch") +async def review_catalog_batch(source_id: str, payload: CatalogBatchReviewIn, + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """批量审阅：逐条结果和请求一一对应。逐表按 if_version 乐观锁，同一张表的几条一次写入；一张表冲突、路径不对、表不存在 + 只记在那几条上（ok 为假，code 是 catalog_conflict / catalog_path_invalid / table_missing），不影响别的。""" + _, source = await _resolved(session, source_id) + items = [catalog_review.BatchItem(i.table_name, i.path, i.action, i.if_version) for i in payload.items] + return {"results": await catalog_review.review_batch(session, source, items, actor=actor_of(x_actor))} + + +@router.get("/{source_id}/catalog/{table}/usage") +async def catalog_table_runs(source_id: str, table: str, + limit: int = Query(20, ge=1, le=catalog_review.RUNS_LIMIT_MAX), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """用过这张表的运行（total 是几次运行），新的在前；取法同使用次数（catalog.table_usage）。""" + _, source = await _resolved(session, source_id) + key, _ = await _table_key(session, source, table) + return await catalog_review.table_runs(session, source, key, limit=limit) + + def _tables_of(source: Any) -> dict[str, Any] | None: """核对提案用的表结构；没有表结构（尚未探查）时不核对列名。""" return ((source.schema_cache or {}).get("tables") or None) if source is not None else None diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 3dd0bb39..cdef28cf 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -2249,27 +2249,27 @@ def query_snapshot_meta(source: Any, sql: str) -> dict[str, Any]: return {"source_id": getattr(source, "id", None), "source": getattr(source, "name", None), "tables": tables} -async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: - """每张表被运行查询过的次数：{schema_cache 的表名: 次数}，没被查过的表不出现。 +async def query_snapshot_tables(session: AsyncSession, source: Any) -> list[tuple[str, frozenset[str]]]: + """这个源上属于运行的每一行查询快照引用：(运行 id, 这条查询用到的表——schema_cache 的表名)，没对上任何表的不收。 - 取法：数据源查询工具每次成功查询都存一份查询快照工件（tools/datasource.py），工件引用表 artifacts 里 - 按 (内容哈希, 运行) 各记一行。数「属于某次运行（runs 里有这次运行）、属于这个源、SQL 里用到了这张表」的 - 行：同一次运行里结果一模一样的重复查询只算一次，工具库里的试用（不属于任何运行）不算。 + 使用次数（table_usage）和「这张表被哪些运行用过」（data/catalog_review.py）共用这一份取法：数据源查询工具每次成功 + 查询都存一份查询快照工件（tools/datasource.py），工件引用表 artifacts 里按 (内容哈希, 运行) 各记一行；只收属于某次运行 + （runs 里有这次运行）、属于这个源的行，工具库里的试用（不属于任何运行）不算。 - 表名在查询当时就记进了引用的 meta（query_snapshot_meta），这里只读 artifacts 一张表；升级前的老引用 - 没有 meta，退回读工件内容里的源名和 SQL，读过的按工件 id 缓存（内容永不改变）。SQL 里写的表名去掉 - schema 前缀后按 name_key（只把 ASCII 字母转小写，和 SQLite 比较标识符的口径一致）对到表结构上，对不上的不计。 + 表名在查询当时就记进了引用的 meta（query_snapshot_meta），这里只读 artifacts 一张表；升级前的老引用没有 meta，退回读 + 工件内容里的源名和 SQL，读过的按工件 id 缓存（内容永不改变）。SQL 里写的表名去掉 schema 前缀后按 name_key（只把 + ASCII 字母转小写，和 SQLite 比较标识符的口径一致）对到表结构上，对不上的不计。 """ rows = (await session.execute( - select(Artifact.id, Artifact.meta).join(Run, Run.id == Artifact.run_id) + select(Artifact.id, Artifact.meta, Artifact.run_id).join(Run, Run.id == Artifact.run_id) .where(Artifact.kind == QUERY_SNAPSHOT_KIND) )).all() tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} lookup: dict[str, str] = {} for name in tables: lookup.setdefault(name_key(name), name) - counts: Counter[str] = Counter() - for artifact_id, meta in rows: + out: list[tuple[str, frozenset[str]]] = [] + for artifact_id, meta, run_id in rows: meta = meta if isinstance(meta, dict) else {} if isinstance(meta.get("tables"), list): source_id, source_name, names = meta.get("source_id"), meta.get("source"), meta["tables"] @@ -2281,8 +2281,21 @@ async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: continue elif source_name != source.name: continue - hit = {lookup.get(name_key(str(n).rsplit(".", 1)[-1])) for n in names} - counts.update(t for t in hit if t is not None) + hit = frozenset(t for t in (lookup.get(name_key(str(n).rsplit(".", 1)[-1])) for n in names) if t is not None) + if hit: + out.append((str(run_id), hit)) + return out + + +async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: + """每张表被运行查询过的次数：{schema_cache 的表名: 次数}，没被查过的表不出现。 + + 按查询快照引用的行数（query_snapshot_tables）：同一次运行里结果一模一样的重复查询只算一次，同一次运行里不同的查询 + 各算一次，工具库里的试用不算。 + """ + counts: Counter[str] = Counter() + for _run, hit in await query_snapshot_tables(session, source): + counts.update(hit) return dict(counts) @@ -2337,7 +2350,8 @@ __all__ = [ "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "is_profile_note", "join_paths", "make_item", "merge_notes", "name_words", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", - "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", + "query_snapshot_meta", "query_snapshot_tables", "read_catalog", "read_entry", "relation_graph", "relation_id", + "render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes", "same_profile_note", "save_human_edit", "save_patch", "status_counts", "structure_relation_ids", "system_notes_source", "table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry", diff --git a/backend/app/data/catalog_review.py b/backend/app/data/catalog_review.py new file mode 100644 index 00000000..b03c9e31 --- /dev/null +++ b/backend/app/data/catalog_review.py @@ -0,0 +1,404 @@ +"""数据目录的审阅队列、批量审阅、表被哪些运行用过、审阅进度。 + +**为什么要有它。** 起草（数据库注释、外键、命名推断、数据剖析、模型）一次写出成百上千项，绝大多数停在「推断」：SQL 检查的 +错误级规则只认已确认、有确证的项，推断的项只作提示。逐表点开确认太慢，也看不出先确认哪几项。队列把推断的项摊平，按 +「这张表常不常用 × 这一项错了影响多大 × 这一项从哪来」排好，最该先看的在前面；勾几项一次确认、驳回或撤销审阅。 + +- **队列**（review_queue，GET /catalog/review-queue）：只列推断（proposed）的项。priority = 使用次数权重 × 风险权重 × + 来源权重，从高到低；同分按表名、路径排，结果稳定。 + - 使用次数权重：1 + log2(1 + 使用次数)。使用次数同目录清单（catalog.table_usage）：按对数增长，常用表在前，但不至于 + 让常用表上的中文名压过所有表上影响数字的项。 + - 风险权重（RISK_WEIGHT）：有效记录条件、去重规则、业务日期、关联关系最高（直接改变数字），业务主键、度量类型次之 + （计数和能不能加总），码值、粒度、表类型再次，中文名、说明、含义、单位最低。 + - 来源权重（SOURCE_WEIGHT）：模型起草、命名推断是猜的，比数据库注释优先；数据剖析（码值候选、业务日期、没核实的关系） + 居中。 + - 两组（kind）：facts 影响数字的项（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）；names 名称和说明 + （中文名、说明、含义、单位、码值、粒度、表类型）。表的「说明」和中文名、含义一样只是说明文字，归 names。 + - 表结构里已经没有的表、列上的项不列（确认了也没有用，目录页上另有清理的提示）。 + - reasons 用中文说为什么排在这里：常用表（使用 N 次）、影响数字的项（有效记录条件）、模型起草，未确认…… +- **批量审阅**（review_batch，POST /catalog/review/batch）：逐表按 if_version 乐观锁，同一张表的几条在一次写入里完成， + 一张表冲突、写不进去不影响别的表；路径不对、表不存在逐条报（results 和请求一一对应）。审阅的规则同单项审阅 + （catalog.review_item）。 +- **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：取法同使用次数（catalog.query_snapshot_tables），按运行 + 去重，新的在前。 +- **进度**（review_progress，GET /catalog/review-progress）：使用次数前 TOP_N 张表（只算被运行用到过的）、全部、影响 + 数字的项各自的确认情况。项的范围和队列一样：表结构里没有的表、列不算。 +""" +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog +from app.db.models import Run + +#: 队列的筛选：全部、影响数字的项、名称和说明 +KINDS = ("all", "facts", "names") +#: 队列一页最多几项；批量审阅一次最多几条；「被哪些运行用过」一次最多几次运行 +QUEUE_LIMIT_MAX = 200 +BATCH_MAX = 200 +RUNS_LIMIT_MAX = 100 +#: 进度里的「常用表」取使用次数前几张 +TOP_N = 20 + +#: 每一项属于哪一组、风险几档：键是 (t, 字段) / (c, 字段) / (r, "")，档 4 最高 +FIELD_GROUP: dict[tuple[str, str], tuple[str, int]] = { + ("t", "valid_filter"): ("facts", 4), ("t", "dedup"): ("facts", 4), ("t", "business_date"): ("facts", 4), + ("r", ""): ("facts", 4), + ("t", "keys"): ("facts", 3), ("c", "measure"): ("facts", 3), + ("c", "codes"): ("names", 2), ("t", "grain"): ("names", 2), ("t", "kind"): ("names", 2), + ("t", "label"): ("names", 1), ("t", "description"): ("names", 1), ("c", "label"): ("names", 1), + ("c", "meaning"): ("names", 1), ("c", "unit"): ("names", 1), +} +RISK_WEIGHT = {4: 4.0, 3: 3.0, 2: 2.0, 1: 1.0} +#: 来源权重：没写的（外键约束、人工填写，它们本来就不是推断）按 1 +SOURCE_WEIGHT = {"llm": 1.5, "name": 1.5, "profile": 1.25, "comment": 1.0} + +#: 来源的界面叫法（和 frontend/src/lib/terms.ts 的 CATALOG_SOURCE_LABEL 一致，有测试核对） +SOURCE_LABEL = {"comment": "数据库注释", "fk": "外键约束", "name": "命名推断", "profile": "数据剖析", "llm": "模型起草", + "human": "人工填写"} +RELATION_LABEL = "关联关系" +#: reasons 的说法 +REASON_TOP = "常用表（使用 {n} 次）" +REASON_USED = "使用 {n} 次" +REASON_RISK = {4: "影响数字的项（{label}）", 3: "影响计数和汇总的项（{label}）", 2: "帮助理解取值的项（{label}）", + 1: "名称和说明（{label}）"} +REASON_PENDING_CODES = "{n} 个码值的含义待填写" +REASON_SOURCE = "{source}，未确认" +#: 码值含义为空（数据剖析写进来的候选）时的说法，同界面的 CODES_TEXT.pending +CODE_PENDING = "含义待填写" + +#: 批量审阅逐条结果的机读码 +CODE_CONFLICT = "catalog_conflict" +CODE_PATH = "catalog_path_invalid" +CODE_INVALID = "catalog_invalid" +CODE_TABLE_MISSING = "table_missing" + +Slot = catalog.Slot + + +# ========================================================================== +# 一项的路径、叫法、显示 +# ========================================================================== + + +def slot_path(key: Slot) -> str: + """槽位 → 审阅接口的路径写法（catalog.parse_path 的反过来）。""" + if key[0] == "t": + return key[1] + if key[0] == "c": + return f"columns.{key[1]}.{key[2]}" + return f"relations.{key[1]}" + + +def field_of(key: Slot) -> tuple[str, str]: + """槽位 → FIELD_GROUP 的键。""" + if key[0] == "r": + return ("r", "") + return (key[0], key[-1]) + + +def field_name(key: Slot) -> str: + """这一项的字段名：表级、列级的项是字段名，关联关系是 relation。""" + return "relation" if key[0] == "r" else key[-1] + + +def field_label(key: Slot) -> str: + """字段的界面叫法：中文名、码值、有效记录条件、关联关系……""" + if key[0] == "t": + return catalog.TABLE_FIELD_LABEL.get(key[1], key[1]) + if key[0] == "c": + return catalog.COLUMN_FIELD_LABEL.get(key[2], key[2]) + return RELATION_LABEL + + +def item_value(key: Slot, item: Mapping[str, Any]) -> Any: + """一项的值：关联关系是两端、基数和覆盖率，别的是 value。""" + if key[0] == "r": + return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")} + return item.get("value") + + +def _percent(ratio: float) -> str: + """覆盖率写成百分数：整数不带小数，其余留一位；不到 100% 的不写成 100%。""" + text = f"{ratio * 100:.1f}".rstrip("0").rstrip(".") + return "99.9%" if text == "100" and ratio < 1 else f"{text}%" + + +def relation_text(rel: Mapping[str, Any]) -> str: + """关联关系写成一行：member_id → members.id（多对一，覆盖率 98%）。""" + left = "、".join(str(c) for c in rel.get("columns") or []) + right = "、".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or []) + extra = [] + if rel.get("cardinality") in catalog.CARDINALITY_LABEL: + extra.append(catalog.CARDINALITY_LABEL[rel["cardinality"]]) + coverage = rel.get("coverage") + if isinstance(coverage, (int, float)) and not isinstance(coverage, bool): + extra.append(f"覆盖率 {_percent(float(coverage))}") + return f"{left} → {right}" + (f"（{'，'.join(extra)}）" if extra else "") + + +def display(key: Slot, value: Any) -> str: + """一项的值写成一行字，用界面上的说法（明细表、可累加、多对一……）。""" + if key[0] == "r": + return relation_text(value if isinstance(value, Mapping) else {}) + field_ = key[-1] + if field_ == "kind": + return catalog.UI_KIND_LABEL.get(value, str(value)) + if field_ == "measure": + return catalog.UI_MEASURE_LABEL.get(value, str(value)) + if field_ == "keys" and isinstance(value, list): + return "、".join(str(v) for v in value) + if field_ == "codes" and isinstance(value, Mapping): + return "、".join(f"{k}={v if str(v).strip() else CODE_PENDING}" for k, v in value.items()) + if field_ == "business_date" and isinstance(value, Mapping): + parts = [str(value.get("column") or "")] + if value.get("rule"): + parts.append(f"规则：{value['rule']}") + if value.get("timezone"): + parts.append(f"时区：{value['timezone']}") + return "；".join(parts) + return value if isinstance(value, str) else str(value) + + +# ========================================================================== +# 排序 +# ========================================================================== + + +def usage_weight(usage: int) -> float: + return 1.0 + math.log2(1 + max(0, int(usage or 0))) + + +def priority(usage: int, key: Slot, source: Any) -> float: + """一项的优先级：使用次数权重 × 风险权重 × 来源权重（不取整；排序和交给界面时取四位小数）。""" + _, tier = FIELD_GROUP[field_of(key)] + return usage_weight(usage) * RISK_WEIGHT[tier] * SOURCE_WEIGHT.get(str(source), 1.0) + + +def top_tables(tables: Mapping[str, Any], usage: Mapping[str, int]) -> list[str]: + """使用次数前 TOP_N 张表：只算被运行用到过的，同样多的按表结构里的顺序。""" + order = {name: i for i, name in enumerate(tables)} + used = [name for name in tables if usage.get(name, 0) > 0] + return sorted(used, key=lambda n: (-usage[n], order[n]))[:TOP_N] + + +def _reasons(key: Slot, item: Mapping[str, Any], usage: int, top: bool) -> list[str]: + out: list[str] = [] + if usage > 0: + out.append((REASON_TOP if top else REASON_USED).format(n=f"{usage:,}")) + _, tier = FIELD_GROUP[field_of(key)] + out.append(REASON_RISK[tier].format(label=field_label(key))) + if key[0] == "c" and key[2] == "codes" and isinstance(item.get("value"), Mapping): + pending = sum(1 for v in item["value"].values() if not str(v).strip()) + if pending: + out.append(REASON_PENDING_CODES.format(n=pending)) + source = str(item.get("source") or "") + out.append(REASON_SOURCE.format(source=SOURCE_LABEL.get(source, source))) + return out + + +def live_slots(notes: Mapping[str, Any] | None, meta: Mapping[str, Any] | None) -> list[tuple[Slot, dict[str, Any]]]: + """一张表目录里还算数的项：表结构里已经没有的列上的项不算（不知道表结构时都算）。""" + columns = catalog._column_names(meta) + return [(key, item) for key, item in catalog._slots(notes).items() + if not (key[0] == "c" and columns is not None and key[1] not in columns)] + + +# ========================================================================== +# 队列 +# ========================================================================== + + +async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", table: str | None = None, + limit: int = 50, offset: int = 0) -> dict[str, Any]: + """审阅队列：{total, items}。source 是读表结构用的源（上传源是绑定当前快照的视图）。""" + cache = getattr(source, "schema_cache", None) or {} + tables = cache.get("tables") or {} + wanted = None + if table: + wanted = catalog.resolve_table_name(cache, table) or table + entries = await catalog.read_catalog(session, source.id) + usage = await catalog.table_usage(session, source) + top = set(top_tables(tables, usage)) + items: list[dict[str, Any]] = [] + for name, entry in entries.items(): + meta = tables.get(name) + if meta is None or (wanted is not None and name != wanted): + continue + used = usage.get(name, 0) + for key, item in live_slots(entry.notes, meta): + if item.get("status") != "proposed": + continue + group, _ = FIELD_GROUP[field_of(key)] + if kind != "all" and group != kind: + continue + value = item_value(key, item) + items.append({ + "table_name": name, + "path": slot_path(key), + "field": field_name(key), + "field_label": field_label(key), + # 列级的项是哪一列（表级的项、关联关系为 null） + "column": key[1] if key[0] == "c" else None, + "value": value, + "display": display(key, value), + "source": item.get("source"), + "status": item.get("status"), + "note": item.get("note") if isinstance(item.get("note"), str) and item.get("note") else None, + "priority": round(priority(used, key, item.get("source")), 4), + "reasons": _reasons(key, item, used, name in top), + "table_usage": used, + "version": entry.version, + }) + items.sort(key=lambda i: (-i["priority"], i["table_name"], i["path"])) + return {"total": len(items), "items": items[offset:offset + limit]} + + +# ========================================================================== +# 批量审阅 +# ========================================================================== + + +@dataclass(frozen=True) +class BatchItem: + table_name: str + path: str + action: str + if_version: int + + +def _result(item: BatchItem, *, ok: bool, version: int | None = None, error: str | None = None, + code: str | None = None) -> dict[str, Any]: + return {"table_name": item.table_name, "path": item.path, "ok": ok, "version": version if ok else None, + "error": error, "code": code} + + +async def review_batch(session: AsyncSession, source: Any, items: Iterable[BatchItem], *, + actor: str | None) -> list[dict[str, Any]]: + """批量审阅，返回逐条结果（和请求一一对应）。 + + 表名按表结构认（全名、大小写不一致都认）；表结构里没有、但还留着目录的也认（同单项审阅）。同一张表的几条按请求里的 + 顺序在当前目录上依次审阅，一次写入：版本不符（if_version 和库里的不一样）、路径不对的那几条不做，其余照做；写入时撞上 + 别人刚改过，这张表的几条都记冲突。""" + items = list(items) + cache = getattr(source, "schema_cache", None) or {} + results: list[dict[str, Any] | None] = [None] * len(items) + groups: dict[str, list[int]] = {} + known: dict[str, str | None] = {} + for i, item in enumerate(items): + if item.table_name not in known: + key = catalog.resolve_table_name(cache, item.table_name) + if key is None and await catalog.read_entry(session, source.id, item.table_name) is not None: + key = item.table_name + known[item.table_name] = key + key = known[item.table_name] + if key is None: + results[i] = _result(item, ok=False, code=CODE_TABLE_MISSING, + error=f"数据源中没有表 {item.table_name}，可能已被删除或尚未探查结构") + continue + groups.setdefault(key, []).append(i) + at = catalog.now_iso() + for key, indexes in groups.items(): + entry = await catalog.read_entry(session, source.id, key) + current = entry.version if entry else 0 + notes: dict[str, Any] = entry.notes if entry else {} + applied: list[int] = [] + for i in indexes: + item = items[i] + if item.if_version != current: + results[i] = _result(item, ok=False, code=CODE_CONFLICT, error=str(catalog.CatalogConflict(key, current))) + continue + try: + notes = catalog.review_item(notes, item.path, item.action, at=at) + except catalog.CatalogPathError as e: + results[i] = _result(item, ok=False, code=CODE_PATH, error=str(e)) + continue + applied.append(i) + if not applied: + continue + try: + written = await catalog.write_entry(session, source.id, key, notes, if_version=current, actor=actor) + except catalog.CatalogConflict as e: + for i in applied: + results[i] = _result(items[i], ok=False, code=CODE_CONFLICT, error=str(e)) + continue + except catalog.CatalogInvalid as e: + for i in applied: + results[i] = _result(items[i], ok=False, code=CODE_INVALID, error=str(e)) + continue + for i in applied: + results[i] = _result(items[i], ok=True, version=written.version) + return [r for r in results if r is not None] + + +# ========================================================================== +# 被哪些运行用过 +# ========================================================================== + + +async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: int = 20) -> dict[str, Any]: + """用过这张表的运行：{total, runs}，新的在前（按开始时间，没开始的按建立时间）。取法同使用次数。""" + run_ids = {run for run, hit in await catalog.query_snapshot_tables(session, source) if table in hit} + if not run_ids: + return {"total": 0, "runs": []} + started = func.coalesce(Run.started_at, Run.created_at) + rows = (await session.execute( + select(Run.id, Run.workflow_id, Run.workflow_name, Run.run_class, Run.status, Run.started_at) + .where(Run.id.in_(run_ids)).order_by(started.desc(), Run.id).limit(limit) + )).all() + return { + "total": len(run_ids), + "runs": [{"run_id": r.id, "workflow_id": r.workflow_id, "workflow_name": r.workflow_name or "", + "run_class": r.run_class, "status": r.status, + "started_at": r.started_at.isoformat() if r.started_at else None} for r in rows], + } + + +# ========================================================================== +# 进度 +# ========================================================================== + + +async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]: + """审阅进度：常用 TOP_N 张表（只算被运行用到过的）、全部、影响数字的项。项按状态计数；verified（有确证）另给一项。""" + tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + entries = await catalog.read_catalog(session, source.id) + usage = await catalog.table_usage(session, source) + top = set(top_tables(tables, usage)) + every: Counter[str] = Counter() + head: Counter[str] = Counter() + facts: Counter[str] = Counter() + for name, entry in entries.items(): + meta = tables.get(name) + if meta is None: + continue + for key, item in live_slots(entry.notes, meta): + status = str(item.get("status")) + every[status] += 1 + if name in top: + head[status] += 1 + if FIELD_GROUP[field_of(key)][0] == "facts": + facts[status] += 1 + return { + "top_n": TOP_N, + "top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"], + "items_proposed": head["proposed"], "items_verified": head["verified"]}, + "all": {"items_total": sum(every.values()), "items_confirmed": every["confirmed"], + "items_proposed": every["proposed"], "items_rejected": every["rejected"], + "items_verified": every["verified"]}, + "facts": {"proposed": facts["proposed"], "confirmed": facts["confirmed"]}, + } + + +__all__ = ["BATCH_MAX", "BatchItem", "CODE_CONFLICT", "CODE_INVALID", "CODE_PATH", "CODE_TABLE_MISSING", + "FIELD_GROUP", "KINDS", "QUEUE_LIMIT_MAX", "RISK_WEIGHT", "RUNS_LIMIT_MAX", "SOURCE_LABEL", "SOURCE_WEIGHT", + "TOP_N", "display", "field_label", "field_name", "field_of", "item_value", "live_slots", "priority", + "relation_text", "review_batch", "review_progress", "review_queue", "slot_path", "table_runs", "top_tables", + "usage_weight"] diff --git a/backend/tests/test_catalog_review_queue.py b/backend/tests/test_catalog_review_queue.py new file mode 100644 index 00000000..1598103c --- /dev/null +++ b/backend/tests/test_catalog_review_queue.py @@ -0,0 +1,357 @@ +"""数据目录的审阅队列、批量审阅、表被哪些运行用过、审阅进度（data/catalog_review.py）。 + +- GET /catalog/review-queue：只列推断（proposed）的项，按「使用次数权重 × 风险权重 × 来源权重」从高到低，同分按表名、 + 路径；facts（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）/ names（中文名、说明、含义、单位、码值、 + 粒度、表类型）两组；按表筛选；分页；reasons 写为什么排在这里；表结构里已经没有的表、列不列 +- POST /catalog/review/batch：逐表按 if_version 乐观锁，一张表冲突不影响别的表；同一张表的几条一次写入；路径不对、表不存在 + 逐条报 +- GET /catalog/{table}/usage：用过这张表的运行，新的在前，取法同 catalog.table_usage +- GET /catalog/review-progress：常用 20 张表、全部、影响数字的项各自的确认情况 + +表名、列名、业务数字都是虚构的。 +""" +from __future__ import annotations + +import re +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from urllib.parse import quote + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data import catalog, catalog_review +from app.data.catalog import make_item +from app.db.base import SessionLocal +from app.db.models import Artifact, DataSource, Run +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +ACTOR = {"X-Actor": quote("林晓")} +TERMS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "terms.ts" + + +def _col(name: str, type_: str = "INTEGER") -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": None} + + +def _tables() -> dict: + return { + "orders": {"qualified": "orders", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col("id"), _col("status"), _col("amount", "REAL"), _col("member_id"), + _col("paid_at", "TEXT")]}, + "members": {"qualified": "members", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col("id"), _col("nick", "TEXT"), _col("tier")]}, + "coupons": {"qualified": "coupons", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": [_col("id"), _col("face_value", "REAL")]}, + } + + +def _member_rel(status: str = "proposed", source: str = "name") -> dict: + return {"id": catalog.relation_id("orders", ["member_id"], "members", ["id"]), "columns": ["member_id"], + "to_table": "members", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": 0.98, + "source": source, "status": status} + + +ORDERS = { + "label": make_item("订单", "llm"), + "grain": make_item("每笔订单一行", "llm", "confirmed"), + "valid_filter": make_item("status <> 9", "llm", note="剖析核对：排除 3.2% 的行（共 12,345 行）"), + "kind": make_item("fact", "llm"), + "columns": { + "status": {"label": make_item("状态", "comment"), "codes": make_item({"1": "已支付", "9": ""}, "profile")}, + "amount": {"measure": make_item("flow", "llm"), "unit": make_item("元", "llm", "rejected")}, + # 表结构里已经没有这一列：不进队列 + "legacy_flag": {"label": make_item("旧标记", "llm")}, + }, + "relations": [_member_rel()], +} +MEMBERS = {"label": make_item("会员", "llm"), "keys": make_item(["id"], "name"), + "columns": {"nick": {"label": make_item("昵称", "comment")}}} +COUPONS = {"label": make_item("优惠券", "comment")} +#: 表结构里已经没有这张表：不进队列 +GONE = {"label": make_item("旧表", "llm")} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _write(sid: str, table: str, notes: dict) -> None: + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, table) + await catalog.write_entry(session, sid, table, notes, if_version=entry.version if entry else 0, actor=None) + + +async def _use(sid: str, name: str, table_runs: dict[str, int]) -> list[str]: + """给每张表造几次运行查询它（每次运行一份查询快照），返回运行 id（先建的在前、开始得早）。""" + runs: list[str] = [] + base = datetime(2026, 9, 1, tzinfo=timezone.utc) + async with SessionLocal() as session: + for table, n in table_runs.items(): + for _ in range(n): + rid = uuid.uuid4().hex + session.add(Run(id=rid, graph={}, workflow_name=f"周报-{table}", run_class="formal", + status="succeeded", started_at=base + timedelta(hours=len(runs)))) + session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": [table]})) + runs.append(rid) + await session.commit() + return runs + + +async def _source(*, usage: dict[str, int] | None = None) -> str: + name = f"rq_{uuid.uuid4().hex[:10]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": _tables()}) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + for table, notes in (("orders", ORDERS), ("members", MEMBERS), ("coupons", COUPONS), ("gone_table", GONE)): + await _write(sid, table, notes) + if usage: + await _use(sid, name, usage) + return sid + + +def _base(sid: str) -> str: + return f"/api/datasources/{sid}/catalog" + + +# -------------------------------------------------------------------------- +# 排序规则（纯函数） +# -------------------------------------------------------------------------- + + +def test_priority_multiplies_usage_risk_and_source_weights(): + p = catalog_review.priority + # 风险：有效记录条件 > 业务主键 > 码值 > 中文名 + assert p(0, ("t", "valid_filter"), "llm") > p(0, ("t", "keys"), "llm") > p(0, ("c", "x", "codes"), "llm") \ + > p(0, ("c", "x", "label"), "llm") + assert p(0, ("r", "r1"), "name") == p(0, ("t", "dedup"), "name") == p(0, ("t", "business_date"), "name") + assert p(0, ("c", "x", "measure"), "llm") == p(0, ("t", "keys"), "llm") + assert p(0, ("t", "grain"), "llm") == p(0, ("t", "kind"), "llm") == p(0, ("c", "x", "codes"), "llm") + assert p(0, ("c", "x", "meaning"), "llm") == p(0, ("c", "x", "unit"), "llm") == p(0, ("t", "label"), "llm") + # 来源：模型起草、命名推断比数据库注释优先 + assert p(0, ("t", "label"), "llm") == p(0, ("t", "label"), "name") > p(0, ("t", "label"), "comment") + # 使用次数：用得多的在前，但按对数增长，不压过风险 + assert p(37, ("t", "label"), "llm") > p(1, ("t", "label"), "llm") > p(0, ("t", "label"), "llm") + assert p(37, ("t", "label"), "llm") == pytest.approx( + p(0, ("t", "label"), "llm") * p(37, ("t", "label"), "comment") / p(0, ("t", "label"), "comment")) + + +def test_source_labels_follow_the_ui(): + text = TERMS.read_text(encoding="utf-8") + block = re.search(r"export const CATALOG_SOURCE_LABEL\b[^=]*=\s*\{(.*?)\n\}", text, re.S) + assert block + ui = dict(re.findall(r"(\w+):\s*'([^']+)'", block.group(1))) + assert ui == catalog_review.SOURCE_LABEL + + +# -------------------------------------------------------------------------- +# 队列 +# -------------------------------------------------------------------------- + + +async def test_queue_lists_only_proposed_items_by_priority(client): + sid = await _source(usage={"orders": 37, "members": 2}) + r = await client.get(f"{_base(sid)}/review-queue") + assert r.status_code == 200, r.text + body = r.json() + items = body["items"] + paths = [(i["table_name"], i["path"]) for i in items] + # 已确认、已驳回的不列；表结构里没有的表、列不列 + assert ("orders", "grain") not in paths and ("orders", "columns.amount.unit") not in paths + assert not any(t == "gone_table" for t, _ in paths) + assert ("orders", "columns.legacy_flag.label") not in paths + assert body["total"] == len(items) == 11 + assert all(i["status"] == "proposed" for i in items) + # 从高到低，同分按表名、路径 + keys = [(-i["priority"], i["table_name"], i["path"]) for i in items] + assert keys == sorted(keys) + first = items[0] + rel_id = _member_rel()["id"] + assert (first["table_name"], first["path"]) in {("orders", "valid_filter"), ("orders", f"relations.{rel_id}")} + vf = next(i for i in items if i["path"] == "valid_filter") + assert vf["field"] == "valid_filter" and vf["field_label"] == "有效记录条件" + assert vf["value"] == "status <> 9" and vf["display"] == "status <> 9" + assert vf["source"] == "llm" and vf["note"] == "剖析核对：排除 3.2% 的行（共 12,345 行）" + assert vf["table_usage"] == 37 and vf["version"] == 1 + assert vf["reasons"] == ["常用表（使用 37 次）", "影响数字的项（有效记录条件）", "模型起草，未确认"] + # 常用表上的中文名排在没人用的表上的中文名前面；同一张表上影响数字的项排在中文名前面 + order = [(i["table_name"], i["path"]) for i in items] + assert order.index(("orders", "label")) < order.index(("coupons", "label")) + assert order.index(("members", "keys")) < order.index(("members", "label")) + + +async def test_queue_values_are_displayed_in_ui_words(client): + sid = await _source() + items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]} + rel = items[("orders", f"relations.{_member_rel()['id']}")] + assert rel["field"] == "relation" and rel["field_label"] == "关联关系" + assert rel["value"] == {"columns": ["member_id"], "to_table": "members", "to_columns": ["id"], + "cardinality": "many_to_one", "coverage": 0.98} + assert rel["display"] == "member_id → members.id（多对一，覆盖率 98%）" + assert rel["reasons"][-1] == "命名推断，未确认" + kind = items[("orders", "kind")] + assert kind["display"] == "明细表" and kind["value"] == "fact" + codes = items[("orders", "columns.status.codes")] + assert codes["field"] == "codes" and codes["field_label"] == "码值" and codes["column"] == "status" + assert codes["display"] == "1=已支付、9=含义待填写" + assert "1 个码值的含义待填写" in codes["reasons"] and "数据剖析，未确认" in codes["reasons"] + assert items[("orders", "columns.amount.measure")]["display"] == "可累加" + assert items[("members", "keys")]["display"] == "id" + assert items[("members", "label")]["reasons"] == ["名称和说明（中文名）", "模型起草，未确认"] + assert items[("coupons", "label")]["reasons"] == ["名称和说明（中文名）", "数据库注释，未确认"] + + +async def test_queue_filters_by_kind_and_table_and_pages(client): + sid = await _source(usage={"orders": 3}) + facts = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "facts"})).json() + assert {i["field"] for i in facts["items"]} == {"valid_filter", "relation", "measure", "keys"} + names = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "names"})).json() + assert {i["field"] for i in names["items"]} == {"label", "codes", "kind"} + assert facts["total"] + names["total"] == 11 + only = (await client.get(f"{_base(sid)}/review-queue", params={"table": "MEMBERS"})).json() + assert {i["table_name"] for i in only["items"]} == {"members"} and only["total"] == 3 + everything = (await client.get(f"{_base(sid)}/review-queue")).json()["items"] + page = (await client.get(f"{_base(sid)}/review-queue", params={"limit": 4, "offset": 4})).json() + assert page["total"] == 11 and page["items"] == everything[4:8] + bad = await client.get(f"{_base(sid)}/review-queue", params={"kind": "other"}) + assert bad.status_code == 422 + assert (await client.get(f"{_base(sid)}/review-queue", params={"limit": 0})).status_code == 422 + missing = await client.get("/api/datasources/nope/catalog/review-queue") + assert missing.status_code == 404 + + +# -------------------------------------------------------------------------- +# 批量审阅 +# -------------------------------------------------------------------------- + + +async def test_batch_review_writes_each_table_once_and_isolates_conflicts(client): + sid = await _source() + rel_path = f"relations.{_member_rel()['id']}" + body = {"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}, + {"table_name": "orders", "path": rel_path, "action": "reject", "if_version": 1}, + {"table_name": "orders", "path": "columns.nothing.label", "action": "confirm", "if_version": 1}, + {"table_name": "members", "path": "label", "action": "confirm", "if_version": 7}, # 版本不对 + {"table_name": "coupons", "path": "label", "action": "reject", "if_version": 1}, + {"table_name": "ghost", "path": "label", "action": "confirm", "if_version": 0}, + ]} + r = await client.post(f"{_base(sid)}/review/batch", json=body, headers=ACTOR) + assert r.status_code == 200, r.text + results = r.json()["results"] + assert [(x["table_name"], x["path"], x["ok"]) for x in results] == [ + ("orders", "valid_filter", True), ("orders", rel_path, True), ("orders", "columns.nothing.label", False), + ("members", "label", False), ("coupons", "label", True), ("ghost", "label", False)] + # 同一张表的两条一次写进去：版本只加 1 + assert results[0]["version"] == results[1]["version"] == 2 + assert results[2]["code"] == "catalog_path_invalid" and "找不到" in results[2]["error"] + assert results[2]["version"] is None + assert results[3]["code"] == "catalog_conflict" and "重新载入" in results[3]["error"] + assert results[4]["version"] == 2 and results[4]["error"] is None and results[4]["code"] is None + assert results[5]["code"] == "table_missing" + async with SessionLocal() as session: + orders = await catalog.read_entry(session, sid, "orders") + members = await catalog.read_entry(session, sid, "members") + coupons = await catalog.read_entry(session, sid, "coupons") + assert orders.version == 2 and orders.updated_by == "林晓" + assert orders.notes["valid_filter"]["status"] == "confirmed" + assert orders.notes["relations"][0]["status"] == "rejected" + assert members.version == 1 and members.notes["label"]["status"] == "proposed" + assert coupons.notes["label"]["status"] == "rejected" + # 撤销审阅：回到来源的初始状态 + again = await client.post(f"{_base(sid)}/review/batch", json={"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "reset", "if_version": 2}]}) + assert again.json()["results"][0]["ok"] is True + async with SessionLocal() as session: + assert (await catalog.read_entry(session, sid, "orders")).notes["valid_filter"]["status"] == "proposed" + + +async def test_batch_review_limits(client): + sid = await _source() + many = [{"table_name": "orders", "path": "label", "action": "confirm", "if_version": 1}] * 201 + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": many})).status_code == 422 + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": []})).status_code == 422 + wrong = [{"table_name": "orders", "path": "label", "action": "approve", "if_version": 1}] + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": wrong})).status_code == 422 + ok = [{"table_name": "orders", "path": "label", "action": "confirm", "if_version": 1}] * 200 + r = await client.post(f"{_base(sid)}/review/batch", json={"items": ok}) + assert r.status_code == 200 and all(x["ok"] for x in r.json()["results"]) + + +# -------------------------------------------------------------------------- +# 被哪些运行用过 +# -------------------------------------------------------------------------- + + +async def test_table_usage_lists_runs_newest_first(client): + sid = await _source() + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + runs = await _use(sid, name, {"orders": 3, "members": 1}) + # 同一次运行查了两次 orders：只算一次运行 + async with SessionLocal() as session: + session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders", "members"]})) + # 别的数据源的同名表不算 + other = uuid.uuid4().hex + session.add(Run(id=other, graph={}, workflow_name="别处", status="succeeded")) + session.add(Artifact(id=uuid.uuid4().hex, run_id=other, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": "elsewhere", "source": "elsewhere", "tables": ["orders"]})) + await session.commit() + r = await client.get(f"{_base(sid)}/orders/usage") + assert r.status_code == 200, r.text + body = r.json() + assert body["total"] == 3 + assert [x["run_id"] for x in body["runs"]] == [runs[2], runs[1], runs[0]] + first = body["runs"][0] + assert first["workflow_name"] == "周报-orders" and first["run_class"] == "formal" + assert first["status"] == "succeeded" and first["started_at"].startswith("2026-09-01T02:00") + assert set(first) == {"run_id", "workflow_id", "workflow_name", "run_class", "status", "started_at"} + limited = (await client.get(f"{_base(sid)}/orders/usage", params={"limit": 1})).json() + assert limited["total"] == 3 and len(limited["runs"]) == 1 + members = (await client.get(f"{_base(sid)}/members/usage")).json() + assert members["total"] == 2 + assert (await client.get(f"{_base(sid)}/ghost/usage")).status_code == 404 + + +# -------------------------------------------------------------------------- +# 进度 +# -------------------------------------------------------------------------- + + +async def test_review_progress_counts_top_tables_all_and_facts(client): + sid = await _source(usage={"orders": 5}) + r = await client.get(f"{_base(sid)}/review-progress") + assert r.status_code == 200, r.text + body = r.json() + assert body["top_n"] == 20 + # 常用表只算被运行用到过的表：这里只有 orders（9 项：表结构里没有的列不算） + assert body["top"] == {"tables": 1, "items_total": 9, "items_confirmed": 1, "items_proposed": 7, + "items_verified": 0} + assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1, + "items_verified": 0} + # 影响数字的项：有效记录条件、关联关系、度量类型、业务主键 + assert body["facts"] == {"proposed": 4, "confirmed": 0} + await client.post(f"{_base(sid)}/review/batch", json={"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}]}) + after = (await client.get(f"{_base(sid)}/review-progress")).json() + assert after["top"]["items_confirmed"] == 2 and after["facts"] == {"proposed": 3, "confirmed": 1} From 2298cf848aa255f76076ac4895140ae2ef5e9e69 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:06:45 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E5=AF=BC=E5=87=BA=E5=AF=BC=E5=85=A5=EF=BC=9A=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E4=B8=BA=E3=80=8C<=E6=95=B0=E6=8D=AE=E6=BA=90=E5=90=8D>-?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95.json=E3=80=8D=E9=99=84?= =?UTF-8?q?=E4=BB=B6=EF=BC=88=E5=85=A8=E9=83=A8=E7=8A=B6=E6=80=81=E7=9A=84?= =?UTF-8?q?=E9=A1=B9=E3=80=81=E8=B5=B7=E8=8D=89=E8=A6=81=E6=B1=82=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E5=AF=BC=E5=85=A5=E5=85=88=E9=A2=84=E8=A7=88=E5=86=8D?= =?UTF-8?q?=E5=86=99=E5=85=A5=EF=BC=8C=E8=A1=A8=E5=90=8D=E3=80=81=E5=88=97?= =?UTF-8?q?=E5=90=8D=E6=8C=89=E8=A1=A8=E7=BB=93=E6=9E=84=E7=9A=84=E5=86=99?= =?UTF-8?q?=E6=B3=95=E5=86=8D=E5=BF=BD=E7=95=A5=E5=A4=A7=E5=B0=8F=E5=86=99?= =?UTF-8?q?=E5=AF=B9=EF=BC=88=E5=B8=A6=20schema=20=E5=89=8D=E7=BC=80?= =?UTF-8?q?=E7=9A=84=E5=8E=BB=E6=8E=89=E5=89=8D=E7=BC=80=E5=86=8D=E5=AF=B9?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB=E4=B8=A4?= =?UTF-8?q?=E7=AB=AF=E6=8D=A2=E6=88=90=E7=9B=AE=E6=A0=87=E7=9A=84=E5=86=99?= =?UTF-8?q?=E6=B3=95=E5=B9=B6=E9=87=8D=E7=AE=97=E7=BC=96=E5=8F=B7=EF=BC=9B?= =?UTF-8?q?=E5=B7=B2=E7=A1=AE=E8=AE=A4=E3=80=81=E5=B7=B2=E9=A9=B3=E5=9B=9E?= =?UTF-8?q?=E7=9A=84=E9=A1=B9=E7=85=A7=E5=B8=A6=EF=BC=88=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E3=80=81=E7=8A=B6=E6=80=81=E4=B8=8D=E5=8F=98=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E7=9B=AE=E6=A0=87=E9=87=8C=E5=B7=B2=E7=A1=AE=E8=AE=A4=E3=80=81?= =?UTF-8?q?=E5=B7=B2=E9=A9=B3=E5=9B=9E=E8=80=8C=E4=B8=8D=E4=B8=80=E6=A0=B7?= =?UTF-8?q?=E7=9A=84=E4=BF=9D=E7=95=99=E7=9B=AE=E6=A0=87=E7=9A=84=E5=B9=B6?= =?UTF-8?q?=E8=AE=B0=E5=86=B2=E7=AA=81=EF=BC=8C=E6=8E=A8=E6=96=AD=E3=80=81?= =?UTF-8?q?=E6=9C=89=E7=A1=AE=E8=AF=81=E7=9A=84=E5=8F=AA=E5=A1=AB=E7=A9=BA?= =?UTF-8?q?=E7=BC=BA=EF=BC=9B=E8=A1=A8=E7=BB=93=E6=9E=84=E9=87=8C=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E7=9A=84=E5=88=97=E3=80=81=E4=B8=A4=E7=AB=AF=E5=AF=B9?= =?UTF-8?q?=E4=B8=8D=E4=B8=8A=E7=9A=84=E5=85=B3=E7=B3=BB=E3=80=81=E7=9B=AE?= =?UTF-8?q?=E6=A0=87=E9=81=AE=E7=BD=A9=E5=88=97=E7=9A=84=E7=A0=81=E5=80=BC?= =?UTF-8?q?=E3=80=81=E4=B8=8D=E5=90=88=E8=A7=84=E7=9A=84=E9=A1=B9=E8=B7=B3?= =?UTF-8?q?=E8=BF=87=E5=B9=B6=E5=86=99=E5=8E=9F=E5=9B=A0=EF=BC=9B=E9=80=90?= =?UTF-8?q?=E8=A1=A8=E4=B9=90=E8=A7=82=E9=94=81=E3=80=81=E4=B8=80=E5=BC=A0?= =?UTF-8?q?=E8=A1=A8=E5=86=99=E4=B8=8D=E8=BF=9B=E5=8E=BB=E4=B8=8D=E5=BD=B1?= =?UTF-8?q?=E5=93=8D=E5=88=AB=E7=9A=84=EF=BC=9B=E6=9C=89=E8=B5=B7=E8=8D=89?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=97=B6=20409=EF=BC=8C=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E4=B8=8D=E5=AF=B9=20422=EF=BC=8C=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E8=A1=A8=E7=BB=93=E6=9E=84=20409=EF=BC=9B=E8=B5=B7?= =?UTF-8?q?=E8=8D=89=E8=A6=81=E6=B1=82=E5=8F=AA=E5=9C=A8=E5=8B=BE=E9=80=89?= =?UTF-8?q?=E4=B8=94=E7=9B=AE=E6=A0=87=E8=BF=98=E6=B2=A1=E6=9C=89=E6=97=B6?= =?UTF-8?q?=E5=86=99=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/catalog.py | 66 +++- backend/app/data/catalog_transfer.py | 484 +++++++++++++++++++++++++ backend/tests/test_catalog_transfer.py | 378 +++++++++++++++++++ 3 files changed, 927 insertions(+), 1 deletion(-) create mode 100644 backend/app/data/catalog_transfer.py create mode 100644 backend/tests/test_catalog_transfer.py diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 7680eb54..b3554756 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -16,6 +16,8 @@ - GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅， GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行 （data/catalog_review.py） +- GET /catalog/export 导出数据目录（JSON 附件）；POST /catalog/import/preview 导入预览（只读）， + POST /catalog/import 导入（data/catalog_transfer.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 @@ -38,7 +40,7 @@ from app.api.coded import ( ) from app.api.runs import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions -from app.data import catalog_review +from app.data import catalog_review, catalog_transfer from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -262,6 +264,68 @@ async def catalog_review_progress(source_id: str, session: AsyncSession = Depend return await catalog_review.review_progress(session, source) +# ---- 导出导入（导出同样要登记在 GET /catalog/{table} 前面） + + +@router.get("/{source_id}/catalog/export") +async def export_catalog(source_id: str, session: AsyncSession = Depends(get_session)) -> Any: + """导出这个数据源的数据目录：JSON 附件「<数据源名>-数据目录.json」，含全部状态的项（data/catalog_transfer.py）。""" + from fastapi.responses import JSONResponse + + row = await _source_row(session, source_id) + body = await catalog_transfer.export_catalog(session, row) + name = catalog_transfer.attachment_name(row.name) + return JSONResponse(body, headers={"Content-Disposition": catalog_transfer.content_disposition(name, row.id)}) + + +class CatalogImportIn(BaseModel): + #: 导出的那个对象（格式在 data/catalog_transfer.read_file 里核对，不对 422，说明是中文） + content: Any = None + #: 目标还没有起草要求时，是否写入文件里的起草要求 + include_guidance: bool = False + + +async def _import_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: + row, source = await _resolved(session, source_id) + cache = source.schema_cache or {} + if not cache.get("tables"): + raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法导入数据目录。请先在数据源卡片上点「探查结构」", + SCHEMA_MISSING) + return row, source + + +@router.post("/{source_id}/catalog/import/preview") +async def preview_catalog_import(source_id: str, payload: CatalogImportIn, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """导入预览（只读）：每张对上的表新增、更新、保留几项，冲突和跳过的逐条列出。文件格式不对 422；没有表结构 409。""" + row, source = await _import_source(session, source_id) + try: + report = await catalog_transfer.plan_import(session, row, source, payload.content, + include_guidance=payload.include_guidance) + except catalog_transfer.TransferInvalid as e: + raise HTTPException(422, str(e)) from e + return report.as_dict() + + +@router.post("/{source_id}/catalog/import") +async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务 + 409 catalog_draft_busy；文件格式不对 422；没有表结构 409。""" + row, source = await _import_source(session, source_id) + try: + await catalog_jobs.ensure_idle(session, row) + except catalog_jobs.DraftBusy as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e + try: + report = await catalog_transfer.run_import(session, row, source, payload.content, + include_guidance=payload.include_guidance, + actor=actor_of(x_actor)) + except catalog_transfer.TransferInvalid as e: + raise HTTPException(422, str(e)) from e + return report.as_dict() + + @router.get("/{source_id}/catalog/{table}") async def get_catalog_table(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py new file mode 100644 index 00000000..34954f8d --- /dev/null +++ b/backend/app/data/catalog_transfer.py @@ -0,0 +1,484 @@ +"""数据目录的导出导入：本机整理好的目录带到部署实例上（GET /catalog/export、POST /catalog/import/preview、/import）。 + +**为什么要有它。** 目录（尤其是人工确认、驳回过的项）是在本机对着开发库一项项审出来的；部署实例连的是同一套业务库， +却没有办法把这些结论带过去，只能在那边从头再审一遍。导出是一个 JSON 文件，导入先预览、再确认。 + +**文件**（export_catalog）：{format: "agentlab-catalog", version: 1, exported_at, source: {name, kind}, guidance, +tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经没有的表也带上（目录里有什么就导出什么）。 + +**导入的规矩**（plan_import，预览和导入共用这一份）： + +- 表按名字对：表结构里的写法、忽略大小写（name_key）、带 schema 前缀的对不上时去掉前缀再对（catalog.resolve_table_name）。 + 只对目标数据源表结构里的表；对不上的列在 unmatched_tables。文件里两张表对上目标的同一张时，写法完全一样的那张优先， + 另一张算对不上。列名同样按表结构的写法、再忽略大小写对；业务主键、业务日期、关联关系两端的列和指向的表都换成目标的写法， + 关联关系的编号按目标的表名、列名重算（catalog.relation_id）。 +- 文件里已确认、已驳回的项照带：来源、状态、备注、修改时间都不变。目标那一格空着就新增；目标是推断、有确证的就更新； + 目标也是已确认、已驳回的，和文件一样就保留，不一样（值或状态不同）就保留目标的、记一条冲突。关联关系的「一样」看两端和 + 基数，覆盖率是测量值、不算。 +- 文件里推断、有确证的项只填目标的空缺，目标那一格有东西（哪怕是驳回）就保留目标的，不算冲突。 +- 跳过并写原因：表结构里没有的列、业务主键和业务日期指着的列没有的、关联关系两端对不上的（指向的表、两端的列）、 + 目标数据源遮罩的列上的码值（码值就是这一列的取值，遮罩了就不该进目录）、不认识的字段、取值不合规的项（说法同 + catalog.validate_notes）。 +- 逐表写（乐观锁，撞上别人刚改过就重读重算，最多三次），一张表写不进去只记在那张表上，不影响别的表。 +- 起草要求：勾了 include_guidance、文件里有、目标还没有时才写（action: replace）；目标已有的不覆盖（keep）；没勾或文件里 + 没有是 none。 +- 文件格式不对（不是对象、格式标记或版本不对、表的目录不是对象、起草要求不是文字或超长）抛 TransferInvalid，接口回 422。 + +有排队或进行中的起草任务时不导入（接口先调 catalog_jobs.ensure_idle，409 catalog_draft_busy）；目标数据源没有表结构时 +接口回 409（什么也对不上）。 +""" +from __future__ import annotations + +import copy +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_jobs +from app.data.catalog_review import field_label, slot_path +from app.data.engine import masked_columns +from app.data.names import name_key + +FORMAT = "agentlab-catalog" +VERSION = 1 +FILE_SUFFIX = "-数据目录.json" +_BAD_FILENAME = re.compile(r'[\\/:*?"<>|\x00-\x1f\x7f]') +_NAME_MAX = 80 +_UNNAMED = "数据源" + +#: 状态：照带的、只填空缺的 +_LOCKED = ("confirmed", "rejected") +_STATUS_LABEL = {"proposed": "推断", "verified": "有确证", "confirmed": "已确认", "rejected": "已驳回"} + +#: 写入一直撞上别人刚改过（重读重算三次仍冲突） +WRITE_CONFLICT_ERROR = "这张表的数据目录正被其他人修改，没有导入，请稍后重新导入" +WRITE_ERROR = "写入这张表的数据目录时出错，没有导入：{line}" + + +class TransferInvalid(ValueError): + """文件格式不对。message 给人看（接口回 422）。""" + + +# ========================================================================== +# 导出 +# ========================================================================== + + +def attachment_name(source_name: str) -> str: + """附件名「<数据源名>-数据目录.json」。文件名里不能有的字换成下划线。""" + name = _BAD_FILENAME.sub("_", source_name or "").strip(" .")[:_NAME_MAX].strip(" .") + return f"{name or _UNNAMED}{FILE_SUFFIX}" + + +def content_disposition(filename: str, source_id: str) -> str: + """附件头：中文名按 RFC 5987 写在 filename* 里；filename 给不认 filename* 的老客户端一个纯 ASCII 的名字。""" + from urllib.parse import quote + + ascii_id = re.sub(r"[^A-Za-z0-9]", "", source_id)[:8] + return f"attachment; filename=\"agentlab-catalog-{ascii_id}.json\"; filename*=UTF-8''{quote(filename, safe='')}" + + +async def export_catalog(session: AsyncSession, row: Any) -> dict[str, Any]: + """导出这个数据源的数据目录（row 是数据源记录，目录按它的 id 存）。""" + entries = await catalog.read_catalog(session, row.id) + return { + "format": FORMAT, + "version": VERSION, + "exported_at": catalog.now_iso(), + "source": {"name": row.name, "kind": row.kind}, + "guidance": catalog_jobs.guidance_of(row), + "tables": {name: {"notes": entry.notes, "version": entry.version} for name, entry in sorted(entries.items())}, + } + + +# ========================================================================== +# 读文件 +# ========================================================================== + + +@dataclass(frozen=True) +class ImportFile: + tables: dict[str, dict[str, Any]] + guidance: str | None + + +def read_file(content: Any, *, include_guidance: bool) -> ImportFile: + """导出的那个对象 → ImportFile。格式不对抛 TransferInvalid（说法给人看）。""" + if not isinstance(content, Mapping): + raise TransferInvalid("文件内容不是数据目录导出的格式，请选择从 AgentLab 导出的数据目录文件") + if content.get("format") != FORMAT: + raise TransferInvalid("这个文件不是 AgentLab 导出的数据目录（文件里的格式标记不对），请重新选择文件") + version = content.get("version") + if version != VERSION: + raise TransferInvalid(f"这个数据目录文件的格式版本是「{version}」，当前只能导入第 {VERSION} 版的文件，" + "请用同一版本的 AgentLab 重新导出") + tables = content.get("tables") + if not isinstance(tables, Mapping): + raise TransferInvalid("数据目录文件里没有表的目录，文件可能不完整，请重新导出") + out: dict[str, dict[str, Any]] = {} + for name, body in tables.items(): + notes = body.get("notes") if isinstance(body, Mapping) else None + if not isinstance(name, str) or not name.strip() or not isinstance(notes, Mapping): + raise TransferInvalid(f"数据目录文件里表「{str(name)[:60]}」的目录格式不正确，文件可能被改动过，请重新导出") + out[name] = copy.deepcopy(dict(notes)) + guidance = content.get("guidance") + if guidance is not None and not isinstance(guidance, str): + raise TransferInvalid("数据目录文件里的起草要求格式不正确，文件可能被改动过，请重新导出") + text = guidance.strip() if isinstance(guidance, str) else "" + if include_guidance and len(text) > catalog_jobs.GUIDANCE_MAX: + raise TransferInvalid(f"数据目录文件里的起草要求有 {len(text)} 字，超过了 {catalog_jobs.GUIDANCE_MAX} 字的上限，" + "请删减后再导入，或者不导入起草要求") + return ImportFile(tables=out, guidance=text or None) + + +# ========================================================================== +# 对表、对列 +# ========================================================================== + + +def match_tables(file_tables: list[str], schema_cache: Mapping[str, Any]) -> tuple[dict[str, str], list[str]]: + """文件里的表 → 目标表结构里的键。返回 ({文件里的表名: 目标表名}, 对不上的表名)。写法完全一样的先认领。""" + tables = (schema_cache or {}).get("tables") or {} + matched: dict[str, str] = {} + claimed: set[str] = set() + ordered = sorted(file_tables, key=lambda n: n not in tables) + for name in ordered: + key = catalog.resolve_table_name(schema_cache, name) + if key is None or key in claimed: + continue + matched[name] = key + claimed.add(key) + unmatched = [name for name in file_tables if name not in matched] + return {name: matched[name] for name in file_tables if name in matched}, unmatched + + +class _Columns: + """一张目标表的列：按表结构的写法认，再忽略大小写认（只认唯一对上的）。""" + + def __init__(self, meta: Mapping[str, Any] | None) -> None: + names = [str(c.get("name")) for c in (meta or {}).get("columns") or [] if isinstance(c, Mapping) + and c.get("name") is not None] + self.names = set(names) + self._loose: dict[str, list[str]] = {} + for name in names: + self._loose.setdefault(name_key(name), []).append(name) + + def find(self, name: Any) -> str | None: + if not isinstance(name, str): + return None + if name in self.names: + return name + found = self._loose.get(name_key(name)) or [] + return found[0] if len(found) == 1 else None + + +# ========================================================================== +# 一张表怎么并 +# ========================================================================== + + +@dataclass +class TablePlan: + """文件里一张表并进目标的计划。writes 是要写进目标的项（目标的槽位 → 项）。""" + + table_name: str + matched_as: str + add: int = 0 + update: int = 0 + keep: int = 0 + conflicts: list[dict[str, Any]] = field(default_factory=list) + skipped: list[dict[str, Any]] = field(default_factory=list) + writes: dict[catalog.Slot, dict[str, Any]] = field(default_factory=dict) + version: int = 0 + error: str | None = None + + def as_dict(self) -> dict[str, Any]: + return {"table_name": self.table_name, "matched_as": self.matched_as, "add": self.add, "update": self.update, + "keep": self.keep, "conflicts": self.conflicts, "skipped": self.skipped, "version": self.version, + "error": self.error} + + +def _raw_items(notes: Mapping[str, Any]) -> list[tuple[str, catalog.Slot | None, Any, str | None]]: + """文件里一张表的每一项：(文件里的路径, 槽位, 项, 不认的原因)。不认识的字段、不是对象的列也逐个列出来（槽位为 None）。""" + out: list[tuple[str, catalog.Slot | None, Any, str | None]] = [] + for name, item in notes.items(): + if name in catalog.TABLE_FIELDS: + out.append((name, ("t", name), item, None)) + elif name not in ("columns", "relations"): + out.append((str(name), None, item, f"数据目录里不认识的字段「{name}」")) + columns = notes.get("columns", {}) + if not isinstance(columns, Mapping): + out.append(("columns", None, columns, "列的目录应为以列名为键的对象")) + columns = {} + for col, items in columns.items(): + if not isinstance(items, Mapping): + out.append((f"columns.{col}", None, items, f"列 {col} 的目录应为对象")) + continue + for name, item in items.items(): + path = f"columns.{col}.{name}" + if name in catalog.COLUMN_FIELDS: + out.append((path, ("c", str(col), name), item, None)) + else: + out.append((path, None, item, f"列 {col} 含有不认识的字段「{name}」")) + relations = notes.get("relations", []) + if not isinstance(relations, list): + out.append(("relations", None, relations, "关联关系应为列表")) + relations = [] + for rel in relations: + rid = rel.get("id") if isinstance(rel, Mapping) else None + out.append((f"relations.{rid}", ("r", str(rid)), rel, None)) + return out + + +def _item_problem(key: catalog.Slot, item: Any) -> str | None: + """这一项本身合不合规（说法同 catalog.validate_notes）。""" + if key[0] == "t": + probe: dict[str, Any] = {key[1]: item} + elif key[0] == "c": + probe = {"columns": {key[1]: {key[2]: item}}} + else: + probe = {"relations": [item]} + problems = catalog.validate_notes(probe) + return problems[0] if problems else None + + +def _decision(key: catalog.Slot, item: Mapping[str, Any]) -> Any: + """判断「一样不一样」看的部分：状态，加上值（码值另看「已列全」）。关联关系的两端由编号定了，再看基数；覆盖率不算。""" + if key[0] == "r": + return item.get("status"), item.get("cardinality") + if key[0] == "c" and key[2] == "codes": + return item.get("status"), item.get("value"), catalog.codes_complete(item) + return item.get("status"), item.get("value") + + +def _conflict_why(file_status: str, target_status: str) -> str: + if target_status == "rejected": + return f"目标里这一项已经驳回，文件里是{_STATUS_LABEL.get(file_status, file_status)}，保留目标的" + if file_status == "rejected": + return "目标里这一项已经人工确认过，文件里是已驳回，保留目标的" + return "目标里这一项已经人工确认过，和文件里的不一样，保留目标的" + + +class _Mapper: + """把文件里的一项换成目标的写法：列名、业务主键和业务日期的列、关联关系两端和编号。对不上返回原因。""" + + def __init__(self, target: str, schema_cache: Mapping[str, Any], masked: set[str]) -> None: + self.target = target + self.cache = schema_cache + self.tables = (schema_cache or {}).get("tables") or {} + self.columns = _Columns(self.tables.get(target)) + self.masked = masked + + def __call__(self, key: catalog.Slot, item: Any) -> tuple[catalog.Slot | None, Any, str | None]: + if not isinstance(item, Mapping): + return None, item, None # 交给合规检查说 + item = copy.deepcopy(dict(item)) + if key[0] == "c": + col = self.columns.find(key[1]) + if col is None: + return None, item, f"目标的表结构里没有列 {key[1]}，跳过" + if key[2] == "codes" and col.lower() in self.masked: + return None, item, f"列 {col} 在目标数据源的设置中被遮罩，码值是这一列的取值，不导入" + return ("c", col, key[2]), item, None + if key[0] == "t": + value = item.get("value") + if key[1] == "keys" and isinstance(value, list): + mapped = [self.columns.find(c) for c in value] + if missing := [str(c) for c, m in zip(value, mapped) if m is None]: + return None, item, f"业务主键里的列「{'、'.join(missing)}」不在目标的表结构里，跳过" + item["value"] = mapped + elif key[1] == "business_date" and isinstance(value, Mapping) and isinstance(value.get("column"), str): + col = self.columns.find(value["column"]) + if col is None: + return None, item, f"业务日期的列「{value['column']}」不在目标的表结构里，跳过" + item["value"] = {**value, "column": col} + return key, item, None + return self._relation(item) + + def _relation(self, rel: dict[str, Any]) -> tuple[catalog.Slot | None, Any, str | None]: + cols, to_table, to_cols = rel.get("columns"), rel.get("to_table"), rel.get("to_columns") + if not (isinstance(cols, list) and isinstance(to_cols, list) and isinstance(to_table, str)): + return None, rel, None + target_table = catalog.resolve_table_name(self.cache, to_table) + if target_table is None: + return None, rel, f"关联关系指向的表 {to_table} 不在目标数据源的表结构里，跳过" + mine = [self.columns.find(c) for c in cols] + if missing := [str(c) for c, m in zip(cols, mine) if m is None]: + return None, rel, f"关联关系的本表字段「{'、'.join(missing)}」不在目标的表结构里，跳过" + theirs_columns = _Columns(self.tables.get(target_table)) + theirs = [theirs_columns.find(c) for c in to_cols] + if missing := [str(c) for c, m in zip(to_cols, theirs) if m is None]: + return None, rel, f"关联关系的目标字段「{'、'.join(missing)}」不在 {target_table} 的表结构里，跳过" + rid = catalog.relation_id(self.target, mine, target_table, theirs) + return ("r", rid), {**rel, "id": rid, "columns": mine, "to_table": target_table, "to_columns": theirs}, None + + +def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], target_notes: Mapping[str, Any] | None, + schema_cache: Mapping[str, Any], masked: set[str]) -> TablePlan: + """文件里一张表并进目标那张表的计划（纯函数，不写库）。""" + plan = TablePlan(table_name=table_name, matched_as=target) + existing = catalog._slots(target_notes) + mapper = _Mapper(target, schema_cache, masked) + seen: set[catalog.Slot] = set() + for path, key, item, why in _raw_items(file_notes): + if why is not None or key is None: + plan.skipped.append({"path": path, "why": why or "无法识别这一项"}) + continue + mapped, item, why = mapper(key, item) + if why is not None: + plan.skipped.append({"path": path, "why": why}) + continue + problem = _item_problem(mapped or key, item) + if problem is not None or mapped is None: + plan.skipped.append({"path": path, "why": problem or "无法识别这一项"}) + continue + if mapped in seen: + plan.skipped.append({"path": path, "why": f"和文件里前面的{catalog.describe_slot(mapped)}是同一项，只导入第一项"}) + continue + seen.add(mapped) + status = item.get("status") + have = existing.get(mapped) + if have is None: + plan.add += 1 + plan.writes[mapped] = item + elif status not in _LOCKED: + plan.keep += 1 # 推断、有确证的只填空缺 + elif have.get("status") in _LOCKED: + if _decision(mapped, have) == _decision(mapped, item): + plan.keep += 1 + else: + plan.conflicts.append({ + "path": slot_path(mapped), "label": _label(mapped, item), + "file_value": _shown_value(mapped, item), "target_value": _shown_value(mapped, have), + "file_status": status, "target_status": have.get("status"), + "why": _conflict_why(str(status), str(have.get("status"))), + }) + elif _decision(mapped, have) == _decision(mapped, item): + plan.keep += 1 + else: + plan.update += 1 + plan.writes[mapped] = item + return plan + + +def _label(key: catalog.Slot, item: Mapping[str, Any]) -> str: + if key[0] == "t": + return f"表的{field_label(key)}" + if key[0] == "c": + return f"列 {key[1]} 的{field_label(key)}" + return f"指向 {item.get('to_table')} 的关联关系" + + +def _shown_value(key: catalog.Slot, item: Mapping[str, Any]) -> Any: + if key[0] == "r": + return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")} + return item.get("value") + + +# ========================================================================== +# 预览、导入 +# ========================================================================== + + +@dataclass +class ImportReport: + tables: list[TablePlan] + unmatched: list[str] + guidance: dict[str, Any] + + def as_dict(self) -> dict[str, Any]: + return { + "tables": [t.as_dict() for t in self.tables], + "unmatched_tables": self.unmatched, + "totals": {"tables": len(self.tables), "add": sum(t.add for t in self.tables), + "update": sum(t.update for t in self.tables), "keep": sum(t.keep for t in self.tables), + "conflicts": sum(len(t.conflicts) for t in self.tables), + "skipped": sum(len(t.skipped) for t in self.tables)}, + "guidance": self.guidance, + } + + +def _guidance(file: ImportFile, row: Any, include: bool) -> dict[str, Any]: + target = catalog_jobs.guidance_of(row) + if not include or file.guidance is None: + action = "none" + elif target: + action = "keep" + else: + action = "replace" + return {"file": file.guidance, "target": target, "action": action} + + +async def plan_import(session: AsyncSession, row: Any, source: Any, content: Any, *, + include_guidance: bool) -> ImportReport: + """预览：逐表算出要新增、更新、保留什么，冲突和跳过的项及原因。不写库。row 是数据源记录（遮罩、起草要求按它）， + source 是读表结构用的源（上传源是绑定当前快照的视图）。格式不对抛 TransferInvalid。""" + file = read_file(content, include_guidance=include_guidance) + cache = getattr(source, "schema_cache", None) or {} + matched, unmatched = match_tables(list(file.tables), cache) + masked = {c.lower() for c in masked_columns(getattr(row, "options", None))} + plans: list[TablePlan] = [] + for name, target in matched.items(): + entry = await catalog.read_entry(session, source.id, target) + plan = plan_table(name, target, file.tables[name], entry.notes if entry else None, cache, masked) + plan.version = entry.version if entry else 0 + plans.append(plan) + return ImportReport(plans, unmatched, _guidance(file, row, include_guidance)) + + +async def run_import(session: AsyncSession, row: Any, source: Any, content: Any, *, include_guidance: bool, + actor: str | None) -> ImportReport: + """导入：按 plan_import 的规矩逐表写入（乐观锁，撞上别人刚改过就重读重算，最多三次），再按需写起草要求。 + 一张表写不进去只记在它的 error 上（新增、更新记 0），不影响别的表。""" + file = read_file(content, include_guidance=include_guidance) + cache = getattr(source, "schema_cache", None) or {} + matched, unmatched = match_tables(list(file.tables), cache) + masked = {c.lower() for c in masked_columns(getattr(row, "options", None))} + plans: list[TablePlan] = [] + for name, target in matched.items(): + plans.append(await _import_table(session, source, name, target, file.tables[name], cache, masked, actor)) + guidance = _guidance(file, row, include_guidance) + if guidance["action"] == "replace": + await catalog_jobs.save_guidance(session, row, file.guidance) + return ImportReport(plans, unmatched, guidance) + + +async def _import_table(session: AsyncSession, source: Any, name: str, target: str, notes: Mapping[str, Any], + cache: Mapping[str, Any], masked: set[str], actor: str | None) -> TablePlan: + plan = TablePlan(table_name=name, matched_as=target) + try: + for _attempt in range(3): + entry = await catalog.read_entry(session, source.id, target) + version = entry.version if entry else 0 + plan = plan_table(name, target, notes, entry.notes if entry else None, cache, masked) + plan.version = version + if not plan.writes: + return plan + slots = catalog._slots(entry.notes if entry else None) + slots.update(plan.writes) + try: + written = await catalog.write_entry(session, source.id, target, catalog._assemble(slots), + if_version=version, actor=actor) + except catalog.CatalogConflict: + continue + plan.version = written.version + return plan + plan.error = WRITE_CONFLICT_ERROR + except catalog.CatalogInvalid as e: + await session.rollback() + plan.error = WRITE_ERROR.format(line=str(e)) + except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 + from app.core.errors import describe_exception, first_line + + await session.rollback() + plan.error = WRITE_ERROR.format(line=first_line(e) or describe_exception(e)) + plan.add = plan.update = 0 + return plan + + +__all__ = ["FORMAT", "ImportFile", "ImportReport", "TablePlan", "TransferInvalid", "VERSION", "attachment_name", + "content_disposition", "export_catalog", "match_tables", "plan_import", "plan_table", "read_file", + "run_import"] diff --git a/backend/tests/test_catalog_transfer.py b/backend/tests/test_catalog_transfer.py new file mode 100644 index 00000000..de5164b5 --- /dev/null +++ b/backend/tests/test_catalog_transfer.py @@ -0,0 +1,378 @@ +"""数据目录的导出导入（data/catalog_transfer.py）：本机整理好的目录带到部署实例上。 + +- GET /catalog/export：附件「<数据源名>-数据目录.json」，{format: agentlab-catalog, version: 1, exported_at, + source: {name, kind}, guidance, tables: {表名: {notes, version}}}，含全部状态的项（表结构里已经没有的表也带） +- POST /catalog/import/preview：只读，报告每张对上的表新增、更新、保留、冲突、跳过了什么 +- POST /catalog/import：同样的报告，加每张表写入后的版本。规则： + - 表按名字对（忽略大小写；带 schema 前缀的对不上时去掉前缀再对），对不上的列在 unmatched_tables + - 文件里已确认、已驳回的项照带（来源、状态不变）；目标里已确认、已驳回的项和文件不一样时保留目标的、记冲突 + - 文件里推断、有确证的项只填目标的空缺，不覆盖目标已有的 + - 表结构里没有的列、关系两端对不上的跳过并写原因；目标数据源遮罩的列，码值不导入 + - 逐表乐观锁，一张表写不进去不影响别的；有排队或进行中的起草任务时 409 catalog_draft_busy；文件格式不对 422 + - 起草要求：勾了导入、目标还没有时才写；目标已有的不覆盖 + +表名、列名、业务数字都是虚构的。 +""" +from __future__ import annotations + +import copy +import json +import uuid +from urllib.parse import quote, unquote + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data import catalog, catalog_jobs, catalog_transfer +from app.data.catalog import make_item +from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION +from app.db.base import SessionLocal +from app.db.models import CatalogDraftJob, DataSource +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +ACTOR = {"X-Actor": quote("沈岚")} + + +def _col(name: str, type_: str = "INTEGER") -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": None} + + +def _tables(*, upper: bool = False) -> dict: + """两边的表结构：目标那边（upper）表名、列名写法不同，还少了一列 tier。""" + def n(name: str) -> str: + return name.upper() if upper else name + + members_cols = [_col(n("id")), _col(n("nick"), "TEXT")] + ([] if upper else [_col("tier")]) + return { + n("orders"): {"qualified": n("orders"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col(n("id")), _col(n("status")), _col(n("amount"), "REAL"), _col(n("member_id")), + _col(n("contact_phone"), "TEXT")]}, + n("members"): {"qualified": n("members"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": members_cols}, + n("refunds"): {"qualified": n("refunds"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": [_col(n("id")), _col(n("order_id"))]}, + } + + +def _rel(to_table: str, columns: list[str], to_columns: list[str], *, status: str, source: str = "name", + table: str = "orders") -> dict: + return {"id": catalog.relation_id(table, columns, to_table, to_columns), "columns": columns, + "to_table": to_table, "to_columns": to_columns, "cardinality": "many_to_one", "coverage": None, + "source": source, "status": status} + + +#: 本机（源）的目录 +SOURCE_ORDERS = { + "label": make_item("订单", "human", "confirmed", at="2026-10-01T08:00:00+00:00"), + "grain": make_item("每笔订单一行", "llm"), + "valid_filter": make_item("status <> 9", "human", "confirmed"), + "dedup": make_item("同一订单号取最后一次", "llm", "rejected"), + "kind": make_item("fact", "llm", "confirmed"), + "columns": { + "status": {"codes": make_item({"1": "已支付", "9": "作废"}, "human", "confirmed", complete=True), + "label": make_item("状态", "llm")}, + "amount": {"unit": make_item("元", "human", "confirmed"), "measure": make_item("flow", "llm")}, + "contact_phone": {"codes": make_item({"13900000001": "测试号"}, "human", "confirmed"), + "label": make_item("联系电话", "human", "confirmed")}, + # 目标表结构里没有这一列 + "gift_flag": {"label": make_item("赠品标记", "human", "confirmed")}, + }, + "relations": [ + _rel("members", ["member_id"], ["id"], status="confirmed"), + # 目标那边没有这张表 + _rel("stores", ["member_id"], ["id"], status="rejected"), + ], +} +SOURCE_MEMBERS = {"label": make_item("会员", "llm"), "keys": make_item(["tier"], "human", "confirmed"), + "columns": {"nick": {"label": make_item("昵称", "llm", "verified")}}} +#: 目标那边没有的表 +SOURCE_GHOST = {"label": make_item("旧表", "human", "confirmed")} + +#: 目标（部署实例）上已有的目录 +TARGET_ORDERS = { + "label": make_item("订单表", "human", "confirmed"), # 和文件不一样、都确认过：冲突，保留目标 + "grain": make_item("一行一单", "llm"), # 文件里是推断：保留目标 + "valid_filter": make_item("STATUS <> 9", "llm"), # 目标是推断、文件确认过：更新 + "kind": make_item("fact", "human", "confirmed"), # 一样：保留 + "columns": {"STATUS": {"label": make_item("状态码", "comment")}}, +} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _write(sid: str, table: str, notes: dict) -> None: + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, table) + await catalog.write_entry(session, sid, table, notes, if_version=entry.version if entry else 0, actor=None) + + +async def _source(name: str, tables: dict, entries: dict[str, dict], **kw) -> str: + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": tables}, **kw) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + for table, notes in entries.items(): + await _write(sid, table, notes) + return sid + + +async def _pair() -> tuple[str, str]: + local = await _source(f"本机库_{uuid.uuid4().hex[:6]}", _tables(), + {"orders": SOURCE_ORDERS, "members": SOURCE_MEMBERS, "ghost_table": SOURCE_GHOST}, + options={CATALOG_DRAFT_GUIDANCE_OPTION: "金额一律按元"}) + remote = await _source(f"deploy_{uuid.uuid4().hex[:6]}", _tables(upper=True), {"ORDERS": TARGET_ORDERS}, + options={"mask_columns": "contact_phone"}) + return local, remote + + +def _base(sid: str) -> str: + return f"/api/datasources/{sid}/catalog" + + +async def _export(client, sid: str) -> dict: + r = await client.get(f"{_base(sid)}/export") + assert r.status_code == 200, r.text + return r.json() + + +# -------------------------------------------------------------------------- +# 导出 +# -------------------------------------------------------------------------- + + +async def test_export_is_a_json_attachment_with_every_item(client): + local, _ = await _pair() + r = await client.get(f"{_base(local)}/export") + assert r.status_code == 200, r.text + assert r.headers["content-type"].startswith("application/json") + disposition = r.headers["content-disposition"] + assert disposition.startswith("attachment;") and "filename=\"agentlab-catalog-" in disposition + async with SessionLocal() as session: + name = (await session.get(DataSource, local)).name + assert unquote(disposition.split("filename*=UTF-8''", 1)[1]) == f"{name}-数据目录.json" + body = r.json() + assert body["format"] == "agentlab-catalog" and body["version"] == 1 + assert body["exported_at"] and body["source"] == {"name": name, "kind": "sqlite"} + assert body["guidance"] == "金额一律按元" + assert set(body["tables"]) == {"orders", "members", "ghost_table"} + assert body["tables"]["orders"]["version"] == 1 + # 含全部状态的项：驳回的也带 + assert body["tables"]["orders"]["notes"]["dedup"]["status"] == "rejected" + assert body["tables"]["orders"]["notes"] == SOURCE_ORDERS + assert body["tables"]["ghost_table"] == {"notes": SOURCE_GHOST, "version": 1} + + +def test_attachment_name_drops_characters_files_cannot_have(): + assert catalog_transfer.attachment_name('景区/客流:库?') == "景区_客流_库_-数据目录.json" + assert catalog_transfer.attachment_name(" ") == "数据源-数据目录.json" + + +# -------------------------------------------------------------------------- +# 预览与导入 +# -------------------------------------------------------------------------- + + +def _by(report: dict) -> dict[str, dict]: + return {t["matched_as"]: t for t in report["tables"]} + + +async def test_preview_reports_and_writes_nothing(client): + local, remote = await _pair() + content = await _export(client, local) + r = await client.post(f"{_base(remote)}/import/preview", json={"content": content, "include_guidance": True}) + assert r.status_code == 200, r.text + report = r.json() + assert report["unmatched_tables"] == ["ghost_table"] + tables = _by(report) + assert set(tables) == {"ORDERS", "MEMBERS"} + orders = tables["ORDERS"] + assert orders["table_name"] == "orders" + conflicts = {c["path"]: c for c in orders["conflicts"]} + assert set(conflicts) == {"label"} + label = conflicts["label"] + assert label["label"] == "表的中文名" and label["file_value"] == "订单" and label["target_value"] == "订单表" + assert "保留" in label["why"] + skipped = {s["path"]: s["why"] for s in orders["skipped"]} + assert "columns.gift_flag.label" in skipped and "表结构里没有列 gift_flag" in skipped["columns.gift_flag.label"] + assert "遮罩" in skipped["columns.contact_phone.codes"] + stores = next(p for p in skipped if p.startswith("relations.")) + assert "stores" in skipped[stores] + # 新增：dedup（驳回的照带）、status 码值、amount 两项、联系电话的中文名、关联关系；更新：有效记录条件 + # 保留：粒度（文件里是推断，目标已有）、表类型（一样）、STATUS 的中文名（文件里是推断，目标已有） + assert (orders["add"], orders["update"], orders["keep"]) == (6, 1, 3) + members = tables["MEMBERS"] + # 业务主键指着的列 tier 目标那边没有：跳过；会员的中文名和昵称填空缺 + assert (members["add"], members["update"], members["keep"]) == (2, 0, 0) + assert [s["path"] for s in members["skipped"]] == ["keys"] + assert report["totals"] == {"tables": 2, "add": 8, "update": 1, "keep": 3, "conflicts": 1, + "skipped": len(orders["skipped"]) + 1} + assert report["guidance"] == {"file": "金额一律按元", "target": None, "action": "replace"} + async with SessionLocal() as session: + assert (await catalog.read_entry(session, remote, "ORDERS")).version == 1 + assert await catalog.read_entry(session, remote, "MEMBERS") is None + assert catalog_jobs.guidance_of(await session.get(DataSource, remote)) is None + + +async def test_import_writes_by_the_rules_and_reports_versions(client): + local, remote = await _pair() + content = await _export(client, local) + r = await client.post(f"{_base(remote)}/import", json={"content": content, "include_guidance": True}, + headers=ACTOR) + assert r.status_code == 200, r.text + report = r.json() + tables = _by(report) + assert tables["ORDERS"]["version"] == 2 and tables["MEMBERS"]["version"] == 1 + assert tables["ORDERS"]["error"] is None + assert report["guidance"]["action"] == "replace" + async with SessionLocal() as session: + orders = await catalog.read_entry(session, remote, "ORDERS") + members = await catalog.read_entry(session, remote, "MEMBERS") + row = await session.get(DataSource, remote) + notes = orders.notes + assert orders.updated_by == "沈岚" + assert notes["label"]["value"] == "订单表" # 冲突：保留目标的 + assert notes["grain"]["value"] == "一行一单" # 推断只填空缺 + assert notes["valid_filter"] == {**SOURCE_ORDERS["valid_filter"]} # 照带：来源、状态不变 + assert notes["dedup"]["status"] == "rejected" and notes["dedup"]["source"] == "llm" + codes = notes["columns"]["STATUS"]["codes"] + assert codes["value"] == {"1": "已支付", "9": "作废"} and codes["complete"] is True + assert notes["columns"]["STATUS"]["label"]["value"] == "状态码" + assert notes["columns"]["AMOUNT"]["measure"] == SOURCE_ORDERS["columns"]["amount"]["measure"] + # 遮罩列：中文名照带，码值不导入 + assert notes["columns"]["CONTACT_PHONE"] == {"label": SOURCE_ORDERS["columns"]["contact_phone"]["label"]} + assert "GIFT_FLAG" not in notes["columns"] and "gift_flag" not in notes["columns"] + [rel] = notes["relations"] + assert rel["id"] == catalog.relation_id("ORDERS", ["MEMBER_ID"], "MEMBERS", ["ID"]) + assert (rel["columns"], rel["to_table"], rel["to_columns"]) == (["MEMBER_ID"], "MEMBERS", ["ID"]) + assert rel["status"] == "confirmed" and rel["source"] == "name" + assert catalog.validate_notes(notes) == [] + assert members.notes["label"]["status"] == "proposed" and "keys" not in members.notes + assert members.notes["columns"]["NICK"]["label"]["status"] == "verified" + assert catalog_jobs.guidance_of(row) == "金额一律按元" + # 再导一次：都对上了，什么都不写；目标已经有起草要求，不覆盖 + again = (await client.post(f"{_base(remote)}/import", json={"content": content, "include_guidance": True})).json() + assert again["totals"]["add"] == again["totals"]["update"] == 0 + assert _by(again)["ORDERS"]["version"] == 2 + assert again["guidance"] == {"file": "金额一律按元", "target": "金额一律按元", "action": "keep"} + + +async def test_target_rejection_beats_file_confirmation_and_guidance_is_optional(client): + local, remote = await _pair() + await _write(remote, "MEMBERS", {"label": make_item("会员", "llm", "rejected")}) + async with SessionLocal() as session: + row = await session.get(DataSource, remote) + await catalog_jobs.save_guidance(session, row, "部署实例自己的要求") + content = await _export(client, local) + content["tables"]["members"]["notes"]["label"] = make_item("会员", "human", "confirmed") + report = (await client.post(f"{_base(remote)}/import", + json={"content": content, "include_guidance": False})).json() + members = _by(report)["MEMBERS"] + [conflict] = members["conflicts"] + assert conflict["path"] == "label" and conflict["target_status"] == "rejected" + assert conflict["file_status"] == "confirmed" and "驳回" in conflict["why"] + assert report["guidance"] == {"file": "金额一律按元", "target": "部署实例自己的要求", "action": "none"} + async with SessionLocal() as session: + entry = await catalog.read_entry(session, remote, "MEMBERS") + assert entry.notes["label"]["status"] == "rejected" + assert catalog_jobs.guidance_of(await session.get(DataSource, remote)) == "部署实例自己的要求" + + +async def test_one_table_failing_does_not_stop_the_others(client, monkeypatch): + local, remote = await _pair() + content = await _export(client, local) + real = catalog.write_entry + + async def flaky(session, source_id, table, notes, **kw): + if table == "ORDERS": + raise catalog.CatalogConflict(table, 99) + return await real(session, source_id, table, notes, **kw) + + monkeypatch.setattr(catalog, "write_entry", flaky) + report = (await client.post(f"{_base(remote)}/import", json={"content": content})).json() + tables = _by(report) + assert tables["ORDERS"]["error"] and "稍后" in tables["ORDERS"]["error"] + assert tables["ORDERS"]["add"] == tables["ORDERS"]["update"] == 0 + assert tables["MEMBERS"]["error"] is None and tables["MEMBERS"]["version"] == 1 + + +async def test_import_waits_for_draft_jobs_and_rejects_bad_files(client): + local, remote = await _pair() + content = await _export(client, local) + async with SessionLocal() as session: + session.add(CatalogDraftJob(id=uuid.uuid4().hex, source_id=remote, status="running")) + await session.commit() + busy = await client.post(f"{_base(remote)}/import", json={"content": content}) + assert busy.status_code == 409 and busy.json()["code"] == "catalog_draft_busy" + # 预览是只读的，照常给 + assert (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).status_code == 200 + bad = [ + "不是对象", + {**content, "format": "something-else"}, + {**content, "version": 2}, + {**content, "tables": []}, + {**content, "tables": {"orders": {"notes": "x"}}}, + {**content, "guidance": 3}, + ] + for body in bad: + r = await client.post(f"{_base(local)}/import/preview", json={"content": body}) + assert r.status_code == 422, body + assert any("一" <= ch <= "鿿" for ch in r.json()["detail"]) + too_long = {**content, "guidance": "长" * (catalog_jobs.GUIDANCE_MAX + 1)} + r = await client.post(f"{_base(local)}/import/preview", json={"content": too_long, "include_guidance": True}) + assert r.status_code == 422 and "起草要求" in r.json()["detail"] + + +async def test_invalid_items_are_skipped_not_fatal(client): + local, remote = await _pair() + content = await _export(client, local) + notes = content["tables"]["orders"]["notes"] + notes["kind"] = {"value": "event", "source": "human", "status": "confirmed"} # 取值不在可选值内 + notes["formula"] = {"value": "sum(amount)", "source": "human", "status": "confirmed"} # 不认识的字段 + notes["columns"]["amount"]["colour"] = {"value": "红", "source": "human", "status": "confirmed"} + report = (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).json() + skipped = {s["path"]: s["why"] for s in _by(report)["ORDERS"]["skipped"]} + assert "明细表" in skipped["kind"] + assert "formula" in skipped["formula"] and "columns.amount.colour" in skipped + + +async def test_import_needs_a_schema(client): + local, _ = await _pair() + content = await _export(client, local) + empty = await _source(f"empty_{uuid.uuid4().hex[:6]}", {}, {}) + r = await client.post(f"{_base(empty)}/import/preview", json={"content": content}) + assert r.status_code == 409 and r.json()["code"] == "schema_missing" + + +async def test_round_trip_into_an_empty_copy_reproduces_the_catalog(client): + local, _ = await _pair() + content = await _export(client, local) + copy_sid = await _source(f"copy_{uuid.uuid4().hex[:6]}", _tables(), {}) + report = (await client.post(f"{_base(copy_sid)}/import", json={"content": json.loads(json.dumps(content)), + "include_guidance": True})).json() + assert report["unmatched_tables"] == ["ghost_table"] + assert all(not t["conflicts"] for t in report["tables"]) + async with SessionLocal() as session: + mine = await catalog.read_entry(session, local, "orders") + theirs = await catalog.read_entry(session, copy_sid, "orders") + # 关系两端都对得上的照带；指向目标那边没有的表的跳过 + expected = copy.deepcopy(mine.notes) + expected["relations"] = [r for r in mine.notes["relations"] if r["to_table"] != "stores"] + del expected["columns"]["gift_flag"] # 表结构里没有这一列 + assert theirs.notes == expected From a5d9ee0a16af9c9ed231b79f7c74380fc37702f7 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:20:02 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=EF=BC=9A?= =?UTF-8?q?=E8=A1=A8=E8=AF=A6=E6=83=85=E9=A1=B5=E9=87=8C=E5=92=8C=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E8=80=85=E5=AF=B9=E8=AF=9D=E5=AE=8C=E5=96=84=E4=B8=80?= =?UTF-8?q?=E5=BC=A0=E8=A1=A8=E7=9A=84=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=EF=BC=88POST=20/catalog/{table}/assistant=EF=BC=8CSSE=EF=BC=9A?= =?UTF-8?q?=E6=AD=A3=E6=96=87=E5=A2=9E=E9=87=8F=E3=80=81=E5=8F=AA=E8=AF=BB?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E3=80=81=E4=BF=AE=E6=94=B9=E6=8F=90=E6=A1=88?= =?UTF-8?q?=E3=80=81=E5=AE=8C=E6=88=90=E3=80=81=E5=87=BA=E9=94=99=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E4=BA=A4=E7=BB=99=E6=A8=A1=E5=9E=8B=E8=A1=A8=E7=BB=93?= =?UTF-8?q?=E6=9E=84=E3=80=81=E7=9B=AE=E5=BD=95=E7=8E=B0=E7=8A=B6=EF=BC=88?= =?UTF-8?q?=E6=AF=8F=E9=A1=B9=E7=9A=84=E6=9D=A5=E6=BA=90=E5=92=8C=E7=8A=B6?= =?UTF-8?q?=E6=80=81=EF=BC=89=E3=80=81=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB?= =?UTF-8?q?=EF=BC=88=E5=90=AB=E5=88=AB=E7=9A=84=E8=A1=A8=E6=8C=87=E8=BF=87?= =?UTF-8?q?=E6=9D=A5=E7=9A=84=EF=BC=89=E3=80=81=E5=89=96=E6=9E=90=E7=BB=93?= =?UTF-8?q?=E8=AE=BA=E3=80=81=E8=B5=B7=E8=8D=89=E8=A6=81=E6=B1=82=E3=80=81?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0=E3=80=81=E6=9C=80=E8=BF=91?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E8=B5=B7=E8=8D=89=E7=95=99=E7=A9=BA=E7=9A=84?= =?UTF-8?q?=E5=88=97=E5=92=8C=E7=95=8C=E9=9D=A2=E4=B8=8A=E5=BE=85=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E7=9A=84=E4=BF=AE=E6=94=B9=EF=BC=8C=E9=81=AE=E7=BD=A9?= =?UTF-8?q?=E5=88=97=E5=8F=AA=E7=BB=99=E5=88=97=E5=90=8D=EF=BC=9B=E6=8F=90?= =?UTF-8?q?=E6=A1=88=E6=8C=89=20plan=5Fpatch=20=E6=A0=B8=E5=AF=B9=EF=BC=88?= =?UTF-8?q?=E5=85=AC=E5=BC=8F=E4=B8=8D=E6=94=B6=E3=80=81=E5=88=97=E5=90=8D?= =?UTF-8?q?=E5=BF=85=E9=A1=BB=E5=9C=A8=E8=A1=A8=E7=BB=93=E6=9E=84=E9=87=8C?= =?UTF-8?q?=E3=80=81=E9=81=AE=E7=BD=A9=E5=88=97=E4=B8=8D=E7=BB=99=E7=A0=81?= =?UTF-8?q?=E5=80=BC=E3=80=81=E5=B7=B2=E7=A1=AE=E8=AE=A4=E7=9A=84=E5=90=8C?= =?UTF-8?q?=E5=80=BC=E4=B8=8D=E6=8F=90=EF=BC=89=EF=BC=8C=E5=BE=85=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E4=BF=AE=E6=94=B9=E9=87=8C=E7=9A=84=E7=A0=81=E5=80=BC?= =?UTF-8?q?=E5=B9=B6=E8=BF=9B=E6=96=B0=E6=8F=90=E7=9A=84=E7=A0=81=E5=80=BC?= =?UTF-8?q?=EF=BC=8C=E5=8F=AA=E4=BA=A4=E7=BB=99=E7=95=8C=E9=9D=A2=E4=B8=8D?= =?UTF-8?q?=E5=86=99=E7=9B=AE=E5=BD=95=EF=BC=9B=E5=8F=AA=E8=AF=BB=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=8F=AA=E5=9C=A8=E5=BC=80=E4=BA=86=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=89=96=E6=9E=90=E7=9A=84=E6=BA=90=E4=B8=8A=E7=BB=99=EF=BC=8C?= =?UTF-8?q?=E7=BB=8F=E5=89=96=E6=9E=90=E7=9A=84=E5=8F=AA=E8=AF=BB=E5=AE=88?= =?UTF-8?q?=E5=8D=AB=E5=92=8C=E5=8D=95=E6=9D=A1=E6=97=B6=E9=99=90=E3=80=81?= =?UTF-8?q?=E6=AF=8F=E6=AC=A1=E8=AF=B7=E6=B1=82=E6=9C=80=E5=A4=9A=205=20?= =?UTF-8?q?=E6=9D=A1=E3=80=8150=20=E8=A1=8C=E4=BA=A4=E7=BB=99=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=EF=BC=8C=E7=BB=93=E6=9E=9C=E5=92=8C=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E5=85=88=E9=81=AE=E5=86=8D=E4=BA=A4=EF=BC=9B=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E5=8F=96=E6=B3=95=E3=80=81=E8=AF=B4=E6=B3=95=E5=90=8C=E8=B5=B7?= =?UTF-8?q?=E8=8D=89=EF=BC=88=E7=94=A8=E4=B8=8D=E4=BA=86=20409=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E4=BF=9D=E7=95=99=E6=80=9D=E8=80=83=E3=80=81=E9=A2=9D?= =?UTF-8?q?=E5=BA=A6=2032768=20=E5=8F=AF=E9=80=80=E5=9B=9E=208192=E3=80=81?= =?UTF-8?q?=E5=8D=95=E6=AC=A1=E6=97=B6=E9=99=90=E5=90=8C=E8=B5=B7=E8=8D=89?= =?UTF-8?q?=EF=BC=8C=E8=BF=98=E6=B2=A1=E8=BE=93=E5=87=BA=E6=97=B6=E7=BD=91?= =?UTF-8?q?=E7=BB=9C=E9=94=99=E8=AF=AF=E5=92=8C=20429=E3=80=815xx=20?= =?UTF-8?q?=E9=87=8D=E8=AF=95=EF=BC=8C=E7=AD=89=E5=BE=85=E6=97=B6=E5=8F=91?= =?UTF-8?q?=20SSE=20=E6=B3=A8=E9=87=8A=E4=BF=9D=E6=B4=BB=EF=BC=8C=E5=81=9C?= =?UTF-8?q?=E6=AD=A2=E6=97=B6=E6=AD=A3=E5=9C=A8=E8=BF=9B=E8=A1=8C=E7=9A=84?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E8=B7=9F=E7=9D=80=E5=81=9C=EF=BC=9BPATCH=20?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E6=9C=80=E5=A4=9A=20200=20=E9=A1=B9=EF=BC=88?= =?UTF-8?q?=E9=A2=84=E8=A7=88=E5=92=8C=E4=BF=9D=E5=AD=98=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/catalog.py | 42 +- backend/app/data/catalog.py | 5 +- backend/app/data/catalog_assistant.py | 878 ++++++++++++++++++++++++ backend/tests/test_catalog_assistant.py | 550 +++++++++++++++ 4 files changed, 1472 insertions(+), 3 deletions(-) create mode 100644 backend/app/data/catalog_assistant.py create mode 100644 backend/tests/test_catalog_assistant.py diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index b3554756..370c412e 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -18,6 +18,8 @@ （data/catalog_review.py） - GET /catalog/export 导出数据目录（JSON 附件）；POST /catalog/import/preview 导入预览（只读）， POST /catalog/import 导入（data/catalog_transfer.py） +- POST /catalog/{table}/assistant 表助手：对话完善这张表的目录（text/event-stream），提案不写目录 + （data/catalog_assistant.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 @@ -40,7 +42,7 @@ from app.api.coded import ( ) from app.api.runs import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions -from app.data import catalog_review, catalog_transfer +from app.data import catalog_assistant, catalog_review, catalog_transfer from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -690,6 +692,44 @@ async def save_catalog_patch(source_id: str, table: str, payload: CatalogPatchIn return await _detail(session, source, key, entry) +class AssistantMessageIn(BaseModel): + role: Literal["user", "assistant"] + content: str = Field(max_length=catalog_assistant.MESSAGE_MAX) + + +class AssistantIn(BaseModel): + #: 整段对话（服务端不存）；没有使用者的话时助手先开口 + messages: list[AssistantMessageIn] = Field(default_factory=list, max_length=catalog_assistant.MAX_MESSAGES) + #: 界面上还没保存的修改，让模型知道 + pending: list[CatalogPatchChange] = Field(default_factory=list, max_length=catalog_assistant.MAX_PENDING) + + +@router.post("/{source_id}/catalog/{table}/assistant") +async def catalog_table_assistant(source_id: str, table: str, payload: AssistantIn, + session: AsyncSession = Depends(get_session)) -> Any: + """表助手：和使用者对话完善这张表的目录，返回 text/event-stream（事件见 data/catalog_assistant.py）。提案只交给界面， + 不写目录；保存走 /catalog/{table}/patch。开始前：表不在 404；对话不合规 422；起草模型用不了 409 + catalog_draft_model_unavailable（说法同起草任务）。""" + from fastapi.responses import StreamingResponse + + row, source = await _resolved(session, source_id) + key = catalog.resolve_table_name(source.schema_cache, table) + if key is None: + raise HTTPException(404, f"数据源中没有表 {table}，可能已被删除或尚未探查结构") + messages = [m.model_dump() for m in payload.messages] + if problem := catalog_assistant.conversation_problem(messages): + raise HTTPException(422, problem) + try: + model, model_id = await catalog_assistant.resolve_model(session) + except catalog.CatalogModelUnavailable as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_MODEL_UNAVAILABLE) from e + context = await catalog_assistant.prepare(session, row, source, key, + pending=[c.model_dump() for c in payload.pending]) + return StreamingResponse(catalog_assistant.stream(model, model_id, context, messages), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + @router.get("/{source_id}/catalog/{table}/impact") async def catalog_table_impact(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index cdef28cf..40bdaab8 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -860,8 +860,9 @@ async def save_human_edit(session: AsyncSession, source_id: str, table: str, sub #: 新增关联关系的路径：relations.new，值写两端；编号按两端算（relation_id），同一条关系不论谁提都是同一个编号 NEW_RELATION_PATH = "relations.new" -#: 一条提案最多几项：用户说的是一两条事实，几十项的「提案」多半是模型在重写整张表 -PATCH_MAX_CHANGES = 20 +#: 一条提案最多几项（预览、保存都按它）。对话里的纠正通常是一两条事实；表助手（data/catalog_assistant.py）会逐列补中文名、 +#: 含义，界面上跨几轮攒下的待保存修改一次保存，宽表上常常几十上百项，所以放到 200 +PATCH_MAX_CHANGES = 200 _REASON_MAX = 500 #: 看起来像计算公式的文字：「转化率=下单数/访问数」「= SUM(amount)」。公式进口径卡，目录只记数据事实。 #: 不认减号：「上线日期=2024-01-01」不是公式。有效记录条件本来就是 SQL 条件，不查它。 diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py new file mode 100644 index 00000000..475a6a5e --- /dev/null +++ b/backend/app/data/catalog_assistant.py @@ -0,0 +1,878 @@ +"""表助手：在表详情页和使用者对话，一起完善一张表的数据目录（POST /catalog/{table}/assistant，SSE）。 + +**为什么要有它。** 模型起草只看表结构，缩写列名、状态码的意思常常拿不准而留空；审阅时人知道答案，却要一项一项填。表助手 +把这张表的结构、目录现状（每项的来源和状态）、关联关系、剖析结果、起草要求、使用次数、上次起草留空的列交给模型，让它主动 +问拿不准的地方；使用者说出一条业务事实，它整理成成批的修改提案（形状同助手流里的 catalog_patch），使用者在界面上逐项看过、 +一次保存（PATCH /catalog/{table}/patch，记为人工确认）。 + +**只改目录、不改业务库。** 模型只有两个工具： +- propose_changes：一批修改提案。按 catalog.plan_patch 核对（取值格式、列名必须在表结构里、公式不收），遮罩列不给码值，已经 + 是这个值又确认过的不提；核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。 + 界面上还没保存的修改（pending）里同一列的码值并进新提的码值：界面上同一路径以后一次为准，不能把先提的码值丢了。 +- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限（catalog_profile._Runner： + guard.check(readonly=True)，单条时限取剖析设置和数据源查询时限中较小的一个），每次请求最多 MAX_QUERIES 条（守卫拒掉的 + 也算），最多 QUERY_ROWS 行交给模型、SHOWN_ROWS 行交给界面。结果交给模型、界面之前先遮（engine/masking.mask_query_result： + 取自遮罩列的结果列整列换成「已遮罩」，改了名字的也认；这一次请求里前面遮掉的值按值换），报错原话里的值也换。没开剖析时 + 提示词里写明只能根据表结构和目录推断。 + +**遮罩列只给列名。** 表结构里遮罩的列只写列名和类型（不给注释），目录里这些列的码值、带着取值的备注一律不给（剖析的备注里 +有取值范围）；界面上还没保存的修改里这些列的码值也不给。 + +**模型。** 取模型的顺序同起草（data/draft_model.py：起草模型 → 助手的模型 → 默认接入），用不了时的说法也同起草 +（先过 catalog.resolve_draft_model，接口回 409 catalog_draft_model_unavailable）。保留思考、温度 0，额度 +catalog.MODEL_MAX_TOKENS，服务方嫌大时退回 catalog.MODEL_FALLBACK_MAX_TOKENS（providers/budget.BudgetFallback）；单次调用 +最多等 catalog.MODEL_TIMEOUT_S（同起草）。网络错误、429、5xx 在还没输出任何东西时重试（最多 len(RETRY_DELAYS_S) 次）。 +一次请求最多调用 MAX_ROUNDS 次模型（查询、提案之后接着答）；倒数第二轮的工具结果里提醒它直接回答。 + +**事件**（每个事件一行 data: ）： +- {"type": "text", "delta"}：回答正文的增量。几轮之间补一个空行； +- {"type": "query", "sql", "columns", "rows"（前 SHOWN_ROWS 行）, "row_count", "truncated", "error"}：跑了一条只读查询； +- {"type": "proposal", "source", "source_id", "table", "table_label", "version", "changes": [{path, label, before, + before_status, after, value, reason, state, note?}]}：一批核对过的修改提案； +- {"type": "done", "usage": {input_tokens, output_tokens}, "queries"}； +- {"type": "error", "message", "code"}：中途出错，流到此为止（之后没有 done）。code：assistant_model_error 调用模型出错、 + assistant_truncated 输出到了额度上限被截断、assistant_timeout 单次调用超过时限、assistant_rounds 调用次数用完还没答完、 + assistant_failed 意料之外的错误。 +等模型的时候每 HEARTBEAT_S 秒发一行 SSE 注释（「: keep-alive」，不是事件），免得中间的代理断开。 + +**对话不存服务端**：每次请求带上整段对话（最多 MAX_MESSAGES 条、每条最多 MESSAGE_MAX 字）。查询结果只在这一次请求里 +看得到，提示词里让模型把要用的结论写进回答。 + +写给模型的文字都在名字带 _SYSTEM / _PROMPT 的常量里；这里的 ERROR_* 是给人看的。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import json +import logging +from collections.abc import AsyncIterator, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_jobs, catalog_profile +from app.data.catalog_review import SOURCE_LABEL, slot_path +from app.data.draft_model import DRAFT_MODEL_PLACE +from app.data.engine import SnapshotTampered, masked_columns + +logger = logging.getLogger(__name__) + +#: 对话最多几条、每条最多几个字；界面上待保存的修改最多几项（同 PATCH 的上限） +MAX_MESSAGES = 40 +MESSAGE_MAX = 4000 +MAX_PENDING = catalog.PATCH_MAX_CHANGES +#: 每次请求最多调用几次模型、最多几条只读查询 +MAX_ROUNDS = 8 +MAX_QUERIES = 5 +#: 一条查询最多几行交给模型、几行交给界面；交给模型时每格最多几个字、一条结果最多几个字 +QUERY_ROWS = 50 +SHOWN_ROWS = 10 +CELL_MAX = 300 +TOOL_CHARS = 12_000 +#: 网络错误、429、5xx 重试前等几秒（依次取用，个数就是最多重试几次） +RETRY_DELAYS_S = (2.0, 5.0) +#: 等模型的时候每隔几秒发一行 SSE 注释 +HEARTBEAT_S = 10.0 +#: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条 +UNNAMED_SHOWN = 30 +INCOMING_SHOWN = 30 + +QUERY_TOOL = "run_query" +PROPOSE_TOOL = "propose_changes" + +#: 错误事件的机读码 +CODE_MODEL = "assistant_model_error" +CODE_TRUNCATED = "assistant_truncated" +CODE_TIMEOUT = "assistant_timeout" +CODE_ROUNDS = "assistant_rounds" +CODE_FAILED = "assistant_failed" + +# ---- 给人看的 +ERROR_MODEL = ("调用模型「{model}」失败：{reason}。请稍后重试；如持续失败，请在{place}中换一个模型，" + "或到「设置 → 模型接入」测试该模型") +ERROR_TRUNCATED = "{cut}，回答没有写完。通常是思考占满了输出额度，可以把问题拆小一点再问，或在{place}中换一个模型" +ERROR_TIMEOUT = "模型 {seconds} 秒内没有回答完，已停止这一轮。可以把问题拆小一点再问，或在{place}中换一个模型" +ERROR_ROUNDS = "这一轮已经调用了 {n} 次模型仍没有答完，已停止。可以换个问法，或者分几次问" +ERROR_FAILED = "表助手出错：{line}" +#: 对话的最后一条是助手说的（接口 422） +LAST_NOT_USER = "对话的最后一条应当是使用者的话，请重新发送" +#: 起草模型刚才还能用、建流式模型时却用不了（期间设置被改了）时的说法 +ERROR_NO_MODEL = "起草模型暂时无法使用，请稍后重试，或到「设置 → 模型接入」检查" +#: 查询事件里的报错 +QUERY_TAMPERED = "数据源「{name}」的{error}。请联系管理员检查数据目录" + +# ---- 写给模型的 +ASSISTANT_SYSTEM = """你是 AgentLab 的「表助手」，和使用者一起完善一张表的数据目录：表和列的中文名、含义、单位、度量类型、码值、\ +粒度、表类型、业务主键、业务日期、有效记录条件、去重规则、关联关系。 + +规矩： +- 你只改数据目录，不改业务库：不能写数据、建表、改结构。 +- 只记数据事实，不写计算公式：「转化率=下单数/访问数」这类指标怎么算的说法属于口径卡，不进数据目录。 +- 拿不准就问，不要猜。一次只问一两件最要紧的事，问得具体（例如「status 为 9 的行是不是作废的单子？」）。目录里留空的列、\ +标着「推断」的项是最值得问的。 +- 使用者说出一条业务事实时，用 propose_changes 把它整理成修改提案，reason 里引用使用者的原话（例如「使用者说：9 是作废」）。\ +一次可以提多项。提案只是建议：使用者在界面上看过、保存之后才写进数据目录，记为人工确认。 +- 表名、列名照原样写，只能用下面表结构里有的列。 +- 标着「已确认」的项由人工确认过，使用者没有明确要改就不要提修改；标着「已驳回」的项使用者驳回过，不要再提。 +- 「界面上还没保存的修改」是使用者已经接受、还没保存的提案：不要重复提；要改其中一项就再提一次，写完整的新值。 +- 遮罩的列只给了列名：不要猜它的取值，也不要给它写码值。 +- 回答用中文，简短直接。查询结果只在这一轮看得到，后面要用的结论请写进回答里。 + +修改提案的写法（propose_changes 的 changes，每项 path、value、reason）： +- path：表级项直接写字段名（label 中文名、description 说明、grain 粒度、keys 业务主键的列名列表、kind 表类型 \ +fact/dimension/snapshot/log/config、business_date {"column","rule","timezone"}、valid_filter 有效记录的 SQL 条件、dedup 去重\ +规则）；列级项写 columns.<列名>.<字段>（label、meaning 含义、unit 单位、measure 度量类型 \ +flow/stock/ratio/identifier/status/attribute、codes 码值 {"码":"含义"}）；已有的关联关系写 relations.<编号>；新增关联关系\ +写 relations.new，value 写 {"columns":[本表列],"to_table":"目标表","to_columns":[目标列],\ +"cardinality":"many_to_one/one_to_one/one_to_many"}。 +- codes 只写要补充或更正的码，原有的码会保留；valid_filter 要写成完整的条件（原有条件也要带上）。""" + +#: 查询工具的说明（开了剖析时接在系统提示后面）、没开剖析时的那句 +ASSISTANT_QUERY_PROMPT = """只读查询：run_query 在这个数据源上执行一条只读 SQL（{dialect} 的写法），最多返回 {rows} 行，\ +这一轮最多 {limit} 条。用它核对码值的取值和分布、日期范围、关联对不对得上；不要取明细的个人信息。遮罩列的值显示为「已遮罩」。""" +ASSISTANT_NO_QUERY_PROMPT = "这个数据源没开剖析，不能查数据，只能根据表结构和目录推断。" +ASSISTANT_DISABLED_PROMPT = "这个数据源已停用，不能查数据，只能根据表结构和目录推断。" +#: 使用者还没说话（刚打开表助手）时补的那句 +ASSISTANT_OPENING_PROMPT = ("（使用者刚打开表助手，还没有说话。）请用一两句话说明这张表的目录现在最缺什么，然后就最要紧的一两项" + "向使用者提问；先不要提修改提案。") +#: 对话从助手的开场白开始时，补在最前面的一句（各家的接口都要求从使用者这一边开始） +ASSISTANT_OPENED_PROMPT = "（使用者打开了表助手。）" + +#: 背景各段的写法 +ASSISTANT_PROMPT_LABELS = { + "head": "下面是要完善的表。", + "table": "数据源「{source}」（{kind}），表 {table}{view}", + "view": "（视图）", + "comment": "数据库注释：{text}", + "system": "系统说明：{text}", + "usage": "这张表被运行用过 {n} 次。", + "description": "数据源说明：{text}", + "guidance": "数据源的起草要求（使用者补充的背景和要求）：{text}", + "structure": "表结构：", + "column": "- {name} {type}{marks}", + "pk": "主键", + "not_null": "非空", + "col_comment": "注释：{text}", + "masked": "遮罩列：只给列名，不给任何取值", + "unique": "唯一约束：{cols}", + "fk": "外键：{cols} → {target}", + "catalog": "这张表现在的数据目录（每项写明来源和状态；路径就是修改提案里的 path）：", + "catalog_empty": "这张表还没有数据目录。", + "item": "- {path}（{label}）：{value}〔{source}，{status}〕", + "item_note": " 备注：{text}", + "relation": "{columns} → {to_table}.{to_columns}{extra}", + "relation_extra": "（{text}）", + "cardinality": "基数 {value}", + "coverage": "覆盖率 {value}", + "incoming": "别的表指向这张表的关联关系：", + "incoming_item": "- {table}.{columns} → {target}.{to_columns}〔{source}，{status}〕", + "blank": "最近一次起草时模型拿不准、留空的列：{cols}（共 {n} 列）", + "unnamed": "目录里还没有中文名和含义的列：{cols}（共 {n} 列{more}）", + "unnamed_more": "，这里列出前 {n} 个", + "pending": "界面上还没保存的修改（使用者已经接受、还没保存）：", + "pending_item": "- {path}：{value}（依据：{reason}）", + "status_proposed": "推断", + "status_verified": "有确证", + "status_confirmed": "已确认", + "status_rejected": "已驳回", + "field_relation": "关联关系", +} + +#: 工具结果的写法（交回模型） +ASSISTANT_TOOL_PROMPT = { + "proposed": "已把 {n} 项修改交给使用者确认（使用者保存后才写进数据目录）：{paths}", + "nothing": "没有可以交给使用者的修改。", + "dropped": "没有收下的项：{reasons}", + "masked_codes": "列 {column} 在数据源设置中被遮罩，不能给它写码值", + "same": "{where}已经是这个值、也确认过了，不用再提", + "no_changes": "提案里没有要修改的项", + "query_failed": "查询失败：{error}", + "query_capped": "未执行：这一轮最多查 {n} 条，不能再查了。请根据已经查到的结果回答", + "query_disabled": "未执行：这个数据源没开数据剖析，不能查数据", + "unknown_tool": "没有名为 {name} 的工具，可用的工具：{tools}", + "last_round": "（这一轮只剩最后一步了：请直接回答使用者，不要再调用工具。）", + "empty_sql": "未执行：没有给出 SQL", +} + +QUERY_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": QUERY_TOOL, + "description": "在这个数据源上执行一条只读 SQL 查询，返回列名和前几行（遮罩列的值显示为「已遮罩」）。", + "parameters": { + "type": "object", + "properties": {"sql": {"type": "string", "description": "一条只读的 SELECT 语句"}}, + "required": ["sql"], + }, + }, +} +PROPOSE_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": PROPOSE_TOOL, + "description": "把使用者说出的业务事实整理成一批数据目录修改提案，交给使用者确认后保存。不会直接写进数据目录。", + "parameters": { + "type": "object", + "properties": { + "changes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "要改的项，写法见系统提示"}, + "value": {"description": "新的值：文字、列名列表、码值对照或关联关系"}, + "reason": {"type": "string", "description": "一句话依据，引用使用者的原话"}, + }, + "required": ["path", "value", "reason"], + }, + }, + }, + "required": ["changes"], + }, + }, +} + + +# ========================================================================== +# 模型 +# ========================================================================== + + +async def resolve_model(session: AsyncSession) -> tuple[Any, str]: + """(模型, 模型 id)。能不能用、用不了时的说法和起草完全一致：先过 catalog.resolve_draft_model（起草任务建任务时也是这样 + 查的），用不了抛 catalog.CatalogModelUnavailable。再按同一个接入、模型建流式对话用的模型：保留思考、温度 0，大小两份额度 + （BudgetFallback），单次时限同起草。只建模型对象，不发请求。""" + from app.data.draft_model import DraftModelUnavailable, draft_unavailable, resolve_draft_provider + from app.providers.budget import BudgetFallback + from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model + + await catalog.resolve_draft_model(session) + try: + provider, choice = await resolve_draft_provider(session, none_reason=ERROR_NO_MODEL, mock_reason=ERROR_NO_MODEL) + except DraftModelUnavailable as e: + raise catalog.CatalogModelUnavailable(str(e)) from e + spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=catalog.MODEL_MAX_TOKENS, temperature=0, + timeout=int(catalog.MODEL_TIMEOUT_S)) + try: + big = build_chat_model(provider, spec) + small = build_chat_model(provider, spec.model_copy(update={"max_tokens": catalog.MODEL_FALLBACK_MAX_TOKENS})) + except ProviderNotConfigured as e: + raise catalog.CatalogModelUnavailable(draft_unavailable(e, choice.origin)) from e + model_id = spec.model or provider.default_model or "" + return BudgetFallback(big, small, limits=(catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS)), model_id + + +# ========================================================================== +# 对话 +# ========================================================================== + + +def conversation_problem(messages: Iterable[Mapping[str, Any]]) -> str | None: + """对话本身的问题（接口 422）：使用者说过话、最后一条却是助手说的。没有使用者的话是「刚打开表助手」，不算问题。""" + said = [m for m in messages if str(m.get("content") or "").strip()] + if any(m.get("role") == "user" for m in said) and said[-1].get("role") != "user": + return LAST_NOT_USER + return None + + +def _history(messages: Iterable[Mapping[str, Any]]) -> list[Any]: + """对话 → 模型的消息：空的不要，同一方接连说的并成一条；从助手的开场白开始的，前面补一句使用者打开了表助手；使用者还没 + 说话的，补一句开场的请求。""" + from langchain_core.messages import AIMessage, HumanMessage + + merged: list[tuple[str, str]] = [] + for m in messages: + text = str(m.get("content") or "").strip() + if not text: + continue + role = "user" if m.get("role") == "user" else "assistant" + if merged and merged[-1][0] == role: + merged[-1] = (role, f"{merged[-1][1]}\n\n{text}") + else: + merged.append((role, text)) + out: list[Any] = [] + if merged and merged[0][0] == "assistant": + out.append(HumanMessage(content=ASSISTANT_OPENED_PROMPT)) + for role, text in merged: + out.append(HumanMessage(content=text) if role == "user" else AIMessage(content=text)) + if not any(role == "user" for role, _ in merged): + out.append(HumanMessage(content=ASSISTANT_OPENING_PROMPT)) + return out + + +# ========================================================================== +# 背景 +# ========================================================================== + + +def _value_text(value: Any) -> str: + return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) + + +def _status_text(status: Any) -> str: + return ASSISTANT_PROMPT_LABELS.get(f"status_{status}", str(status)) + + +def _source_text(source: Any) -> str: + return SOURCE_LABEL.get(str(source), str(source)) + + +def _relation_line(rel: Mapping[str, Any]) -> str: + labels = ASSISTANT_PROMPT_LABELS + extra = [] + if rel.get("cardinality"): + extra.append(labels["cardinality"].format(value=rel["cardinality"])) + coverage = rel.get("coverage") + if isinstance(coverage, (int, float)) and not isinstance(coverage, bool): + extra.append(labels["coverage"].format(value=f"{coverage:.0%}")) + return labels["relation"].format( + columns=", ".join(str(c) for c in rel.get("columns") or []), to_table=rel.get("to_table"), + to_columns=", ".join(str(c) for c in rel.get("to_columns") or []), + extra=labels["relation_extra"].format(text="，".join(extra)) if extra else "") + + +def _touches_masked(key: catalog.Slot, item: Mapping[str, Any], masked: set[str]) -> bool: + """这一项说的是不是遮罩列：列级的项看列名，业务主键、业务日期看指着的列，关联关系看本表这一端。""" + if key[0] == "c": + return key[1].lower() in masked + if key[0] == "r": + return any(str(c).lower() in masked for c in item.get("columns") or []) + value = item.get("value") + if key == ("t", "business_date") and isinstance(value, Mapping): + return str(value.get("column") or "").lower() in masked + if key == ("t", "keys") and isinstance(value, list): + return any(str(c).lower() in masked for c in value) + return False + + +def _catalog_lines(notes: Mapping[str, Any] | None, masked: set[str]) -> list[str]: + """这张表的目录，每项一行：路径、字段、值、来源和状态，有备注的另起一行。遮罩列上的码值不给，说到遮罩列的项不给备注 + （剖析的备注里有取值范围）。""" + labels = ASSISTANT_PROMPT_LABELS + lines: list[str] = [] + for key, item in catalog._slots(notes).items(): + hidden = _touches_masked(key, item, masked) + if hidden and key[0] == "c" and key[2] == "codes": + continue + if key[0] == "r": + label, value = labels["field_relation"], _relation_line(item) + else: + label = (catalog.TABLE_FIELD_LABEL if key[0] == "t" else catalog.COLUMN_FIELD_LABEL).get(key[-1], key[-1]) + value = _value_text(item.get("value")) + lines.append(labels["item"].format(path=slot_path(key), label=label, value=value, + source=_source_text(item.get("source")), + status=_status_text(item.get("status")))) + note = item.get("note") + if isinstance(note, str) and note.strip() and not hidden: + lines.append(labels["item_note"].format(text=note.strip())) + return lines + + +def _structure_lines(meta: Mapping[str, Any], masked: set[str], *, system: bool) -> list[str]: + labels = ASSISTANT_PROMPT_LABELS + pk = {str(c) for c in meta.get("primary_key") or []} + lines = [labels["structure"]] + for col in meta.get("columns") or []: + if not isinstance(col, Mapping) or col.get("name") is None: + continue + name = str(col["name"]) + marks: list[str] = [] + if name.lower() in masked: + marks.append(labels["masked"]) + else: + if name in pk: + marks.append(labels["pk"]) + if not col.get("nullable", True): + marks.append(labels["not_null"]) + text = str(col.get("comment") or "").strip() + if text: + marks.append((labels["system"] if system else labels["col_comment"]).format(text=text)) + lines.append(labels["column"].format(name=name, type=col.get("type") or "", + marks=("，" + "，".join(marks)) if marks else "")) + for group in meta.get("unique") or []: + if isinstance(group, list) and group: + lines.append(labels["unique"].format(cols="、".join(str(c) for c in group))) + for fk in meta.get("foreign_keys") or []: + if isinstance(fk, Mapping) and fk.get("columns") and fk.get("to_table"): + target = ", ".join(f"{fk['to_table']}.{c}" for c in fk.get("to_columns") or []) or str(fk["to_table"]) + lines.append(labels["fk"].format(cols=", ".join(str(c) for c in fk["columns"]), target=target)) + return lines + + +async def _last_blank(session: AsyncSession, source_id: str, table: str) -> tuple[int, list[str]]: + """最近一次结束于 ok / partial 的起草里这张表模型留空的列：(几列, 最多 5 个列名)。停止、中断的那次不算（那次没起草完， + 留空数是 0）。没有是 (0, [])。""" + from app.db.models import CatalogDraftJob, CatalogDraftTask + + row = (await session.execute( + select(CatalogDraftTask.blank_columns, CatalogDraftTask.blank_sample) + .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) + .where(CatalogDraftJob.source_id == source_id, CatalogDraftTask.table_name == table, + CatalogDraftTask.status.in_(("ok", "partial"))) + .order_by(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq).limit(1) + )).first() + if row is None: + return 0, [] + return int(row[0] or 0), [str(c) for c in row[1] or []] + + +@dataclass +class AssistantContext: + """一次请求要用到的东西，开流之前读好（流里不再用请求的会话）。""" + + source_id: str + source_name: str + kind: str + table: str + tables: dict[str, Any] + #: 读表结构、发查询用的源（上传源是绑定当前快照的视图） + query_source: Any + options: dict[str, Any] + masked: set[str] + can_query: bool + settings: catalog_profile.ProfileSettings + system: str + pending_codes: dict[str, dict[str, Any]] = field(default_factory=dict) + + def tools(self) -> list[dict[str, Any]]: + return [QUERY_TOOL_PROMPT, PROPOSE_TOOL_PROMPT] if self.can_query else [PROPOSE_TOOL_PROMPT] + + +async def prepare(session: AsyncSession, row: Any, source: Any, table: str, + pending: Iterable[Mapping[str, Any]] = ()) -> AssistantContext: + """读好这次请求要用的背景：表结构、目录、别的表指过来的关联关系、使用次数、起草要求、留空的列、待保存的修改，拼成系统 + 提示。row 是数据源记录（遮罩、剖析开关、起草要求按它），source 是读表结构和发查询用的源。table 是表结构里的键。""" + labels = ASSISTANT_PROMPT_LABELS + cache = getattr(source, "schema_cache", None) or {} + tables = cache.get("tables") or {} + meta = tables.get(table) or {} + options = dict(getattr(row, "options", None) or {}) + masked = {c.lower() for c in masked_columns(options)} + settings = catalog_profile.profile_settings(options) + enabled = getattr(row, "enabled", True) is not False + can_query = settings.enabled and enabled + system_notes = catalog.system_notes_source(source) + entries = await catalog.read_catalog(session, source.id) + entry = entries.get(table) + notes = entry.notes if entry else {} + usage = (await catalog.table_usage(session, source)).get(table, 0) + blank_n, blank_cols = await _last_blank(session, source.id, table) + + lines = [labels["head"], labels["table"].format(source=row.name, kind=row.kind, table=table, + view=labels["view"] if meta.get("is_view") else "")] + comment = str(meta.get("comment") or "").strip() + if comment: + lines.append((labels["system"] if system_notes else labels["comment"]).format(text=comment)) + lines.append(labels["usage"].format(n=usage)) + description = str(getattr(row, "description", "") or "").strip() + if description: + lines.append(labels["description"].format(text=description)) + if guidance := catalog_jobs.guidance_of(row): + lines.append(labels["guidance"].format(text=guidance)) + lines += ["", *_structure_lines(meta, masked, system=system_notes), ""] + current = _catalog_lines(notes, masked) + lines += [labels["catalog"], *current] if current else [labels["catalog_empty"]] + incoming = [] + for other, other_entry in sorted(entries.items()): + if other == table: + continue + for key, rel in catalog._slots(other_entry.notes).items(): + if key[0] == "r" and rel.get("to_table") == table: + incoming.append(labels["incoming_item"].format( + table=other, columns=", ".join(str(c) for c in rel.get("columns") or []), target=table, + to_columns=", ".join(str(c) for c in rel.get("to_columns") or []), + source=_source_text(rel.get("source")), status=_status_text(rel.get("status")))) + if incoming: + lines += ["", labels["incoming"], *incoming[:INCOMING_SHOWN]] + named = catalog.visible_notes(notes).get("columns") or {} + columns = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, Mapping) and c.get("name") is not None] + unnamed = [c for c in columns if not any(f in (named.get(c) or {}) for f in ("label", "meaning"))] + blank = [c for c in blank_cols if c in unnamed] + if blank_n and blank: + lines += ["", labels["blank"].format(cols="、".join(blank), n=blank_n)] + if unnamed: + more = labels["unnamed_more"].format(n=UNNAMED_SHOWN) if len(unnamed) > UNNAMED_SHOWN else "" + lines.append(labels["unnamed"].format(cols="、".join(unnamed[:UNNAMED_SHOWN]), n=len(unnamed), more=more)) + pending = list(pending) + pending_codes: dict[str, dict[str, Any]] = {} + shown_pending: list[str] = [] + for change in pending: + path = str(change.get("path") or "").strip() + value = change.get("value") + try: + key = catalog.parse_path(path) + except catalog.CatalogPathError: + continue + if key[0] == "c" and key[2] == "codes": + if key[1].lower() in masked: + continue + if isinstance(value, Mapping): + pending_codes[path] = dict(value) + reason = str(change.get("reason") or "").strip() + shown_pending.append(labels["pending_item"].format(path=path, value=_value_text(value), reason=reason or "—")) + if shown_pending: + lines += ["", labels["pending"], *shown_pending] + if can_query: + tools = ASSISTANT_QUERY_PROMPT.format(dialect=row.kind, rows=QUERY_ROWS, limit=MAX_QUERIES) + else: + tools = ASSISTANT_NO_QUERY_PROMPT if enabled or not settings.enabled else ASSISTANT_DISABLED_PROMPT + system = "\n\n".join([ASSISTANT_SYSTEM, tools, "\n".join(lines)]) + return AssistantContext(source_id=source.id, source_name=row.name, kind=str(row.kind or ""), table=table, + tables=tables, query_source=source, options=options, masked=masked, can_query=can_query, + settings=settings, system=system, pending_codes=pending_codes) + + +# ========================================================================== +# 工具 +# ========================================================================== + + +def _cell(value: Any, limit: int | None = None) -> Any: + """一格交出去的样子：数、布尔、空照原样，别的写成文字（日期、Decimal……），长的截断。""" + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, (datetime, date)): + text = value.isoformat() + elif isinstance(value, Decimal): + text = str(value) + elif isinstance(value, (bytes, bytearray)): + text = f"<{len(value)} 字节>" + else: + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) + if limit is not None and len(text) > limit: + return text[:limit] + "…" + return text + + +def _rows(rows: Any, n: int, limit: int | None) -> list[list[Any]]: + out: list[list[Any]] = [] + for row in (rows or [])[:n]: + cells = row if isinstance(row, list) else list(row.values()) if isinstance(row, dict) else [row] + out.append([_cell(v, limit) for v in cells]) + return out + + +class _Tools: + """一次请求里的工具：查询次数、这一次遮掉的值（后面的查询按值换）。""" + + def __init__(self, ctx: AssistantContext) -> None: + from app.engine.masking import SeenValues + + self.ctx = ctx + self.queries = 0 + self.seen = SeenValues() + + # ---------------------------------------------------------------- 查询 + + async def query(self, sql: Any) -> tuple[dict[str, Any] | None, str]: + """(给界面的 query 事件, 交回模型的结果)。""" + prompts = ASSISTANT_TOOL_PROMPT + if not self.ctx.can_query: + return None, prompts["query_disabled"] + if self.queries >= MAX_QUERIES: + return None, prompts["query_capped"].format(n=MAX_QUERIES) + sql = sql.strip() if isinstance(sql, str) else "" + if not sql: + return None, prompts["empty_sql"] + self.queries += 1 + mask = masked_columns(self.ctx.options) + known = await asyncio.to_thread(self.seen.pooled) + known = known if known.strong else None + error: str | None = None + result: Any = None + settings = dataclasses.replace(self.ctx.settings, max_queries=1) + try: + runner = catalog_profile._Runner(self.ctx.query_source, settings, + catalog_profile.SqlDialect(self.ctx.kind)) + result = await runner(sql, max_rows=QUERY_ROWS) + except catalog_profile._Skip as e: + error = e.detail + except catalog_profile._Stop as e: + error = str(e) + except SnapshotTampered as e: + error = QUERY_TAMPERED.format(name=self.ctx.source_name, error=e) + except ValueError as e: # 不支持的数据库类型 + error = str(e) + if error is not None: + if mask or known is not None: + from app.engine.masking import mask_error_text + + error = mask_error_text(error, sql, known) + event = {"type": "query", "sql": sql, "columns": [], "rows": [], "row_count": 0, "truncated": False, + "error": error} + return event, prompts["query_failed"].format(error=error) + payload = result.to_payload() + if mask or known is not None: + from app.engine.masking import mask_query_result + + schema = getattr(self.ctx.query_source, "schema_cache", None) + done = await asyncio.to_thread(mask_query_result, payload, mask, self.ctx.kind, known, schema=schema) + payload = done.payload + await asyncio.to_thread(self.seen.add, done.secrets) + columns = [str(c) for c in payload.get("columns") or []] + rows = payload.get("rows") or [] + event = {"type": "query", "sql": str(payload.get("sql") or sql), "columns": columns, + "rows": _rows(rows, SHOWN_ROWS, None), "row_count": int(payload.get("row_count") or len(rows)), + "truncated": bool(payload.get("truncated")), "error": None} + given = {"columns": columns, "rows": _rows(rows, QUERY_ROWS, CELL_MAX), "row_count": event["row_count"], + "truncated": event["truncated"]} + text = json.dumps(given, ensure_ascii=False, default=str) + while len(text) > TOOL_CHARS and given["rows"]: + given["rows"] = given["rows"][:max(1, len(given["rows"]) // 2)] if len(given["rows"]) > 1 else [] + given["rows_shown"] = len(given["rows"]) + text = json.dumps(given, ensure_ascii=False, default=str) + return event, text + + # ---------------------------------------------------------------- 提案 + + def _with_pending(self, change: Any) -> Any: + """界面上还没保存的修改里同一列的码值，并进新提的码值（新提的同一个码为准）。""" + if not isinstance(change, Mapping): + return change + path = str(change.get("path") or "").strip() + before = self.ctx.pending_codes.get(path) + value = change.get("value") + if before is None or not isinstance(value, Mapping): + return change + return {**change, "value": {**before, **value}} + + async def propose(self, changes: Any) -> tuple[dict[str, Any] | None, str]: + """(给界面的 proposal 事件, 交回模型的结果)。提案不写目录。""" + from app.db.base import SessionLocal + + prompts = ASSISTANT_TOOL_PROMPT + if not isinstance(changes, list) or not changes: + return None, prompts["no_changes"] + changes = [self._with_pending(c) for c in changes] + async with SessionLocal() as session: + entry = await catalog.read_entry(session, self.ctx.source_id, self.ctx.table) + notes = entry.notes if entry else {} + plan = catalog.plan_patch(notes, changes, table=self.ctx.table, tables=self.ctx.tables) + dropped = list(plan.problems) + kept: list[catalog.PatchChange] = [] + for change in plan.changes: + key = catalog.parse_path(change.path) + if key[0] == "c" and key[2] == "codes" and key[1].lower() in self.ctx.masked: + dropped.append(prompts["masked_codes"].format(column=key[1])) + continue + if change.state == "same": + dropped.append(prompts["same"].format(where=_where(key, change))) + continue + kept.append(change) + said: list[str] = [] + event: dict[str, Any] | None = None + if kept: + label = notes.get("label") if isinstance(notes.get("label"), Mapping) else {} + event = { + "type": "proposal", "source": self.ctx.source_name, "source_id": self.ctx.source_id, + "table": self.ctx.table, + "table_label": label.get("value") if label and label.get("status") != "rejected" else None, + "version": entry.version if entry else 0, + "changes": [{**c.as_dict(), "label": _where(catalog.parse_path(c.path), c)} for c in kept], + } + said.append(prompts["proposed"].format(n=len(kept), paths="、".join(c.path for c in kept))) + else: + said.append(prompts["nothing"]) + if dropped: + said.append(prompts["dropped"].format(reasons="；".join(dropped))) + return event, "\n".join(said) + + +def _where(key: catalog.Slot, change: catalog.PatchChange) -> str: + """提案里一项的中文说法：「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」。""" + value = change.after if isinstance(change.after, Mapping) else change.before + if key[0] == "r" and isinstance(value, Mapping) and value.get("to_table"): + return f"指向 {value['to_table']} 的关联关系" + return catalog.describe_slot(key) + + +# ========================================================================== +# 流 +# ========================================================================== + + +#: 上一轮的正文以这些结尾（说完了一句话）时，下一轮另起一段 +_SENTENCE_END = ("。", "！", "？", "；", "：", ".", "!", "?", ";", ":") + + +def _sse(obj: Mapping[str, Any]) -> str: + return f"data: {json.dumps(obj, ensure_ascii=False, default=str)}\n\n" + + +class _CallFailed(Exception): + """一次模型调用没成：code 是错误事件的机读码，message 给人看。""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _seconds(value: Any) -> str: + number = float(value) + return str(int(number)) if number.is_integer() else f"{number:g}" + + +def _transient(e: BaseException) -> bool: + from app.core.errors import NETWORK, RATE_LIMIT, SERVER, classify + + return classify(e) in (RATE_LIMIT, SERVER, NETWORK) + + +async def _pump(runner: Any, history: list[Any], queue: asyncio.Queue[tuple[str, Any]]) -> None: + """在单独的任务里读模型的流，一块一块放进队列；读完放 end，出错放 error。""" + try: + async for chunk in runner.astream(history): + await queue.put(("chunk", chunk)) + await queue.put(("end", None)) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 交给读队列的一方按原因处理 + await queue.put(("error", e)) + + +async def _round(runner: Any, history: list[Any], model_id: str) -> AsyncIterator[tuple[str, Any]]: + """调一次模型：交出 ("text", 增量)、("beat", None)（等得久了，发心跳）、最后 ("message", (合起来的消息, 截断了没有))。 + 网络错误、429、5xx 在还没输出任何正文时重试；超时、别的错误抛 _CallFailed。""" + from app.core.errors import TIMEOUT, classify, describe_exception + from app.engine.state import message_text + from app.providers.budget import cut_off + + loop = asyncio.get_running_loop() + attempt = 0 + while True: + queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + task = asyncio.ensure_future(_pump(runner, history, queue)) + deadline = loop.time() + float(catalog.MODEL_TIMEOUT_S) + message: Any = None + cut = False + started = False + failure: BaseException | None = None + try: + while True: + remaining = deadline - loop.time() + if remaining <= 0: + raise _CallFailed(CODE_TIMEOUT, ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), + place=DRAFT_MODEL_PLACE)) + try: + kind, value = await asyncio.wait_for(queue.get(), timeout=min(remaining, HEARTBEAT_S)) + except asyncio.TimeoutError: + yield "beat", None + continue + if kind == "end": + break + if kind == "error": + failure = value + break + message = value if message is None else message + value + cut = cut or bool(cut_off(value)) + text = message_text(value) + if text: + started = True + yield "text", text + finally: + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + if failure is None: + yield "message", (message, cut) + return + if not started and attempt < len(RETRY_DELAYS_S) and _transient(failure): + logger.info("表助手：调用模型出错（%s），%s 秒后重试", type(failure).__name__, RETRY_DELAYS_S[attempt]) + await asyncio.sleep(RETRY_DELAYS_S[attempt]) + attempt += 1 + continue + if isinstance(failure, TimeoutError) or classify(failure) == TIMEOUT: + raise _CallFailed(CODE_TIMEOUT, ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), + place=DRAFT_MODEL_PLACE)) from failure + raise _CallFailed(CODE_MODEL, ERROR_MODEL.format(model=model_id or "", reason=describe_exception(failure), + place=DRAFT_MODEL_PLACE)) from failure + + +async def stream(model: Any, model_id: str, ctx: AssistantContext, + messages: Iterable[Mapping[str, Any]]) -> AsyncIterator[str]: + """表助手的一次请求：交出 SSE 的每一行（事件、心跳注释）。""" + from langchain_core.messages import SystemMessage, ToolMessage, message_chunk_to_message + + from app.providers.budget import cut_words + + history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)] + tools = _Tools(ctx) + runner = model.bind_tools(ctx.tools()) + usage = {"input_tokens": 0, "output_tokens": 0} + wrote = "" + names = [t["function"]["name"] for t in ctx.tools()] + try: + for round_no in range(MAX_ROUNDS): + fresh = True + message: Any = None + cut = False + # 流在半路被关掉（使用者点了停止、断开）时，正在读的那次调用跟着停：aclosing 让 _round 的收尾马上执行 + async with contextlib.aclosing(_round(runner, history, model_id)) as rounds: + async for kind, value in rounds: + if kind == "beat": + yield ": keep-alive\n\n" + elif kind == "text": + if fresh and wrote.endswith(_SENTENCE_END): + # 上一轮（查询、提案之前）说完了一句话：这一轮另起一段 + yield _sse({"type": "text", "delta": "\n\n"}) + fresh = False + wrote = value + yield _sse({"type": "text", "delta": value}) + else: + message, cut = value + meta = getattr(message, "usage_metadata", None) or {} + usage["input_tokens"] += int(meta.get("input_tokens") or 0) + usage["output_tokens"] += int(meta.get("output_tokens") or 0) + if cut: + limit = getattr(model, "max_tokens", None) + yield _sse({"type": "error", "code": CODE_TRUNCATED, + "message": ERROR_TRUNCATED.format(cut=cut_words("表助手", limit), place=DRAFT_MODEL_PLACE)}) + return + calls = list(getattr(message, "tool_calls", None) or []) if message is not None else [] + if not calls: + yield _sse({"type": "done", "usage": usage, "queries": tools.queries}) + return + if round_no == MAX_ROUNDS - 1: + yield _sse({"type": "error", "code": CODE_ROUNDS, "message": ERROR_ROUNDS.format(n=MAX_ROUNDS)}) + return + # 合起来的流式块换成普通消息再交回模型（不带半截的工具调用块） + history.append(message_chunk_to_message(message)) + for call in calls: + name = str(call.get("name") or "") + args = call.get("args") if isinstance(call.get("args"), Mapping) else {} + call_id = str(call.get("id") or f"{name}-{round_no}-{len(history)}") + event: dict[str, Any] | None = None + if name == QUERY_TOOL: + event, content = await tools.query(args.get("sql")) + elif name == PROPOSE_TOOL: + event, content = await tools.propose(args.get("changes")) + else: + content = ASSISTANT_TOOL_PROMPT["unknown_tool"].format(name=name, tools="、".join(names)) + if event is not None: + yield _sse(event) + if round_no == MAX_ROUNDS - 2: + content = f"{content}\n{ASSISTANT_TOOL_PROMPT['last_round']}" + history.append(ToolMessage(content=content, tool_call_id=call_id)) + except _CallFailed as e: + yield _sse({"type": "error", "code": e.code, "message": e.message}) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 流已经开了，照实交一条错误事件 + from app.core.errors import describe_exception, first_line + + logger.exception("表助手出错：数据源 %s 表 %s", ctx.source_id, ctx.table) + yield _sse({"type": "error", "code": CODE_FAILED, + "message": ERROR_FAILED.format(line=first_line(e) or describe_exception(e))}) + + +__all__ = ["AssistantContext", "CODE_FAILED", "CODE_MODEL", "CODE_ROUNDS", "CODE_TIMEOUT", "CODE_TRUNCATED", + "HEARTBEAT_S", "MAX_MESSAGES", "MAX_PENDING", "MAX_QUERIES", "MAX_ROUNDS", "MESSAGE_MAX", "PROPOSE_TOOL", + "QUERY_ROWS", "QUERY_TOOL", "SHOWN_ROWS", "conversation_problem", "prepare", "resolve_model", "stream"] diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py new file mode 100644 index 00000000..2d268bb0 --- /dev/null +++ b/backend/tests/test_catalog_assistant.py @@ -0,0 +1,550 @@ +"""表助手（data/catalog_assistant.py，POST /api/datasources/{id}/catalog/{table}/assistant）。 + +- 开始前：表不在 404；起草模型用不了 409 catalog_draft_model_unavailable（说法同起草任务）；对话超过 40 条、一条超过 + 4000 字、待保存的修改超过 200 项 422 +- 流：text 增量、query（跑过的只读查询，前 10 行，遮罩列写「已遮罩」）、proposal（按 plan_patch 核对过的提案，形状同助手流 + 的 catalog_patch：公式不收、列名必须在表结构里、遮罩列不给码值）、done（用量、查询条数）、error（中途出错，之后没有 done） +- 交给模型的背景：表结构、这张表的目录（每项的来源和状态）、关联关系（含别的表指过来的）、使用次数、数据源的起草要求、 + 最近一次起草留空的列、界面上还没保存的修改；遮罩列只给列名、不给任何值 +- 只读查询：只有开了数据剖析的源才给这个工具；经剖析的只读守卫和单条时限；每次请求最多 5 条；最多 50 行交给模型；结果 + 先遮再交给模型；没开剖析时提示词里写明不能查数据 +- 对话不存服务端；提案不写目录（保存走 PATCH，最多 200 项） + +表名、列名、业务数字都是虚构的；号码是 139000000xx 这样的虚构号码。 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessageChunk, HumanMessage, SystemMessage, ToolMessage + +from app.data import catalog, catalog_assistant +from app.data.catalog import make_item +from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION +from app.db.base import SessionLocal +from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +PHONES = [f"139000000{i:02d}" for i in range(1, 31)] + + +# -------------------------------------------------------------------------- +# 假模型：按剧本一轮一轮回 +# -------------------------------------------------------------------------- + + +@dataclass +class Turn: + text: list[str] = field(default_factory=list) + calls: list[tuple[str, dict]] = field(default_factory=list) + cut: bool = False + usage: tuple[int, int] = (100, 20) + error: BaseException | None = None + hang: bool = False + + +class Scripted: + """bind_tools 记下交给它的工具；astream 每调一次按剧本回一轮，记下这一轮收到的消息。""" + + def __init__(self, turns: list[Turn]) -> None: + self.turns = list(turns) + self.seen: list[list[Any]] = [] + self.tools: list[Any] | None = None + + def bind_tools(self, tools: list[Any], **_: Any) -> "Scripted": + self.tools = tools + return self + + async def astream(self, messages: list[Any]): + self.seen.append(list(messages)) + turn = self.turns.pop(0) + if turn.error is not None: + raise turn.error + if turn.hang: + await asyncio.sleep(3600) + for piece in turn.text: + yield AIMessageChunk(content=piece) + for i, (name, args) in enumerate(turn.calls): + yield AIMessageChunk(content="", tool_call_chunks=[ + {"name": name, "args": json.dumps(args, ensure_ascii=False), "id": f"c{len(self.seen)}_{i}", + "index": i}]) + yield AIMessageChunk(content="", usage_metadata={"input_tokens": turn.usage[0], "output_tokens": turn.usage[1], + "total_tokens": sum(turn.usage)}, + response_metadata={"finish_reason": "length" if turn.cut else "stop"}) + + def tool_names(self) -> list[str]: + return [t["function"]["name"] for t in self.tools or []] + + +class _Unavailable(Exception): + status_code = 503 + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _shop_db(path) -> str: + conn = sqlite3.connect(path) + conn.executescript(""" + CREATE TABLE members (id INTEGER PRIMARY KEY, nick TEXT); + CREATE TABLE orders (id INTEGER PRIMARY KEY, order_no TEXT NOT NULL, status INTEGER NOT NULL, + amount REAL, contact_phone TEXT, created_at TEXT, + member_id INTEGER REFERENCES members(id)); + """) + conn.executemany("INSERT INTO members VALUES (?, ?)", [(i, f"会员{i}") for i in range(1, 6)]) + conn.executemany("INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?, ?)", [ + (i, f"NO{i:04d}", 9 if i % 10 == 0 else 1, 10.0 * i, PHONES[i - 1], f"2026-09-{i:02d}", i % 5 + 1) + for i in range(1, 31)]) + conn.commit() + conn.close() + return str(path) + + +ORDERS_NOTES = { + "label": make_item("订单", "llm"), + "grain": make_item("每笔订单一行", "human", "confirmed"), + "kind": make_item("fact", "llm", "rejected"), + "columns": { + "status": {"codes": make_item({"1": "已支付"}, "profile", note="数据剖析：全表 30 行，2 个取值")}, + "contact_phone": {"label": make_item("联系电话", "llm"), + "codes": make_item({PHONES[0]: "测试号"}, "human", "confirmed")}, + }, +} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _source(client, tmp_path, *, profile: bool = True, guidance: str | None = "金额一律按元", + notes: dict | None = None) -> str: + options: dict[str, Any] = {"mask_columns": "contact_phone"} + if profile: + options["catalog_profile"] = {"enabled": True} + if guidance: + options[CATALOG_DRAFT_GUIDANCE_OPTION] = guidance + async with SessionLocal() as session: + row = DataSource(name=f"shop_{uuid.uuid4().hex[:8]}", kind="sqlite", database=_shop_db(tmp_path / "shop.db"), + options=options, description="虚构的网店库") + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + r = await client.post(f"/api/datasources/{sid}/introspect") + assert r.status_code == 200, r.text + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", notes if notes is not None else ORDERS_NOTES, + if_version=0, actor=None) + return sid + + +def use(monkeypatch, model: Scripted) -> None: + async def resolve(session): + return model, "fake-chat" + + monkeypatch.setattr(catalog_assistant, "resolve_model", resolve) + + +def _url(sid: str, table: str = "orders") -> str: + return f"/api/datasources/{sid}/catalog/{table}/assistant" + + +async def _talk(client, sid: str, body: dict, table: str = "orders") -> tuple[list[dict], list[str]]: + """发一轮，返回 (事件, 原始行)。""" + events: list[dict] = [] + lines: list[str] = [] + async with client.stream("POST", _url(sid, table), json=body) as r: + assert r.status_code == 200, await r.aread() + assert r.headers["content-type"].startswith("text/event-stream") + async for line in r.aiter_lines(): + lines.append(line) + if line.startswith("data: "): + events.append(json.loads(line[6:])) + return events, lines + + +def _text(events: list[dict]) -> str: + return "".join(e["delta"] for e in events if e["type"] == "text") + + +def _ask(text: str, **extra) -> dict: + return {"messages": [{"role": "user", "content": text}], **extra} + + +# -------------------------------------------------------------------------- +# 开始前的检查 +# -------------------------------------------------------------------------- + + +async def test_refusals_before_the_stream(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + use(monkeypatch, Scripted([])) + assert (await client.post(_url(sid, "ghost"), json=_ask("你好"))).status_code == 404 + assert (await client.post("/api/datasources/nope/catalog/orders/assistant", json=_ask("x"))).status_code == 404 + too_many = {"messages": [{"role": "user", "content": "x"}] * 41} + assert (await client.post(_url(sid), json=too_many)).status_code == 422 + too_long = _ask("长" * 4001) + assert (await client.post(_url(sid), json=too_long)).status_code == 422 + pending = [{"path": "label", "value": "订单", "reason": "x"}] * 201 + assert (await client.post(_url(sid), json=_ask("x", pending=pending))).status_code == 422 + wrong_role = {"messages": [{"role": "system", "content": "x"}]} + assert (await client.post(_url(sid), json=wrong_role)).status_code == 422 + last_is_answer = {"messages": [{"role": "user", "content": "x"}, {"role": "assistant", "content": "y"}]} + r = await client.post(_url(sid), json=last_is_answer) + assert r.status_code == 422 and "最后一条" in r.json()["detail"] + + +async def test_unavailable_draft_model_is_409_with_the_draft_wording(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + + async def unavailable(session): + raise catalog.CatalogModelUnavailable("未配置模型接入，无法由模型起草数据目录。请到「设置 → 模型接入」添加") + + monkeypatch.setattr(catalog, "resolve_draft_model", unavailable) + r = await client.post(_url(sid), json=_ask("你好")) + assert r.status_code == 409 + assert r.json()["code"] == "catalog_draft_model_unavailable" and "模型接入" in r.json()["detail"] + + +# -------------------------------------------------------------------------- +# 背景 +# -------------------------------------------------------------------------- + + +async def _blank_draft(sid: str) -> None: + """最近一次结束于 ok 的起草任务记下留空的列；之后一次停止了的不算。""" + async with SessionLocal() as session: + done = CatalogDraftJob(id=uuid.uuid4().hex, source_id=sid, status="succeeded", + created_at=datetime(2026, 10, 1, tzinfo=timezone.utc)) + stopped = CatalogDraftJob(id=uuid.uuid4().hex, source_id=sid, status="cancelled", + created_at=datetime(2026, 10, 2, tzinfo=timezone.utc)) + session.add_all([done, stopped]) + await session.flush() + session.add_all([ + CatalogDraftTask(job_id=done.id, seq=0, table_name="orders", status="ok", blank_columns=2, + blank_sample=["order_no", "created_at"]), + CatalogDraftTask(job_id=stopped.id, seq=0, table_name="orders", status="cancelled", blank_columns=0), + ]) + await session.commit() + + +async def test_background_has_structure_catalog_and_hides_masked_values(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + await _blank_draft(sid) + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "members", {"relations": [ + {"id": catalog.relation_id("members", ["id"], "orders", ["member_id"]), "columns": ["id"], + "to_table": "orders", "to_columns": ["member_id"], "cardinality": "one_to_many", "coverage": None, + "source": "human", "status": "confirmed"}]}, if_version=0, actor=None) + model = Scripted([Turn(text=["这张表还缺 order_no 的含义。"])]) + use(monkeypatch, model) + pending = [{"path": "columns.amount.unit", "value": "元", "reason": "用户说：金额是元"}] + events, _ = await _talk(client, sid, _ask("看看还缺什么", pending=pending)) + assert _text(events) == "这张表还缺 order_no 的含义。" + assert events[-1] == {"type": "done", "usage": {"input_tokens": 100, "output_tokens": 20}, "queries": 0} + [messages] = model.seen + assert isinstance(messages[0], SystemMessage) and isinstance(messages[-1], HumanMessage) + system = messages[0].content + # 规矩 + for rule in ("只改数据目录", "不改业务库", "公式", "拿不准", "原话"): + assert rule in system + # 表结构、目录（来源和状态）、使用次数、起草要求、数据源说明 + for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认", + "模型起草", "已驳回", "被运行用过 0 次", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"): + assert fact in system, fact + # 别的表指过来的关联关系 + assert "members.id → orders.member_id" in system + # 最近一次结束于 ok 的起草留空的列（之后停止了的那次不算） + assert "order_no、created_at" in system + # 界面上还没保存的修改 + assert "columns.amount.unit" in system and "用户说：金额是元" in system + # 遮罩列只给列名：码值、取值一个都不给 + assert "contact_phone" in system + assert not any(p in system for p in PHONES) and "测试号" not in system + # 没开剖析的源才写不能查数据；这里开了，给了查询工具 + assert model.tool_names() == ["run_query", "propose_changes"] + assert "没开剖析" not in system + + +async def test_without_profiling_there_is_no_query_tool(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path, profile=False) + model = Scripted([Turn(text=["只能根据表结构推断。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("状态 9 是什么")) + assert model.tool_names() == ["propose_changes"] + assert "这个数据源没开剖析，不能查数据，只能根据表结构和目录推断" in model.seen[0][0].content + + +async def test_opening_without_user_messages(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(text=["order_no 是什么意思？"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, {"messages": []}) + assert _text(events) == "order_no 是什么意思？" + assert isinstance(model.seen[0][-1], HumanMessage) + # 界面先显示了一句开场白（助手说的）：照样从使用者这一边开始 + model = Scripted([Turn(text=["好的。"])]) + use(monkeypatch, model) + await _talk(client, sid, {"messages": [{"role": "assistant", "content": "有两列留空了"}, + {"role": "user", "content": "order_no 是订单号"}]}) + kinds = [type(m).__name__ for m in model.seen[0]] + assert kinds == ["SystemMessage", "HumanMessage", "AIMessage", "HumanMessage"] + + +# -------------------------------------------------------------------------- +# 提案 +# -------------------------------------------------------------------------- + + +async def test_proposals_are_checked_and_never_written(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + changes = [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}, + {"path": "valid_filter", "value": "status <> 9", "reason": "用户说：统计时排除作废"}, + {"path": "columns.amount.meaning", "value": "实收=单价*数量", "reason": "x"}, # 公式：不收 + {"path": "columns.ghost.label", "value": "幽灵", "reason": "x"}, # 列不存在：不收 + {"path": "columns.contact_phone.codes", "value": {PHONES[1]: "客服"}, "reason": "x"}, # 遮罩列：不给码值 + {"path": "grain", "value": "每笔订单一行", "reason": "x"}, # 已经是这个值：不用提 + ] + model = Scripted([Turn(text=["好的，"], calls=[("propose_changes", {"changes": changes})]), + Turn(text=["已经整理成提案。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("9 是作废，统计时排除")) + assert _text(events) == "好的，已经整理成提案。" + [proposal] = [e for e in events if e["type"] == "proposal"] + assert proposal["source_id"] == sid and proposal["table"] == "orders" and proposal["version"] == 1 + assert proposal["source"].startswith("shop_") and proposal["table_label"] == "订单" + by = {c["path"]: c for c in proposal["changes"]} + assert set(by) == {"columns.status.codes", "valid_filter"} + codes = by["columns.status.codes"] + assert codes["label"] == "列 status 的码值" and codes["reason"] == "用户说：9 是作废" + assert codes["before"] == {"1": "已支付"} and codes["after"] == {"1": "已支付", "9": "作废"} + assert codes["value"] == {"9": "作废"} and codes["state"] == "change" + assert by["valid_filter"]["label"] == "表的有效记录条件" and by["valid_filter"]["before"] is None + # 没收的几项照实告诉模型（不带遮罩列的任何取值） + tool = model.seen[1][-1] + assert isinstance(tool, ToolMessage) + for said in ("2 项", "公式", "ghost", "遮罩", "已经是这个值"): + assert said in tool.content, said + assert not any(p in tool.content for p in PHONES) + assert events[-1]["type"] == "done" and events[-1]["usage"] == {"input_tokens": 200, "output_tokens": 40} + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, "orders") + assert entry.version == 1 and "valid_filter" not in entry.notes + + +async def test_proposal_codes_build_on_the_pending_ones(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(calls=[("propose_changes", {"changes": [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}]})]), + Turn(text=["好。"])]) + use(monkeypatch, model) + pending = [{"path": "columns.status.codes", "value": {"2": "已退款"}, "reason": "用户说：2 是退款"}] + events, _ = await _talk(client, sid, _ask("9 是作废", pending=pending)) + [proposal] = [e for e in events if e["type"] == "proposal"] + [change] = proposal["changes"] + assert change["value"] == {"2": "已退款", "9": "作废"} + assert change["after"] == {"1": "已支付", "2": "已退款", "9": "作废"} + + +async def test_nothing_acceptable_means_no_proposal_event(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]), + Turn(text=["那一列不存在。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("ghost 列叫什么")) + assert not [e for e in events if e["type"] == "proposal"] + assert "没有可以交给使用者的修改" in model.seen[1][-1].content + + +# -------------------------------------------------------------------------- +# 只读查询 +# -------------------------------------------------------------------------- + + +async def test_queries_are_read_only_masked_and_capped(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + queries = [ + ("run_query", {"sql": "SELECT status, COUNT(*) AS n FROM orders GROUP BY status ORDER BY status"}), + ("run_query", {"sql": "SELECT id, contact_phone AS phone, amount FROM orders ORDER BY id"}), + ("run_query", {"sql": "DELETE FROM orders"}), + ] + model = Scripted([Turn(text=["我查一下。"], calls=queries), + Turn(calls=[("run_query", {"sql": "SELECT 1"})] * 3), + Turn(text=["9 占一成。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("状态分布")) + shown = [e for e in events if e["type"] == "query"] + assert len(shown) == 5 # 每次请求最多 5 条 + first, second, third = shown[:3] + assert first["columns"] == ["status", "n"] and first["rows"] == [[1, 27], [9, 3]] + assert first["row_count"] == 2 and first["error"] is None + # 遮罩列改了名字也认得出：值写「已遮罩」；界面只给前 10 行 + assert second["columns"] == ["id", "phone", "amount"] and len(second["rows"]) == 10 + assert {row[1] for row in second["rows"]} == {"已遮罩"} and second["row_count"] == 30 + # 只读守卫：写语句不执行 + assert third["error"] and "守卫" in third["error"] and third["rows"] == [] + # 交给模型的：最多 50 行、遮罩值看不到；第 6 条没执行 + second_round = model.seen[1] + results = [m for m in second_round if isinstance(m, ToolMessage)] + assert len(results) == 3 + assert json.loads(results[1].content)["rows"][29][1] == "已遮罩" + assert not any(p in m.content for m in second_round for p in PHONES) + capped = [m for m in model.seen[2] if isinstance(m, ToolMessage)][-1] + assert "未执行" in capped.content and "5 条" in capped.content + assert events[-1] == {"type": "done", "usage": {"input_tokens": 300, "output_tokens": 60}, "queries": 5} + with sqlite3.connect(tmp_path / "shop.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0] == 30 + + +async def test_query_results_given_to_the_model_stop_at_50_rows(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + sql = "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 80) SELECT i FROM n" + model = Scripted([Turn(calls=[("run_query", {"sql": sql})]), Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("数一下")) + [query] = [e for e in events if e["type"] == "query"] + assert query["row_count"] == 50 and query["truncated"] is True and len(query["rows"]) == 10 + result = json.loads([m for m in model.seen[1] if isinstance(m, ToolMessage)][0].content) + assert len(result["rows"]) == 50 and result["truncated"] is True + + +# -------------------------------------------------------------------------- +# 中途出错 +# -------------------------------------------------------------------------- + + +async def test_errors_end_the_stream_without_done(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + use(monkeypatch, Scripted([Turn(error=RuntimeError("接口不认这个模型"))])) + events, _ = await _talk(client, sid, _ask("你好")) + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_model_error" + assert "fake-chat" in events[-1]["message"] and not [e for e in events if e["type"] == "done"] + use(monkeypatch, Scripted([Turn(text=["写到一半"], cut=True)])) + events, _ = await _talk(client, sid, _ask("你好")) + assert _text(events) == "写到一半" + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_truncated" + assert "上限" in events[-1]["message"] + + +async def test_transient_errors_are_retried_before_any_output(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "RETRY_DELAYS_S", (0.0, 0.0)) + model = Scripted([Turn(error=_Unavailable("服务暂时不可用")), Turn(text=["好了。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("你好")) + assert _text(events) == "好了。" and events[-1]["type"] == "done" + assert len(model.seen) == 2 + + +async def test_a_call_over_the_time_limit_is_stopped(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.3) + monkeypatch.setattr(catalog_assistant, "HEARTBEAT_S", 0.1) + use(monkeypatch, Scripted([Turn(hang=True)])) + events, lines = await _talk(client, sid, _ask("你好")) + assert events == [events[-1]] and events[-1]["code"] == "assistant_timeout" + assert "0.3 秒" in events[-1]["message"] + # 等模型的时候按拍发 SSE 注释（不是事件），免得中间的代理断开 + assert any(line.startswith(":") for line in lines) + + +async def test_steps_are_bounded(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "MAX_ROUNDS", 3) + again = Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]) + model = Scripted([again, again, again]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("你好")) + assert len(model.seen) == 3 + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_rounds" + # 倒数第二轮的工具结果里提醒模型直接回答 + assert "直接回答" in [m for m in model.seen[2] if isinstance(m, ToolMessage)][-1].content + + +class _Slow(Scripted): + """说了半句就一直想：记下这次调用有没有被停掉。""" + + def __init__(self) -> None: + super().__init__([]) + self.cancelled = False + + async def astream(self, messages: list[Any]): + self.seen.append(list(messages)) + yield AIMessageChunk(content="我想一想") + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + self.cancelled = True + raise + yield AIMessageChunk(content="……") + + +async def test_stopping_the_stream_stops_the_model_call(): + """使用者点停止（连接断开、流被关掉）：正在进行的模型调用跟着停，不在后台接着花额度。""" + from app.data import catalog_profile + + model = _Slow() + ctx = catalog_assistant.AssistantContext( + source_id="s", source_name="虚构库", kind="sqlite", table="orders", tables={}, query_source=None, options={}, + masked=set(), can_query=False, settings=catalog_profile.ProfileSettings(), system="系统提示") + events = catalog_assistant.stream(model, "fake-chat", ctx, [{"role": "user", "content": "你好"}]) + first = await events.__anext__() + assert json.loads(first[len("data: "):]) == {"type": "text", "delta": "我想一想"} + await events.aclose() + assert model.cancelled + + +# -------------------------------------------------------------------------- +# 保存：PATCH 一次最多 200 项 +# -------------------------------------------------------------------------- + + +async def test_patch_accepts_up_to_200_changes(client): + """表助手一次能提很多项（逐列补中文名），保存不再卡在 20 项：预览、保存都收 200 项，201 项 422。""" + assert catalog.PATCH_MAX_CHANGES == 200 + columns = [f"c{i:03d}" for i in range(200)] + table = {"qualified": "wide", "primary_key": [], "comment": None, "is_view": False, "foreign_keys": [], + "unique": [], "columns": [{"name": c, "type": "TEXT", "nullable": True, "comment": None} for c in columns]} + async with SessionLocal() as session: + row = DataSource(name=f"wide_{uuid.uuid4().hex[:8]}", kind="sqlite", database=":memory:", + schema_cache={"tables": {"wide": table}}) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + base = f"/api/datasources/{sid}/catalog/wide" + changes = [{"path": f"columns.{c}.label", "value": f"第 {i} 列", "reason": "用户逐列说明"} + for i, c in enumerate(columns)] + preview = await client.post(f"{base}/patch/preview", json={"changes": changes}) + assert preview.status_code == 200, preview.text + assert len(preview.json()["changes"]) == 200 and preview.json()["problems"] == [] + saved = await client.post(f"{base}/patch", json={"changes": changes, "if_version": 0}) + assert saved.status_code == 200, saved.text + assert len(saved.json()["notes"]["columns"]) == 200 + assert saved.json()["notes"]["columns"]["c199"]["label"]["value"] == "第 199 列" + too_many = changes + [{"path": "label", "value": "宽表"}] + assert (await client.post(f"{base}/patch/preview", json={"changes": too_many})).status_code == 422 + assert (await client.post(f"{base}/patch", json={"changes": too_many, "if_version": 1})).status_code == 422 From 65369b8a6e9366cc3caab25c29a6f12c39a3164e Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:23:08 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=EF=BC=9ASDK=20?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E8=87=AA=E5=B7=B1=E9=87=8D=E8=AF=95=EF=BC=88?= =?UTF-8?q?=E5=90=8C=E8=B5=B7=E8=8D=89=EF=BC=8C=E7=BD=91=E7=BB=9C=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E3=80=81429=E3=80=815xx=20=E5=8F=AA=E7=94=B1=E8=A1=A8?= =?UTF-8?q?=E5=8A=A9=E6=89=8B=E5=9C=A8=E8=BF=98=E6=B2=A1=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E6=97=B6=E9=87=8D=E8=AF=95=EF=BC=8C=E9=80=80=E9=81=BF=203?= =?UTF-8?q?=E3=80=8110=20=E7=A7=92=EF=BC=89=EF=BC=9B=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E8=A1=A5=E4=B8=8A=E5=8F=96=E6=A8=A1=E5=9E=8B=E7=9A=84=E8=A7=84?= =?UTF-8?q?=E6=A0=BC=EF=BC=88=E8=B5=B7=E8=8D=89=E8=AE=BE=E7=BD=AE=E3=80=81?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E6=80=9D=E8=80=83=E3=80=81=E9=A2=9D=E5=BA=A6?= =?UTF-8?q?=2032768=20/=208192=E3=80=81=E5=8D=95=E6=AC=A1=E6=97=B6?= =?UTF-8?q?=E9=99=90=E3=80=81=E4=B8=8D=E9=87=8D=E8=AF=95=EF=BC=89=E5=92=8C?= =?UTF-8?q?=E4=B8=A4=E4=B8=AA=E5=B7=A5=E5=85=B7=E5=9C=A8=20OpenAI=20?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E3=80=81Anthropic=20=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E4=B8=8A=E9=83=BD=E7=BB=91=E5=BE=97=E4=B8=8A=EF=BC=9Bquery=5Fs?= =?UTF-8?q?napshot=5Ftables=20=E4=B8=8D=E8=BF=9B=20catalog=20=E7=9A=84=20?= =?UTF-8?q?=5F=5Fall=5F=5F=EF=BC=88=E6=8C=89=E6=A8=A1=E5=9D=97=E5=B1=9E?= =?UTF-8?q?=E6=80=A7=E5=8F=96=E7=94=A8=EF=BC=8C=E5=B0=91=E5=92=8C=E8=B5=B7?= =?UTF-8?q?=E8=8D=89=E9=82=A3=E8=BE=B9=E7=9A=84=E6=94=B9=E5=8A=A8=E6=92=9E?= =?UTF-8?q?=E5=9C=A8=E4=B8=80=E8=B5=B7=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog.py | 3 +- backend/app/data/catalog_assistant.py | 7 +-- backend/tests/test_catalog_assistant.py | 63 +++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 40bdaab8..36d481d3 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -2351,8 +2351,7 @@ __all__ = [ "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "is_profile_note", "join_paths", "make_item", "merge_notes", "name_words", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", - "query_snapshot_meta", "query_snapshot_tables", "read_catalog", "read_entry", "relation_graph", "relation_id", - "render_table_index", + "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes", "same_profile_note", "save_human_edit", "save_patch", "status_counts", "structure_relation_ids", "system_notes_source", "table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry", diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index 475a6a5e..f8ac5e4e 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -75,8 +75,8 @@ QUERY_ROWS = 50 SHOWN_ROWS = 10 CELL_MAX = 300 TOOL_CHARS = 12_000 -#: 网络错误、429、5xx 重试前等几秒（依次取用，个数就是最多重试几次） -RETRY_DELAYS_S = (2.0, 5.0) +#: 网络错误、429、5xx 重试前等几秒（依次取用，个数就是最多重试几次）。在对话里等，比起草（10、30 秒）短 +RETRY_DELAYS_S = (3.0, 10.0) #: 等模型的时候每隔几秒发一行 SSE 注释 HEARTBEAT_S = 10.0 #: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条 @@ -257,8 +257,9 @@ async def resolve_model(session: AsyncSession) -> tuple[Any, str]: provider, choice = await resolve_draft_provider(session, none_reason=ERROR_NO_MODEL, mock_reason=ERROR_NO_MODEL) except DraftModelUnavailable as e: raise catalog.CatalogModelUnavailable(str(e)) from e + # SDK 自己不重试（同起草）：网络错误、429、5xx 由这里在还没输出时重试，两层叠起来实际次数就对不上了 spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=catalog.MODEL_MAX_TOKENS, temperature=0, - timeout=int(catalog.MODEL_TIMEOUT_S)) + timeout=int(catalog.MODEL_TIMEOUT_S), max_retries=0) try: big = build_chat_model(provider, spec) small = build_chat_model(provider, spec.model_copy(update={"max_tokens": catalog.MODEL_FALLBACK_MAX_TOKENS})) diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index 2d268bb0..89f29bd8 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -228,6 +228,69 @@ async def test_unavailable_draft_model_is_409_with_the_draft_wording(client, tmp assert r.json()["code"] == "catalog_draft_model_unavailable" and "模型接入" in r.json()["detail"] +@pytest.fixture +async def mem(monkeypatch): + """单独的内存库：接入和设置只在这个用例里。""" + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + from sqlalchemy.pool import StaticPool + + from app.db.base import Base + + for key in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(key, raising=False) + engine = create_async_engine("sqlite+aiosqlite:///:memory:", poolclass=StaticPool) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + session = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)() + yield session + await session.close() + await engine.dispose() + + +async def _provider(session, kind: str) -> None: + from app.core.crypto import encrypt + from app.data import draft_model + from app.db.models import Provider, Setting + + session.add(Provider(name="起草网关", kind=kind, base_url="http://gw.example.test/v1" if kind != "anthropic" else None, + api_key=encrypt("sk-test"), models=[{"id": "模型甲"}], default_model="模型甲", enabled=True, + extra={})) + session.add(Setting(key=draft_model.CATALOG_DRAFT_SETTING_KEY, value={"provider": "起草网关", "model": None})) + await session.commit() + + +async def test_model_follows_the_draft_setting_keeps_thinking_and_a_big_budget(mem, monkeypatch): + from app.providers import factory + + await _provider(mem, "openai_compatible") + specs: list[Any] = [] + real = factory.build_chat_model + + def spy(provider, spec): + specs.append(spec) + return real(provider, spec) + + monkeypatch.setattr(factory, "build_chat_model", spy) + model, model_id = await catalog_assistant.resolve_model(mem) + assert model_id == "模型甲" and model.max_tokens == catalog.MODEL_MAX_TOKENS == 32768 + mine = specs[-2:] # 前两份是 catalog.resolve_draft_model 查能不能用时建的 + assert [s.max_tokens for s in mine] == [catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS] + assert all(s.thinking is None and s.temperature == 0 and s.max_retries == 0 for s in mine) + assert all(s.timeout == catalog.MODEL_TIMEOUT_S for s in mine) + + +@pytest.mark.parametrize("kind", ["openai_compatible", "anthropic"]) +async def test_tools_bind_on_real_chat_models(mem, kind): + """两个工具的定义各家的模型都收（只绑定，不发请求）。""" + await _provider(mem, kind) + model, _ = await catalog_assistant.resolve_model(mem) + tools = [catalog_assistant.QUERY_TOOL_PROMPT, catalog_assistant.PROPOSE_TOOL_PROMPT] + bound = model.bind_tools(tools) + given = bound.model.kwargs["tools"] + names = [t.get("name") or t["function"]["name"] for t in given] + assert names == ["run_query", "propose_changes"] + + # -------------------------------------------------------------------------- # 背景 # -------------------------------------------------------------------------- From 77ddbe96ea303b2b86e0bc8f7b35c05f2279f965 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:26:05 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=E7=9A=84?= =?UTF-8?q?=E8=83=8C=E6=99=AF=E5=9C=A8=E5=AE=BD=E8=A1=A8=E4=B8=8A=E4=B8=8D?= =?UTF-8?q?=E8=87=B3=E4=BA=8E=E5=A4=AA=E9=95=BF=EF=BC=9A=E8=BF=99=E5=BC=A0?= =?UTF-8?q?=E8=A1=A8=E7=9A=84=E7=9B=AE=E5=BD=95=E6=9C=80=E5=A4=9A=E5=88=97?= =?UTF-8?q?=20600=20=E9=A1=B9=EF=BC=88=E8=A1=A8=E7=BA=A7=E7=9A=84=E9=A1=B9?= =?UTF-8?q?=E3=80=81=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB=E5=9C=A8=E5=89=8D?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E5=85=B6=E4=BD=99=E5=86=99=E6=98=8E=E8=BF=98?= =?UTF-8?q?=E6=9C=89=E5=87=A0=E9=A1=B9=E6=B2=A1=E5=88=97=E5=87=BA=EF=BC=9B?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E6=97=B6=E7=95=8C=E9=9D=A2=E7=9B=B4=E6=8E=A5?= =?UTF-8?q?=E4=BA=A4=E6=9D=A5=E7=9A=84=E6=96=87=E4=BB=B6=E5=8E=9F=E6=96=87?= =?UTF-8?q?=EF=BC=88JSON=20=E6=96=87=E5=AD=97=EF=BC=89=E4=B9=9F=E8=AE=A4?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E6=98=AF=20JSON=20=E7=9A=84=20422=EF=BC=9B?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E5=86=B2=E7=AA=81=E8=AF=B4=E6=98=8E=E9=87=8C?= =?UTF-8?q?=E7=9A=84=E7=8A=B6=E6=80=81=E7=94=A8=E7=95=8C=E9=9D=A2=E7=9A=84?= =?UTF-8?q?=E5=8F=AB=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog_assistant.py | 15 +++++++++++---- backend/app/data/catalog_transfer.py | 11 +++++++++-- backend/tests/test_catalog_assistant.py | 12 ++++++++++++ backend/tests/test_catalog_transfer.py | 5 +++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index f8ac5e4e..9dadbd98 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -79,9 +79,11 @@ TOOL_CHARS = 12_000 RETRY_DELAYS_S = (3.0, 10.0) #: 等模型的时候每隔几秒发一行 SSE 注释 HEARTBEAT_S = 10.0 -#: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条 +#: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条；这张表的目录最多列几项 +#: （几百列的宽表全部起草过有上千项，全列出来提示词就太长了：表级的项、关联关系在前，列的项在后） UNNAMED_SHOWN = 30 INCOMING_SHOWN = 30 +CATALOG_ITEMS_SHOWN = 600 QUERY_TOOL = "run_query" PROPOSE_TOOL = "propose_changes" @@ -166,6 +168,7 @@ ASSISTANT_PROMPT_LABELS = { "catalog_empty": "这张表还没有数据目录。", "item": "- {path}（{label}）：{value}〔{source}，{status}〕", "item_note": " 备注：{text}", + "catalog_more": "（还有 {n} 项没有列出：这张表的目录太长。要改的项照上面的路径写法提，核对时按当前目录算）", "relation": "{columns} → {to_table}.{to_columns}{extra}", "relation_extra": "（{text}）", "cardinality": "基数 {value}", @@ -357,10 +360,14 @@ def _catalog_lines(notes: Mapping[str, Any] | None, masked: set[str]) -> list[st （剖析的备注里有取值范围）。""" labels = ASSISTANT_PROMPT_LABELS lines: list[str] = [] - for key, item in catalog._slots(notes).items(): + # 表级的项、关联关系在前，列的项在后（稳定排序，各自照目录里的顺序）；遮罩列上的码值不算 + slots = [(key, item) for key, item in sorted(catalog._slots(notes).items(), key=lambda kv: kv[0][0] == "c") + if not (key[0] == "c" and key[2] == "codes" and key[1].lower() in masked)] + for shown, (key, item) in enumerate(slots): + if shown >= CATALOG_ITEMS_SHOWN: + lines.append(labels["catalog_more"].format(n=len(slots) - shown)) + break hidden = _touches_masked(key, item, masked) - if hidden and key[0] == "c" and key[2] == "codes": - continue if key[0] == "r": label, value = labels["field_relation"], _relation_line(item) else: diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py index 34954f8d..6dccd7ec 100644 --- a/backend/app/data/catalog_transfer.py +++ b/backend/app/data/catalog_transfer.py @@ -30,6 +30,7 @@ tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经 from __future__ import annotations import copy +import json import re from collections.abc import Mapping from dataclasses import dataclass, field @@ -51,7 +52,8 @@ _UNNAMED = "数据源" #: 状态：照带的、只填空缺的 _LOCKED = ("confirmed", "rejected") -_STATUS_LABEL = {"proposed": "推断", "verified": "有确证", "confirmed": "已确认", "rejected": "已驳回"} +#: 状态的界面叫法（同 frontend/src/lib/terms.ts 的 CATALOG_STATUS_LABEL） +_STATUS_LABEL = {"proposed": "推断", "verified": "已验证", "confirmed": "已确认", "rejected": "已驳回"} #: 写入一直撞上别人刚改过（重读重算三次仍冲突） WRITE_CONFLICT_ERROR = "这张表的数据目录正被其他人修改，没有导入，请稍后重新导入" @@ -106,7 +108,12 @@ class ImportFile: def read_file(content: Any, *, include_guidance: bool) -> ImportFile: - """导出的那个对象 → ImportFile。格式不对抛 TransferInvalid（说法给人看）。""" + """导出的那个对象 → ImportFile。界面直接交来文件的原文（一段 JSON 文字）也认。格式不对抛 TransferInvalid（说法给人看）。""" + if isinstance(content, str): + try: + content = json.loads(content) + except ValueError as e: + raise TransferInvalid("文件内容不是合法的 JSON，请选择从 AgentLab 导出的数据目录文件") from e if not isinstance(content, Mapping): raise TransferInvalid("文件内容不是数据目录导出的格式，请选择从 AgentLab 导出的数据目录文件") if content.get("format") != FORMAT: diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index 89f29bd8..f8f90b6f 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -351,6 +351,18 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client, assert "没开剖析" not in system +async def test_long_catalogs_are_cut_with_a_note(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "CATALOG_ITEMS_SHOWN", 2) + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("看看")) + system = model.seen[0][0].content + # 表级的项在前；遮罩列的码值本来就不算；剩下的说明还有几项 + assert "- label（中文名）：订单" in system and "- grain（粒度）" in system and "- kind（表类型）" not in system + assert "还有 3 项没有列出" in system + + async def test_without_profiling_there_is_no_query_tool(client, tmp_path, monkeypatch): sid = await _source(client, tmp_path, profile=False) model = Scripted([Turn(text=["只能根据表结构推断。"])]) diff --git a/backend/tests/test_catalog_transfer.py b/backend/tests/test_catalog_transfer.py index de5164b5..e9e50b98 100644 --- a/backend/tests/test_catalog_transfer.py +++ b/backend/tests/test_catalog_transfer.py @@ -334,6 +334,11 @@ async def test_import_waits_for_draft_jobs_and_rejects_bad_files(client): r = await client.post(f"{_base(local)}/import/preview", json={"content": body}) assert r.status_code == 422, body assert any("一" <= ch <= "鿿" for ch in r.json()["detail"]) + # 界面直接交来文件原文也认；不是 JSON 的 422 + as_text = await client.post(f"{_base(local)}/import/preview", json={"content": json.dumps(content)}) + assert as_text.status_code == 200 and as_text.json()["totals"]["tables"] == 2 + not_json = await client.post(f"{_base(local)}/import/preview", json={"content": "{不是 JSON"}) + assert not_json.status_code == 422 and "JSON" in not_json.json()["detail"] too_long = {**content, "guidance": "长" * (catalog_jobs.GUIDANCE_MAX + 1)} r = await client.post(f"{_base(local)}/import/preview", json={"content": too_long, "include_guidance": True}) assert r.status_code == 422 and "起草要求" in r.json()["detail"] From 8c270987bd489afeace001349f83bb13366ede55 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:28:30 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=EF=BC=9A?= =?UTF-8?q?=E7=BB=91=E5=AE=9A=E5=B7=A5=E5=85=B7=E5=87=BA=E9=94=99=EF=BC=88?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E4=B8=8D=E6=94=AF=E6=8C=81=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E4=B9=8B=E7=B1=BB=EF=BC=89=E4=B9=9F=E5=9C=A8?= =?UTF-8?q?=E6=B5=81=E9=87=8C=E7=85=A7=E5=AE=9E=E4=BA=A4=E4=B8=80=E6=9D=A1?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E4=BA=8B=E4=BB=B6=EF=BC=8C=E4=B8=8D=E8=AE=A9?= =?UTF-8?q?=E6=B5=81=E5=8D=8A=E8=B7=AF=E6=96=AD=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog_assistant.py | 3 ++- backend/tests/test_catalog_assistant.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index 9dadbd98..c6e5d9fa 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -813,11 +813,12 @@ async def stream(model: Any, model_id: str, ctx: AssistantContext, history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)] tools = _Tools(ctx) - runner = model.bind_tools(ctx.tools()) usage = {"input_tokens": 0, "output_tokens": 0} wrote = "" names = [t["function"]["name"] for t in ctx.tools()] try: + # 绑工具也放在这里面：响应头已经发出去了，出了错要照实交一条错误事件，不能让流半路断掉 + runner = model.bind_tools(ctx.tools()) for round_no in range(MAX_ROUNDS): fresh = True message: Any = None diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index f8f90b6f..8013dd75 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -524,6 +524,19 @@ async def test_errors_end_the_stream_without_done(client, tmp_path, monkeypatch) assert "上限" in events[-1]["message"] +async def test_a_model_without_tools_reports_an_error_event(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + + class NoTools(Scripted): + def bind_tools(self, tools, **_): + raise NotImplementedError("这个模型不支持工具调用") + + use(monkeypatch, NoTools([])) + events, _ = await _talk(client, sid, _ask("你好")) + assert events == [{"type": "error", "code": "assistant_failed", "message": events[0]["message"]}] + assert "工具调用" in events[0]["message"] + + async def test_transient_errors_are_retried_before_any_output(client, tmp_path, monkeypatch): sid = await _source(client, tmp_path) monkeypatch.setattr(catalog_assistant, "RETRY_DELAYS_S", (0.0, 0.0)) From 864e63daddc3e351845fe336169a51295d7dd83f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 09:51:49 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E5=AE=A1=E9=98=85=E9=98=9F=E5=88=97?= =?UTF-8?q?=E4=B8=8E=E8=BF=9B=E5=BA=A6=E6=8C=89=E7=BA=A6=E5=AE=9A=E6=94=B9?= =?UTF-8?q?=EF=BC=9Afield=5Flabel=20=E5=86=99=E7=BB=99=E4=BA=BA=E7=9C=8B?= =?UTF-8?q?=E7=9A=84=E5=AE=8C=E6=95=B4=E4=BD=8D=E7=BD=AE=EF=BC=88=E3=80=8C?= =?UTF-8?q?=E8=A1=A8=E7=9A=84=E6=9C=89=E6=95=88=E8=AE=B0=E5=BD=95=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=E3=80=8D=E3=80=8C=E5=88=97=20status=20=E7=9A=84?= =?UTF-8?q?=E7=A0=81=E5=80=BC=E3=80=8D=E3=80=8C=E6=8C=87=E5=90=91=20member?= =?UTF-8?q?s=20=E7=9A=84=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB=E3=80=8D?= =?UTF-8?q?=EF=BC=8C=E5=92=8C=E8=A1=A8=E5=8A=A9=E6=89=8B=E6=8F=90=E6=A1=88?= =?UTF-8?q?=E3=80=81=E5=AF=BC=E5=85=A5=E5=86=B2=E7=AA=81=E7=9A=84=E8=AF=B4?= =?UTF-8?q?=E6=B3=95=E5=90=8C=E4=B8=80=E4=B8=AA=E5=87=BD=E6=95=B0=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E5=88=97=E7=BA=A7=E7=9A=84=E9=A1=B9=E5=8F=A6=E5=B8=A6?= =?UTF-8?q?=20column=EF=BC=9B=E6=8E=92=E5=9C=A8=E8=BF=99=E9=87=8C=E7=9A=84?= =?UTF-8?q?=E5=8E=9F=E5=9B=A0=E5=86=99=E3=80=8C=E8=A2=AB=20N=20=E6=AC=A1?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=94=A8=E5=88=B0=E3=80=8D=EF=BC=8C=E6=95=B0?= =?UTF-8?q?=E7=9A=84=E6=98=AF=E8=BF=90=E8=A1=8C=EF=BC=88=E5=90=8C=E3=80=8C?= =?UTF-8?q?=E7=94=A8=E8=BF=87=E8=BF=99=E5=BC=A0=E8=A1=A8=E7=9A=84=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E3=80=8D=E7=9A=84=20total=EF=BC=89=EF=BC=8C=E5=90=8C?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E8=BF=90=E8=A1=8C=E9=87=8C=E6=9F=A5=E4=BA=86?= =?UTF-8?q?=E5=87=A0=E6=AC=A1=E3=80=81=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0?= =?UTF-8?q?=E6=AF=94=E8=BF=90=E8=A1=8C=E6=AC=A1=E6=95=B0=E5=A4=9A=E6=97=B6?= =?UTF-8?q?=E4=B8=A4=E4=B8=AA=E9=83=BD=E5=86=99=EF=BC=88=E3=80=8C=E8=A2=AB?= =?UTF-8?q?=202=20=E6=AC=A1=E8=BF=90=E8=A1=8C=E7=94=A8=E5=88=B0=EF=BC=8C?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=205=20=E6=AC=A1=E3=80=8D=EF=BC=89=EF=BC=8Cta?= =?UTF-8?q?ble=5Fusage=20=E5=92=8C=E4=BC=98=E5=85=88=E7=BA=A7=E7=85=A7?= =?UTF-8?q?=E6=97=A7=E6=8C=89=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0=EF=BC=88?= =?UTF-8?q?=E5=90=8C=E7=9B=AE=E5=BD=95=E6=B8=85=E5=8D=95=E7=9A=84=E3=80=8C?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=20N=20=E6=AC=A1=E3=80=8D=E5=92=8C=E6=8E=92?= =?UTF-8?q?=E5=BA=8F=EF=BC=89=EF=BC=9B=E8=BF=9B=E5=BA=A6=E9=87=8C=E7=9A=84?= =?UTF-8?q?=E5=B8=B8=E7=94=A8=2020=20=E5=BC=A0=E8=A1=A8=E5=92=8C=E8=B5=B7?= =?UTF-8?q?=E8=8D=89=E7=9A=84=E9=BB=98=E8=AE=A4=E8=8C=83=E5=9B=B4=E6=98=AF?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E6=89=B9=EF=BC=88=E4=BD=BF=E7=94=A8=E6=AC=A1?= =?UTF-8?q?=E6=95=B0=E4=BB=8E=E5=A4=9A=E5=88=B0=E5=B0=91=EF=BC=8C=E7=94=A8?= =?UTF-8?q?=E5=88=B0=E7=9A=84=E4=B8=8D=E5=A4=9F=2020=20=E5=BC=A0=E6=97=B6?= =?UTF-8?q?=E6=8C=89=E8=A1=A8=E7=BB=93=E6=9E=84=E7=9A=84=E9=A1=BA=E5=BA=8F?= =?UTF-8?q?=E8=A1=A5=E9=BD=90=EF=BC=89=EF=BC=8Ctop=20=E5=8F=AA=E7=BB=99=20?= =?UTF-8?q?tables=E3=80=81items=5Ftotal=E3=80=81items=5Fconfirmed=EF=BC=8C?= =?UTF-8?q?all=20=E5=8F=AA=E7=BB=99=E7=BA=A6=E5=AE=9A=E7=9A=84=E5=9B=9B?= =?UTF-8?q?=E9=A1=B9=EF=BC=9B=E8=A1=A8=E5=8A=A9=E6=89=8B=E8=83=8C=E6=99=AF?= =?UTF-8?q?=E9=87=8C=E5=86=99=E8=BF=99=E5=BC=A0=E8=A1=A8=E8=A2=AB=E5=87=A0?= =?UTF-8?q?=E6=AC=A1=E8=BF=90=E8=A1=8C=E7=94=A8=E5=88=B0=E3=80=81=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=87=A0=E6=AC=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog_assistant.py | 16 ++-- backend/app/data/catalog_review.py | 105 +++++++++++++++------ backend/app/data/catalog_transfer.py | 12 +-- backend/tests/test_catalog_assistant.py | 22 ++++- backend/tests/test_catalog_review_queue.py | 66 ++++++++++--- 5 files changed, 158 insertions(+), 63 deletions(-) diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index c6e5d9fa..e8fd3b28 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -57,7 +57,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.data import catalog, catalog_jobs, catalog_profile -from app.data.catalog_review import SOURCE_LABEL, slot_path +from app.data.catalog_review import SOURCE_LABEL, item_label, slot_path, table_usage_counts from app.data.draft_model import DRAFT_MODEL_PLACE from app.data.engine import SnapshotTampered, masked_columns @@ -153,7 +153,8 @@ ASSISTANT_PROMPT_LABELS = { "view": "（视图）", "comment": "数据库注释：{text}", "system": "系统说明：{text}", - "usage": "这张表被运行用过 {n} 次。", + "usage": "这张表被 {runs} 次运行用到（查询 {n} 次）。", + "usage_none": "这张表还没有被运行用到过。", "description": "数据源说明：{text}", "guidance": "数据源的起草要求（使用者补充的背景和要求）：{text}", "structure": "表结构：", @@ -469,7 +470,8 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, entries = await catalog.read_catalog(session, source.id) entry = entries.get(table) notes = entry.notes if entry else {} - usage = (await catalog.table_usage(session, source)).get(table, 0) + usage = await table_usage_counts(session, source) + used, runs = usage.queries.get(table, 0), usage.runs.get(table, 0) blank_n, blank_cols = await _last_blank(session, source.id, table) lines = [labels["head"], labels["table"].format(source=row.name, kind=row.kind, table=table, @@ -477,7 +479,7 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, comment = str(meta.get("comment") or "").strip() if comment: lines.append((labels["system"] if system_notes else labels["comment"]).format(text=comment)) - lines.append(labels["usage"].format(n=usage)) + lines.append(labels["usage"].format(runs=runs, n=used) if used else labels["usage_none"]) description = str(getattr(row, "description", "") or "").strip() if description: lines.append(labels["description"].format(text=description)) @@ -693,11 +695,9 @@ class _Tools: def _where(key: catalog.Slot, change: catalog.PatchChange) -> str: - """提案里一项的中文说法：「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」。""" + """提案里一项的中文说法：「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」（同审阅队列的 field_label）。""" value = change.after if isinstance(change.after, Mapping) else change.before - if key[0] == "r" and isinstance(value, Mapping) and value.get("to_table"): - return f"指向 {value['to_table']} 的关联关系" - return catalog.describe_slot(key) + return item_label(key, value if isinstance(value, Mapping) else None) # ========================================================================== diff --git a/backend/app/data/catalog_review.py b/backend/app/data/catalog_review.py index b03c9e31..5e7fba68 100644 --- a/backend/app/data/catalog_review.py +++ b/backend/app/data/catalog_review.py @@ -6,8 +6,8 @@ - **队列**（review_queue，GET /catalog/review-queue）：只列推断（proposed）的项。priority = 使用次数权重 × 风险权重 × 来源权重，从高到低；同分按表名、路径排，结果稳定。 - - 使用次数权重：1 + log2(1 + 使用次数)。使用次数同目录清单（catalog.table_usage）：按对数增长，常用表在前，但不至于 - 让常用表上的中文名压过所有表上影响数字的项。 + - 使用次数权重：1 + log2(1 + 使用次数)。使用次数同目录清单（catalog.table_usage，清单的「使用 N 次」和排序、起草的默认 + 范围都按它）：按对数增长，常用表在前，但不至于让常用表上的中文名压过所有表上影响数字的项。 - 风险权重（RISK_WEIGHT）：有效记录条件、去重规则、业务日期、关联关系最高（直接改变数字），业务主键、度量类型次之 （计数和能不能加总），码值、粒度、表类型再次，中文名、说明、含义、单位最低。 - 来源权重（SOURCE_WEIGHT）：模型起草、命名推断是猜的，比数据库注释优先；数据剖析（码值候选、业务日期、没核实的关系） @@ -15,14 +15,19 @@ - 两组（kind）：facts 影响数字的项（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）；names 名称和说明 （中文名、说明、含义、单位、码值、粒度、表类型）。表的「说明」和中文名、含义一样只是说明文字，归 names。 - 表结构里已经没有的表、列上的项不列（确认了也没有用，目录页上另有清理的提示）。 - - reasons 用中文说为什么排在这里：常用表（使用 N 次）、影响数字的项（有效记录条件）、模型起草，未确认…… + - field_label 是这一项给人看的位置：「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」（item_label， + 和表助手提案、导入冲突里的说法一致）；列级的项另给 column。 + - table_usage 是所在表的使用次数（同清单）。reasons 用中文说为什么排在这里：常用表（被 N 次运行用到）、影响数字的项 + （有效记录条件）、模型起草，未确认……「被 N 次运行用到」数的是运行（同 /usage 的 total）；同一次运行里查了几次、 + 使用次数比运行次数多时两个都写（「被 2 次运行用到，使用 5 次」），和清单、运行列表上的数都对得上。 - **批量审阅**（review_batch，POST /catalog/review/batch）：逐表按 if_version 乐观锁，同一张表的几条在一次写入里完成， 一张表冲突、写不进去不影响别的表；路径不对、表不存在逐条报（results 和请求一一对应）。审阅的规则同单项审阅 （catalog.review_item）。 - **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：取法同使用次数（catalog.query_snapshot_tables），按运行 - 去重，新的在前。 -- **进度**（review_progress，GET /catalog/review-progress）：使用次数前 TOP_N 张表（只算被运行用到过的）、全部、影响 - 数字的项各自的确认情况。项的范围和队列一样：表结构里没有的表、列不算。 + 去重，新的在前；total 是运行次数。 +- **进度**（review_progress，GET /catalog/review-progress）：常用表（使用次数前 TOP_N 张，和起草对话框「使用次数最多的 + 20 张表」是同一批：catalog.default_tables 的取法，用到的不够 TOP_N 张时按表结构的顺序补齐）、全部、影响数字的项各自的 + 确认情况。项的范围和队列一样：表结构里没有的表、列不算。 """ from __future__ import annotations @@ -44,7 +49,7 @@ KINDS = ("all", "facts", "names") QUEUE_LIMIT_MAX = 200 BATCH_MAX = 200 RUNS_LIMIT_MAX = 100 -#: 进度里的「常用表」取使用次数前几张 +#: 进度里的「常用表」取使用次数前几张（同起草的默认范围 catalog.DRAFT_DEFAULT_TABLES） TOP_N = 20 #: 每一项属于哪一组、风险几档：键是 (t, 字段) / (c, 字段) / (r, "")，档 4 最高 @@ -64,9 +69,10 @@ SOURCE_WEIGHT = {"llm": 1.5, "name": 1.5, "profile": 1.25, "comment": 1.0} SOURCE_LABEL = {"comment": "数据库注释", "fk": "外键约束", "name": "命名推断", "profile": "数据剖析", "llm": "模型起草", "human": "人工填写"} RELATION_LABEL = "关联关系" -#: reasons 的说法 -REASON_TOP = "常用表（使用 {n} 次）" -REASON_USED = "使用 {n} 次" +#: reasons 的说法：常用表（被几次运行用到）；使用次数比运行次数多（同一次运行里查了几次）时两个都写 +REASON_TOP = "常用表（{used}）" +USED_RUNS = "被 {runs} 次运行用到" +USED_RUNS_AND_QUERIES = "被 {runs} 次运行用到，使用 {n} 次" REASON_RISK = {4: "影响数字的项（{label}）", 3: "影响计数和汇总的项（{label}）", 2: "帮助理解取值的项（{label}）", 1: "名称和说明（{label}）"} REASON_PENDING_CODES = "{n} 个码值的含义待填写" @@ -109,8 +115,8 @@ def field_name(key: Slot) -> str: return "relation" if key[0] == "r" else key[-1] -def field_label(key: Slot) -> str: - """字段的界面叫法：中文名、码值、有效记录条件、关联关系……""" +def field_text(key: Slot) -> str: + """字段的界面叫法（不带列名）：中文名、码值、有效记录条件、关联关系……""" if key[0] == "t": return catalog.TABLE_FIELD_LABEL.get(key[1], key[1]) if key[0] == "c": @@ -118,6 +124,13 @@ def field_label(key: Slot) -> str: return RELATION_LABEL +def item_label(key: Slot, item: Mapping[str, Any] | None = None) -> str: + """一项给人看的位置：「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」（不知道指向哪张表时写编号）。""" + if key[0] == "r" and isinstance(item, Mapping) and item.get("to_table"): + return f"指向 {item['to_table']} 的{RELATION_LABEL}" + return catalog.describe_slot(key) + + def item_value(key: Slot, item: Mapping[str, Any]) -> Any: """一项的值：关联关系是两端、基数和覆盖率，别的是 value。""" if key[0] == "r": @@ -183,18 +196,49 @@ def priority(usage: int, key: Slot, source: Any) -> float: def top_tables(tables: Mapping[str, Any], usage: Mapping[str, int]) -> list[str]: - """使用次数前 TOP_N 张表：只算被运行用到过的，同样多的按表结构里的顺序。""" + """使用次数前 TOP_N 张表（同 catalog.default_tables，起草对话框的「使用次数最多的 20 张表」）：使用次数从多到少， + 一样多的（包括都没用过的）按表结构里的顺序。表结构里不到 TOP_N 张时就是全部。""" order = {name: i for i, name in enumerate(tables)} - used = [name for name in tables if usage.get(name, 0) > 0] - return sorted(used, key=lambda n: (-usage[n], order[n]))[:TOP_N] + return sorted(tables, key=lambda n: (-usage.get(n, 0), order[n]))[:TOP_N] -def _reasons(key: Slot, item: Mapping[str, Any], usage: int, top: bool) -> list[str]: +@dataclass(frozen=True) +class Usage: + """每张表用得多不多（没被用过的表不出现）。 + + queries：使用次数，同 catalog.table_usage（目录清单的「使用 N 次」、清单的排序、起草的默认范围）； + runs：用到它的运行有几次，同「被哪些运行用过」（table_runs）的 total。同一次运行里查了几次这张表，queries 比 runs 多。 + """ + + queries: dict[str, int] + runs: dict[str, int] + + +async def table_usage_counts(session: AsyncSession, source: Any) -> Usage: + """一次读出每张表的使用次数和运行次数（同一份查询快照引用：catalog.query_snapshot_tables）。""" + queries: Counter[str] = Counter() + runs: dict[str, set[str]] = {} + for run, hit in await catalog.query_snapshot_tables(session, source): + queries.update(hit) + for name in hit: + runs.setdefault(name, set()).add(run) + return Usage(dict(queries), {name: len(ids) for name, ids in runs.items()}) + + +def usage_text(queries: int, runs: int) -> str: + """「被 2 次运行用到」；使用次数比运行次数多时「被 2 次运行用到，使用 5 次」。""" + if queries > runs: + return USED_RUNS_AND_QUERIES.format(runs=f"{runs:,}", n=f"{queries:,}") + return USED_RUNS.format(runs=f"{runs:,}") + + +def _reasons(key: Slot, item: Mapping[str, Any], usage: int, runs: int, top: bool) -> list[str]: out: list[str] = [] if usage > 0: - out.append((REASON_TOP if top else REASON_USED).format(n=f"{usage:,}")) + used = usage_text(usage, runs) + out.append(REASON_TOP.format(used=used) if top else used) _, tier = FIELD_GROUP[field_of(key)] - out.append(REASON_RISK[tier].format(label=field_label(key))) + out.append(REASON_RISK[tier].format(label=field_text(key))) if key[0] == "c" and key[2] == "codes" and isinstance(item.get("value"), Mapping): pending = sum(1 for v in item["value"].values() if not str(v).strip()) if pending: @@ -225,14 +269,14 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", if table: wanted = catalog.resolve_table_name(cache, table) or table entries = await catalog.read_catalog(session, source.id) - usage = await catalog.table_usage(session, source) - top = set(top_tables(tables, usage)) + usage = await table_usage_counts(session, source) + top = set(top_tables(tables, usage.queries)) items: list[dict[str, Any]] = [] for name, entry in entries.items(): meta = tables.get(name) if meta is None or (wanted is not None and name != wanted): continue - used = usage.get(name, 0) + used = usage.queries.get(name, 0) for key, item in live_slots(entry.notes, meta): if item.get("status") != "proposed": continue @@ -244,7 +288,7 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", "table_name": name, "path": slot_path(key), "field": field_name(key), - "field_label": field_label(key), + "field_label": item_label(key, item), # 列级的项是哪一列（表级的项、关联关系为 null） "column": key[1] if key[0] == "c" else None, "value": value, @@ -253,7 +297,7 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", "status": item.get("status"), "note": item.get("note") if isinstance(item.get("note"), str) and item.get("note") else None, "priority": round(priority(used, key, item.get("source")), 4), - "reasons": _reasons(key, item, used, name in top), + "reasons": _reasons(key, item, used, usage.runs.get(name, 0), name in top), "table_usage": used, "version": entry.version, }) @@ -367,7 +411,8 @@ async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: i async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]: - """审阅进度：常用 TOP_N 张表（只算被运行用到过的）、全部、影响数字的项。项按状态计数；verified（有确证）另给一项。""" + """审阅进度：常用 TOP_N 张表（top_tables）、全部、影响数字的项，项按状态计数。全部的 items_total 里还有有确证的项 + （不单列）。""" tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} entries = await catalog.read_catalog(session, source.id) usage = await catalog.table_usage(session, source) @@ -388,17 +433,15 @@ async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]: facts[status] += 1 return { "top_n": TOP_N, - "top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"], - "items_proposed": head["proposed"], "items_verified": head["verified"]}, + "top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"]}, "all": {"items_total": sum(every.values()), "items_confirmed": every["confirmed"], - "items_proposed": every["proposed"], "items_rejected": every["rejected"], - "items_verified": every["verified"]}, + "items_proposed": every["proposed"], "items_rejected": every["rejected"]}, "facts": {"proposed": facts["proposed"], "confirmed": facts["confirmed"]}, } __all__ = ["BATCH_MAX", "BatchItem", "CODE_CONFLICT", "CODE_INVALID", "CODE_PATH", "CODE_TABLE_MISSING", "FIELD_GROUP", "KINDS", "QUEUE_LIMIT_MAX", "RISK_WEIGHT", "RUNS_LIMIT_MAX", "SOURCE_LABEL", "SOURCE_WEIGHT", - "TOP_N", "display", "field_label", "field_name", "field_of", "item_value", "live_slots", "priority", - "relation_text", "review_batch", "review_progress", "review_queue", "slot_path", "table_runs", "top_tables", - "usage_weight"] + "TOP_N", "Usage", "display", "field_name", "field_of", "field_text", "item_label", "item_value", + "live_slots", "priority", "relation_text", "review_batch", "review_progress", "review_queue", "slot_path", + "table_runs", "table_usage_counts", "top_tables", "usage_text", "usage_weight"] diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py index 6dccd7ec..6a5b3e5d 100644 --- a/backend/app/data/catalog_transfer.py +++ b/backend/app/data/catalog_transfer.py @@ -39,7 +39,7 @@ from typing import Any from sqlalchemy.ext.asyncio import AsyncSession from app.data import catalog, catalog_jobs -from app.data.catalog_review import field_label, slot_path +from app.data.catalog_review import item_label, slot_path from app.data.engine import masked_columns from app.data.names import name_key @@ -358,7 +358,7 @@ def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], targ plan.keep += 1 else: plan.conflicts.append({ - "path": slot_path(mapped), "label": _label(mapped, item), + "path": slot_path(mapped), "label": item_label(mapped, item), "file_value": _shown_value(mapped, item), "target_value": _shown_value(mapped, have), "file_status": status, "target_status": have.get("status"), "why": _conflict_why(str(status), str(have.get("status"))), @@ -371,14 +371,6 @@ def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], targ return plan -def _label(key: catalog.Slot, item: Mapping[str, Any]) -> str: - if key[0] == "t": - return f"表的{field_label(key)}" - if key[0] == "c": - return f"列 {key[1]} 的{field_label(key)}" - return f"指向 {item.get('to_table')} 的关联关系" - - def _shown_value(key: catalog.Slot, item: Mapping[str, Any]) -> Any: if key[0] == "r": return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")} diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index 8013dd75..09a4a769 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -335,7 +335,7 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client, assert rule in system # 表结构、目录（来源和状态）、使用次数、起草要求、数据源说明 for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认", - "模型起草", "已驳回", "被运行用过 0 次", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"): + "模型起草", "已驳回", "还没有被运行用到过", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"): assert fact in system, fact # 别的表指过来的关联关系 assert "members.id → orders.member_id" in system @@ -351,6 +351,26 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client, assert "没开剖析" not in system +async def test_background_says_how_many_runs_used_the_table(client, tmp_path, monkeypatch): + """使用次数的口径同审阅队列：被几次运行用到（同「用过这张表的运行」的 total），查询几次（同目录清单的「使用 N 次」）。""" + from app.db.models import Artifact, Run + + sid = await _source(client, tmp_path) + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + for i, n in enumerate((2, 1)): + rid = uuid.uuid4().hex + session.add(Run(id=rid, graph={}, workflow_name=f"周报{i}", status="succeeded")) + for _ in range(n): + session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("看看")) + assert "这张表被 2 次运行用到（查询 3 次）。" in model.seen[0][0].content + + async def test_long_catalogs_are_cut_with_a_note(client, tmp_path, monkeypatch): sid = await _source(client, tmp_path) monkeypatch.setattr(catalog_assistant, "CATALOG_ITEMS_SHOWN", 2) diff --git a/backend/tests/test_catalog_review_queue.py b/backend/tests/test_catalog_review_queue.py index 1598103c..239694d2 100644 --- a/backend/tests/test_catalog_review_queue.py +++ b/backend/tests/test_catalog_review_queue.py @@ -2,11 +2,13 @@ - GET /catalog/review-queue：只列推断（proposed）的项，按「使用次数权重 × 风险权重 × 来源权重」从高到低，同分按表名、 路径；facts（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）/ names（中文名、说明、含义、单位、码值、 - 粒度、表类型）两组；按表筛选；分页；reasons 写为什么排在这里；表结构里已经没有的表、列不列 + 粒度、表类型）两组；按表筛选；分页；field_label 是给人看的位置（「列 status 的码值」）；reasons 写为什么排在这里 + （被几次运行用到，和 /usage 的 total 一致）；table_usage 是使用次数（同目录清单）；表结构里已经没有的表、列不列 - POST /catalog/review/batch：逐表按 if_version 乐观锁，一张表冲突不影响别的表；同一张表的几条一次写入；路径不对、表不存在 逐条报 - GET /catalog/{table}/usage：用过这张表的运行，新的在前，取法同 catalog.table_usage -- GET /catalog/review-progress：常用 20 张表、全部、影响数字的项各自的确认情况 +- GET /catalog/review-progress：常用 20 张表（使用次数前 20 张，同起草的默认范围：用到的不够 20 张时按表结构的顺序 + 补齐）、全部、影响数字的项各自的确认情况 表名、列名、业务数字都是虚构的。 """ @@ -188,11 +190,11 @@ async def test_queue_lists_only_proposed_items_by_priority(client): rel_id = _member_rel()["id"] assert (first["table_name"], first["path"]) in {("orders", "valid_filter"), ("orders", f"relations.{rel_id}")} vf = next(i for i in items if i["path"] == "valid_filter") - assert vf["field"] == "valid_filter" and vf["field_label"] == "有效记录条件" + assert vf["field"] == "valid_filter" and vf["field_label"] == "表的有效记录条件" assert vf["value"] == "status <> 9" and vf["display"] == "status <> 9" assert vf["source"] == "llm" and vf["note"] == "剖析核对：排除 3.2% 的行（共 12,345 行）" assert vf["table_usage"] == 37 and vf["version"] == 1 - assert vf["reasons"] == ["常用表（使用 37 次）", "影响数字的项（有效记录条件）", "模型起草，未确认"] + assert vf["reasons"] == ["常用表（被 37 次运行用到）", "影响数字的项（有效记录条件）", "模型起草，未确认"] # 常用表上的中文名排在没人用的表上的中文名前面；同一张表上影响数字的项排在中文名前面 order = [(i["table_name"], i["path"]) for i in items] assert order.index(("orders", "label")) < order.index(("coupons", "label")) @@ -203,7 +205,7 @@ async def test_queue_values_are_displayed_in_ui_words(client): sid = await _source() items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]} rel = items[("orders", f"relations.{_member_rel()['id']}")] - assert rel["field"] == "relation" and rel["field_label"] == "关联关系" + assert rel["field"] == "relation" and rel["field_label"] == "指向 members 的关联关系" assert rel["value"] == {"columns": ["member_id"], "to_table": "members", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": 0.98} assert rel["display"] == "member_id → members.id（多对一，覆盖率 98%）" @@ -211,15 +213,39 @@ async def test_queue_values_are_displayed_in_ui_words(client): kind = items[("orders", "kind")] assert kind["display"] == "明细表" and kind["value"] == "fact" codes = items[("orders", "columns.status.codes")] - assert codes["field"] == "codes" and codes["field_label"] == "码值" and codes["column"] == "status" + assert codes["field"] == "codes" and codes["field_label"] == "列 status 的码值" and codes["column"] == "status" assert codes["display"] == "1=已支付、9=含义待填写" assert "1 个码值的含义待填写" in codes["reasons"] and "数据剖析，未确认" in codes["reasons"] assert items[("orders", "columns.amount.measure")]["display"] == "可累加" - assert items[("members", "keys")]["display"] == "id" + assert items[("members", "keys")]["display"] == "id" and items[("members", "keys")]["field_label"] == "表的业务主键" + assert items[("orders", "columns.amount.measure")]["field_label"] == "列 amount 的度量类型" assert items[("members", "label")]["reasons"] == ["名称和说明（中文名）", "模型起草，未确认"] assert items[("coupons", "label")]["reasons"] == ["名称和说明（中文名）", "数据库注释，未确认"] +async def test_queue_reasons_count_runs_like_the_usage_list(client): + """同一次运行里查了几次这张表：原因写「被几次运行用到」（和 /usage 的 total 一样），table_usage 是使用次数（和目录清单的 + 「使用 N 次」一样），两个数不一样时原因里两个都写；优先级按使用次数（同清单的排序）。""" + sid = await _source() + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + runs = await _use(sid, name, {"orders": 2, "members": 1}) + async with SessionLocal() as session: + for _ in range(3): + session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + usage = await catalog.table_usage(session, await session.get(DataSource, sid)) + assert usage["orders"] == 5 + items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]} + vf = items[("orders", "valid_filter")] + assert vf["table_usage"] == 5 + assert vf["reasons"][0] == "常用表（被 2 次运行用到，使用 5 次）" + assert items[("members", "label")]["reasons"][0] == "常用表（被 1 次运行用到）" + assert vf["priority"] == round(catalog_review.priority(5, ("t", "valid_filter"), "llm"), 4) + assert (await client.get(f"{_base(sid)}/orders/usage")).json()["total"] == 2 + + async def test_queue_filters_by_kind_and_table_and_pages(client): sid = await _source(usage={"orders": 3}) facts = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "facts"})).json() @@ -338,20 +364,34 @@ async def test_table_usage_lists_runs_newest_first(client): # -------------------------------------------------------------------------- -async def test_review_progress_counts_top_tables_all_and_facts(client): +async def test_review_progress_counts_top_tables_all_and_facts(client, monkeypatch): sid = await _source(usage={"orders": 5}) r = await client.get(f"{_base(sid)}/review-progress") assert r.status_code == 200, r.text body = r.json() assert body["top_n"] == 20 - # 常用表只算被运行用到过的表：这里只有 orders（9 项：表结构里没有的列不算） - assert body["top"] == {"tables": 1, "items_total": 9, "items_confirmed": 1, "items_proposed": 7, - "items_verified": 0} - assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1, - "items_verified": 0} + # 表结构里只有 3 张表：都算常用表（使用次数前 20 张）；表结构里没有的表、列不算 + assert body["top"] == {"tables": 3, "items_total": 13, "items_confirmed": 1} + assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1} # 影响数字的项：有效记录条件、关联关系、度量类型、业务主键 assert body["facts"] == {"proposed": 4, "confirmed": 0} await client.post(f"{_base(sid)}/review/batch", json={"items": [ {"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}]}) after = (await client.get(f"{_base(sid)}/review-progress")).json() assert after["top"]["items_confirmed"] == 2 and after["facts"] == {"proposed": 3, "confirmed": 1} + # 只取前 2 张：用得多的在前，没人用的按表结构的顺序补（同起草的默认范围） + monkeypatch.setattr(catalog_review, "TOP_N", 2) + used = await _source(usage={"coupons": 1}) + top = (await client.get(f"{_base(used)}/review-progress")).json()["top"] + assert top == {"tables": 2, "items_total": 1 + 9, "items_confirmed": 1} + + +def test_top_tables_are_the_draft_default_scope(): + """常用表和起草对话框「使用次数最多的 20 张表」是同一批：使用次数从多到少，一样多（包括都没用过）的按表结构里的顺序。""" + tables = {f"t{i:02d}": {} for i in range(30)} + usage = {"t29": 4, "t05": 4, "t17": 9} + top = catalog_review.top_tables(tables, usage) + assert catalog_review.TOP_N == catalog.DRAFT_DEFAULT_TABLES == 20 + assert top[:3] == ["t17", "t05", "t29"] and len(top) == 20 + assert top[3:] == [n for n in tables if n not in usage][:17] + assert catalog_review.top_tables({"a": {}, "b": {}}, {}) == ["a", "b"] From 9b4663afa851000dadacc00b12c3b10aa9dae2ab Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 09:52:13 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E5=AF=BC=E5=85=A5=E6=A0=B8=E5=AF=B9?= =?UTF-8?q?=E6=9C=89=E6=95=88=E8=AE=B0=E5=BD=95=E6=9D=A1=E4=BB=B6=E7=94=A8?= =?UTF-8?q?=E5=88=B0=E7=9A=84=E5=88=97=EF=BC=9A=E6=9C=89=E6=95=88=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E6=9D=A1=E4=BB=B6=E6=98=AF=E8=A1=A8=E7=BA=A7=E7=9A=84?= =?UTF-8?q?=E4=B8=80=E9=A1=B9=E3=80=81=E5=8D=B4=E5=86=99=E7=9D=80=E5=88=97?= =?UTF-8?q?=E5=90=8D=EF=BC=8C=E6=8C=89=E5=88=97=E8=B7=B3=E8=BF=87=E7=9A=84?= =?UTF-8?q?=E8=A7=84=E7=9F=A9=E6=8B=A6=E4=B8=8D=E4=BD=8F=E5=AE=83=E3=80=82?= =?UTF-8?q?=E6=8C=89=E7=9B=AE=E6=A0=87=E6=95=B0=E6=8D=AE=E6=BA=90=E7=9A=84?= =?UTF-8?q?=E6=96=B9=E8=A8=80=E8=A7=A3=E6=9E=90=E5=87=BA=E6=9D=A1=E4=BB=B6?= =?UTF-8?q?=E9=87=8C=E7=94=A8=E5=88=B0=E7=9A=84=E5=88=97=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=9C=A8=E7=9B=AE=E6=A0=87=E7=9A=84=E8=A1=A8=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E9=87=8C=E3=80=81=E6=98=AF=E5=88=AB=E7=9A=84=E8=A1=A8=E7=9A=84?= =?UTF-8?q?=E5=88=97=E3=80=81=E5=B8=A6=E5=AD=90=E6=9F=A5=E8=AF=A2=EF=BC=88?= =?UTF-8?q?=E6=B2=A1=E6=B3=95=E6=A0=B8=E5=AF=B9=E5=AD=90=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E7=94=A8=E5=88=B0=E7=9A=84=E8=A1=A8=E5=92=8C=E5=88=97=EF=BC=89?= =?UTF-8?q?=E3=80=81=E8=A7=A3=E6=9E=90=E4=B8=8D=E4=BA=86=E6=88=96=E4=B8=8D?= =?UTF-8?q?=E6=AD=A2=E4=B8=80=E5=8F=A5=E7=9A=84=E8=B7=B3=E8=BF=87=E5=B9=B6?= =?UTF-8?q?=E5=86=99=E5=8E=9F=E5=9B=A0=EF=BC=9B=E4=BA=BA=E5=B7=A5=E5=86=99?= =?UTF-8?q?=E7=9A=84=E6=9D=A1=E4=BB=B6=E4=B8=8D=E9=99=90=E5=87=BD=E6=95=B0?= =?UTF-8?q?=EF=BC=8C=E5=88=97=E5=90=8D=E5=A4=A7=E5=B0=8F=E5=86=99=E5=92=8C?= =?UTF-8?q?=E7=9B=AE=E6=A0=87=E4=B8=8D=E5=90=8C=E3=80=81=E5=B8=A6=E6=9C=AC?= =?UTF-8?q?=E8=A1=A8=E8=A1=A8=E5=90=8D=EF=BC=88=E6=96=87=E4=BB=B6=E9=87=8C?= =?UTF-8?q?=E7=9A=84=E5=86=99=E6=B3=95=E3=80=81=E7=9B=AE=E6=A0=87=E7=9A=84?= =?UTF-8?q?=E5=86=99=E6=B3=95=E9=83=BD=E8=AE=A4=EF=BC=89=E7=9A=84=E7=85=A7?= =?UTF-8?q?=E5=B8=A6=E3=80=82=E6=A0=B8=E5=AF=B9=E7=94=A8=E7=9A=84=20filter?= =?UTF-8?q?=5Fcolumns=5Fproblem=20=E6=94=BE=E5=9C=A8=20catalog=5Ftransfer?= =?UTF-8?q?=EF=BC=8C=E8=A1=A8=E5=8A=A9=E6=89=8B=E5=9C=A8=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E8=B5=B7=E8=8D=89=E7=9A=84=E6=A0=A1=E9=AA=8C=E8=A7=84=E7=9F=A9?= =?UTF-8?q?=E6=97=B6=E4=B9=9F=E7=94=A8=E5=AE=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog_transfer.py | 82 ++++++++++++++++++++++---- backend/tests/test_catalog_transfer.py | 30 +++++++++- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py index 6a5b3e5d..ad2dd3c5 100644 --- a/backend/app/data/catalog_transfer.py +++ b/backend/app/data/catalog_transfer.py @@ -19,6 +19,9 @@ tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经 - 跳过并写原因：表结构里没有的列、业务主键和业务日期指着的列没有的、关联关系两端对不上的（指向的表、两端的列）、 目标数据源遮罩的列上的码值（码值就是这一列的取值，遮罩了就不该进目录）、不认识的字段、取值不合规的项（说法同 catalog.validate_notes）。 +- 有效记录条件是表级的一项、却写着列名，按列跳过的规矩拦不住它：按目标数据源的方言解析出用到的列 + （filter_columns_problem），不在目标的表结构里、是别的表的列、解析不了或不是一个条件、带子查询（没法核对子查询用到的表 + 和列）的跳过。人工写的条件不限函数（模型起草时的那套白名单只管模型写的条件）；列名大小写和目标不同、带本表表名的照带。 - 逐表写（乐观锁，撞上别人刚改过就重读重算，最多三次），一张表写不进去只记在那张表上，不影响别的表。 - 起草要求：勾了 include_guidance、文件里有、目标还没有时才写（action: replace）；目标已有的不覆盖（keep）；没勾或文件里 没有是 none。 @@ -32,7 +35,7 @@ from __future__ import annotations import copy import json import re -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any @@ -42,6 +45,7 @@ from app.data import catalog, catalog_jobs from app.data.catalog_review import item_label, slot_path from app.data.engine import masked_columns from app.data.names import name_key +from app.data.sqlcheck import dialect_of FORMAT = "agentlab-catalog" VERSION = 1 @@ -182,6 +186,50 @@ class _Columns: return found[0] if len(found) == 1 else None +def filter_columns_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, table: str, + also: Iterable[str] = (), where: str = "表结构") -> str | None: + """有效记录条件用到的列是不是都在这张表的表结构里（给人看的一句话；没问题返回 None）。 + + 按数据源的方言（sqlcheck.dialect_of）解析成一个条件：解析不了、不止一句、是一条查询、带子查询（没法核对子查询用到的 + 表和列）的不收；列不带表名，或带这张表的表名（table、表结构里的 qualified、also 里的写法，比如文件里的表名；忽略大小写、 + 去掉 schema 前缀），列名忽略大小写在表结构里。不限函数、运算。不是文字的交给合规检查说（返回 None）；数据源类型 + 没有对应的方言时没法解析，不核对。where 是说法里「表结构」的叫法（导入时是「目标的表结构」）。""" + if not isinstance(condition, str) or not condition.strip(): + return None + dialect = dialect_of(kind) + if dialect is None: + return None + import sqlglot + from sqlglot import exp + + try: + parsed = [s for s in sqlglot.parse(condition.strip(), read=dialect) if s is not None] + except Exception: # noqa: BLE001 - 语法错、方言不支持的写法 + parsed = [] + if len(parsed) != 1 or isinstance(parsed[0], exp.Query) or not isinstance(parsed[0], exp.Condition): + return f"有效记录条件不是一个能解析的 SQL 条件（按 {kind} 的写法），没法核对它用到的列" + expression = parsed[0] + if any(isinstance(node, (exp.Query, exp.Subquery)) for node in expression.walk()): + return "有效记录条件里有子查询，没法核对子查询用到的表和列" + own = {name_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified"), *also) if n} + columns = {str(c.get("name")).lower() for c in (meta or {}).get("columns") or [] + if isinstance(c, Mapping) and c.get("name") is not None} + foreign: list[str] = [] + missing: list[str] = [] + for column in expression.find_all(exp.Column): + if column.is_star: + continue + if column.table and name_key(column.table) not in own: + foreign.append(column.sql(dialect=dialect)) + elif column.name.lower() not in columns: + missing.append(column.name) + if foreign: + return f"有效记录条件引用了别的表的列（{'、'.join(dict.fromkeys(foreign))}），只能用这张表的列" + if missing: + return f"有效记录条件里的列「{'、'.join(dict.fromkeys(missing))}」不在{where}里" + return None + + # ========================================================================== # 一张表怎么并 # ========================================================================== @@ -270,14 +318,18 @@ def _conflict_why(file_status: str, target_status: str) -> str: class _Mapper: - """把文件里的一项换成目标的写法：列名、业务主键和业务日期的列、关联关系两端和编号。对不上返回原因。""" + """把文件里的一项换成目标的写法：列名、业务主键和业务日期的列、关联关系两端和编号；有效记录条件核对用到的列。对不上 + 返回原因。name 是文件里的表名（条件里带它当表名也认），kind 是目标数据源的类型（解析条件用它的方言）。""" - def __init__(self, target: str, schema_cache: Mapping[str, Any], masked: set[str]) -> None: + def __init__(self, name: str, target: str, schema_cache: Mapping[str, Any], masked: set[str], + kind: str | None) -> None: + self.name = name self.target = target self.cache = schema_cache self.tables = (schema_cache or {}).get("tables") or {} self.columns = _Columns(self.tables.get(target)) self.masked = masked + self.kind = kind def __call__(self, key: catalog.Slot, item: Any) -> tuple[catalog.Slot | None, Any, str | None]: if not isinstance(item, Mapping): @@ -302,6 +354,11 @@ class _Mapper: if col is None: return None, item, f"业务日期的列「{value['column']}」不在目标的表结构里，跳过" item["value"] = {**value, "column": col} + elif key[1] == "valid_filter": + problem = filter_columns_problem(value, self.tables.get(self.target), kind=self.kind, + table=self.target, also=(self.name,), where="目标的表结构") + if problem is not None: + return None, item, f"{problem}，跳过" return key, item, None return self._relation(item) @@ -324,11 +381,11 @@ class _Mapper: def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], target_notes: Mapping[str, Any] | None, - schema_cache: Mapping[str, Any], masked: set[str]) -> TablePlan: - """文件里一张表并进目标那张表的计划（纯函数，不写库）。""" + schema_cache: Mapping[str, Any], masked: set[str], kind: str | None = None) -> TablePlan: + """文件里一张表并进目标那张表的计划（纯函数，不写库）。kind 是目标数据源的类型（解析有效记录条件用）。""" plan = TablePlan(table_name=table_name, matched_as=target) existing = catalog._slots(target_notes) - mapper = _Mapper(target, schema_cache, masked) + mapper = _Mapper(table_name, target, schema_cache, masked, kind) seen: set[catalog.Slot] = set() for path, key, item, why in _raw_items(file_notes): if why is not None or key is None: @@ -422,7 +479,7 @@ async def plan_import(session: AsyncSession, row: Any, source: Any, content: Any plans: list[TablePlan] = [] for name, target in matched.items(): entry = await catalog.read_entry(session, source.id, target) - plan = plan_table(name, target, file.tables[name], entry.notes if entry else None, cache, masked) + plan = plan_table(name, target, file.tables[name], entry.notes if entry else None, cache, masked, row.kind) plan.version = entry.version if entry else 0 plans.append(plan) return ImportReport(plans, unmatched, _guidance(file, row, include_guidance)) @@ -438,7 +495,8 @@ async def run_import(session: AsyncSession, row: Any, source: Any, content: Any, masked = {c.lower() for c in masked_columns(getattr(row, "options", None))} plans: list[TablePlan] = [] for name, target in matched.items(): - plans.append(await _import_table(session, source, name, target, file.tables[name], cache, masked, actor)) + plans.append(await _import_table(session, source, name, target, file.tables[name], cache, masked, row.kind, + actor)) guidance = _guidance(file, row, include_guidance) if guidance["action"] == "replace": await catalog_jobs.save_guidance(session, row, file.guidance) @@ -446,13 +504,13 @@ async def run_import(session: AsyncSession, row: Any, source: Any, content: Any, async def _import_table(session: AsyncSession, source: Any, name: str, target: str, notes: Mapping[str, Any], - cache: Mapping[str, Any], masked: set[str], actor: str | None) -> TablePlan: + cache: Mapping[str, Any], masked: set[str], kind: str | None, actor: str | None) -> TablePlan: plan = TablePlan(table_name=name, matched_as=target) try: for _attempt in range(3): entry = await catalog.read_entry(session, source.id, target) version = entry.version if entry else 0 - plan = plan_table(name, target, notes, entry.notes if entry else None, cache, masked) + plan = plan_table(name, target, notes, entry.notes if entry else None, cache, masked, kind) plan.version = version if not plan.writes: return plan @@ -479,5 +537,5 @@ async def _import_table(session: AsyncSession, source: Any, name: str, target: s __all__ = ["FORMAT", "ImportFile", "ImportReport", "TablePlan", "TransferInvalid", "VERSION", "attachment_name", - "content_disposition", "export_catalog", "match_tables", "plan_import", "plan_table", "read_file", - "run_import"] + "content_disposition", "export_catalog", "filter_columns_problem", "match_tables", "plan_import", + "plan_table", "read_file", "run_import"] diff --git a/backend/tests/test_catalog_transfer.py b/backend/tests/test_catalog_transfer.py index e9e50b98..61822085 100644 --- a/backend/tests/test_catalog_transfer.py +++ b/backend/tests/test_catalog_transfer.py @@ -7,7 +7,8 @@ - 表按名字对（忽略大小写；带 schema 前缀的对不上时去掉前缀再对），对不上的列在 unmatched_tables - 文件里已确认、已驳回的项照带（来源、状态不变）；目标里已确认、已驳回的项和文件不一样时保留目标的、记冲突 - 文件里推断、有确证的项只填目标的空缺，不覆盖目标已有的 - - 表结构里没有的列、关系两端对不上的跳过并写原因；目标数据源遮罩的列，码值不导入 + - 表结构里没有的列、关系两端对不上的跳过并写原因；目标数据源遮罩的列，码值不导入；有效记录条件用到的列（按目标的方言 + 解析）不在目标的表结构里、引用别的表的列、带子查询的跳过（人工写的条件不限函数） - 逐表乐观锁，一张表写不进去不影响别的；有排队或进行中的起草任务时 409 catalog_draft_busy；文件格式不对 422 - 起草要求：勾了导入、目标还没有时才写；目标已有的不覆盖 @@ -357,6 +358,33 @@ async def test_invalid_items_are_skipped_not_fatal(client): assert "formula" in skipped["formula"] and "columns.amount.colour" in skipped +async def test_valid_filter_columns_must_be_in_the_target_table(client): + """有效记录条件是表级的一项，按列跳过的规矩拦不住它：解析出用到的列，不在目标的表结构里、是别的表的列、带子查询的都跳过 + 并写原因。人工写的条件不限函数；带本表表名（文件里的写法、目标的写法都认）、大小写不同的照带。""" + local, remote = await _pair() + content = await _export(client, local) + + async def outcome(condition: str) -> tuple[str | None, int]: + body = copy.deepcopy(content) + body["tables"]["orders"]["notes"]["valid_filter"] = make_item(condition, "human", "confirmed") + report = (await client.post(f"{_base(remote)}/import/preview", json={"content": body})).json() + orders = _by(report)["ORDERS"] + why = {s["path"]: s["why"] for s in orders["skipped"]}.get("valid_filter") + return why, orders["update"] + + why, update = await outcome("gift_flag = 0 AND status <> 9") + assert why and "gift_flag" in why and "目标的表结构" in why and update == 0 + why, _ = await outcome("members.tier = 1") + assert why and "members.tier" in why and "别的表" in why + why, _ = await outcome("member_id IN (SELECT id FROM members)") + assert why and "子查询" in why + why, _ = await outcome("status <> 9; DELETE FROM orders") + assert why and "条件" in why + for fine in ("orders.status <> 9", "ORDERS.STATUS <> 9 AND ABS(amount) > 0", "contact_phone IS NOT NULL"): + why, update = await outcome(fine) + assert why is None and update == 1, fine + + async def test_import_needs_a_schema(client): local, _ = await _pair() content = await _export(client, local) From 813522ec7b16b9d9e45706974f0b417534820a6a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 09:52:37 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=EF=BC=9A?= =?UTF-8?q?=E5=BE=85=E4=BF=9D=E5=AD=98=E7=9A=84=E4=BF=AE=E6=94=B9=E4=B8=8D?= =?UTF-8?q?=E8=AE=BE=E4=B8=8A=E9=99=90=E3=80=81=E6=9C=89=E6=95=88=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E6=9D=A1=E4=BB=B6=E6=8C=89=E8=B5=B7=E8=8D=89=E7=9A=84?= =?UTF-8?q?=E8=A7=84=E7=9F=A9=E6=A0=B8=E5=AF=B9=E3=80=82=E5=AE=BD=E8=A1=A8?= =?UTF-8?q?=E4=B8=8A=E9=80=90=E5=88=97=E8=A1=A5=E4=B8=AD=E6=96=87=E5=90=8D?= =?UTF-8?q?=E3=80=81=E5=90=AB=E4=B9=89=EF=BC=8C=E5=87=A0=E8=BD=AE=E4=B8=8B?= =?UTF-8?q?=E6=9D=A5=E5=BE=85=E4=BF=9D=E5=AD=98=E7=9A=84=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=B8=B8=E5=B8=B8=E8=B6=85=E8=BF=87=E4=B8=80=E6=AC=A1=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E7=9A=84=20200=20=E9=A1=B9=EF=BC=8C=E4=BB=A5=E5=89=8D?= =?UTF-8?q?=E8=AF=B7=E6=B1=82=E9=87=8C=E8=B6=85=E8=BF=87=20200=20=E9=A1=B9?= =?UTF-8?q?=E5=B0=B1=20422=E3=80=81=E5=AF=B9=E8=AF=9D=E7=94=A8=E4=B8=8D?= =?UTF-8?q?=E4=BA=86=EF=BC=9B=E7=8E=B0=E5=9C=A8=E4=BA=A4=E5=A4=9A=E5=B0=91?= =?UTF-8?q?=E9=83=BD=E6=94=B6=EF=BC=8C=E8=83=8C=E6=99=AF=E9=87=8C=E5=8F=AA?= =?UTF-8?q?=E5=88=97=E6=9C=80=E8=BF=91=E7=9A=84=20200=20=E9=A1=B9=EF=BC=88?= =?UTF-8?q?=E7=95=8C=E9=9D=A2=E4=B8=8A=E6=96=B0=E7=9A=84=E6=8F=90=E6=A1=88?= =?UTF-8?q?=E6=8E=A5=E5=9C=A8=E5=90=8E=E9=9D=A2=EF=BC=89=E5=B9=B6=E5=86=99?= =?UTF-8?q?=E6=98=8E=E5=8F=A6=E6=9C=89=E5=87=A0=E9=A1=B9=EF=BC=8C=E7=A0=81?= =?UTF-8?q?=E5=80=BC=E7=85=A7=E5=85=A8=E9=83=A8=E5=BE=85=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E7=9A=84=E4=BF=AE=E6=94=B9=E5=90=88=E5=B9=B6=E3=80=82=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E6=8F=90=E7=9A=84=E6=9C=89=E6=95=88=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E6=98=AF=20SQL=EF=BC=9A=E6=8C=89=20sqlcheck.?= =?UTF-8?q?valid=5Ffilter=5Fproblem=20=E6=A0=B8=E5=AF=B9=EF=BC=88=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E6=9D=A1=E4=BB=B6=E3=80=81=E5=8F=AA=E7=94=A8=E8=BF=99?= =?UTF-8?q?=E5=BC=A0=E8=A1=A8=E7=9A=84=E5=88=97=E3=80=81=E4=B8=8D=E8=83=BD?= =?UTF-8?q?=E6=9C=89=E5=AD=90=E6=9F=A5=E8=AF=A2=E3=80=81=E8=81=9A=E5=90=88?= =?UTF-8?q?=E3=80=81=E7=AA=97=E5=8F=A3=E3=80=81=E5=88=86=E5=8F=B7=EF=BC=8C?= =?UTF-8?q?=E5=8F=AA=E8=83=BD=E7=94=A8=E5=87=A0=E4=B8=AA=E5=87=BD=E6=95=B0?= =?UTF-8?q?=EF=BC=89=EF=BC=8Csqlcheck=20=E9=87=8C=E8=BF=98=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E8=BF=99=E6=9D=A1=E8=A7=84=E7=9F=A9=E6=97=B6=E8=87=B3?= =?UTF-8?q?=E5=B0=91=E6=A0=B8=E5=AF=B9=E7=94=A8=E5=88=B0=E7=9A=84=E5=88=97?= =?UTF-8?q?=E9=83=BD=E6=98=AF=E8=BF=99=E5=BC=A0=E8=A1=A8=E7=9A=84=E3=80=81?= =?UTF-8?q?=E9=83=BD=E5=9C=A8=E8=A1=A8=E7=BB=93=E6=9E=84=E9=87=8C=E3=80=81?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E5=AD=90=E6=9F=A5=E8=AF=A2=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E8=BF=87=E7=9A=84=E4=B8=8D=E4=BA=A4=E7=BB=99=E7=95=8C=E9=9D=A2?= =?UTF-8?q?=E3=80=81=E5=8E=9F=E5=9B=A0=E4=BA=A4=E5=9B=9E=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=EF=BC=9B=E7=B3=BB=E7=BB=9F=E6=8F=90=E7=A4=BA=E9=87=8C=E5=85=88?= =?UTF-8?q?=E5=86=99=E6=98=8E=E8=BF=99=E5=87=A0=E6=9D=A1=E5=86=99=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/catalog.py | 5 +- backend/app/data/catalog_assistant.py | 42 +++++++++--- backend/tests/test_catalog_assistant.py | 87 +++++++++++++++++++++++-- 3 files changed, 119 insertions(+), 15 deletions(-) diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 370c412e..6a7ee7ef 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -700,8 +700,9 @@ class AssistantMessageIn(BaseModel): class AssistantIn(BaseModel): #: 整段对话（服务端不存）；没有使用者的话时助手先开口 messages: list[AssistantMessageIn] = Field(default_factory=list, max_length=catalog_assistant.MAX_MESSAGES) - #: 界面上还没保存的修改，让模型知道 - pending: list[CatalogPatchChange] = Field(default_factory=list, max_length=catalog_assistant.MAX_PENDING) + #: 界面上还没保存的修改，让模型知道。不设上限：宽表上几轮攒下来常常超过一次保存的 200 项，不能因此不让问（背景里只列 + #: 最近的 catalog_assistant.PENDING_SHOWN 项，码值照全部合并） + pending: list[CatalogPatchChange] = Field(default_factory=list) @router.post("/{source_id}/catalog/{table}/assistant") diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index e8fd3b28..27d848f5 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -7,7 +7,8 @@ **只改目录、不改业务库。** 模型只有两个工具： - propose_changes：一批修改提案。按 catalog.plan_patch 核对（取值格式、列名必须在表结构里、公式不收），遮罩列不给码值，已经 - 是这个值又确认过的不提；核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。 + 是这个值又确认过的不提；有效记录条件是模型写的 SQL，另按起草的规矩核对（_Tools.filter_problem：一个条件、只用这张表的 + 列、不能有子查询……）。核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。 界面上还没保存的修改（pending）里同一列的码值并进新提的码值：界面上同一路径以后一次为准，不能把先提的码值丢了。 - run_query：只读查询，只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限（catalog_profile._Runner： guard.check(readonly=True)，单条时限取剖析设置和数据源查询时限中较小的一个），每次请求最多 MAX_QUERIES 条（守卫拒掉的 @@ -36,7 +37,9 @@ catalog.MODEL_MAX_TOKENS，服务方嫌大时退回 catalog.MODEL_FALLBACK_MAX_T 等模型的时候每 HEARTBEAT_S 秒发一行 SSE 注释（「: keep-alive」，不是事件），免得中间的代理断开。 **对话不存服务端**：每次请求带上整段对话（最多 MAX_MESSAGES 条、每条最多 MESSAGE_MAX 字）。查询结果只在这一次请求里 -看得到，提示词里让模型把要用的结论写进回答。 +看得到，提示词里让模型把要用的结论写进回答。界面上待保存的修改不设上限（宽表上逐列补中文名、含义，几轮下来常常超过一次 +保存的 200 项，不能因此不让问）：背景里只列最近的 PENDING_SHOWN 项（界面上新的提案接在后面），写明另有几项；码值照全部 +待保存的修改合并。 写给模型的文字都在名字带 _SYSTEM / _PROMPT 的常量里；这里的 ERROR_* 是给人看的。 """ @@ -56,17 +59,18 @@ from typing import Any from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.data import catalog, catalog_jobs, catalog_profile +from app.data import catalog, catalog_jobs, catalog_profile, sqlcheck from app.data.catalog_review import SOURCE_LABEL, item_label, slot_path, table_usage_counts +from app.data.catalog_transfer import filter_columns_problem from app.data.draft_model import DRAFT_MODEL_PLACE from app.data.engine import SnapshotTampered, masked_columns logger = logging.getLogger(__name__) -#: 对话最多几条、每条最多几个字；界面上待保存的修改最多几项（同 PATCH 的上限） +#: 对话最多几条、每条最多几个字；界面上待保存的修改在背景里最多列几项（最近的；请求里交多少都收） MAX_MESSAGES = 40 MESSAGE_MAX = 4000 -MAX_PENDING = catalog.PATCH_MAX_CHANGES +PENDING_SHOWN = 200 #: 每次请求最多调用几次模型、最多几条只读查询 MAX_ROUNDS = 8 MAX_QUERIES = 5 @@ -133,7 +137,9 @@ fact/dimension/snapshot/log/config、business_date {"column","rule","timezone"} flow/stock/ratio/identifier/status/attribute、codes 码值 {"码":"含义"}）；已有的关联关系写 relations.<编号>；新增关联关系\ 写 relations.new，value 写 {"columns":[本表列],"to_table":"目标表","to_columns":[目标列],\ "cardinality":"many_to_one/one_to_one/one_to_many"}。 -- codes 只写要补充或更正的码，原有的码会保留；valid_filter 要写成完整的条件（原有条件也要带上）。""" +- codes 只写要补充或更正的码，原有的码会保留；valid_filter 要写成完整的条件（原有条件也要带上）。 +- valid_filter 只能用这张表的列（不带表名），由比较、IN、BETWEEN、LIKE、IS NULL 用 AND / OR / NOT 连起来，例如 \ +status NOT IN ('C', 'X')；不能有子查询、聚合、窗口函数、分号，函数只能用 UPPER、LOWER、TRIM、COALESCE。""" #: 查询工具的说明（开了剖析时接在系统提示后面）、没开剖析时的那句 ASSISTANT_QUERY_PROMPT = """只读查询：run_query 在这个数据源上执行一条只读 SQL（{dialect} 的写法），最多返回 {rows} 行，\ @@ -180,6 +186,7 @@ ASSISTANT_PROMPT_LABELS = { "unnamed": "目录里还没有中文名和含义的列：{cols}（共 {n} 列{more}）", "unnamed_more": "，这里列出前 {n} 个", "pending": "界面上还没保存的修改（使用者已经接受、还没保存）：", + "pending_more": "（另有 {n} 项更早的待保存修改没有列出：太多了。它们同样已经接受，不要重复提）", "pending_item": "- {path}：{value}（依据：{reason}）", "status_proposed": "推断", "status_verified": "有确证", @@ -509,7 +516,6 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, if unnamed: more = labels["unnamed_more"].format(n=UNNAMED_SHOWN) if len(unnamed) > UNNAMED_SHOWN else "" lines.append(labels["unnamed"].format(cols="、".join(unnamed[:UNNAMED_SHOWN]), n=len(unnamed), more=more)) - pending = list(pending) pending_codes: dict[str, dict[str, Any]] = {} shown_pending: list[str] = [] for change in pending: @@ -527,7 +533,10 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, reason = str(change.get("reason") or "").strip() shown_pending.append(labels["pending_item"].format(path=path, value=_value_text(value), reason=reason or "—")) if shown_pending: - lines += ["", labels["pending"], *shown_pending] + lines += ["", labels["pending"]] + if len(shown_pending) > PENDING_SHOWN: + lines.append(labels["pending_more"].format(n=len(shown_pending) - PENDING_SHOWN)) + lines += shown_pending[-PENDING_SHOWN:] if can_query: tools = ASSISTANT_QUERY_PROMPT.format(dialect=row.kind, rows=QUERY_ROWS, limit=MAX_QUERIES) else: @@ -652,6 +661,18 @@ class _Tools: return change return {**change, "value": {**before, **value}} + def filter_problem(self, condition: Any) -> str | None: + """模型提的有效记录条件哪里不合规（交回模型的一句话），合规是 None。 + + 按模型起草时的规矩核对（sqlcheck.valid_filter_problem：一个条件、只用这张表的列、不能有子查询、聚合、窗口、分号， + 只能用比较、IN、BETWEEN、LIKE、IS NULL 和几个函数）；sqlcheck 里还没有这条规矩时，至少核对用到的列都是这张表的、 + 都在表结构里、没有子查询（catalog_transfer.filter_columns_problem，导入时核对人工写的条件也用它）。""" + meta = self.ctx.tables.get(self.ctx.table) or {} + strict = getattr(sqlcheck, "valid_filter_problem", None) + if strict is not None: + return strict(condition, meta, kind=self.ctx.kind, table=self.ctx.table) + return filter_columns_problem(condition, meta, kind=self.ctx.kind, table=self.ctx.table) + async def propose(self, changes: Any) -> tuple[dict[str, Any] | None, str]: """(给界面的 proposal 事件, 交回模型的结果)。提案不写目录。""" from app.db.base import SessionLocal @@ -674,6 +695,9 @@ class _Tools: if change.state == "same": dropped.append(prompts["same"].format(where=_where(key, change))) continue + if key == ("t", "valid_filter") and (problem := self.filter_problem(change.after)) is not None: + dropped.append(problem) + continue kept.append(change) said: list[str] = [] event: dict[str, Any] | None = None @@ -883,5 +907,5 @@ async def stream(model: Any, model_id: str, ctx: AssistantContext, __all__ = ["AssistantContext", "CODE_FAILED", "CODE_MODEL", "CODE_ROUNDS", "CODE_TIMEOUT", "CODE_TRUNCATED", - "HEARTBEAT_S", "MAX_MESSAGES", "MAX_PENDING", "MAX_QUERIES", "MAX_ROUNDS", "MESSAGE_MAX", "PROPOSE_TOOL", + "HEARTBEAT_S", "MAX_MESSAGES", "MAX_QUERIES", "MAX_ROUNDS", "MESSAGE_MAX", "PENDING_SHOWN", "PROPOSE_TOOL", "QUERY_ROWS", "QUERY_TOOL", "SHOWN_ROWS", "conversation_problem", "prepare", "resolve_model", "stream"] diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index 09a4a769..8c4b4470 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -1,9 +1,11 @@ """表助手（data/catalog_assistant.py，POST /api/datasources/{id}/catalog/{table}/assistant）。 - 开始前：表不在 404；起草模型用不了 409 catalog_draft_model_unavailable（说法同起草任务）；对话超过 40 条、一条超过 - 4000 字、待保存的修改超过 200 项 422 + 4000 字 422。界面上待保存的修改不设上限（宽表上攒到 200 项以上也照常对话）：背景里只列最近的 200 项、写明另有几项， + 码值照全部待保存的修改合并 - 流：text 增量、query（跑过的只读查询，前 10 行，遮罩列写「已遮罩」）、proposal（按 plan_patch 核对过的提案，形状同助手流 - 的 catalog_patch：公式不收、列名必须在表结构里、遮罩列不给码值）、done（用量、查询条数）、error（中途出错，之后没有 done） + 的 catalog_patch：公式不收、列名必须在表结构里、遮罩列不给码值；有效记录条件要是只用这张表的列的一个条件，有起草的 + 校验规矩 sqlcheck.valid_filter_problem 时按它）、done（用量、查询条数）、error（中途出错，之后没有 done） - 交给模型的背景：表结构、这张表的目录（每项的来源和状态）、关联关系（含别的表指过来的）、使用次数、数据源的起草要求、 最近一次起草留空的列、界面上还没保存的修改；遮罩列只给列名、不给任何值 - 只读查询：只有开了数据剖析的源才给这个工具；经剖析的只读守卫和单条时限；每次请求最多 5 条；最多 50 行交给模型；结果 @@ -207,8 +209,6 @@ async def test_refusals_before_the_stream(client, tmp_path, monkeypatch): assert (await client.post(_url(sid), json=too_many)).status_code == 422 too_long = _ask("长" * 4001) assert (await client.post(_url(sid), json=too_long)).status_code == 422 - pending = [{"path": "label", "value": "订单", "reason": "x"}] * 201 - assert (await client.post(_url(sid), json=_ask("x", pending=pending))).status_code == 422 wrong_role = {"messages": [{"role": "system", "content": "x"}]} assert (await client.post(_url(sid), json=wrong_role)).status_code == 422 last_is_answer = {"messages": [{"role": "user", "content": "x"}, {"role": "assistant", "content": "y"}]} @@ -333,6 +333,8 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client, # 规矩 for rule in ("只改数据目录", "不改业务库", "公式", "拿不准", "原话"): assert rule in system + # 有效记录条件的写法（同起草时的校验规矩），免得提了再被退回 + assert "只能用这张表的列" in system and "不能有子查询" in system and "UPPER、LOWER、TRIM、COALESCE" in system # 表结构、目录（来源和状态）、使用次数、起草要求、数据源说明 for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认", "模型起草", "已驳回", "还没有被运行用到过", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"): @@ -464,6 +466,83 @@ async def test_proposal_codes_build_on_the_pending_ones(client, tmp_path, monkey assert change["after"] == {"1": "已支付", "2": "已退款", "9": "作废"} +async def test_many_pending_changes_do_not_block_the_conversation(client, tmp_path, monkeypatch): + """宽表上逐列补中文名、含义，待保存的修改攒到 200 项以上：照常对话（不 422）。背景里只列最近的 200 项、写明另有几项； + 码值照全部待保存的修改合并（最早那条码值没列进背景，新提的码值也不能把它丢了）。""" + sid = await _source(client, tmp_path) + oldest = {"path": "columns.status.codes", "value": {"2": "已退款"}, "reason": "用户说：2 是退款"} + labels = [{"path": f"columns.c{i:03d}.label", "value": f"第 {i} 列", "reason": f"用户说：第 {i} 列"} + for i in range(201)] + model = Scripted([Turn(calls=[("propose_changes", {"changes": [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}]})]), + Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("9 是作废", pending=[oldest, *labels])) + assert events[-1]["type"] == "done" + system = model.seen[0][0].content + # 最近的 200 项列出来了，最早的两项没列，写明另有 2 项 + assert "columns.c200.label" in system and "columns.c001.label" in system + assert "columns.c000.label" not in system and "用户说：2 是退款" not in system + assert "另有 2 项" in system + [proposal] = [e for e in events if e["type"] == "proposal"] + [change] = proposal["changes"] + assert change["value"] == {"2": "已退款", "9": "作废"} + # 再多也收 + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + many = [{"path": f"columns.c{i:04d}.meaning", "value": "说明", "reason": "x"} for i in range(1200)] + events, _ = await _talk(client, sid, _ask("接着看", pending=many)) + assert events[-1]["type"] == "done" and "另有 1000 项" in model.seen[0][0].content + + +async def _propose_filters(client, sid: str, monkeypatch, filters: list[str]) -> tuple[list[dict], Scripted]: + """模型一轮提一条有效记录条件（同一条提案里同一个路径只认第一次），最后一轮回答。""" + turns = [Turn(calls=[("propose_changes", {"changes": [ + {"path": "valid_filter", "value": f, "reason": "用户说：统计时排除作废"}]})]) for f in filters] + model = Scripted([*turns, Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("统计时排除作废的单子")) + return events, model + + +async def test_valid_filter_proposals_must_be_a_condition_on_this_table(client, tmp_path, monkeypatch): + """有效记录条件是模型写的 SQL：引用表结构里没有的列、别的表的列、带子查询的不交给界面，原因交回模型；合规的照常提。""" + sid = await _source(client, tmp_path) + events, model = await _propose_filters(client, sid, monkeypatch, [ + "status <> 9 AND ghost_col = 1", + "members.nick IS NOT NULL", + "member_id IN (SELECT id FROM members)", + "status <> 9", + ]) + proposals = [e for e in events if e["type"] == "proposal"] + assert [[c["value"] for c in p["changes"]] for p in proposals] == [["status <> 9"]] + said = [[m for m in seen if isinstance(m, ToolMessage)][-1].content for seen in model.seen[1:5]] + assert "ghost_col" in said[0] and "不在表结构里" in said[0] and "没有可以交给使用者的修改" in said[0] + assert "members.nick" in said[1] + assert "子查询" in said[2] + assert "已把 1 项修改交给使用者确认" in said[3] + + +async def test_valid_filter_proposals_follow_the_drafting_rules(client, tmp_path, monkeypatch): + """有起草的校验规矩（sqlcheck.valid_filter_problem：只能用几个函数、不能有聚合……）时按它核对，说法照它的。""" + from app.data import sqlcheck + + sid = await _source(client, tmp_path) + calls: list[tuple] = [] + + def strict(condition, meta, *, kind, table=None): + calls.append((condition, [c["name"] for c in meta["columns"]], kind, table)) + return "有效行过滤里只能用 UPPER、LOWER、TRIM、COALESCE 这几个函数，不能用 ABS" if "ABS" in condition else None + + monkeypatch.setattr(sqlcheck, "valid_filter_problem", strict, raising=False) + events, model = await _propose_filters(client, sid, monkeypatch, ["ABS(amount) > 0", "status <> 9"]) + proposals = [e for e in events if e["type"] == "proposal"] + assert [[c["value"] for c in p["changes"]] for p in proposals] == [["status <> 9"]] + assert "不能用 ABS" in [m for m in model.seen[1] if isinstance(m, ToolMessage)][-1].content + assert calls[0][0] == "ABS(amount) > 0" and calls[0][2:] == ("sqlite", "orders") + assert "status" in calls[0][1] and "member_id" in calls[0][1] + + async def test_nothing_acceptable_means_no_proposal_event(client, tmp_path, monkeypatch): sid = await _source(client, tmp_path) model = Scripted([Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]),