From 864e63daddc3e351845fe336169a51295d7dd83f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 09:51:49 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=A1=E9=98=85=E9=98=9F=E5=88=97=E4=B8=8E?= =?UTF-8?q?=E8=BF=9B=E5=BA=A6=E6=8C=89=E7=BA=A6=E5=AE=9A=E6=94=B9=EF=BC=9A?= =?UTF-8?q?field=5Flabel=20=E5=86=99=E7=BB=99=E4=BA=BA=E7=9C=8B=E7=9A=84?= =?UTF-8?q?=E5=AE=8C=E6=95=B4=E4=BD=8D=E7=BD=AE=EF=BC=88=E3=80=8C=E8=A1=A8?= =?UTF-8?q?=E7=9A=84=E6=9C=89=E6=95=88=E8=AE=B0=E5=BD=95=E6=9D=A1=E4=BB=B6?= =?UTF-8?q?=E3=80=8D=E3=80=8C=E5=88=97=20status=20=E7=9A=84=E7=A0=81?= =?UTF-8?q?=E5=80=BC=E3=80=8D=E3=80=8C=E6=8C=87=E5=90=91=20members=20?= =?UTF-8?q?=E7=9A=84=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB=E3=80=8D=EF=BC=8C?= =?UTF-8?q?=E5=92=8C=E8=A1=A8=E5=8A=A9=E6=89=8B=E6=8F=90=E6=A1=88=E3=80=81?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E5=86=B2=E7=AA=81=E7=9A=84=E8=AF=B4=E6=B3=95?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E4=B8=AA=E5=87=BD=E6=95=B0=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E5=88=97=E7=BA=A7=E7=9A=84=E9=A1=B9=E5=8F=A6=E5=B8=A6=20column?= =?UTF-8?q?=EF=BC=9B=E6=8E=92=E5=9C=A8=E8=BF=99=E9=87=8C=E7=9A=84=E5=8E=9F?= =?UTF-8?q?=E5=9B=A0=E5=86=99=E3=80=8C=E8=A2=AB=20N=20=E6=AC=A1=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=94=A8=E5=88=B0=E3=80=8D=EF=BC=8C=E6=95=B0=E7=9A=84?= =?UTF-8?q?=E6=98=AF=E8=BF=90=E8=A1=8C=EF=BC=88=E5=90=8C=E3=80=8C=E7=94=A8?= =?UTF-8?q?=E8=BF=87=E8=BF=99=E5=BC=A0=E8=A1=A8=E7=9A=84=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E3=80=8D=E7=9A=84=20total=EF=BC=89=EF=BC=8C=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E6=AC=A1=E8=BF=90=E8=A1=8C=E9=87=8C=E6=9F=A5=E4=BA=86=E5=87=A0?= =?UTF-8?q?=E6=AC=A1=E3=80=81=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0=E6=AF=94?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=AC=A1=E6=95=B0=E5=A4=9A=E6=97=B6=E4=B8=A4?= =?UTF-8?q?=E4=B8=AA=E9=83=BD=E5=86=99=EF=BC=88=E3=80=8C=E8=A2=AB=202=20?= =?UTF-8?q?=E6=AC=A1=E8=BF=90=E8=A1=8C=E7=94=A8=E5=88=B0=EF=BC=8C=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=205=20=E6=AC=A1=E3=80=8D=EF=BC=89=EF=BC=8Ctable=5Fusa?= =?UTF-8?q?ge=20=E5=92=8C=E4=BC=98=E5=85=88=E7=BA=A7=E7=85=A7=E6=97=A7?= =?UTF-8?q?=E6=8C=89=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0=EF=BC=88=E5=90=8C?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E6=B8=85=E5=8D=95=E7=9A=84=E3=80=8C=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=20N=20=E6=AC=A1=E3=80=8D=E5=92=8C=E6=8E=92=E5=BA=8F?= =?UTF-8?q?=EF=BC=89=EF=BC=9B=E8=BF=9B=E5=BA=A6=E9=87=8C=E7=9A=84=E5=B8=B8?= =?UTF-8?q?=E7=94=A8=2020=20=E5=BC=A0=E8=A1=A8=E5=92=8C=E8=B5=B7=E8=8D=89?= =?UTF-8?q?=E7=9A=84=E9=BB=98=E8=AE=A4=E8=8C=83=E5=9B=B4=E6=98=AF=E5=90=8C?= =?UTF-8?q?=E4=B8=80=E6=89=B9=EF=BC=88=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0?= =?UTF-8?q?=E4=BB=8E=E5=A4=9A=E5=88=B0=E5=B0=91=EF=BC=8C=E7=94=A8=E5=88=B0?= =?UTF-8?q?=E7=9A=84=E4=B8=8D=E5=A4=9F=2020=20=E5=BC=A0=E6=97=B6=E6=8C=89?= =?UTF-8?q?=E8=A1=A8=E7=BB=93=E6=9E=84=E7=9A=84=E9=A1=BA=E5=BA=8F=E8=A1=A5?= =?UTF-8?q?=E9=BD=90=EF=BC=89=EF=BC=8Ctop=20=E5=8F=AA=E7=BB=99=20tables?= =?UTF-8?q?=E3=80=81items=5Ftotal=E3=80=81items=5Fconfirmed=EF=BC=8Call=20?= =?UTF-8?q?=E5=8F=AA=E7=BB=99=E7=BA=A6=E5=AE=9A=E7=9A=84=E5=9B=9B=E9=A1=B9?= =?UTF-8?q?=EF=BC=9B=E8=A1=A8=E5=8A=A9=E6=89=8B=E8=83=8C=E6=99=AF=E9=87=8C?= =?UTF-8?q?=E5=86=99=E8=BF=99=E5=BC=A0=E8=A1=A8=E8=A2=AB=E5=87=A0=E6=AC=A1?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=94=A8=E5=88=B0=E3=80=81=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E5=87=A0=E6=AC=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog_assistant.py | 16 ++-- backend/app/data/catalog_review.py | 105 +++++++++++++++------ backend/app/data/catalog_transfer.py | 12 +-- backend/tests/test_catalog_assistant.py | 22 ++++- backend/tests/test_catalog_review_queue.py | 66 ++++++++++--- 5 files changed, 158 insertions(+), 63 deletions(-) diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index c6e5d9fa..e8fd3b28 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -57,7 +57,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.data import catalog, catalog_jobs, catalog_profile -from app.data.catalog_review import SOURCE_LABEL, slot_path +from app.data.catalog_review import SOURCE_LABEL, item_label, slot_path, table_usage_counts from app.data.draft_model import DRAFT_MODEL_PLACE from app.data.engine import SnapshotTampered, masked_columns @@ -153,7 +153,8 @@ ASSISTANT_PROMPT_LABELS = { "view": "（视图）", "comment": "数据库注释：{text}", "system": "系统说明：{text}", - "usage": "这张表被运行用过 {n} 次。", + "usage": "这张表被 {runs} 次运行用到（查询 {n} 次）。", + "usage_none": "这张表还没有被运行用到过。", "description": "数据源说明：{text}", "guidance": "数据源的起草要求（使用者补充的背景和要求）：{text}", "structure": "表结构：", @@ -469,7 +470,8 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, entries = await catalog.read_catalog(session, source.id) entry = entries.get(table) notes = entry.notes if entry else {} - usage = (await catalog.table_usage(session, source)).get(table, 0) + usage = await table_usage_counts(session, source) + used, runs = usage.queries.get(table, 0), usage.runs.get(table, 0) blank_n, blank_cols = await _last_blank(session, source.id, table) lines = [labels["head"], labels["table"].format(source=row.name, kind=row.kind, table=table, @@ -477,7 +479,7 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, comment = str(meta.get("comment") or "").strip() if comment: lines.append((labels["system"] if system_notes else labels["comment"]).format(text=comment)) - lines.append(labels["usage"].format(n=usage)) + lines.append(labels["usage"].format(runs=runs, n=used) if used else labels["usage_none"]) description = str(getattr(row, "description", "") or "").strip() if description: lines.append(labels["description"].format(text=description)) @@ -693,11 +695,9 @@ class _Tools: def _where(key: catalog.Slot, change: catalog.PatchChange) -> str: - """提案里一项的中文说法：「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」。""" + """提案里一项的中文说法：「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」（同审阅队列的 field_label）。""" value = change.after if isinstance(change.after, Mapping) else change.before - if key[0] == "r" and isinstance(value, Mapping) and value.get("to_table"): - return f"指向 {value['to_table']} 的关联关系" - return catalog.describe_slot(key) + return item_label(key, value if isinstance(value, Mapping) else None) # ========================================================================== diff --git a/backend/app/data/catalog_review.py b/backend/app/data/catalog_review.py index b03c9e31..5e7fba68 100644 --- a/backend/app/data/catalog_review.py +++ b/backend/app/data/catalog_review.py @@ -6,8 +6,8 @@ - **队列**（review_queue，GET /catalog/review-queue）：只列推断（proposed）的项。priority = 使用次数权重 × 风险权重 × 来源权重，从高到低；同分按表名、路径排，结果稳定。 - - 使用次数权重：1 + log2(1 + 使用次数)。使用次数同目录清单（catalog.table_usage）：按对数增长，常用表在前，但不至于 - 让常用表上的中文名压过所有表上影响数字的项。 + - 使用次数权重：1 + log2(1 + 使用次数)。使用次数同目录清单（catalog.table_usage，清单的「使用 N 次」和排序、起草的默认 + 范围都按它）：按对数增长，常用表在前，但不至于让常用表上的中文名压过所有表上影响数字的项。 - 风险权重（RISK_WEIGHT）：有效记录条件、去重规则、业务日期、关联关系最高（直接改变数字），业务主键、度量类型次之 （计数和能不能加总），码值、粒度、表类型再次，中文名、说明、含义、单位最低。 - 来源权重（SOURCE_WEIGHT）：模型起草、命名推断是猜的，比数据库注释优先；数据剖析（码值候选、业务日期、没核实的关系） @@ -15,14 +15,19 @@ - 两组（kind）：facts 影响数字的项（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）；names 名称和说明 （中文名、说明、含义、单位、码值、粒度、表类型）。表的「说明」和中文名、含义一样只是说明文字，归 names。 - 表结构里已经没有的表、列上的项不列（确认了也没有用，目录页上另有清理的提示）。 - - reasons 用中文说为什么排在这里：常用表（使用 N 次）、影响数字的项（有效记录条件）、模型起草，未确认…… + - field_label 是这一项给人看的位置：「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」（item_label， + 和表助手提案、导入冲突里的说法一致）；列级的项另给 column。 + - table_usage 是所在表的使用次数（同清单）。reasons 用中文说为什么排在这里：常用表（被 N 次运行用到）、影响数字的项 + （有效记录条件）、模型起草，未确认……「被 N 次运行用到」数的是运行（同 /usage 的 total）；同一次运行里查了几次、 + 使用次数比运行次数多时两个都写（「被 2 次运行用到，使用 5 次」），和清单、运行列表上的数都对得上。 - **批量审阅**（review_batch，POST /catalog/review/batch）：逐表按 if_version 乐观锁，同一张表的几条在一次写入里完成， 一张表冲突、写不进去不影响别的表；路径不对、表不存在逐条报（results 和请求一一对应）。审阅的规则同单项审阅 （catalog.review_item）。 - **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：取法同使用次数（catalog.query_snapshot_tables），按运行 - 去重，新的在前。 -- **进度**（review_progress，GET /catalog/review-progress）：使用次数前 TOP_N 张表（只算被运行用到过的）、全部、影响 - 数字的项各自的确认情况。项的范围和队列一样：表结构里没有的表、列不算。 + 去重，新的在前；total 是运行次数。 +- **进度**（review_progress，GET /catalog/review-progress）：常用表（使用次数前 TOP_N 张，和起草对话框「使用次数最多的 + 20 张表」是同一批：catalog.default_tables 的取法，用到的不够 TOP_N 张时按表结构的顺序补齐）、全部、影响数字的项各自的 + 确认情况。项的范围和队列一样：表结构里没有的表、列不算。 """ from __future__ import annotations @@ -44,7 +49,7 @@ KINDS = ("all", "facts", "names") QUEUE_LIMIT_MAX = 200 BATCH_MAX = 200 RUNS_LIMIT_MAX = 100 -#: 进度里的「常用表」取使用次数前几张 +#: 进度里的「常用表」取使用次数前几张（同起草的默认范围 catalog.DRAFT_DEFAULT_TABLES） TOP_N = 20 #: 每一项属于哪一组、风险几档：键是 (t, 字段) / (c, 字段) / (r, "")，档 4 最高 @@ -64,9 +69,10 @@ SOURCE_WEIGHT = {"llm": 1.5, "name": 1.5, "profile": 1.25, "comment": 1.0} SOURCE_LABEL = {"comment": "数据库注释", "fk": "外键约束", "name": "命名推断", "profile": "数据剖析", "llm": "模型起草", "human": "人工填写"} RELATION_LABEL = "关联关系" -#: reasons 的说法 -REASON_TOP = "常用表（使用 {n} 次）" -REASON_USED = "使用 {n} 次" +#: reasons 的说法：常用表（被几次运行用到）；使用次数比运行次数多（同一次运行里查了几次）时两个都写 +REASON_TOP = "常用表（{used}）" +USED_RUNS = "被 {runs} 次运行用到" +USED_RUNS_AND_QUERIES = "被 {runs} 次运行用到，使用 {n} 次" REASON_RISK = {4: "影响数字的项（{label}）", 3: "影响计数和汇总的项（{label}）", 2: "帮助理解取值的项（{label}）", 1: "名称和说明（{label}）"} REASON_PENDING_CODES = "{n} 个码值的含义待填写" @@ -109,8 +115,8 @@ def field_name(key: Slot) -> str: return "relation" if key[0] == "r" else key[-1] -def field_label(key: Slot) -> str: - """字段的界面叫法：中文名、码值、有效记录条件、关联关系……""" +def field_text(key: Slot) -> str: + """字段的界面叫法（不带列名）：中文名、码值、有效记录条件、关联关系……""" if key[0] == "t": return catalog.TABLE_FIELD_LABEL.get(key[1], key[1]) if key[0] == "c": @@ -118,6 +124,13 @@ def field_label(key: Slot) -> str: return RELATION_LABEL +def item_label(key: Slot, item: Mapping[str, Any] | None = None) -> str: + """一项给人看的位置：「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」（不知道指向哪张表时写编号）。""" + if key[0] == "r" and isinstance(item, Mapping) and item.get("to_table"): + return f"指向 {item['to_table']} 的{RELATION_LABEL}" + return catalog.describe_slot(key) + + def item_value(key: Slot, item: Mapping[str, Any]) -> Any: """一项的值：关联关系是两端、基数和覆盖率，别的是 value。""" if key[0] == "r": @@ -183,18 +196,49 @@ def priority(usage: int, key: Slot, source: Any) -> float: def top_tables(tables: Mapping[str, Any], usage: Mapping[str, int]) -> list[str]: - """使用次数前 TOP_N 张表：只算被运行用到过的，同样多的按表结构里的顺序。""" + """使用次数前 TOP_N 张表（同 catalog.default_tables，起草对话框的「使用次数最多的 20 张表」）：使用次数从多到少， + 一样多的（包括都没用过的）按表结构里的顺序。表结构里不到 TOP_N 张时就是全部。""" order = {name: i for i, name in enumerate(tables)} - used = [name for name in tables if usage.get(name, 0) > 0] - return sorted(used, key=lambda n: (-usage[n], order[n]))[:TOP_N] + return sorted(tables, key=lambda n: (-usage.get(n, 0), order[n]))[:TOP_N] -def _reasons(key: Slot, item: Mapping[str, Any], usage: int, top: bool) -> list[str]: +@dataclass(frozen=True) +class Usage: + """每张表用得多不多（没被用过的表不出现）。 + + queries：使用次数，同 catalog.table_usage（目录清单的「使用 N 次」、清单的排序、起草的默认范围）； + runs：用到它的运行有几次，同「被哪些运行用过」（table_runs）的 total。同一次运行里查了几次这张表，queries 比 runs 多。 + """ + + queries: dict[str, int] + runs: dict[str, int] + + +async def table_usage_counts(session: AsyncSession, source: Any) -> Usage: + """一次读出每张表的使用次数和运行次数（同一份查询快照引用：catalog.query_snapshot_tables）。""" + queries: Counter[str] = Counter() + runs: dict[str, set[str]] = {} + for run, hit in await catalog.query_snapshot_tables(session, source): + queries.update(hit) + for name in hit: + runs.setdefault(name, set()).add(run) + return Usage(dict(queries), {name: len(ids) for name, ids in runs.items()}) + + +def usage_text(queries: int, runs: int) -> str: + """「被 2 次运行用到」；使用次数比运行次数多时「被 2 次运行用到，使用 5 次」。""" + if queries > runs: + return USED_RUNS_AND_QUERIES.format(runs=f"{runs:,}", n=f"{queries:,}") + return USED_RUNS.format(runs=f"{runs:,}") + + +def _reasons(key: Slot, item: Mapping[str, Any], usage: int, runs: int, top: bool) -> list[str]: out: list[str] = [] if usage > 0: - out.append((REASON_TOP if top else REASON_USED).format(n=f"{usage:,}")) + used = usage_text(usage, runs) + out.append(REASON_TOP.format(used=used) if top else used) _, tier = FIELD_GROUP[field_of(key)] - out.append(REASON_RISK[tier].format(label=field_label(key))) + out.append(REASON_RISK[tier].format(label=field_text(key))) if key[0] == "c" and key[2] == "codes" and isinstance(item.get("value"), Mapping): pending = sum(1 for v in item["value"].values() if not str(v).strip()) if pending: @@ -225,14 +269,14 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", if table: wanted = catalog.resolve_table_name(cache, table) or table entries = await catalog.read_catalog(session, source.id) - usage = await catalog.table_usage(session, source) - top = set(top_tables(tables, usage)) + usage = await table_usage_counts(session, source) + top = set(top_tables(tables, usage.queries)) items: list[dict[str, Any]] = [] for name, entry in entries.items(): meta = tables.get(name) if meta is None or (wanted is not None and name != wanted): continue - used = usage.get(name, 0) + used = usage.queries.get(name, 0) for key, item in live_slots(entry.notes, meta): if item.get("status") != "proposed": continue @@ -244,7 +288,7 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", "table_name": name, "path": slot_path(key), "field": field_name(key), - "field_label": field_label(key), + "field_label": item_label(key, item), # 列级的项是哪一列（表级的项、关联关系为 null） "column": key[1] if key[0] == "c" else None, "value": value, @@ -253,7 +297,7 @@ async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", "status": item.get("status"), "note": item.get("note") if isinstance(item.get("note"), str) and item.get("note") else None, "priority": round(priority(used, key, item.get("source")), 4), - "reasons": _reasons(key, item, used, name in top), + "reasons": _reasons(key, item, used, usage.runs.get(name, 0), name in top), "table_usage": used, "version": entry.version, }) @@ -367,7 +411,8 @@ async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: i async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]: - """审阅进度：常用 TOP_N 张表（只算被运行用到过的）、全部、影响数字的项。项按状态计数；verified（有确证）另给一项。""" + """审阅进度：常用 TOP_N 张表（top_tables）、全部、影响数字的项，项按状态计数。全部的 items_total 里还有有确证的项 + （不单列）。""" tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} entries = await catalog.read_catalog(session, source.id) usage = await catalog.table_usage(session, source) @@ -388,17 +433,15 @@ async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]: facts[status] += 1 return { "top_n": TOP_N, - "top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"], - "items_proposed": head["proposed"], "items_verified": head["verified"]}, + "top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"]}, "all": {"items_total": sum(every.values()), "items_confirmed": every["confirmed"], - "items_proposed": every["proposed"], "items_rejected": every["rejected"], - "items_verified": every["verified"]}, + "items_proposed": every["proposed"], "items_rejected": every["rejected"]}, "facts": {"proposed": facts["proposed"], "confirmed": facts["confirmed"]}, } __all__ = ["BATCH_MAX", "BatchItem", "CODE_CONFLICT", "CODE_INVALID", "CODE_PATH", "CODE_TABLE_MISSING", "FIELD_GROUP", "KINDS", "QUEUE_LIMIT_MAX", "RISK_WEIGHT", "RUNS_LIMIT_MAX", "SOURCE_LABEL", "SOURCE_WEIGHT", - "TOP_N", "display", "field_label", "field_name", "field_of", "item_value", "live_slots", "priority", - "relation_text", "review_batch", "review_progress", "review_queue", "slot_path", "table_runs", "top_tables", - "usage_weight"] + "TOP_N", "Usage", "display", "field_name", "field_of", "field_text", "item_label", "item_value", + "live_slots", "priority", "relation_text", "review_batch", "review_progress", "review_queue", "slot_path", + "table_runs", "table_usage_counts", "top_tables", "usage_text", "usage_weight"] diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py index 6dccd7ec..6a5b3e5d 100644 --- a/backend/app/data/catalog_transfer.py +++ b/backend/app/data/catalog_transfer.py @@ -39,7 +39,7 @@ from typing import Any from sqlalchemy.ext.asyncio import AsyncSession from app.data import catalog, catalog_jobs -from app.data.catalog_review import field_label, slot_path +from app.data.catalog_review import item_label, slot_path from app.data.engine import masked_columns from app.data.names import name_key @@ -358,7 +358,7 @@ def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], targ plan.keep += 1 else: plan.conflicts.append({ - "path": slot_path(mapped), "label": _label(mapped, item), + "path": slot_path(mapped), "label": item_label(mapped, item), "file_value": _shown_value(mapped, item), "target_value": _shown_value(mapped, have), "file_status": status, "target_status": have.get("status"), "why": _conflict_why(str(status), str(have.get("status"))), @@ -371,14 +371,6 @@ def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], targ return plan -def _label(key: catalog.Slot, item: Mapping[str, Any]) -> str: - if key[0] == "t": - return f"表的{field_label(key)}" - if key[0] == "c": - return f"列 {key[1]} 的{field_label(key)}" - return f"指向 {item.get('to_table')} 的关联关系" - - def _shown_value(key: catalog.Slot, item: Mapping[str, Any]) -> Any: if key[0] == "r": return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")} diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index 8013dd75..09a4a769 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -335,7 +335,7 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client, assert rule in system # 表结构、目录（来源和状态）、使用次数、起草要求、数据源说明 for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认", - "模型起草", "已驳回", "被运行用过 0 次", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"): + "模型起草", "已驳回", "还没有被运行用到过", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"): assert fact in system, fact # 别的表指过来的关联关系 assert "members.id → orders.member_id" in system @@ -351,6 +351,26 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client, assert "没开剖析" not in system +async def test_background_says_how_many_runs_used_the_table(client, tmp_path, monkeypatch): + """使用次数的口径同审阅队列：被几次运行用到（同「用过这张表的运行」的 total），查询几次（同目录清单的「使用 N 次」）。""" + from app.db.models import Artifact, Run + + sid = await _source(client, tmp_path) + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + for i, n in enumerate((2, 1)): + rid = uuid.uuid4().hex + session.add(Run(id=rid, graph={}, workflow_name=f"周报{i}", status="succeeded")) + for _ in range(n): + session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("看看")) + assert "这张表被 2 次运行用到（查询 3 次）。" in model.seen[0][0].content + + async def test_long_catalogs_are_cut_with_a_note(client, tmp_path, monkeypatch): sid = await _source(client, tmp_path) monkeypatch.setattr(catalog_assistant, "CATALOG_ITEMS_SHOWN", 2) diff --git a/backend/tests/test_catalog_review_queue.py b/backend/tests/test_catalog_review_queue.py index 1598103c..239694d2 100644 --- a/backend/tests/test_catalog_review_queue.py +++ b/backend/tests/test_catalog_review_queue.py @@ -2,11 +2,13 @@ - GET /catalog/review-queue：只列推断（proposed）的项，按「使用次数权重 × 风险权重 × 来源权重」从高到低，同分按表名、 路径；facts（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）/ names（中文名、说明、含义、单位、码值、 - 粒度、表类型）两组；按表筛选；分页；reasons 写为什么排在这里；表结构里已经没有的表、列不列 + 粒度、表类型）两组；按表筛选；分页；field_label 是给人看的位置（「列 status 的码值」）；reasons 写为什么排在这里 + （被几次运行用到，和 /usage 的 total 一致）；table_usage 是使用次数（同目录清单）；表结构里已经没有的表、列不列 - POST /catalog/review/batch：逐表按 if_version 乐观锁，一张表冲突不影响别的表；同一张表的几条一次写入；路径不对、表不存在 逐条报 - GET /catalog/{table}/usage：用过这张表的运行，新的在前，取法同 catalog.table_usage -- GET /catalog/review-progress：常用 20 张表、全部、影响数字的项各自的确认情况 +- GET /catalog/review-progress：常用 20 张表（使用次数前 20 张，同起草的默认范围：用到的不够 20 张时按表结构的顺序 + 补齐）、全部、影响数字的项各自的确认情况 表名、列名、业务数字都是虚构的。 """ @@ -188,11 +190,11 @@ async def test_queue_lists_only_proposed_items_by_priority(client): rel_id = _member_rel()["id"] assert (first["table_name"], first["path"]) in {("orders", "valid_filter"), ("orders", f"relations.{rel_id}")} vf = next(i for i in items if i["path"] == "valid_filter") - assert vf["field"] == "valid_filter" and vf["field_label"] == "有效记录条件" + assert vf["field"] == "valid_filter" and vf["field_label"] == "表的有效记录条件" assert vf["value"] == "status <> 9" and vf["display"] == "status <> 9" assert vf["source"] == "llm" and vf["note"] == "剖析核对：排除 3.2% 的行（共 12,345 行）" assert vf["table_usage"] == 37 and vf["version"] == 1 - assert vf["reasons"] == ["常用表（使用 37 次）", "影响数字的项（有效记录条件）", "模型起草，未确认"] + assert vf["reasons"] == ["常用表（被 37 次运行用到）", "影响数字的项（有效记录条件）", "模型起草，未确认"] # 常用表上的中文名排在没人用的表上的中文名前面；同一张表上影响数字的项排在中文名前面 order = [(i["table_name"], i["path"]) for i in items] assert order.index(("orders", "label")) < order.index(("coupons", "label")) @@ -203,7 +205,7 @@ async def test_queue_values_are_displayed_in_ui_words(client): sid = await _source() items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]} rel = items[("orders", f"relations.{_member_rel()['id']}")] - assert rel["field"] == "relation" and rel["field_label"] == "关联关系" + assert rel["field"] == "relation" and rel["field_label"] == "指向 members 的关联关系" assert rel["value"] == {"columns": ["member_id"], "to_table": "members", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": 0.98} assert rel["display"] == "member_id → members.id（多对一，覆盖率 98%）" @@ -211,15 +213,39 @@ async def test_queue_values_are_displayed_in_ui_words(client): kind = items[("orders", "kind")] assert kind["display"] == "明细表" and kind["value"] == "fact" codes = items[("orders", "columns.status.codes")] - assert codes["field"] == "codes" and codes["field_label"] == "码值" and codes["column"] == "status" + assert codes["field"] == "codes" and codes["field_label"] == "列 status 的码值" and codes["column"] == "status" assert codes["display"] == "1=已支付、9=含义待填写" assert "1 个码值的含义待填写" in codes["reasons"] and "数据剖析，未确认" in codes["reasons"] assert items[("orders", "columns.amount.measure")]["display"] == "可累加" - assert items[("members", "keys")]["display"] == "id" + assert items[("members", "keys")]["display"] == "id" and items[("members", "keys")]["field_label"] == "表的业务主键" + assert items[("orders", "columns.amount.measure")]["field_label"] == "列 amount 的度量类型" assert items[("members", "label")]["reasons"] == ["名称和说明（中文名）", "模型起草，未确认"] assert items[("coupons", "label")]["reasons"] == ["名称和说明（中文名）", "数据库注释，未确认"] +async def test_queue_reasons_count_runs_like_the_usage_list(client): + """同一次运行里查了几次这张表：原因写「被几次运行用到」（和 /usage 的 total 一样），table_usage 是使用次数（和目录清单的 + 「使用 N 次」一样），两个数不一样时原因里两个都写；优先级按使用次数（同清单的排序）。""" + sid = await _source() + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + runs = await _use(sid, name, {"orders": 2, "members": 1}) + async with SessionLocal() as session: + for _ in range(3): + session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + usage = await catalog.table_usage(session, await session.get(DataSource, sid)) + assert usage["orders"] == 5 + items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]} + vf = items[("orders", "valid_filter")] + assert vf["table_usage"] == 5 + assert vf["reasons"][0] == "常用表（被 2 次运行用到，使用 5 次）" + assert items[("members", "label")]["reasons"][0] == "常用表（被 1 次运行用到）" + assert vf["priority"] == round(catalog_review.priority(5, ("t", "valid_filter"), "llm"), 4) + assert (await client.get(f"{_base(sid)}/orders/usage")).json()["total"] == 2 + + async def test_queue_filters_by_kind_and_table_and_pages(client): sid = await _source(usage={"orders": 3}) facts = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "facts"})).json() @@ -338,20 +364,34 @@ async def test_table_usage_lists_runs_newest_first(client): # -------------------------------------------------------------------------- -async def test_review_progress_counts_top_tables_all_and_facts(client): +async def test_review_progress_counts_top_tables_all_and_facts(client, monkeypatch): sid = await _source(usage={"orders": 5}) r = await client.get(f"{_base(sid)}/review-progress") assert r.status_code == 200, r.text body = r.json() assert body["top_n"] == 20 - # 常用表只算被运行用到过的表：这里只有 orders（9 项：表结构里没有的列不算） - assert body["top"] == {"tables": 1, "items_total": 9, "items_confirmed": 1, "items_proposed": 7, - "items_verified": 0} - assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1, - "items_verified": 0} + # 表结构里只有 3 张表：都算常用表（使用次数前 20 张）；表结构里没有的表、列不算 + assert body["top"] == {"tables": 3, "items_total": 13, "items_confirmed": 1} + assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1} # 影响数字的项：有效记录条件、关联关系、度量类型、业务主键 assert body["facts"] == {"proposed": 4, "confirmed": 0} await client.post(f"{_base(sid)}/review/batch", json={"items": [ {"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}]}) after = (await client.get(f"{_base(sid)}/review-progress")).json() assert after["top"]["items_confirmed"] == 2 and after["facts"] == {"proposed": 3, "confirmed": 1} + # 只取前 2 张：用得多的在前，没人用的按表结构的顺序补（同起草的默认范围） + monkeypatch.setattr(catalog_review, "TOP_N", 2) + used = await _source(usage={"coupons": 1}) + top = (await client.get(f"{_base(used)}/review-progress")).json()["top"] + assert top == {"tables": 2, "items_total": 1 + 9, "items_confirmed": 1} + + +def test_top_tables_are_the_draft_default_scope(): + """常用表和起草对话框「使用次数最多的 20 张表」是同一批：使用次数从多到少，一样多（包括都没用过）的按表结构里的顺序。""" + tables = {f"t{i:02d}": {} for i in range(30)} + usage = {"t29": 4, "t05": 4, "t17": 9} + top = catalog_review.top_tables(tables, usage) + assert catalog_review.TOP_N == catalog.DRAFT_DEFAULT_TABLES == 20 + assert top[:3] == ["t17", "t05", "t29"] and len(top) == 20 + assert top[3:] == [n for n in tables if n not in usage][:17] + assert catalog_review.top_tables({"a": {}, "b": {}}, {}) == ["a", "b"]