diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 10a26f87..75159525 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -8,9 +8,10 @@ - POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认 - GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py） - POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型） +- POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 -不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。 +不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 上传表格的源按当前快照的表结构回答（table_versions.resolve_source），和查询实际用的库是同一个版本。 """ @@ -23,7 +24,8 @@ from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from app.api.runs import actor_of -from app.data import catalog, catalog_impact, introspect, table_versions +from app.data import catalog, catalog_impact, catalog_profile, introspect, table_versions +from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -65,6 +67,11 @@ class CatalogDraftIn(BaseModel): use_model: bool = False +class CatalogProfileIn(BaseModel): + #: 要剖析的表；不给或为空时按使用次数取前 10 张目录里有待核实关系的表 + tables: list[str] | None = Field(default=None, max_length=200) + + async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: """(数据源记录, 用来读表结构的源)：上传源解析成绑定当前快照的视图。""" row = await session.get(DataSource, source_id) @@ -227,6 +234,62 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn, } +def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]: + return { + "table_name": t.table, + "queries": t.queries, + "row_estimate": t.row_estimate.to_dict() if t.row_estimate else None, + "findings": t.findings, + "skipped": [s.to_dict() for s in t.skipped], + "date_ranges": t.date_ranges, + "added": t.added, + "updated": t.updated, + "removed": t.removed, + "version": t.version, + "error": t.error, + } + + +@router.post("/{source_id}/catalog/profile") +async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn, + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。 + + 默认关闭：数据源设置里没开启数据剖析、数据源停用，409；同一个数据源已经在剖析，409。 + """ + row, source = await _resolved(session, source_id) + try: + catalog_profile.ensure_enabled(row) + except catalog_profile.ProfileDisabled as e: + raise HTTPException(409, str(e)) from e + cache = source.schema_cache or {} + if not cache.get("tables"): + raise HTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」") + actor = actor_of(x_actor) + try: + report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None, + actor=actor) + except (catalog_profile.ProfileDisabled, catalog_profile.ProfileBusy) as e: + raise HTTPException(409, str(e)) from e + except SnapshotTampered as e: + raise HTTPException(409, str(e)) from e + rows = [_profile_table_out(t) for t in report.tables] + note = None + if not rows: + note = "目录里还没有待核实的关联关系，请先起草数据目录，或指定要剖析的表" + return { + "profiled_at": report.profiled_at, + "actor": actor, + "settings": report.settings.to_dict(), + "queries_used": report.queries_used, + "stopped": report.stopped, + "tables": rows, + "total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")}, + "note": note, + } + + @router.post("/{source_id}/catalog/{table}/review") async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn, x_actor: str | None = Header(default=None), diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index bea73628..64af636e 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -23,6 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw from app.core.config import settings from app.core.crypto import encrypt, mask from app.data import catalog as data_catalog +from app.data.catalog_profile import profile_settings_problem from app.data import introspect as introspect_mod from app.data import table_versions from app.data.engine import ( @@ -336,7 +337,8 @@ _MASK_FIELD = { def _refuse_bad_options(options: dict[str, Any] | None) -> None: - problem = query_timeout_problem(options) or mask_columns_problem(options) + problem = (query_timeout_problem(options) or mask_columns_problem(options) + or profile_settings_problem(options)) if problem: raise HTTPException(422, problem) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 53585abe..8b9b9ad6 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -15,13 +15,14 @@ notes 结构（每个「项」都是 {value, source, status}，可选 note、upd "valid_filter": 项(str，SQL 条件片段), "dedup": 项(str), "columns": {列名: {"label": 项, "meaning": 项, "unit": 项, "measure": 项("flow"|"stock"|"ratio"|"identifier"|"status"|"attribute"), - "codes": 项(dict[str, str])}}, + "codes": 项(dict[码值, 含义]；含义可为空串，表示待填写)}}, "relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage", "source", "status", "note"?}]} -来源（source）：comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析（阶段 2）、llm 模型起草、 -human 人工填写。状态（status）：proposed 推断（命名推断、模型起草、数据库注释）、verified 有确证（外键 -约束；阶段 2 的数据剖析也写这个状态）、confirmed 人工确认、rejected 人工驳回。只有 confirmed 和 verified +来源（source）：comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析（data/catalog_profile.py）、 +llm 模型起草、human 人工填写。状态（status）：proposed 推断（命名推断、模型起草、数据库注释）、verified 有确证 +（外键约束；数据剖析核实了覆盖率和父键唯一的关系也是，见 profile_relation_holds）、confirmed 人工确认、 +rejected 人工驳回。只有 confirmed 和 verified 将来会触发错误级检查；给模型看时 proposed 的项标「推断，未确认」；rejected 的项留着（防止下次起草又提出 来），但模型和检查都看不到。 @@ -98,6 +99,33 @@ def initial_status(source: str) -> str: return _INITIAL_STATUS.get(source, "proposed") +#: 数据剖析把关系升为有确证的抽样覆盖率下限（data/catalog_profile.py 判定，撤销审阅时按它判回剖析的结论） +PROFILE_VERIFY_COVERAGE = 0.95 + + +def profile_relation_holds(rel: Mapping[str, Any]) -> bool: + """一条数据剖析得出的关系站不站得住：抽样覆盖率不低于 PROFILE_VERIFY_COVERAGE，而且被指向列核实唯一。 + + 剖析只在被指向列核实唯一时才写基数（多对一或一对一），所以「基数有值」就是「父键唯一」。结论全在关系 + 自己的字段里，撤销审阅时照原样判得回来，不必另存一份「审阅前的状态」。 + """ + coverage = rel.get("coverage") + return (rel.get("source") == "profile" and isinstance(coverage, (int, float)) and not isinstance(coverage, bool) + and coverage >= PROFILE_VERIFY_COVERAGE and rel.get("cardinality") in ("many_to_one", "one_to_one")) + + +def unreviewed_status(item: Mapping[str, Any]) -> str: + """撤销审阅后回到的状态：来源的初始状态。 + + 数据剖析是例外：它的状态随数据而定（覆盖率够、父键唯一的关系是有确证，其余是推断），不能一律回到 + initial_status("profile") 的推断——那样人工确认过再撤销，有确证就无故丢了，要等下一次剖析才回来。 + 剖析提的码值候选、业务日期永远是推断。 + """ + if item.get("source") == "profile" and "to_table" in item: + return "verified" if profile_relation_holds(item) else "proposed" + return initial_status(item.get("source")) + + def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None, at: str | None = None) -> dict[str, Any]: """构造一个项。status 不给时按来源取初始状态（initial_status）。""" @@ -153,8 +181,10 @@ def _business_date_ok(value: Any) -> bool: def _codes_ok(value: Any) -> bool: + """码值对照：{码值: 含义}，都是文字。含义可以是空串——数据剖析只知道列里出现过哪些取值，含义要等人填； + 以前要求含义非空，剖析得到的码值候选就写不进来。""" return (isinstance(value, dict) and bool(value) - and all(isinstance(k, str) and isinstance(v, str) and v.strip() for k, v in value.items())) + and all(isinstance(k, str) and isinstance(v, str) for k, v in value.items())) #: 表类型、度量类型在界面上的叫法（和 frontend/src/lib/terms.ts 的 CATALOG_KIND_LABEL / CATALOG_MEASURE_LABEL 一致）。 @@ -587,7 +617,7 @@ def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at: # 人工填写的项没有「推断时的样子」可回：撤销就是删掉，下次起草可以重新提出 del slots[key] return _assemble(slots) - item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or initial_status(item.get("source")) + item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or unreviewed_status(item) item["updated_at"] = at or now_iso() return _assemble(slots) @@ -652,7 +682,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str, continue kept = dict(prev) status = item.get("status") - if status != prev.get("status") and status in ("confirmed", "rejected", initial_status(prev.get("source"))): + if status != prev.get("status") and status in ("confirmed", "rejected", unreviewed_status(prev)): kept["status"] = status kept["updated_at"] = at if item.get("note") != prev.get("note"): @@ -779,6 +809,17 @@ def _patch_relation(table: str, raw: Any, path_id: str | None, tables: Mapping[s return value, None +def _merge_codes(before: Mapping[str, str], value: Mapping[str, str]) -> dict[str, str]: + """码值的补充：给出的码加进去或改掉原来的含义。含义为空表示「只知道有这个码、含义待填写」（数据剖析的码值 + 候选就是这样），所以提案里含义为空的码不覆盖已有的含义，只在原来没有这个码时加进去。""" + out = dict(before) + for code, meaning in value.items(): + if not str(meaning).strip() and str(out.get(code) or "").strip(): + continue + out[code] = meaning + return out + + def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, tables: Mapping[str, Any] | None = None) -> PatchPlan: """逐项核对一条目录修改提案，算出改前、改后。不改入参、不抛异常：不合法的项写进 problems。 @@ -790,7 +831,8 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, 规则： - 取值格式和 validate_notes 同一套（_TABLE_VALUE_RULES / _COLUMN_VALUE_RULES / _relation_problems）。 - 码值是补充：在现有（没被驳回的）码值上加或改给出的码，不整份替换——「status=9 表示作废」不该把已确认的 - 其余码值删掉。 + 其余码值删掉。含义为空的码（数据剖析写进来的「含义待填写」）可以被补上含义；提案里含义为空的码只在原来 + 没有这个码时加进去，不会把已有的含义抹掉（_merge_codes）。 - 看起来像计算公式的文字不收（_FORMULA）：公式进口径卡。 - 同一个路径出现两次只认第一次。 """ @@ -864,7 +906,7 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, before = _value_of(key, live) if live is not None else None after = value if key[0] == "c" and key[2] == "codes" and isinstance(before, dict): - after = {**before, **value} + after = _merge_codes(before, value) if before is not None and before == after: state = "same" if live.get("status") == "confirmed" else "confirm" else: @@ -1094,6 +1136,15 @@ def infer_name_relations(schema_cache: Mapping[str, Any] | None, table: str, *, candidates: set[str] = set() for form in _singulars(words[-1]): candidates |= index.get((prefix, form), set()) + if not candidates: + # 直接对不上：库里的表常带业务前缀（store_tags），列名却只写词根（tagId）。 + # 只认和引用它的表共用前缀的那张，从最长的共用前缀试起；仍然不止一张就不推 + own = _words(table) + for k in range(len(own) - 1, 0, -1): + for form in _singulars(words[-1]): + candidates |= index.get(("".join(own[:k]) + prefix, form), set()) + if candidates: + break if len(candidates) != 1: continue target = next(iter(candidates)) @@ -1625,6 +1676,17 @@ def _date_text(value: Mapping[str, Any]) -> str: return "；".join(parts) +def _codes_text(codes: Mapping[str, Any]) -> str: + """码值对照的文字：有含义的写「1=有效」，含义待填写的单独说明——不能写成「1=」，模型会当成含义就是空。""" + filled = [f"{k}={v}" for k, v in codes.items() if str(v).strip()] + pending = [str(k) for k, v in codes.items() if not str(v).strip()] + if not pending: + return "、".join(filled) + if not filled: + return f"{'、'.join(pending)}（含义待填写）" + return f"{'、'.join(filled)}，{'、'.join(pending)} 的含义待填写" + + def _relation_text(rel: Mapping[str, Any]) -> str: left = ", ".join(rel.get("columns") or []) right = ", ".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or []) @@ -1675,7 +1737,7 @@ def _notes_lines(notes: Mapping[str, Any] | None, *, meta: Mapping[str, Any] | N if name == "measure": text = MEASURE_LABEL.get(value, str(value)) elif name == "codes": - text = "、".join(f"{k}={v}" for k, v in value.items()) + text = _codes_text(value) else: text = str(value) # 第一段是中文名（没有中文名时是含义），不加前缀；其余写明是什么 @@ -1932,13 +1994,13 @@ __all__ = [ "CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict", "CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES", "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", - "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", - "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult", - "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot", "draft_catalog", - "draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", + "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_VERIFY_COVERAGE", "PatchChange", "PatchPlan", + "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", + "TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot", + "draft_catalog", "draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths", "make_item", "merge_notes", "now_iso", "parse_path", "plan_patch", - "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", - "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes", - "save_human_edit", "save_patch", "status_counts", "system_notes_source", "table_usage", "validate_notes", - "visible_notes", "write_entry", + "profile_relation_holds", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", + "render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", + "review_item", "same_notes", "save_human_edit", "save_patch", "status_counts", "system_notes_source", + "table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry", ] diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py new file mode 100644 index 00000000..c4d51f0a --- /dev/null +++ b/backend/app/data/catalog_profile.py @@ -0,0 +1,1228 @@ +"""数据剖析：对业务库发少量只读查询，验证数据目录里推断出来的关联关系，补上基数、覆盖率和码值候选。 + +**为什么要有它。** 命名推断只看列名，可能推错，也给不出基数和覆盖率；而只有 verified、confirmed 的关系才会 +触发错误级的 SQL 检查。剖析拿数据说话：从子表抽一批不同的键值去父表里对，对得上、父键唯一的升为 verified +（来源记为 profile，和外键约束的 fk 分开），对不上的保持 proposed 并在备注里写明覆盖率。顺带给状态类的列 +取一次取值分布（码值候选，含义留给人填），给只有一个日期列的表提议业务日期。 + +**这是数据目录里唯一会对业务库发查询的地方**，约束都是硬的： + +- 默认关闭，按数据源在 options.catalog_profile 里开启（profile_settings）； +- 每条查询先过守卫的只读判定（不论数据源本身可写与否），再交给查询层 run_query：同一套连接、守卫和 + 时限，不另开连接； +- 每条查询有时限（不超过数据源自己的查询时限），每次剖析有查询次数上限和总时长上限； +- 表有多大先看数据库的统计信息，没有统计信息就数到上限为止，不做全表 COUNT；超过上限的表不做 + COUNT(DISTINCT)、MIN/MAX 这类整表统计，抽样和取值分布也只看前若干行； +- 数据源设置里遮罩的列不取样，也不出现在任何一条剖析查询里； +- 只取低基数列的取值分布（最多 21 个取值），不取明细行。 +""" +from __future__ import annotations + +import copy +import logging +import re +import time +from collections.abc import Awaitable, Callable +from dataclasses import asdict, dataclass, field +from decimal import Decimal +from typing import Any + +from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, guard +from app.data import engine as data_engine +from app.data.engine import CATALOG_PROFILE_OPTION, QueryResult, SnapshotTampered +from app.data.guard import QueryLimits, SqlRejected + +logger = logging.getLogger(__name__) + +#: 数据源 options 里剖析设置的键 +PROFILE_OPTION = CATALOG_PROFILE_OPTION + + +# ========================================================================== +# 设置 +# ========================================================================== + + +@dataclass(frozen=True) +class ProfileSettings: + """一个数据源的剖析设置。缺省即「关闭」：剖析会对业务库发查询，必须有人按源明确打开。""" + + enabled: bool = False + #: 一次剖析最多发几条查询（读统计信息、抽样、核对都算）。用完就停，剩下的记为「查询次数用完」 + max_queries: int = 60 + #: 单条查询的时限（秒）。实际取它和数据源自己的查询时限中较小的一个 + query_timeout_s: float = 10 + #: 每条关系从子表抽多少个不同的键值去父表里对 + sample_size: int = 2000 + #: 行数不超过它的表才做 COUNT(DISTINCT)、MIN/MAX 这类整表统计；更大的、或者不知道多大的表只看前若干行。 + #: 填 0 表示一律不做整表统计 + max_scan_rows: int = 100_000 + #: 一次剖析的总时长上限（秒）。接口是同步的，不能让一个请求无限地挂着 + max_total_s: float = 120 + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +#: 数值项：(中文叫法, 下限, 上限, 是否必须是整数)。叫法和将来设置界面上的标签一致，报错里只写它 +_NUMBER_FIELDS: dict[str, tuple[str, float, float, bool]] = { + "max_queries": ("查询次数上限", 1, 500, True), + "query_timeout_s": ("单条查询时限（秒）", 1, 60, False), + "sample_size": ("抽样键值数", 10, 10_000, True), + "max_scan_rows": ("整表统计行数上限", 0, 10_000_000, True), + "max_total_s": ("总时长上限（秒）", 10, 600, False), +} +_ENABLED_LABEL = "开启数据剖析" +_KNOWN_KEYS = frozenset({"enabled", *_NUMBER_FIELDS}) + + +def _fmt(value: float) -> str: + return str(int(value)) if float(value).is_integer() else f"{value:g}" + + +def _number(value: Any, low: float, high: float, integer: bool) -> int | float | None: + """设置里的一个数：数字和数字写成的文字都认（表单存的常是文字）。不合规返回 None。""" + if value is None or isinstance(value, bool): + return None + try: + number = float(str(value).strip()) + except ValueError: + return None + if number != number or not low <= number <= high: # NaN 也在这里挡掉 + return None + if number.is_integer(): + return int(number) + return None if integer else number + + +def profile_settings(options: dict[str, Any] | None) -> ProfileSettings: + """options → 剖析设置。读的时候宽松：库里存着的某一项不对（老数据、手改过），那一项用缺省，不报错； + 开关只认 true，写成别的一律当作没开。""" + raw = (options or {}).get(PROFILE_OPTION) + if not isinstance(raw, dict): + return ProfileSettings() + values: dict[str, Any] = {"enabled": raw.get("enabled") is True} + for key, (_, low, high, integer) in _NUMBER_FIELDS.items(): + number = _number(raw.get(key), low, high, integer) + if number is not None: + values[key] = number + return ProfileSettings(**values) + + +def profile_settings_problem(options: dict[str, Any] | None) -> str | None: + """保存数据源时查一下剖析设置；没填、或者填对了返回 None。写的时候严格：不认识的项、超出范围的值都拒。""" + raw = (options or {}).get(PROFILE_OPTION) + if raw is None: + return None + if not isinstance(raw, dict): + return "数据剖析设置的格式不正确，请在数据源设置中重新填写" + if unknown := sorted(set(raw) - _KNOWN_KEYS): + return f"数据剖析设置中有无法识别的项「{'、'.join(map(str, unknown))}」，请删除后重试" + if "enabled" in raw and not isinstance(raw["enabled"], bool): + return f"数据剖析的「{_ENABLED_LABEL}」只能是开启或关闭" + for key, (label, low, high, integer) in _NUMBER_FIELDS.items(): + if key in raw and _number(raw[key], low, high, integer) is None: + kind = "整数" if integer else "数" + return f"数据剖析的「{label}」需要填写 {_fmt(low)} 到 {_fmt(high)} 之间的{kind}；当前为「{raw[key]}」" + return None + + +# ========================================================================== +# SQL：字面量与方言 +# ========================================================================== + +#: 查询层不支持参数绑定（run_query 只收一段 SQL），核对键值只能把值拼进 IN 列表。所以只拼「自己刚从库里 +#: 取回的原值」，并且按类型生成字面量：整数原样写，文字加单引号、单引号写两遍。下面几种文字不拼，跳过： +#: - 带反斜杠的：MySQL 在 NO_BACKSLASH_ESCAPES 开与不开时读法不同，守卫会整条拒掉； +#: - 带「:名字」的：查询层用 SQLAlchemy 的 text() 执行，它不认引号，会把字符串里的 :b 当成绑定参数； +#: - 带控制字符的、太长的：键值不该长这样，多半不是键。 +_BIND_LIKE = re.compile(r"(? str | None: + """一个取回的值 → SQL 字面量；不安全或类型不支持时返回 None（调用方跳过这个值）。 + + kind 是查询层记下的列类型（QueryResult.column_types：number / text / date …），None 表示拿不准。 + DECIMAL 经查询层已经转成了字符串（保精度），所以 number 列的整数文字原样写成数字。只认整数和文字： + 非整数的数做相等比较靠不住；日期、时间的字面量各家写法不同，不猜；布尔不当键。 + """ + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return str(int(value)) if value.is_integer() and abs(value) < 2 ** 53 else None + if isinstance(value, Decimal): + return str(int(value)) if value.is_finite() and value == value.to_integral_value() else None + if not isinstance(value, str): + return None + if kind == "number": + return value if _INT_TEXT.fullmatch(value) else None + if kind not in ("text", None): + return None + if len(value) > MAX_LITERAL_LEN or "\\" in value or _CONTROL.search(value) or _BIND_LIKE.search(value): + return None + return "'" + value.replace("'", "''") + "'" + + +_DIALECT_MODULES = {"sqlite": sqlite, "postgres": postgresql, "postgresql": postgresql, "mysql": mysql, + "mariadb": mysql, "oracle": oracle} + + +class SqlDialect: + """剖析 SQL 的方言差异：标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里。只拼字符串，不连库。 + + 标识符交给 SQLAlchemy 各方言的 identifier_preparer：需要时才加引号（memberTags 加、visits 不加）， + MySQL 用反引号。Oracle 尤其不能一律加双引号：探查拿到的表名是 SQLAlchemy 规整过的小写，库里其实是 + 大写，"visits" 加了引号就成了另一张不存在的表；不加引号 Oracle 自己转大写才对得上。 + """ + + def __init__(self, kind: str | None) -> None: + self.kind = (kind or "").lower() + module = _DIALECT_MODULES.get(self.kind) + if module is None: + raise ValueError(f"不支持的数据库类型：{kind}") + self._dialect = module.dialect() + self._prep = self._dialect.identifier_preparer + self.oracle = self.kind == "oracle" + + def quote(self, name: str) -> str: + return self._prep.quote(name) + + def table(self, meta: dict[str, Any] | None, name: str) -> str: + """写进 FROM 的表名：探查时指定了 schema 的带上它（Oracle 只读账号名下没有对象，全靠 schema 前缀）。""" + schema = (meta or {}).get("schema") + return f"{self._prep.quote_schema(schema)}.{self.quote(name)}" if schema else self.quote(name) + + def limit(self, sql: str, n: int) -> str: + """只取前 n 行。Oracle 11g 没有 LIMIT 也没有 FETCH FIRST，一律套一层 ROWNUM。""" + if self.oracle: + return f"SELECT * FROM ({sql}) WHERE ROWNUM <= {int(n)}" + return f"{sql} LIMIT {int(n)}" + + def _first_rows(self, table: str, column: str, scan_cap: int | None) -> str: + """FROM 子句：整张表，或者只看前 scan_cap 行非空值的子查询（大表不做整表扫描）。""" + if scan_cap is None: + return table + return f"({self.limit(f'SELECT {column} FROM {table} WHERE {column} IS NOT NULL', scan_cap)}) s" + + def stats_sql(self, meta: dict[str, Any] | None, name: str) -> str | None: + """读这张表行数估算值的查询：MySQL information_schema.TABLES.TABLE_ROWS、PostgreSQL pg_class.reltuples、 + Oracle ALL_TABLES.NUM_ROWS。SQLite 没有可靠的统计信息，返回 None（改为数到上限为止）。""" + schema = (meta or {}).get("schema") + if self.kind in ("mysql", "mariadb"): + where_schema = sql_literal(schema, "text") if schema else "DATABASE()" + table = sql_literal(name, "text") + if not (where_schema and table): + return None + return (f"SELECT TABLE_ROWS FROM information_schema.TABLES " + f"WHERE TABLE_SCHEMA = {where_schema} AND TABLE_NAME = {table}") + if self.kind in ("postgres", "postgresql"): + where_schema = sql_literal(schema, "text") if schema else "current_schema()" + table = sql_literal(name, "text") + if not (where_schema and table): + return None + return ("SELECT c.reltuples FROM pg_catalog.pg_class c " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + f"WHERE n.nspname = {where_schema} AND c.relname = {table}") + if self.oracle: + denorm = self._dialect.denormalize_name + owner = sql_literal(denorm(schema), "text") if schema else "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')" + table = sql_literal(denorm(name), "text") + if not (owner and table): + return None + return f"SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = {owner} AND TABLE_NAME = {table}" + return None + + def bounded_count_sql(self, table: str, cap: int) -> str: + """数到 cap 行为止：小表得到准确行数，大表只知道「至少 cap 行」，不做全表 COUNT。""" + return f"SELECT COUNT(*) AS n FROM ({self.limit(f'SELECT 1 AS one FROM {table}', cap)}) s" + + def distinct_sample_sql(self, table: str, column: str, n: int, *, scan_cap: int | None) -> str: + """至多 n 个不同的非空键值。scan_cap 给了时只在前 scan_cap 行里找：键的取值少时 DISTINCT … LIMIT + 凑不满 n 个，会一直扫到表尾。""" + if scan_cap is None: + return self.limit(f"SELECT DISTINCT {column} FROM {table} WHERE {column} IS NOT NULL", n) + return self.limit(f"SELECT DISTINCT {column} FROM {self._first_rows(table, column, scan_cap)}", n) + + def match_count_sql(self, table: str, column: str, literals: list[str]) -> str: + """被指向表里能对上这些键值的个数（去重：被指向列有重复值时不多算）。""" + return f"SELECT COUNT(DISTINCT {column}) AS n FROM {table} WHERE {column} IN ({', '.join(literals)})" + + def unique_check_sql(self, table: str, column: str) -> str: + """非空值个数和不同值个数：相等即这一列（非空部分）唯一。整表统计，只对小表用。""" + return f"SELECT COUNT({column}) AS n, COUNT(DISTINCT {column}) AS d FROM {table}" + + def value_counts_sql(self, table: str, column: str, n: int, *, scan_cap: int | None) -> str: + """取值分布：出现最多的 n 个取值和各自的行数。只拿取值和计数，不拿明细行。""" + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {self._first_rows(table, column, scan_cap)} " + f"WHERE {column} IS NOT NULL GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) + + def min_max_sql(self, table: str, columns: list[str]) -> str: + return f"SELECT {', '.join(f'MIN({c}), MAX({c})' for c in columns)} FROM {table}" + + +# ========================================================================== +# 发查询：守卫、查询次数、时限 +# ========================================================================== + +#: 总时长只剩这么一点时不再发新查询：发出去的时限也只剩零点几秒，白占一次查询次数 +_MIN_WINDOW_S = 0.5 +#: 连续失败这么多条就停：多半是连接断了、账号没有权限，再发只是一条条重复同一个错误 +_MAX_CONSECUTIVE_ERRORS = 3 + + +class _Stop(Exception): + """整次剖析停下：budget 查询次数用完、deadline 总时长用完、failed 连续失败。剩下的检查都不再发查询。""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +class _Skip(Exception): + """这一项没查成或不查：timeout 超时、error 查询失败、rejected 未通过守卫、masked 被遮罩……记下原因，接着查别的。 + + carry：目录里剖析以前对这一项写的结论要不要原样留着。没查成的（超时、表太大……）留着——这次没查不等于 + 结论错了；被遮罩的列上取过值的结论（码值、日期范围）不留：遮罩之后，带着原值的结论也不该再给人看。 + """ + + def __init__(self, reason: str, detail: str, *, carry: bool = True) -> None: + super().__init__(detail) + self.reason = reason + self.detail = detail + self.carry = carry + + +def _brief(e: BaseException) -> str: + """驱动报错的第一行，去掉 SQLAlchemy 加的「(模块.类名)」前缀和后面附的 SQL、文档链接。""" + lines = str(getattr(e, "orig", None) or e).strip().splitlines() + first = re.sub(r"^\([\w.]+\)\s*", "", lines[0] if lines else "") + return first[:160] or type(e).__name__ + + +class _Runner: + """剖析发的每一条查询都从这里走：先按只读过守卫，再交给查询层 run_query；记次数、管时限。 + + - 只读：不论数据源本身可写与否，先用 guard.check(readonly=True) 判一遍，再进 run_query——那里还会按 + 数据源自己的设置判一遍。两遍是同一个守卫，不为剖析放宽，也不另开连接。 + - 查询次数：每发一条记一次（超时、失败的也算），到上限就停。没发出去的（守卫拒了）不算。 + - 时限：单条取设置和数据源查询时限中较小的一个，并且不超过总时长剩下的部分。 + """ + + def __init__(self, source: Any, settings: ProfileSettings, dialect: SqlDialect) -> None: + self.source = source + self.settings = settings + self.kind = dialect.kind + self.timeout = min(float(settings.query_timeout_s), float(data_engine.query_timeout(source))) + self.ends = time.monotonic() + float(settings.max_total_s) + self.used = 0 + self.stopped: str | None = None + self._errors = 0 + + def _stop(self, reason: str) -> None: + self.stopped = reason + raise _Stop(reason) + + async def __call__(self, sql: str, *, max_rows: int = 1) -> QueryResult: + if self.stopped: + raise _Stop(self.stopped) + if self.used >= self.settings.max_queries: + self._stop("budget") + remaining = self.ends - time.monotonic() + if remaining <= _MIN_WINDOW_S: + self._stop("deadline") + try: + guard.check(sql, readonly=True, source_name=getattr(self.source, "name", ""), dialect=self.kind) + except SqlRejected as e: + raise _Skip("rejected", f"剖析查询未通过安全守卫：{e}") from e + self.used += 1 + cut = remaining < self.timeout + seconds = round(min(self.timeout, remaining), 3) + limits = QueryLimits(max_rows=max(1, int(max_rows)), timeout_seconds=seconds) + try: + result = await data_engine.run_query(self.source, sql, limits=limits) + except SqlRejected as e: + # 上面已经按只读过了同一个守卫，查询层再拒只可能是时限到了 + if cut: + self.stopped = "deadline" + raise _Skip("timeout", f"查询超时：超过 {_fmt(seconds)} 秒被中断，已跳过") from e + except SnapshotTampered: + raise + except Exception as e: # noqa: BLE001 - 一条查询失败不该拖垮整次剖析，原因照实记下 + self._errors += 1 + if self._errors >= _MAX_CONSECUTIVE_ERRORS: + self.stopped = "failed" + raise _Skip("error", f"查询失败：{_brief(e)}") from e + self._errors = 0 + return result + + +# ========================================================================== +# 表有多大 +# ========================================================================== + + +@dataclass(frozen=True) +class TableSize: + """一张表的行数。rows 为 None 表示不知道（没有统计信息、也没数）或者超过了整表统计的上限。""" + + rows: int | None + #: stats 数据库的统计信息（估算值）；count 数到上限为止；unknown 没能得到 + method: str + #: 数到上限也没数完：至少这么多行 + at_least: int | None = None + + def within(self, limit: int) -> bool: + """行数已知且不超过 limit：可以做整表统计。不知道多大的表按大表对待。""" + return self.rows is not None and self.rows <= limit + + def to_dict(self) -> dict[str, Any]: + return {"rows": self.rows, "method": self.method, "at_least": self.at_least} + + +def _first(result: QueryResult) -> Any: + return result.rows[0][0] if result.rows and result.rows[0] else None + + +def _as_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + try: + number = float(str(value)) + except ValueError: + return None + return int(number) if number == number and abs(number) != float("inf") else None + + +async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any], name: str, *, + max_scan_rows: int) -> tuple[TableSize, list[_Skip]]: + """先读统计信息；读不到（SQLite、没分析过的表）再数到 max_scan_rows + 1 行为止。 + + 统计信息为 0 或负数（PostgreSQL 没分析过的表是 -1，MySQL 的估算值可能是 0）当作不知道：一张其实很大的表 + 要是被当成空表，后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。查询次数、总时长用完时抛 _Stop。 + """ + skips: list[_Skip] = [] + sql = dialect.stats_sql(meta, name) + if sql: + try: + rows = _as_int(_first(await run(sql))) + except _Skip as e: + skips.append(e) + else: + if rows is not None and rows > 0: + return TableSize(rows, "stats"), skips + if max_scan_rows <= 0: + return TableSize(None, "unknown"), skips + cap = max_scan_rows + 1 + try: + counted = _as_int(_first(await run(dialect.bounded_count_sql(dialect.table(meta, name), cap)))) + except _Skip as e: + skips.append(e) + return TableSize(None, "unknown"), skips + if counted is None: + return TableSize(None, "unknown"), skips + if counted <= max_scan_rows: + return TableSize(counted, "count"), skips + return TableSize(None, "count", at_least=cap), skips + + +# ========================================================================== +# 一次剖析：上下文、结果 +# ========================================================================== + +#: 不指定表时剖析几张：按使用次数取目录里有待核实关系的表 +PROFILE_DEFAULT_TABLES = 10 +#: IN 列表每批最多几个值：Oracle 的 IN 列表不能超过 1000 项（ORA-01795），各家统一按它分批 +IN_CHUNK = 1000 +#: 剖析写的备注都以它开头：人工确认过的关系，只覆盖剖析自己写的备注，不碰人写的 +NOTE_PREFIX = "数据剖析（" +#: 整次剖析停下的原因 → 剩下没查的项的说明 +_STOP_DETAIL = { + "budget": "本次剖析的查询次数已用完（上限 {max_queries} 条），未检查", + "deadline": "本次剖析的总时长已用完（上限 {max_total_s} 秒），未检查", + "failed": "连续多条查询失败，已停止剖析，未检查", +} + + +class ProfileDisabled(Exception): + """这个数据源没有开启数据剖析。message 给人看。""" + + +class ProfileBusy(Exception): + """这个数据源正在剖析。message 给人看。""" + + +@dataclass +class ProfileSkip: + """没做的一项和原因。 + + kind：relation 关系、codes 码值、date 日期列、row_estimate 行数估算、table 整张表。 + reason：budget 查询次数用完、deadline 总时长用完、failed 连续失败、timeout 超时、error 查询失败、 + rejected 未通过守卫、masked 字段被遮罩、too_large 表太大、view 视图、unsupported 暂不支持、 + no_data 没有数据、missing 表结构里没有、high_cardinality 取值太多（不像码值）。detail 是给人看的整句。 + """ + + kind: str + target: str + reason: str + detail: str + path: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"kind": self.kind, "target": self.target, "path": self.path, "reason": self.reason, + "detail": self.detail} + + +@dataclass +class TableProfile: + """一张表这次剖析的结果。findings 是写进目录的发现（给界面看，形状见 _finding_*）。""" + + table: str + queries: int = 0 + row_estimate: TableSize | None = None + findings: list[dict[str, Any]] = field(default_factory=list) + skipped: list[ProfileSkip] = field(default_factory=list) + #: 日期类列的取值范围（MIN、MAX）。只有一个日期列时它还写进了业务日期的备注；多个时只在这里 + date_ranges: list[dict[str, Any]] = field(default_factory=list) + added: int = 0 + updated: int = 0 + removed: int = 0 + version: int = 0 + #: 这张表没剖析（表结构里没有）或结果没写进去（写入一直冲突） + error: str | None = None + + +@dataclass +class ProfileReport: + """一次剖析的结果。stopped：整次剖析中途停下的原因（budget / deadline / failed），做完了为 None。""" + + profiled_at: str + settings: ProfileSettings + tables: list[TableProfile] = field(default_factory=list) + queries_used: int = 0 + stopped: str | None = None + + +@dataclass +class _RelationOutcome: + """一条关系核对完的结论。写不写、怎么写由 _apply 按写入时目录里那条关系的状态决定。""" + + rid: str + target: str + coverage: float + cardinality: str | None + holds: bool + sample: int + matched: int + note: str + + +@dataclass +class _CodesOutcome: + """一列的取值分布：[(码值, 行数)]，按行数从多到少。""" + + column: str + values: list[tuple[str, int]] + rows: int + note: str + + +@dataclass +class _DateOutcome: + """表里唯一一个日期类列，提议作为业务日期。""" + + column: str + low: str + high: str + note: str + + +@dataclass +class _TableOutcome: + """一张表所有检查的结论，以及没查成、要原样保留旧结论的槽位。""" + + relations: list[_RelationOutcome] = field(default_factory=list) + codes: list[_CodesOutcome] = field(default_factory=list) + business_date: _DateOutcome | None = None + #: 没查成的项：目录里剖析以前写的结论原样留着（不然会被 covered={"profile"} 当成「这次没有」删掉） + keep_relations: set[str] = field(default_factory=set) + keep_codes: set[str] = field(default_factory=set) + keep_business_date: bool = False + + +@dataclass +class _Context: + source: Any + settings: ProfileSettings + dialect: SqlDialect + run: _Runner + tables: dict[str, Any] + #: 遮罩的列（小写）。遮罩按列名生效、不分大小写、不分表，和证据面板的比法一致 + masked: set[str] + #: 写进备注的剖析日期（UTC） + day: str + sizes: dict[str, TableSize] = field(default_factory=dict) + + async def size(self, table: str, result: TableProfile) -> TableSize: + """表有多大，一次剖析里每张表只估一次。途中没查成的记进 result（行数估算这一项）。""" + if table not in self.sizes: + size, skips = await estimate_size(self.run, self.dialect, self.tables[table], table, + max_scan_rows=self.settings.max_scan_rows) + self.sizes[table] = size + result.skipped += [ProfileSkip("row_estimate", table, s.reason, s.detail) for s in skips] + return self.sizes[table] + + async def size_with_data(self, table: str, result: TableProfile) -> TableSize: + """同 size，但确知是空表时直接跳过这一项（抛 _Skip），不再为它发查询。""" + size = await self.size(table, result) + if size.rows == 0: + raise _Skip("no_data", f"{table} 是空表，无法剖析") + return size + + def is_masked(self, column: str) -> bool: + return column.lower() in self.masked + + def scan_cap(self, size: TableSize) -> int | None: + """大表（或不知道多大）抽样、取值分布只看前多少行；小表整表看，返回 None。""" + if size.within(self.settings.max_scan_rows): + return None + return max(self.settings.sample_size, self.settings.max_scan_rows) + + +def _column(meta: dict[str, Any], name: str) -> str | None: + """表结构里这一列的写法（目录里的列名大小写可能和表结构不同）；没有返回 None。""" + lowered = name.lower() + return next((str(c["name"]) for c in meta.get("columns") or [] + if isinstance(c, dict) and str(c.get("name", "")).lower() == lowered), None) + + +def _constraint(meta: dict[str, Any], column: str) -> str | None: + """这一列单独是主键或某个唯一约束时返回「主键」/「唯一约束」，否则 None。""" + wanted = [column.lower()] + if [c.lower() for c in meta.get("primary_key") or []] == wanted: + return "主键" + for group in meta.get("unique") or []: + if isinstance(group, list) and [str(c).lower() for c in group] == wanted: + return "唯一约束" + return None + + +def _percent(ratio: float) -> str: + """覆盖率写成百分数：整数不带小数，其余留一位；不到 100% 的不四舍五入成 100%。""" + text = f"{ratio * 100:.1f}".rstrip("0").rstrip(".") + return "99.9%" if text == "100" and ratio < 1 else f"{text}%" + + +def _relation_target(rel: dict[str, Any]) -> str: + left = ", ".join(rel.get("columns") or []) + right = ", ".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or []) + return f"{left} → {right}" + + +# ========================================================================== +# 检查：关联关系 +# ========================================================================== + + +def _relation_candidates(notes: dict[str, Any] | None) -> list[dict[str, Any]]: + """要核对的关系：没被驳回、不是外键约束来的（外键由数据库保证，剖析也顶不掉它）。人工确认过的也核对， + 只补覆盖率和基数。""" + return [r for r in (notes or {}).get("relations") or [] + if isinstance(r, dict) and r.get("id") and r.get("status") != "rejected" and r.get("source") != "fk"] + + +async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str, + result: TableProfile) -> tuple[bool | None, str]: + """这一列（非空部分）唯一吗：(True / False / None 不知道, 依据)。有主键或唯一约束直接认；否则只对行数不超过 + 整表统计上限的表数一次 COUNT 和 COUNT(DISTINCT)，大表不扫。""" + how = _constraint(meta, column) + if how: + return True, how + size = await cx.size(table, result) + if not size.within(cx.settings.max_scan_rows): + return None, "too_large" + d = cx.dialect + res = await cx.run(d.unique_check_sql(d.table(meta, table), d.quote(column))) + row = res.rows[0] if res.rows else [None, None] + total, distinct = _as_int(row[0]), _as_int(row[1]) + if total is None or distinct is None: + return None, "count" + return total == distinct, "count" + + +def _relation_note(day: str, *, sample: int, matched: int, coverage: float, scan_cap: int | None, + skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None, + holds: bool) -> str: + """关系的剖析备注：日期、抽样规模、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。""" + parts: list[str] = [] + low = coverage < catalog.PROFILE_VERIFY_COVERAGE + if low: + parts.append(f"抽样覆盖率 {_percent(coverage)}，可能不是这条关系") + scope = f"（只看前 {scan_cap} 行）" if scan_cap else "" + sampled = f"子表抽样 {sample} 个不同键值{scope}，父表对上 {matched} 个" + parts.append(sampled if low else f"{sampled}，覆盖率 {_percent(coverage)}") + if skipped_values: + parts.append(f"另有 {skipped_values} 个键值的类型不支持核对，未计入") + unique, how = parent + if unique is True: + parts.append(f"被指向列是{how}" if how in ("主键", "唯一约束") else "被指向列经计数核对唯一") + elif unique is False: + parts.append("被指向列有重复值，不能作为关联的被指向键") + elif how == "too_large": + parts.append("被指向的表超过整表统计的行数上限，未核对被指向列是否唯一") + else: + parts.append("未能核对被指向列是否唯一") + if child is not None: + child_unique, child_how = child + if child_unique is True: + basis = child_how if child_how in ("主键", "唯一约束") else "计数核对" + parts.append(f"子表一侧也唯一（{basis}），一对一") + elif child_unique is False: + parts.append("子表一侧有重复值，多对一") + elif child_how == "too_large": + parts.append("子表超过整表统计的行数上限，未核对子表一侧是否唯一，按多对一记") + else: + parts.append("未能核对子表一侧是否唯一，按多对一记") + if not holds and not low: + parts.append("暂不升为有确证") + return f"{NOTE_PREFIX}{day}）：" + "；".join(parts) + "。" + + +async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _TableOutcome, + result: TableProfile) -> None: + """从子表取至多 sample_size 个不同的非空键值，去父表里数对上几个，得到覆盖率；再看父键、子键唯不唯一。""" + cols, to_cols = list(rel.get("columns") or []), list(rel.get("to_columns") or []) + to_table = str(rel.get("to_table") or "") + if len(cols) != 1 or len(to_cols) != 1: + raise _Skip("unsupported", "多列组成的关联关系暂不剖析") + meta = cx.tables[table] + parent = to_table if to_table in cx.tables else catalog.resolve_table_name({"tables": cx.tables}, to_table) + if parent is None: + raise _Skip("missing", f"表结构里没有被指向的表 {to_table}，请先重新探查结构") + pmeta = cx.tables[parent] + col, pcol = _column(meta, cols[0]), _column(pmeta, to_cols[0]) + if col is None or pcol is None: + missing = f"{table}.{cols[0]}" if col is None else f"{parent}.{to_cols[0]}" + raise _Skip("missing", f"表结构里没有列 {missing}，请先重新探查结构") + if pmeta.get("is_view"): + raise _Skip("view", f"被指向的 {parent} 是视图，不剖析") + for name, owner in ((col, table), (pcol, parent)): + if cx.is_masked(name): + raise _Skip("masked", f"{owner}.{name} 在数据源设置中被遮罩，不取样", carry=True) + + d, s = cx.dialect, cx.settings + size = await cx.size_with_data(table, result) + scan_cap = cx.scan_cap(size) + child_table, parent_table = d.table(meta, table), d.table(pmeta, parent) + res = await cx.run(d.distinct_sample_sql(child_table, d.quote(col), s.sample_size, scan_cap=scan_cap), + max_rows=s.sample_size) + kind = (res.column_types or {}).get(res.columns[0]) if res.columns else None + values = [row[0] for row in res.rows if row] + literals = list(dict.fromkeys(lit for v in values if (lit := sql_literal(v, kind)) is not None)) + if not values: + raise _Skip("no_data", f"{table}.{col} 没有非空值，无法核对") + if not literals: + raise _Skip("unsupported", f"{table}.{col} 的取值类型不支持核对（只核对整数和文字）") + matched = 0 + for start in range(0, len(literals), IN_CHUNK): + sql = d.match_count_sql(parent_table, d.quote(pcol), literals[start:start + IN_CHUNK]) + matched += _as_int(_first(await cx.run(sql))) or 0 + matched = min(matched, len(literals)) + coverage = round(matched / len(literals), 4) + + parent_unique = await _unique(cx, parent, pmeta, pcol, result) + child_unique = await _unique(cx, table, meta, col, result) if parent_unique[0] else None + cardinality = None + if child_unique is not None: + cardinality = "one_to_one" if child_unique[0] is True else "many_to_one" + holds = coverage >= catalog.PROFILE_VERIFY_COVERAGE and cardinality is not None + note = _relation_note(cx.day, sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap, + skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique, + holds=holds) + out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=coverage, + cardinality=cardinality, holds=holds, sample=len(literals), + matched=matched, note=note)) + + +# ========================================================================== +# 检查：码值候选、日期列 +# ========================================================================== + +#: 码值候选最多几个取值。查询取 CODES_MAX + 1 个：取满了说明不是低基数列 +CODES_MAX = 20 +#: 码值的长度上限。更长的不像状态码，像说明文字 +MAX_CODE_LEN = 64 +#: 像状态、类型、渠道的列名里会有的词 +_CODE_WORDS = frozenset({"status", "state", "type", "kind", "category", "channel", "level", "method", "mode", + "reason", "stage", "phase", "grade", "flag", "source", "result"}) +#: 以这些词结尾的列不是码值：标识、名称、说明、时间、金额、数量（channel_id、level_name、status_time……） +_NOT_CODE_TAILS = frozenset({"id", "no", "num", "number", "name", "desc", "description", "text", "remark", "note", + "at", "date", "time", "on", "amount", "price", "count", "qty", "rate", "url"}) +#: 文字类型的列名以这些词结尾时，按日期类列看待（SQLite 里日期都存成文字：visit_time、ordered_at、joined_on） +_DATE_TAILS = frozenset({"at", "date", "time", "on", "dt", "day"}) +#: 像日期的取值：2026-07-01、2026/07/01，后面可以跟时间 +_DATE_TEXT = re.compile(r"^\d{4}[-/]\d{2}[-/]\d{2}") +_WORDS = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|\d+") + + +def _words(name: str) -> list[str]: + """列名切成小写的词：下划线和驼峰都是分隔（和目录的命名推断同一个切法）。""" + return [w.lower() for part in name.split("_") for w in _WORDS.findall(part)] + + +def _code_type_ok(type_name: str | None) -> bool: + """整数，或者短文字（定长不超过 MAX_CODE_LEN；SQLite 的 TEXT 不带长度，也算）。""" + t = (type_name or "").upper() + if not t or any(k in t for k in ("CLOB", "BLOB", "BINARY", "JSON", "DATE", "TIME", "INTERVAL", "REAL", "FLOAT", + "DOUBLE", "BOOL")): + return False + if "INT" in t: + return True + size = re.search(r"\(\s*(\d+)\s*(?:,\s*(\d+)\s*)?\)", t) + if any(k in t for k in ("NUMBER", "NUMERIC", "DECIMAL")): + return size is None or not size.group(2) or int(size.group(2)) == 0 + if any(k in t for k in ("CHAR", "TEXT", "STRING")): + return size is None or int(size.group(1)) <= MAX_CODE_LEN + return False + + +def _looks_like_code(name: str) -> bool: + words = _words(name) + return bool(words) and words[-1] not in _NOT_CODE_TAILS and any(w in _CODE_WORDS for w in words) + + +def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: + """要取值分布的列：目录里度量类型记为状态的、剖析以前提过码值候选的（再核一次）、或者类型和命名像状态、 + 类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算；码值人工确认或驳回过的不再取。 + 被遮罩的列也列出来，由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" + pk = [c.lower() for c in meta.get("primary_key") or []] + unique = {str(g[0]).lower() for g in meta.get("unique") or [] if isinstance(g, list) and len(g) == 1} + keys = {str(c).lower() for r in notes.get("relations") or [] if isinstance(r, dict) for c in r.get("columns") or []} + described = notes.get("columns") if isinstance(notes.get("columns"), dict) else {} + out: list[str] = [] + for col in meta.get("columns") or []: + name = str(col.get("name") or "") + items = described.get(name) if isinstance(described.get(name), dict) else {} + codes, measure = items.get("codes"), items.get("measure") + if isinstance(codes, dict) and codes.get("status") in ("confirmed", "rejected"): + continue + status_measure = (isinstance(measure, dict) and measure.get("value") == "status" + and measure.get("status") != "rejected") + profiled = isinstance(codes, dict) and codes.get("source") == "profile" + if not (status_measure or profiled or _looks_like_code(name)): + continue + lowered = name.lower() + if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type")): + continue + out.append(name) + return out + + +def _code_key(value: Any) -> str | None: + """取值分布里的一个取值 → 码值（文字）。只认整数和不太长的文字。""" + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return str(int(value)) if value.is_integer() else None + if isinstance(value, str) and value.strip() and len(value) <= MAX_CODE_LEN and not _CONTROL.search(value): + return value + return None + + +def _share(count: int, total: int) -> str: + if total <= 0: + return "0%" + ratio = count / total + return "不足 1%" if 0 < ratio < 0.01 else f"{round(ratio * 100)}%" + + +async def _check_codes(cx: _Context, table: str, column: str, out: _TableOutcome, result: TableProfile) -> None: + """GROUP BY 取出现最多的 CODES_MAX + 1 个取值：不超过 CODES_MAX 个就是码值候选（值有了，含义等人填）。""" + if cx.is_masked(column): + raise _Skip("masked", f"{table}.{column} 在数据源设置中被遮罩，不取值", carry=False) + d, meta = cx.dialect, cx.tables[table] + size = await cx.size_with_data(table, result) + scan_cap = cx.scan_cap(size) + res = await cx.run(d.value_counts_sql(d.table(meta, table), d.quote(column), CODES_MAX + 1, scan_cap=scan_cap), + max_rows=CODES_MAX + 1) + if not res.rows: + raise _Skip("no_data", f"{table}.{column} 没有非空值，无法取值分布") + if len(res.rows) > CODES_MAX: + raise _Skip("high_cardinality", f"{table}.{column} 的取值超过 {CODES_MAX} 个，不作为码值候选", carry=False) + values: list[tuple[str, int]] = [] + for row in res.rows: + key = _code_key(row[0]) + if key is None: + raise _Skip("unsupported", f"{table}.{column} 的取值不是整数或短文字，不作为码值候选", carry=False) + values.append((key, _as_int(row[1]) or 0)) + if len(values) >= 3 and all(n == 1 for _, n in values): + # 每个取值只出现一次（票种编码 T01…T09）：这是一列标识，不是状态码 + raise _Skip("high_cardinality", f"{table}.{column} 的取值各不相同，像标识而不是码值，不作为码值候选", + carry=False) + rows = sum(n for _, n in values) + scope = f"按前 {scan_cap} 行统计" if scan_cap else "统计全表" + spread = "、".join(f"{k}（{_share(n, rows)}）" for k, n in values) + note = f"{NOTE_PREFIX}{cx.day}）：{scope}，{rows} 行非空值共 {len(values)} 个取值：{spread}。" + out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note)) + + +def _is_date_column(col: dict[str, Any]) -> bool: + """日期类列：类型是日期、时间戳；或者类型是文字、列名像日期（SQLite 的日期都存成文字）。""" + t = str(col.get("type") or "").upper() + if "DATE" in t or "TIMESTAMP" in t: + return True + if t and not any(k in t for k in ("CHAR", "TEXT", "STRING")): + return False + words = _words(str(col.get("name") or "")) + return bool(words) and words[-1] in _DATE_TAILS and (len(words) > 1 or words[0] in ("date", "time", "day")) + + +async def _check_dates(cx: _Context, table: str, columns: list[str], total: int, out: _TableOutcome, + result: TableProfile) -> None: + """日期类列的 MIN、MAX，作为业务日期的参考。整表统计，只对行数不超过上限的表做。表里只有一个日期类列 + （total 是表结构里日期类列的个数，含被遮罩的）时提议它作为业务日期。""" + d, meta = cx.dialect, cx.tables[table] + size = await cx.size_with_data(table, result) + if not size.within(cx.settings.max_scan_rows): + raise _Skip("too_large", f"{table} 的行数超过整表统计的行数上限（{cx.settings.max_scan_rows} 行），" + "不取日期列的最小值和最大值") + res = await cx.run(d.min_max_sql(d.table(meta, table), [d.quote(c) for c in columns])) + row = res.rows[0] if res.rows else [] + for i, column in enumerate(columns): + low, high = (row[2 * i], row[2 * i + 1]) if len(row) >= 2 * i + 2 else (None, None) + if low is None or high is None or not (_DATE_TEXT.match(str(low)) and _DATE_TEXT.match(str(high))): + continue + result.date_ranges.append({"column": column, "min": str(low), "max": str(high)}) + if total != 1: + return + if not result.date_ranges: + raise _Skip("no_data", f"{table}.{columns[0]} 没有可识别的日期值") + found = result.date_ranges[0] + note = (f"{NOTE_PREFIX}{cx.day}）：表里只有这一个日期类列，取值从 {str(found['min'])[:19]} 到 " + f"{str(found['max'])[:19]}，可作为业务日期的参考。") + out.business_date = _DateOutcome(column=found["column"], low=found["min"], high=found["max"], note=note) + + +# ========================================================================== +# 结论进目录 +# ========================================================================== + + +def _relation_finding(o: _RelationOutcome, rel: dict[str, Any], *, status: str, confirmed: bool) -> dict[str, Any]: + if confirmed: + summary = "已人工确认，只补充覆盖率和基数" + elif status == "verified": + summary = f"覆盖率 {_percent(o.coverage)}，升为有确证" + elif o.coverage < catalog.PROFILE_VERIFY_COVERAGE: + summary = f"抽样覆盖率 {_percent(o.coverage)}，保持推断" + else: + summary = "被指向列未核实唯一，保持推断" + return {"kind": "relation", "path": f"relations.{o.rid}", "target": o.target, + "columns": list(rel.get("columns") or []), "to_table": rel.get("to_table"), + "to_columns": list(rel.get("to_columns") or []), "status": status, "confirmed": confirmed, + "coverage": o.coverage, "cardinality": o.cardinality, "sample": o.sample, "matched": o.matched, + "summary": summary} + + +def _codes_value(c: _CodesOutcome, item: dict[str, Any] | None) -> dict[str, str]: + """码值候选的值：观察到的取值，含义先空着。原来有别的来源（模型起草、数据库注释）写过含义的，对得上的取值 + 沿用它的含义；它写了、数据里没出现的取值也留着——码值是「可能的取值」，没出现不等于不存在。""" + old = item.get("value") if isinstance(item, dict) and isinstance(item.get("value"), dict) else {} + value = {key: str(old.get(key) or "") for key, _ in c.values} + for key, meaning in old.items(): + if key not in value and isinstance(meaning, str) and meaning.strip(): + value[key] = meaning + return value + + +def _codes_finding(c: _CodesOutcome, value: dict[str, str]) -> dict[str, Any]: + pending = sum(1 for v in value.values() if not v.strip()) + summary = f"{len(c.values)} 个取值" + (f"，{pending} 个含义待填写" if pending else "") + return {"kind": "codes", "path": f"columns.{c.column}.codes", "column": c.column, + "values": [{"value": k, "rows": n} for k, n in c.values], "rows": c.rows, "status": "proposed", + "summary": summary} + + +def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: + """人工确认过的关系只补覆盖率和基数：覆盖率是测量值，照新的写；基数只在原来没有时补，不改人定的； + 备注只在原来为空、或者是剖析自己写的时候换成新的。状态、来源不动。返回改没改。""" + changed = False + if rel.get("coverage") != o.coverage: + rel["coverage"] = o.coverage + changed = True + if rel.get("cardinality") is None and o.cardinality is not None: + rel["cardinality"] = o.cardinality + changed = True + note = rel.get("note") + if (not note or str(note).startswith(NOTE_PREFIX)) and note != o.note: + rel["note"] = o.note + changed = True + if changed: + rel["updated_at"] = at + return changed + + +def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, + at: str) -> tuple[dict[str, Any], list[dict[str, Any]], catalog.MergeStats]: + """把一张表的结论并进写入时的目录：(新目录, 写进去的发现, 计数)。纯函数，写入冲突重来时再调一次。 + + - 没被确认、驳回的关系：记为 source=profile，覆盖率够、父键唯一的 verified，否则 proposed；按 merge_notes + 的规则并入（剖析顶得掉命名推断、模型起草，顶不掉外键和人工）。 + - 人工确认过的关系：只补覆盖率和基数（_patch_confirmed）。 + - 码值候选：profile / proposed，含义沿用别的来源写过的（_codes_value）；人工确认、驳回过的码值不碰。 + - 业务日期：只有一个日期类列时提议，profile / proposed；已有别的来源定的业务日期不顶掉。 + - covered={"profile"}：这次完整检查过、却没再得出的剖析结论删掉（比如取值变多、不再像码值的列，列被遮罩 + 之后它的码值）；没查成的检查（超时、预算用完……）把剖析以前写的结论原样带上，不因为这次没查成就删。 + """ + existing = existing or {} + current = {str(r["id"]): r for r in existing.get("relations") or [] if isinstance(r, dict) and r.get("id")} + relations: list[dict[str, Any]] = [] + confirmed: dict[str, _RelationOutcome] = {} + findings: list[dict[str, Any]] = [] + for o in out.relations: + rel = current.get(o.rid) + if rel is None or rel.get("status") == "rejected" or rel.get("source") == "fk": + continue # 核对期间被删、被驳回了：不写 + if rel.get("status") == "confirmed": + confirmed[o.rid] = o + findings.append(_relation_finding(o, rel, status="confirmed", confirmed=True)) + continue + status = "verified" if o.holds else "proposed" + relations.append({"id": o.rid, "columns": list(rel["columns"]), "to_table": rel["to_table"], + "to_columns": list(rel["to_columns"]), "cardinality": o.cardinality, + "coverage": o.coverage, "source": "profile", "status": status, "note": o.note}) + findings.append(_relation_finding(o, rel, status=status, confirmed=False)) + for rid in out.keep_relations: + rel = current.get(rid) + if rel is not None and rel.get("source") == "profile": + relations.append(copy.deepcopy(rel)) + draft: dict[str, Any] = {"relations": relations} if relations else {} + + described = existing.get("columns") if isinstance(existing.get("columns"), dict) else {} + columns: dict[str, dict[str, Any]] = {} + for c in out.codes: + item = (described.get(c.column) or {}).get("codes") if isinstance(described.get(c.column), dict) else None + if isinstance(item, dict) and item.get("status") in ("confirmed", "rejected"): + continue + columns[c.column] = {"codes": catalog.make_item(_codes_value(c, item), "profile", "proposed", note=c.note)} + findings.append(_codes_finding(c, columns[c.column]["codes"]["value"])) + for column in out.keep_codes: + item = (described.get(column) or {}).get("codes") if isinstance(described.get(column), dict) else None + if isinstance(item, dict) and item.get("source") == "profile": + columns[column] = {"codes": copy.deepcopy(item)} + if columns: + draft["columns"] = columns + + current_date = existing.get("business_date") if isinstance(existing.get("business_date"), dict) else None + ours = current_date is None or (current_date.get("source") == "profile" + and current_date.get("status") not in ("confirmed", "rejected")) + if out.business_date is not None and ours: + # 已有别的来源（模型起草、人工）定的业务日期不顶掉：按来源的可信程度剖析排得更前，但它只知道 + # 「表里只有这一个日期列」，不比写了规则和时区的那条更懂业务 + b = out.business_date + draft["business_date"] = catalog.make_item({"column": b.column}, "profile", "proposed", note=b.note) + findings.append({"kind": "business_date", "path": "business_date", "column": b.column, "min": b.low, + "max": b.high, "status": "proposed", "summary": f"提议按 {b.column} 作为业务日期"}) + elif out.keep_business_date and current_date is not None and current_date.get("source") == "profile": + draft["business_date"] = copy.deepcopy(current_date) + + merged, stats = catalog.merge_notes(existing, draft, covered={"profile"}, at=at) + for rel in merged.get("relations") or []: + o = confirmed.get(str(rel.get("id"))) + if o is not None and rel.get("status") == "confirmed" and _patch_confirmed(rel, o, at): + stats += catalog.MergeStats(updated=1) + return merged, findings, stats + + +# ========================================================================== +# 入口 +# ========================================================================== + +#: 正在剖析的数据源。检查和登记之间没有 await，在一个事件循环里是原子的；服务端是单进程部署， +#: 多进程部署时要换成库里的锁 +_RUNNING: set[str] = set() + + +@dataclass +class _Check: + kind: str + target: str + path: str | None + run: Callable[[], Awaitable[None]] + #: 没查成时调用：把目录里剖析以前对这一项写的结论标记为原样保留 + keep: Callable[[], None] + + +def _plan(cx: _Context, table: str, notes: dict[str, Any], out: _TableOutcome, + result: TableProfile) -> list[_Check]: + """这张表要做的检查，按价值排：关系、码值、日期。先列全再执行：中途停下时，没做的每一项都要记下原因。""" + meta = cx.tables[table] + checks: list[_Check] = [] + for rel in _relation_candidates(notes): + rid = str(rel["id"]) + checks.append(_Check("relation", _relation_target(rel), f"relations.{rid}", + lambda rel=rel: _check_relation(cx, table, rel, out, result), + lambda rid=rid: out.keep_relations.add(rid))) + for column in _code_columns(meta, notes): + checks.append(_Check("codes", column, f"columns.{column}.codes", + lambda column=column: _check_codes(cx, table, column, out, result), + lambda column=column: out.keep_codes.add(column))) + dates = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, dict) and _is_date_column(c)] + for column in dates: + if cx.is_masked(column): + result.skipped.append(ProfileSkip("date", column, "masked", + f"{table}.{column} 在数据源设置中被遮罩，不取最小值和最大值")) + usable = [c for c in dates if not cx.is_masked(c)] + if usable: + checks.append(_Check("date", "、".join(usable), "business_date" if len(dates) == 1 else None, + lambda: _check_dates(cx, table, usable, len(dates), out, result), + lambda: setattr(out, "keep_business_date", True))) + return checks + + +async def _profile_table(cx: _Context, table: str, notes: dict[str, Any], result: TableProfile) -> _TableOutcome: + out = _TableOutcome() + if cx.tables[table].get("is_view"): + result.skipped.append(ProfileSkip("table", table, "view", "视图没有统计信息，剖析可能触发整个视图的计算，已跳过")) + return out + checks = _plan(cx, table, notes, out, result) + for i, check in enumerate(checks): + try: + await check.run() + except _Skip as e: + result.skipped.append(ProfileSkip(check.kind, check.target, e.reason, e.detail, check.path)) + if e.carry: + check.keep() + except _Stop as e: + detail = _STOP_DETAIL[e.reason].format(max_queries=cx.settings.max_queries, + max_total_s=_fmt(cx.settings.max_total_s)) + for rest in checks[i:]: + result.skipped.append(ProfileSkip(rest.kind, rest.target, e.reason, detail, rest.path)) + rest.keep() + break + if table in cx.sizes: + result.row_estimate = cx.sizes[table] + return out + + +async def _write(session: AsyncSession, source_id: str, table: str, out: _TableOutcome, result: TableProfile, *, + at: str, actor: str | None) -> None: + """并入并写入（乐观锁）。撞上别人刚改过就重读重并，最多三次。""" + for _attempt in range(3): + entry = await catalog.read_entry(session, source_id, table) + version = entry.version if entry else 0 + notes, findings, stats = _apply(entry.notes if entry else {}, out, at=at) + result.findings = findings + result.added, result.updated, result.removed = stats.added, stats.updated, stats.removed + result.version = version + if entry is None and not notes: + return # 什么也没得出，不建空目录 + try: + written = await catalog.write_entry(session, source_id, table, notes, if_version=version, actor=actor) + except catalog.CatalogConflict: + continue + result.version = written.version + return + result.error = "这张表的数据目录正被其他人修改，剖析结果未能写入，请稍后重新剖析" + result.findings = [] + result.added = result.updated = result.removed = 0 + + +async def _pick_tables(session: AsyncSession, source: Any, tables: list[str] | None, + entries: dict[str, catalog.CatalogEntry], report: ProfileReport) -> list[str]: + cache = getattr(source, "schema_cache", None) or {} + all_tables = cache.get("tables") or {} + if tables: + picked: list[str] = [] + for name in dict.fromkeys(str(t) for t in tables): + key = catalog.resolve_table_name(cache, name) + if key is None: + report.tables.append(TableProfile(table=name, error=f"表结构里没有 {name}，请先重新探查结构")) + elif key not in picked: + picked.append(key) + return picked + usage = await catalog.table_usage(session, source) + order = {name: i for i, name in enumerate(all_tables)} + candidates = [name for name, meta in all_tables.items() + if not (meta or {}).get("is_view") and name in entries + and _relation_candidates(entries[name].notes)] + return sorted(candidates, key=lambda n: (-usage.get(n, 0), order[n]))[:PROFILE_DEFAULT_TABLES] + + +async def profile_catalog(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None = None, + actor: str | None = None, settings: ProfileSettings | None = None) -> ProfileReport: + """剖析这些表并把结论写进数据目录，同步完成。 + + - row：数据源记录（开关、遮罩按它的 options）；source：用来查询和读表结构的源（上传源是绑定当前快照的视图）。 + - settings 不给就按 row.options 读；没开启抛 ProfileDisabled，这个源正在剖析抛 ProfileBusy。 + - tables 为空时按使用次数取前 PROFILE_DEFAULT_TABLES 张目录里有待核实关系的表。 + - 每张表做完就写入，后面的表停下（预算、总时长）不影响前面已经写进去的。 + """ + settings = ensure_enabled(row, settings) + if source.id in _RUNNING: + raise ProfileBusy(f"数据源「{getattr(row, 'name', '')}」正在进行数据剖析，请等待完成后再试") + _RUNNING.add(source.id) + try: + return await _profile(session, row, source, tables, actor, settings) + finally: + _RUNNING.discard(source.id) + + +def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> ProfileSettings: + """这个数据源能不能剖析：数据源停用了、或者没开启数据剖析，抛 ProfileDisabled（message 给人看）。 + 返回要用的设置（不给就按 row.options 读）。接口在做别的检查之前先调它：开关是第一道门。""" + settings = settings or profile_settings(getattr(row, "options", None)) + name = getattr(row, "name", "") or "" + if getattr(row, "enabled", True) is False: + raise ProfileDisabled(f"数据源「{name}」已停用，无法进行数据剖析。请先在数据源设置中启用") + if not settings.enabled: + raise ProfileDisabled(f"数据源「{name}」未开启数据剖析。剖析会对业务库发查询，请先在数据源设置中开启数据剖析") + return settings + + +async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[str] | None, actor: str | None, + settings: ProfileSettings) -> ProfileReport: + at = catalog.now_iso() + report = ProfileReport(profiled_at=at, settings=settings) + dialect = SqlDialect(source.kind) + run = _Runner(source, settings, dialect) + cx = _Context(source=source, settings=settings, dialect=dialect, run=run, + tables=(getattr(source, "schema_cache", None) or {}).get("tables") or {}, + masked={c.lower() for c in data_engine.masked_columns(getattr(row, "options", None))}, + day=at[:10]) + entries = await catalog.read_catalog(session, source.id) + for table in await _pick_tables(session, source, tables, entries, report): + result = TableProfile(table=table) + report.tables.append(result) + if run.stopped: + detail = _STOP_DETAIL[run.stopped].format(max_queries=settings.max_queries, + max_total_s=_fmt(settings.max_total_s)) + result.skipped.append(ProfileSkip("table", table, run.stopped, detail)) + continue + before = run.used + entry = entries.get(table) + out = await _profile_table(cx, table, entry.notes if entry else {}, result) + result.queries = run.used - before + await _write(session, source.id, table, out, result, at=at, actor=actor) + report.queries_used = run.used + report.stopped = run.stopped + logger.info("数据剖析 源=%s 表=%s 查询=%d 停止=%s 署名=%s", source.id, + ",".join(t.table for t in report.tables), run.used, run.stopped, actor) + return report + + +__all__ = ["CODES_MAX", "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", + "PROFILE_OPTION", "ProfileBusy", "ProfileDisabled", "ProfileReport", "ProfileSettings", "ProfileSkip", + "SqlDialect", "TableProfile", "TableSize", "ensure_enabled", "estimate_size", "profile_catalog", + "profile_settings", "profile_settings_problem", "sql_literal"] diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index 04a0f6eb..b419f2a1 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -58,11 +58,16 @@ MAX_QUERY_TIMEOUT_S = 600 #: 完整快照仍能按工件 id 取到，SQL 里给列起个别名也能绕开 MASK_COLUMNS_OPTION = "mask_columns" +#: 数据源 options 里的数据剖析设置（一个对象：开关、查询次数上限、单条时限……）。取值和校验见 +#: data/catalog_profile.py；放在这里只为让拼连接串时摘掉它 +CATALOG_PROFILE_OPTION = "catalog_profile" + # options 里这些 key 是 AgentLab 自己的配置，不是驱动参数，拼 URL 时要摘掉。 # schema：探查哪个 schema（企业库里只读账号名下常常什么都没有，数据在别处） # query_timeout_s：查询时限，由数据层按语句下发给数据库，见 _server_deadline # mask_columns：证据面板遮罩的列，驱动不认识它，拼进连接串会被当成未知参数拒掉 -_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION}) +# catalog_profile：数据剖析设置，同上；它还是个对象，拼进去连 URL 都不成形 +_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION, CATALOG_PROFILE_OPTION}) _MASK_SPLIT = re.compile(r"[,，、;；\n]+") diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index e1fd9ae0..d5207577 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -169,13 +169,17 @@ class SqlChecker: 和助手挑表看到的是同一张图（catalog.relation_graph）：目录里的关系，加上没起草过的表按外键、命名现推的； 被驳回的关系不进图，也不会被现推回来。 """ + return [e for e in self.edges_from(a) if e.to_table.lower() == b.lower()] + + def edges_from(self, a: str) -> list[catalog.JoinEdge]: + """从 a 出发的全部边。关系图第一次用到时才建，之后复用。""" if self._edges is None: graph = catalog.relation_graph(self._raw, schema_cache=self.schema_cache) index: dict[str, list[catalog.JoinEdge]] = {} for table, edges in graph.items(): index.setdefault(table.lower(), []).extend(edges) self._edges = index - return [e for e in self._edges.get(a.lower(), []) if e.to_table.lower() == b.lower()] + return list(self._edges.get(a.lower(), [])) # ---- 检查 @@ -557,10 +561,46 @@ class _ScopeCheck: continue # 两张表都没写目录：没有可对照的，不判 condition = " AND ".join(filter(None, (self.excerpt(c) for c in group["conds"]))) if best is None: - self._join_checks.append(self._join_unknown(a, b, candidates, condition)) + parent = self._shared_parent(a, b, pairs) + self._join_checks.append(self._join_sibling(a, b, parent, condition) if parent + else self._join_unknown(a, b, candidates, condition)) elif best.status not in _STRONG: self._join_checks.append(self._join_proposed(a, b, best, condition)) + def _shared_parent(self, a: _Table, b: _Table, pairs: set[tuple[str, str]]) -> str | None: + """兄弟关联：每一对连接列都是两边各自指向同一张父表的同一组列（a.member_id → members.id ← b.member_id）。 + 返回父表名；不是这种连法返回 None。 + + 目录里不会记 a 和 b 之间的关系，但这不是「连错了列」：比如评价表和标签关联表各自按门店编号指向门店、 + 彼此按门店编号相连。照「对不上任何关系」报警告是误报。 + """ + def parents(table: _Table, column: str) -> set[tuple[str, tuple[str, ...]]]: + return {(e.to_table.lower(), tuple(c.lower() for c in e.to_columns)) + for e in self.checker.edges_from(table.name) + if tuple(c.lower() for c in e.from_columns) == (column,) + and e.cardinality in (None, "many_to_one", "one_to_one")} + + shared: set[tuple[str, tuple[str, ...]]] | None = None + for left, right in pairs: + common = parents(a, left) & parents(b, right) + shared = common if shared is None else shared & common + if not shared: + return None + names = {table for table, _ in shared or ()} + return next(iter(names)) if len(names) == 1 else None + + def _join_sibling(self, a: _Table, b: _Table, parent: str, condition: str) -> SqlCheck: + item = (self.checker.notes_of(parent) or {}).get("label") + label = str(item["value"]) if isinstance(item, dict) and isinstance(item.get("value"), str) \ + and item["value"].strip() else parent + return SqlCheck( + code="join_unconfirmed", level="info", table=b.name, sql_excerpt=condition or None, + message=(f"「{a.label()}」与「{b.label()}」之间没有直接的关系，这里经共同的「{label}」相连。" + "关联后行数可能成倍增加：确认其中一侧每个键只有一行，或先各自汇总再关联"), + for_model=(f"{a.name} 和 {b.name} 按 {condition or '这组条件'} 关联：两边都指向 {parent}，彼此是一对多对多。" + "如果其中一侧每个键不止一行，关联后另一侧的行会被重复，求和、计数会放大；" + "先把一侧筛到每个键一行（例如只取主记录），或分别汇总到这个键再关联")) + def _join_unknown(self, a: _Table, b: _Table, candidates: list[catalog.JoinEdge], condition: str) -> SqlCheck: known = "；".join(dict.fromkeys(e.condition() for e in candidates)) hint = f"目录里这两张表的关系是：{known}。照它改关联条件" if known else \ diff --git a/backend/tests/fixtures/catalog/scenic.py b/backend/tests/fixtures/catalog/scenic.py index fd7ffb87..b5269a5b 100644 --- a/backend/tests/fixtures/catalog/scenic.py +++ b/backend/tests/fixtures/catalog/scenic.py @@ -9,8 +9,9 @@ 从 visits 到 channels 要经过它，正好是两跳的连接路径。group_visits 同理（旅行团与入园记录）。 - **命名推断必须放过的列**（NOT_INFERRED）：自引用（employees.employee_id 是工号，不是指向自己）、 多个候选（guide 和 guides 两张表都在，tour_assignments.guide_id 不知道指哪张）、没有对应的表 - （parking_records.lot_id、operation_logs.operator_id、categories.parentId）、类型对不上 - （feedback_forms.store_id 是文字，stores.id 是整数）。 + （operation_logs.operator_id、categories.parentId）、类型对不上（feedback_forms.store_id 是文字，stores.id 是整数）。 + parking_records.lot_id 没有直接对上的 lot / lots，但和引用它的表共用 parking 前缀的 parking_lots 只有一张， + 推得出来（在 NAME_RELATIONS 里）。 - **唯一约束 / 唯一索引。** orders.order_no、channels.channel_code 是列上的 UNIQUE；members.member_no、 visits.ticket_no 是单独建的唯一索引。 @@ -188,6 +189,8 @@ NAME_RELATIONS = frozenset({ ("tour_groups", ("agency_id",), "agencies", ("id",)), ("tour_assignments", ("tour_group_id",), "tour_groups", ("id",)), ("lost_and_found", ("park_id",), "parks", ("id",)), + # 没有 lot / lots，按引用表共用的 parking 前缀对上 parking_lots + ("parking_records", ("lot_id",), "parking_lots", ("id",)), # 视图没有约束，按列名同样推得出来：按日汇总的入园人数照样能按 park_id 连到景区 ("v_daily_visits", ("park_id",), "parks", ("id",)), }) @@ -196,7 +199,6 @@ NAME_RELATIONS = frozenset({ NOT_INFERRED = { ("employees", "employee_id"): "自引用：工号，不是指向本表", ("tour_assignments", "guide_id"): "多个候选：guide 与 guides", - ("parking_records", "lot_id"): "没有名为 lot / lots 的表", ("operation_logs", "operator_id"): "没有名为 operator / operators 的表", ("categories", "parentId"): "没有名为 parent / parents 的表", ("feedback_forms", "store_id"): "类型对不上：文字列指向整数主键", diff --git a/backend/tests/fixtures/sources.py b/backend/tests/fixtures/sources.py new file mode 100644 index 00000000..7b6f6fb3 --- /dev/null +++ b/backend/tests/fixtures/sources.py @@ -0,0 +1,26 @@ +"""测试里建的数据源，用完就删。 + +数据源表在每个测试进程的测试库里是共用的：留下来的源会出现在助手的数据源清单里（50 张表的合成库还会 +触发挑表、多一次模型调用），别的用例的结果就跟着执行顺序变。阶段 1B 就是这样被绊过一次。 +删数据源时它的数据目录随外键级联删除（测试库开着 PRAGMA foreign_keys=ON）。 +""" +from __future__ import annotations + +from sqlalchemy import delete, select + +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource + + +async def drop_source(source_id: str | None = None, *, name: str | None = None) -> None: + """按 id 或名字删一个数据源，并作废它的连接池。不存在就什么都不做。""" + async with SessionLocal() as session: + if source_id is None: + source_id = (await session.execute( + select(DataSource.id).where(DataSource.name == name))).scalar_one_or_none() + if source_id is None: + return + await session.execute(delete(DataSource).where(DataSource.id == source_id)) + await session.commit() + await engines.invalidate(source_id) diff --git a/backend/tests/test_catalog_api.py b/backend/tests/test_catalog_api.py index a66bb1d8..2747cb61 100644 --- a/backend/tests/test_catalog_api.py +++ b/backend/tests/test_catalog_api.py @@ -11,12 +11,12 @@ import pytest from httpx import ASGITransport, AsyncClient from app.data import catalog -from app.data.engine import engines from app.db.base import SessionLocal from app.db.models import DataSource from app.main import app as _fastapi_app from tests.fixtures.catalog import scenic from tests.fixtures.catalog.fake_model import FakeModel +from tests.fixtures.sources import drop_source ACTOR = {"X-Actor": quote("王敏")} @@ -38,7 +38,7 @@ async def source_id(client, scenic_db): r = await client.post(f"/api/datasources/{sid}/introspect") assert r.status_code == 200, r.text yield sid - await engines.invalidate(sid) + await drop_source(sid) def _base(sid: str) -> str: @@ -184,9 +184,12 @@ async def test_draft_without_schema_is_refused(client): session.add(row) await session.commit() sid = row.id - r = await client.post(f"{_base(sid)}/draft", json={}) - assert r.status_code == 409 and "探查结构" in r.json()["detail"] - assert (await client.get(_base(sid))).json()["tables"] == [] + try: + r = await client.post(f"{_base(sid)}/draft", json={}) + assert r.status_code == 409 and "探查结构" in r.json()["detail"] + assert (await client.get(_base(sid))).json()["tables"] == [] + finally: + await drop_source(sid) async def test_schema_endpoint_detail_carries_catalog(client, source_id): diff --git a/backend/tests/test_catalog_draft.py b/backend/tests/test_catalog_draft.py index a80d4282..2989d92f 100644 --- a/backend/tests/test_catalog_draft.py +++ b/backend/tests/test_catalog_draft.py @@ -15,6 +15,7 @@ from app.data.catalog import make_item from app.db.base import SessionLocal from app.db.models import DataSource from tests.fixtures.catalog.fake_model import FakeModel +from tests.fixtures.sources import drop_source def _col(name: str, type_: str = "INTEGER", comment: str | None = None, nullable: bool = True) -> dict: @@ -156,6 +157,31 @@ def test_name_inference_negative(tables, why): assert _infer(tables, next(iter(tables))) == set(), why +def test_name_inference_falls_back_to_a_table_sharing_the_prefix(): + """库里没有 tags，只有带前缀的 store_tags 和 menu_tag：和引用它的表共用 store 前缀的那张才是它要指的。 + 以前这种命名推不出关系，SQL 检查跟着误报「对不上任何关系」。""" + tables = { + "store_tag_links": _table(_col("id"), _col("tagId"), _col("storeID")), + "store_tags": _table(_col("id")), + "menu_tag": _table(_col("id")), + "store": _table(_col("id")), + } + assert _infer(tables, "store_tag_links") == { + (("tagId",), "store_tags", ("id",)), (("storeID",), "store", ("id",))} + + +@pytest.mark.parametrize("tables, why", [ + # 直接对得上就用直接的，不去找带前缀的 + ({"shop_category_links": _table(_col("id"), _col("category_id")), "categories": _table(_col("id")), + "shop_categories": _table(_col("id"))}, {(("category_id",), "categories", ("id",))}), + # 引用它的表没有可共用的前缀：带前缀的表不止一张时照旧不推 + ({"links": _table(_col("id"), _col("tag_id")), "store_tags": _table(_col("id")), + "menu_tags": _table(_col("id"))}, set()), +]) +def test_prefix_fallback_only_when_direct_match_is_missing(tables, why): + assert _infer(tables, next(iter(tables))) == why + + def test_name_inference_one_to_one_when_column_is_own_key(): tables = {"member_profiles": _table(_col("member_id"), pk=("member_id",)), "members": _table(_col("id"))} rel = catalog.infer_name_relations(_source(tables).schema_cache, "member_profiles")[0] @@ -255,6 +281,16 @@ async def test_model_draft_respects_system_notes_of_imported_tables(): # ---------------------------------------------------------------- 起草并写入 +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _drop_created_sources(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + async def _db_source(tables: dict, **kw) -> DataSource: for name, meta in tables.items(): meta["qualified"] = name @@ -263,6 +299,7 @@ async def _db_source(tables: dict, **kw) -> DataSource: schema_cache={"tables": tables}, **kw) session.add(row) await session.commit() + _CREATED.append(row.id) return row diff --git a/backend/tests/test_catalog_impact.py b/backend/tests/test_catalog_impact.py index dd3476d8..a25d23f3 100644 --- a/backend/tests/test_catalog_impact.py +++ b/backend/tests/test_catalog_impact.py @@ -19,6 +19,7 @@ from app.data.engine import engines from app.data.introspect import introspect # 模型要在建表之前注册进 metadata：conftest 的建表只导入了 app.db.base，这个文件单独跑时没有别的模块先导入它们 from app.db import models # noqa: F401 +from tests.fixtures.sources import drop_source _CACHE: dict[str, dict] = {} @@ -106,10 +107,9 @@ async def world(scenic_db): for wid in created: if (wf := await session.get(Workflow, wid)) is not None: await session.delete(wf) - for sid in (ids.source, ids.other): - if (row := await session.get(DataSource, sid)) is not None: - await session.delete(row) await session.commit() + for sid in (ids.source, ids.other): + await drop_source(sid) async def _impact(source_id: str, table: str) -> dict: diff --git a/backend/tests/test_catalog_patch.py b/backend/tests/test_catalog_patch.py index 939ea0f5..5520313b 100644 --- a/backend/tests/test_catalog_patch.py +++ b/backend/tests/test_catalog_patch.py @@ -25,6 +25,7 @@ from app.data import catalog from app.data.engine import engines from app.data.introspect import introspect from tests.fixtures.catalog import scenic_notes +from tests.fixtures.sources import drop_source _CACHE: dict[str, dict] = {} @@ -76,6 +77,32 @@ async def test_codes_are_supplemented_not_replaced(scenic_db): assert change.value == {"9": "作废"} +async def test_codes_with_pending_meanings(scenic_db): + """数据剖析写进来的码值候选含义为空（界面上写「含义待填写」）：提案可以补上含义；提案里含义为空的码只在原来 + 没有时加进去，不抹掉已有的含义；含义为空的码合法，保存不报错。""" + tables = (await _schema(scenic_db))["tables"] + profiled = _visits() + profiled["columns"]["status"]["codes"] = catalog.make_item({"1": "有效", "0": "", "9": ""}, "profile", "proposed") + plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废", "1": ""}}], + table="visits", tables=tables) + assert plan.problems == [] + [change] = plan.changes + assert change.after == {"1": "有效", "0": "", "9": "作废"} and change.state == "change" + # 只给了含义为空的新码：照样加进去，含义待人填 + plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"7": ""}}], table="visits", + tables=tables) + assert plan.changes[0].after == {"1": "有效", "0": "", "9": "", "7": ""} + # 只给了含义为空的已有码：什么都没变，这一项是「保存即确认」（原来是推断） + plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"1": ""}}], table="visits", + tables=tables) + assert plan.changes[0].state == "confirm" + out = catalog.apply_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废"}}], table="visits", + tables=tables) + codes = out["columns"]["status"]["codes"] + assert codes["value"] == {"1": "有效", "0": "", "9": "作废"} and (codes["source"], codes["status"]) == ( + "human", "confirmed") + + async def test_new_relation_gets_its_id_from_both_ends(scenic_db): tables = (await _schema(scenic_db))["tables"] value = {"columns": ["visit_id"], "to_table": "visits", "to_columns": ["id"], "cardinality": "many_to_one"} @@ -190,11 +217,7 @@ async def scenic_source(scenic_db): await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏") source_id = row.id yield SimpleNamespace(id=source_id, name=name) - async with SessionLocal() as session: - row = await session.get(DataSource, source_id) - if row is not None: - await session.delete(row) - await session.commit() + await drop_source(source_id) async def _stream(monkeypatch, model, source, base_graph=None): diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py new file mode 100644 index 00000000..c88d1c82 --- /dev/null +++ b/backend/tests/test_catalog_profile.py @@ -0,0 +1,430 @@ +"""数据剖析接口：POST /api/datasources/{id}/catalog/profile（阶段 2A）。 + +剖析是数据目录里唯一对业务库发查询的地方，这里逐条钉住方案定的约束：默认关闭、只读、每条查询都过守卫、 +查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析； +以及剖析的结论怎么进目录：命名推断对上数据的升为有确证（来源记为数据剖析，和外键分开）、对不上的写明 +覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。 + +所有测试都在合成的景区库上做（tests/fixtures/catalog/scenic.py），数据和名称全部虚构。 +""" +from __future__ import annotations + +import asyncio +import sqlite3 +import uuid +from pathlib import Path +from types import SimpleNamespace +from urllib.parse import quote + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import event +from sqlalchemy.engine import Engine + +from app.data import catalog, catalog_profile +from app.data import engine as data_engine +from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.main import app as _fastapi_app +from tests.fixtures.catalog import scenic +from tests.fixtures.sources import drop_source + +ACTOR = {"X-Actor": quote("周宁")} + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +def _on(**settings) -> dict: + return {PROFILE_OPTION: {"enabled": True, **settings}} + + +@pytest.fixture +async def make_source(client): + """登记一个接上 path 的数据源并走真实的探查接口；draft 给了就先起草这些表的目录。""" + made: list[str] = [] + + async def _make(path: str, *, options: dict | None = None, draft: list[str] | None = None) -> str: + async with SessionLocal() as session: + row = DataSource(name=f"prof_{uuid.uuid4().hex[:10]}", kind="sqlite", database=str(path), + options=options or {}) + session.add(row) + await session.commit() + sid = row.id + made.append(sid) + r = await client.post(f"/api/datasources/{sid}/introspect") + assert r.status_code == 200, r.text + if draft is not None: + r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": draft}) + assert r.status_code == 200, r.text + return sid + + yield _make + for sid in made: + await drop_source(sid) + + +class _Spy: + """记下两样东西：守卫放行（guard.check 返回）的语句，和真正发到这个业务库的语句。 + + 后者挂在 SQLAlchemy 的 before_cursor_execute 上，任何一条到达驱动的语句都逃不掉——包括绕开查询层 + 另开连接发的。断言「发到库的 ⊆ 守卫放行的」就是「所有查询都经过守卫」。 + """ + + def __init__(self, db_name: str) -> None: + self.db_name = db_name + self.checked: list[str] = [] + self.executed: list[str] = [] + + def clear(self) -> None: + self.checked.clear() + self.executed.clear() + + def all_guarded(self) -> bool: + return bool(self.executed) and all(s in self.checked for s in self.executed) + + def mentions(self, word: str) -> list[str]: + return [s for s in self.executed if word.lower() in s.lower()] + + +@pytest.fixture +def spy(monkeypatch, scenic_db): + holder = SimpleNamespace(spy=None) + + def watch(path: str) -> _Spy: + holder.spy = _Spy(Path(path).name) + return holder.spy + + real_check = data_engine.check + + def checking(sql, **kw): + out = real_check(sql, **kw) + if holder.spy is not None: + holder.spy.checked.append(out) + return out + + def on_execute(conn, cursor, statement, parameters, context, executemany): + if holder.spy is not None and holder.spy.db_name in str(conn.engine.url): + holder.spy.executed.append(statement) + + monkeypatch.setattr(data_engine, "check", checking) + event.listen(Engine, "before_cursor_execute", on_execute) + yield watch + event.remove(Engine, "before_cursor_execute", on_execute) + + +def _url(sid: str) -> str: + return f"/api/datasources/{sid}/catalog/profile" + + +async def _notes(client, sid: str, table: str) -> dict: + r = await client.get(f"/api/datasources/{sid}/catalog/{table}") + assert r.status_code == 200, r.text + return r.json() + + +def _relation(notes: dict, column: str) -> dict: + return next(r for r in notes.get("relations") or [] if r["columns"] == [column]) + + +def _table(body: dict, name: str) -> dict: + return next(t for t in body["tables"] if t["table_name"] == name) + + +def _reasons(table: dict) -> set[str]: + return {s["reason"] for s in table["skipped"]} + + +# ---------------------------------------------------------------- 开关与设置 + + +async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, make_source, scenic_db, spy): + sid = await make_source(scenic_db, draft=["visits"]) + watch = spy(scenic_db) + watch.clear() + r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + assert r.status_code == 409 + detail = r.json()["detail"] + assert "数据源设置" in detail and "开启数据剖析" in detail + assert watch.executed == [] # 一条查询都没发 + + +async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db): + sid = await make_source(scenic_db) + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)}) + assert r.status_code == 422 and "查询次数上限" in r.json()["detail"] + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)}) + assert r.status_code == 200, r.text + assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500} + # 设好以后就能剖析；返回里带着这次实际用的设置 + r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR) + assert r.status_code == 200, r.text + assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500 + + +# ---------------------------------------------------------------- 剖析的结论 + + +async def test_profile_verifies_name_relations_and_proposes_codes_and_business_date(client, make_source, scenic_db, + spy): + sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"]) + watch = spy(scenic_db) + watch.clear() + r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) + assert r.status_code == 200, r.text + body = r.json() + + # 所有发到业务库的语句都经过了守卫 + assert watch.all_guarded(), watch.executed + assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"] + assert body["queries_used"] == sum(t["queries"] for t in body["tables"]) + assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"] + + notes = (await _notes(client, sid, "visits")) + assert notes["updated_by"] == "周宁" + notes = notes["notes"] + # 命名推断出来的两条关系：数据对得上、父表主键唯一 → 有确证，来源记为数据剖析 + for column, parent in (("gate_id", "gates"), ("member_id", "members")): + rel = _relation(notes, column) + assert rel["to_table"] == parent + assert (rel["source"], rel["status"]) == ("profile", "verified") + assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" + assert "数据剖析" in rel["note"] and "抽样" in rel["note"] + # 外键约束来的关系不动：外键是外键，剖析是剖析 + park = _relation(notes, "park_id") + assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None) + + # 状态列：码值候选，值有了、含义留空，推断 + codes = notes["columns"]["status"]["codes"] + assert codes["value"] == {"1": "", "0": ""} + assert (codes["source"], codes["status"]) == ("profile", "proposed") and "1500" in codes["note"] + # 只有一个日期类列：提议为业务日期，备注里写明取值范围 + bdate = notes["business_date"] + assert bdate["value"] == {"column": "visit_time"} and (bdate["source"], bdate["status"]) == ("profile", "proposed") + assert "2026-07" in bdate["note"] and "2026-09" in bdate["note"] + + channels = (await _notes(client, sid, "channels"))["notes"] + assert set(channels["columns"]["channel_type"]["codes"]["value"]) == {"线上直销", "线下", "分销"} + assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值 + + visits = _table(body, "visits") + kinds = sorted(f["kind"] for f in visits["findings"]) + assert kinds == ["business_date", "codes", "relation", "relation"] + gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"]) + assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}" + assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"] + assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None} + assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None + + +async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path): + """推错的命名关系：parking_records.lot_id 按名字对上了新建的 lots 表，但停车记录里的区号大多不在里面。""" + path = scenic.build(tmp_path / "lots.db") + db = sqlite3.connect(path) + try: + db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)]) + db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)", + [(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)]) + db.commit() + finally: + db.close() + sid = await make_source(path, options=_on(), draft=["parking_records"]) + before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") + assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed") + + r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR) + assert r.status_code == 200, r.text + rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") + assert rel["status"] == "proposed" and rel["source"] == "profile" + assert rel["coverage"] == 0.15 + assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] + finding = _table(r.json(), "parking_records")["findings"][0] + assert finding["status"] == "proposed" and finding["coverage"] == 0.15 + # 两个日期类列：不提议业务日期，取值范围只在返回里 + ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]} + assert ranges == {"entered_at"} # exited_at 全是空值 + assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"] + + +async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db): + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + detail = await _notes(client, sid, "visits") + gate = _relation(detail["notes"], "gate_id") + r = await client.post(f"/api/datasources/{sid}/catalog/visits/review", + json={"path": f"relations.{gate['id']}", "action": "confirm", + "if_version": detail["version"]}) + assert r.status_code == 200, r.text + + r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + assert r.status_code == 200, r.text + rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") + assert (rel["source"], rel["status"]) == ("name", "confirmed") + assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" + finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"]) + assert finding["status"] == "confirmed" and finding["confirmed"] is True + + +async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db): + """同一天、数据没变：第二次剖析不改目录、不升版本（冻结进快照的目录不会无故变）。""" + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + first = (await client.post(_url(sid), json={"tables": ["visits"]})).json() + second = (await client.post(_url(sid), json={"tables": ["visits"]})).json() + a, b = _table(first, "visits"), _table(second, "visits") + assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0) + assert b["version"] == a["version"] + + +# ---------------------------------------------------------------- 安全约束 + + +async def test_masked_columns_are_never_queried(client, make_source, scenic_db, spy): + sid = await make_source(scenic_db, options={**_on(), "mask_columns": "member_id, Channel_Type"}, + draft=["visits", "channels"]) + watch = spy(scenic_db) + watch.clear() + r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) + assert r.status_code == 200, r.text + assert watch.all_guarded() + assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == [] + visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels") + masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"] + assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"} + assert all("遮罩" in s["detail"] for s in masked) + notes = (await _notes(client, sid, "visits"))["notes"] + assert _relation(notes, "member_id")["source"] == "name" # 没动 + assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常 + assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {}) + + +async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy): + sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"]) + watch = spy(scenic_db) + watch.clear() + r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) + assert r.status_code == 200, r.text + body = r.json() + assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget" + visits, channels = _table(body, "visits"), _table(body, "channels") + assert "budget" in _reasons(visits) + assert channels["queries"] == 0 and _reasons(channels) == {"budget"} + assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget") + # 没查完的关系不改：原来的命名推断原样留着 + rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id") + assert (rel["source"], rel["status"]) == ("name", "proposed") + + +async def test_single_query_timeout_is_recorded_and_profiling_continues(client, make_source, scenic_db, monkeypatch): + """闸口的抽样查询换成一条停不下来的递归查询：1 秒时限到点由数据库掐断，记为超时，其余照常。""" + sid = await make_source(scenic_db, options=_on(query_timeout_s=1), draft=["visits"]) + real = data_engine.run_query + endless = "WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM r) SELECT COUNT(*) FROM r" + + async def slow_gate(source, sql, *, limits=None): + if "gate_id" in sql and "DISTINCT" in sql and "COUNT" not in sql: + assert limits is not None and limits.timeout_seconds == 1 + return await real(source, endless, limits=limits) + return await real(source, sql, limits=limits) + + monkeypatch.setattr(data_engine, "run_query", slow_gate) + r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + assert r.status_code == 200, r.text + visits = _table(r.json(), "visits") + timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"] + assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"] + assert r.json()["stopped"] is None + notes = (await _notes(client, sid, "visits"))["notes"] + assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改 + assert _relation(notes, "member_id")["status"] == "verified" # 其余照常 + + +async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, draft=["visits"]) + real = data_engine.run_query + + async def slow(source, sql, *, limits=None): + await asyncio.sleep(0.5) + return await real(source, sql, limits=limits) + + monkeypatch.setattr(data_engine, "run_query", slow) + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + settings = ProfileSettings(enabled=True, max_total_s=1.4) + report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None, + settings=settings) + assert report.stopped == "deadline" + assert 1 <= report.queries_used <= 3 + reasons = {s.reason for t in report.tables for s in t.skipped} + assert "deadline" in reasons + + +async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy): + """行数超过整表统计上限（这里设成 100 行）：不做 COUNT(DISTINCT)、MIN/MAX，抽样和取值分布都带上限。""" + sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"]) + watch = spy(scenic_db) + watch.clear() + r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + assert r.status_code == 200, r.text + visits = _table(r.json(), "visits") + assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101} + on_visits = [s for s in watch.executed if "from visits" in s.lower()] + assert on_visits and all("LIMIT" in s for s in on_visits), on_visits + assert not any("MIN(" in s for s in on_visits) + assert "too_large" in _reasons(visits) + assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large") + # 父表有主键，覆盖率又够：照样有确证；子表一侧没核对唯一，按多对一记 + rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") + assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one" + + +# ---------------------------------------------------------------- 接口 + + +async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + real = data_engine.run_query + started, release = asyncio.Event(), asyncio.Event() + + async def held(source, sql, *, limits=None): + started.set() + await release.wait() + return await real(source, sql, limits=limits) + + monkeypatch.setattr(data_engine, "run_query", held) + first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)) + await asyncio.wait_for(started.wait(), 10) + second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"] + release.set() + done = await asyncio.wait_for(first, 30) + assert done.status_code == 200, done.text + # 做完以后可以再来 + again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR) + assert again.status_code == 200, again.text + + +async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db): + sid = await make_source(scenic_db, options=_on(), draft=[]) + r = await client.post(_url(sid), json={}, headers=ACTOR) + assert r.status_code == 200, r.text + names = [t["table_name"] for t in r.json()["tables"]] + assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES + async with SessionLocal() as session: + entries = await catalog.read_catalog(session, sid) + for name in names: + rels = entries[name].notes.get("relations") or [] + assert any(rel["source"] not in ("fk",) for rel in rels), name + assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选 + + +async def test_unknown_table_and_missing_source(client, make_source, scenic_db): + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR) + assert r.status_code == 200 + assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0 + r = await client.post(_url("0" * 32), json={}, headers=ACTOR) + assert r.status_code == 404 diff --git a/backend/tests/test_catalog_profile_units.py b/backend/tests/test_catalog_profile_units.py new file mode 100644 index 00000000..d41ecf2d --- /dev/null +++ b/backend/tests/test_catalog_profile_units.py @@ -0,0 +1,205 @@ +"""数据剖析（阶段 2A）里不连库的部分：设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、 +目录对剖析结果的容纳（待填含义的码值、撤销审阅回到剖析的结论）。""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from app.data import catalog, catalog_profile, guard +from app.data.catalog import make_item +from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem +from app.data.engine import build_url + +AT = "2026-10-03T08:00:00+00:00" + + +# ---------------------------------------------------------------- 设置 + + +def test_settings_default_off_and_parse_leniently(): + s = profile_settings({}) + assert s == ProfileSettings() + assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000 + assert s.max_scan_rows == 100_000 and s.max_total_s == 120 + on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}}) + assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500 + # 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错 + odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}}) + assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10 + assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings() + + +@pytest.mark.parametrize("value, label", [ + ({"max_queries": 0}, "查询次数上限"), + ({"max_queries": 501}, "查询次数上限"), + ({"max_queries": 2.5}, "查询次数上限"), + ({"query_timeout_s": 61}, "单条查询时限"), + ({"sample_size": 5}, "抽样键值数"), + ({"max_scan_rows": -1}, "整表统计行数上限"), + ({"max_total_s": 5}, "总时长上限"), + ({"enabled": "true"}, "开启数据剖析"), + ({"max_queries": True}, "查询次数上限"), +]) +def test_settings_problem_names_the_field_in_chinese(value, label): + problem = profile_settings_problem({PROFILE_OPTION: value}) + assert problem and label in problem and "数据剖析" in problem + + +def test_settings_problem_rejects_unknown_keys_and_wrong_shape(): + assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}}) + assert profile_settings_problem({PROFILE_OPTION: ["enabled"]}) + assert profile_settings_problem({}) is None + assert profile_settings_problem({PROFILE_OPTION: None}) is None + assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5, + "sample_size": 10, "max_scan_rows": 0, + "max_total_s": 600}}) is None + + +def test_profile_option_never_reaches_the_connection_url(): + """剖析设置是 AgentLab 自己的配置：拼进连接串，驱动会把它当未知参数拒掉。""" + source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro", + password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}}) + url = build_url(source) + assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url + + +# ---------------------------------------------------------------- 字面量 + + +@pytest.mark.parametrize("value, kind, expected", [ + (42, "number", "42"), + (-7, None, "-7"), + (3.0, "number", "3"), + ("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串 + ("A-01", "text", "'A-01'"), + ("O'Neil", "text", "'O''Neil'"), + ("12:30", "text", "'12:30'"), +]) +def test_sql_literal_quotes_safe_values(value, kind, expected): + assert catalog_profile.sql_literal(value, kind) == expected + + +@pytest.mark.parametrize("value, kind", [ + (True, "boolean"), # 布尔不当键 + (2.5, "number"), # 非整数的数不当键：浮点相等比较靠不住 + ("12.50", "number"), + ("1e3", "number"), + ("2026-07-01", "date"), # 日期的字面量各家写法不同，不猜 + ("a\\b", "text"), # 反斜杠：MySQL 不同设置下读法不同，守卫会拒 + ("a :b", "text"), # 「:名字」会被查询层当成绑定参数 + (":b", "text"), + ("a\nb", "text"), # 控制字符 + ("x" * 201, "text"), + (None, "text"), + ([1], None), +]) +def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind): + assert catalog_profile.sql_literal(value, kind) is None + + +# ---------------------------------------------------------------- 各方言的剖析 SQL + + +_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]}, + "orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}} + + +@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"]) +def test_profile_sql_passes_readonly_guard_on_every_dialect(kind): + """剖析的每一种查询（含读统计信息的系统视图）都得过只读守卫——守卫不为剖析放宽。""" + d = catalog_profile.SqlDialect(kind) + sqls = [] + for table in ("visits", "orders"): + stats = d.stats_sql(_META[table], table) + if kind == "sqlite": + assert stats is None # SQLite 没有可读的统计信息，改为数到上限 + else: + assert stats + sqls.append(stats) + tbl = d.table(_META[table], table) + col = d.quote(_META[table]["columns"][0]["name"]) + sqls += [ + d.bounded_count_sql(tbl, 100_001), + d.distinct_sample_sql(tbl, col, 2000, scan_cap=None), + d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000), + d.match_count_sql(tbl, col, ["1", "'A'"]), + d.unique_check_sql(tbl, col), + d.value_counts_sql(tbl, col, 21, scan_cap=None), + d.value_counts_sql(tbl, col, 21, scan_cap=100_000), + d.min_max_sql(tbl, [col]), + ] + for sql in sqls: + assert guard.check(sql, readonly=True, dialect=kind) == sql, sql + + +def test_dialect_quoting_and_row_limits(): + my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql")) + assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"' + assert my.table(_META["orders"], "orders") == "sales.orders" + assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"' + # Oracle 没有 LIMIT（11g 也没有 FETCH FIRST）：一律用 ROWNUM + sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None) + assert "LIMIT" not in sample and "ROWNUM <= 50" in sample + assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None) + # 统计信息按方言读各自的系统视图 + assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits") + assert "pg_class" in pg.stats_sql(_META["visits"], "visits") + stats = ora.stats_sql(_META["orders"], "orders") + assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats + + +# ---------------------------------------------------------------- 目录对剖析结果的容纳 + + +def test_codes_with_pending_meanings_are_valid_and_render_as_pending(): + """剖析得到的码值只有取值、含义待人填写：目录要收得下，给模型看时写明含义待填写。""" + notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")}, + "channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}} + assert catalog.validate_notes(notes) == [] + text = catalog.render_table_notes(notes) + assert "1、0（含义待填写）" in text + assert "WEB=官网，BOX 的含义待填写" in text + # 码值仍得是对照表：值不是文字的照样拒收 + assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}}) + + +def _profile_rel(coverage, cardinality, status): + return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"], + "to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage, + "source": "profile", "status": status} + + +def test_reset_returns_profile_relation_to_what_profiling_concluded(): + """撤销审阅回到「来源的初始状态」。剖析的结论随数据而定：覆盖率够、父键唯一（基数有值）就是有确证。""" + verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]} + rid = verified["relations"][0]["id"] + out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT) + assert out["relations"][0]["status"] == "confirmed" + out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT) + assert out["relations"][0]["status"] == "verified" + out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT), + f"relations.{rid}", "reset", at=AT) + assert out["relations"][0]["status"] == "verified" + # 覆盖率低、父键没核实唯一的：撤销后仍是推断 + for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)): + low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]} + out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT), + f"relations.{rid}", "reset", at=AT) + assert out["relations"][0]["status"] == "proposed" + # 剖析提的码值候选、业务日期永远是推断 + codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}} + out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT), + "columns.status.codes", "reset", at=AT) + assert out["columns"]["status"]["codes"]["status"] == "proposed" + + +def test_human_edit_may_restore_profile_conclusion_but_not_invent_one(): + rel = _profile_rel(1.0, "many_to_one", "confirmed") + existing = {"relations": [rel]} + back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT) + assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile" + low = _profile_rel(0.4, "many_to_one", "proposed") + kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]}, + table="visits", at=AT) + assert kept["relations"][0]["status"] == "proposed" diff --git a/backend/tests/test_catalog_publish_drift.py b/backend/tests/test_catalog_publish_drift.py index 062e0102..d08ae51e 100644 --- a/backend/tests/test_catalog_publish_drift.py +++ b/backend/tests/test_catalog_publish_drift.py @@ -24,6 +24,7 @@ from app.db.models import DataSource, Run, RunEvent, Workflow, WorkflowVersion from app.engine.runner import run_manager from app.main import app from tests.fixtures.catalog import scenic_notes +from tests.fixtures.sources import drop_source _CACHE: dict[str, dict] = {} SQL = "SELECT COUNT(*) AS n FROM visits v JOIN parks p ON p.id = v.park_id WHERE v.status = 1" @@ -72,9 +73,8 @@ async def source(scenic_db): for wid in made: if (wf := await session.get(Workflow, wid)) is not None: await session.delete(wf) - if (row := await session.get(DataSource, sid)) is not None: - await session.delete(row) await session.commit() + await drop_source(sid) def node(nid, ntype, **config): diff --git a/backend/tests/test_catalog_render.py b/backend/tests/test_catalog_render.py index ce7d3091..54d7850e 100644 --- a/backend/tests/test_catalog_render.py +++ b/backend/tests/test_catalog_render.py @@ -7,6 +7,7 @@ from app.data import catalog from app.data.catalog import make_item from app.db.base import SessionLocal from app.db.models import Artifact, DataSource, Run +from tests.fixtures.sources import drop_source MARK = "（推断，未确认）" @@ -184,4 +185,5 @@ async def test_table_usage_counts_run_queries_per_table(monkeypatch): await session.commit() source = await session.get(DataSource, sid) usage = await catalog.table_usage(session, source) + await drop_source(sid) assert usage == {"visits": 4, "明细": 2, "parks": 1} diff --git a/backend/tests/test_catalog_runtime.py b/backend/tests/test_catalog_runtime.py index d51d8846..ab5c877e 100644 --- a/backend/tests/test_catalog_runtime.py +++ b/backend/tests/test_catalog_runtime.py @@ -14,12 +14,12 @@ from types import SimpleNamespace from app.core.artifact_store import canonical_json, content_hash, load from app.data import catalog from app.data.catalog import make_item -from app.data.engine import engines from app.data.introspect import describe_table, introspect from app.db.base import SessionLocal from app.db.models import Artifact, DataSource from app.tools.datasource import _store_schema, build_datasource_tools from app.tools.registry import ToolContext +from tests.fixtures.sources import drop_source MARK = "（推断，未确认）" @@ -182,4 +182,4 @@ async def test_tools_show_and_freeze_catalog_and_record_query_tables(scenic_db): ref = await session.get(Artifact, (result["artifact"], run_id)) assert ref.meta == {"source_id": row.id, "source": row.name, "tables": ["visits", "parks"]} finally: - await engines.invalidate(row.id) + await drop_source(row.id) diff --git a/backend/tests/test_copilot_sqlcheck.py b/backend/tests/test_copilot_sqlcheck.py index 5d0d0348..efdd24df 100644 --- a/backend/tests/test_copilot_sqlcheck.py +++ b/backend/tests/test_copilot_sqlcheck.py @@ -20,6 +20,7 @@ from app.data.engine import engines from app.data.introspect import introspect from app.data.sqlcheck import SqlChecker from tests.fixtures.catalog import scenic_notes +from tests.fixtures.sources import drop_source FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id " "WHERE o.status = 1") @@ -177,10 +178,7 @@ async def scenic_source(scenic_db): await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏") source_id = row.id yield name - async with SessionLocal() as session: - row = await session.get(DataSource, source_id) - row.enabled = False - await session.commit() + await drop_source(source_id) async def _stream(monkeypatch, model, source): diff --git a/backend/tests/test_publish_sqlcheck.py b/backend/tests/test_publish_sqlcheck.py index 69cb1432..4323af65 100644 --- a/backend/tests/test_publish_sqlcheck.py +++ b/backend/tests/test_publish_sqlcheck.py @@ -23,6 +23,7 @@ from app.engine.governance import lint_for_publish, publish_issues from app.engine.schema import GraphSpec from app.main import app from tests.fixtures.catalog import scenic_notes +from tests.fixtures.sources import drop_source FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id " "WHERE o.status = 1") @@ -133,10 +134,7 @@ async def scenic_source(scenic_db): await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏") source_id = row.id yield name - async with SessionLocal() as session: - row = await session.get(DataSource, source_id) - row.enabled = False - await session.commit() + await drop_source(source_id) @pytest.fixture diff --git a/backend/tests/test_runtime_sqlcheck.py b/backend/tests/test_runtime_sqlcheck.py index 5a8029b4..b84c9bb2 100644 --- a/backend/tests/test_runtime_sqlcheck.py +++ b/backend/tests/test_runtime_sqlcheck.py @@ -32,6 +32,7 @@ from app.main import app from app.tools.datasource import build_datasource_tools from app.tools.registry import ToolContext from tests.fixtures.catalog import scenic_notes +from tests.fixtures.sources import drop_source FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id " "WHERE o.status = 1") @@ -54,6 +55,16 @@ async def _schema(path: str) -> dict: return _CACHE[path] +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _drop_created_sources(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]: name = f"scenic_{uuid.uuid4().hex[:8]}" async with SessionLocal() as session: @@ -61,6 +72,7 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]: row.schema_cache = await _schema(path) session.add(row) await session.commit() + _CREATED.append(row.id) if with_catalog: for table, notes in scenic_notes.notes().items(): await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏") @@ -69,9 +81,8 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]: @pytest.fixture async def scenic_source(scenic_db): - source_id, name = await _source(scenic_db) + _, name = await _source(scenic_db) yield name - await engines.invalidate(source_id) async def _query_tool(name: str): diff --git a/backend/tests/test_sqlcheck.py b/backend/tests/test_sqlcheck.py index 79c896e7..329087b8 100644 --- a/backend/tests/test_sqlcheck.py +++ b/backend/tests/test_sqlcheck.py @@ -254,6 +254,16 @@ async def test_join_unconfirmed_negative(cache, sql): assert "join_unconfirmed" not in codes(cache, sql) +async def test_sibling_join_through_a_shared_parent_is_only_a_note(cache): + """两张表各自按 member_id 指向会员（都是外键），彼此按 member_id 相连：目录里没有这两张表之间的关系， + 但这是经共同父表推得出来的连法，不是「连错了列」。只提示一句：关联后行数可能成倍增加。""" + sql = "SELECT COUNT(*) FROM orders o JOIN member_tag_links t ON t.member_id = o.member_id WHERE o.status = 1" + [check] = [c for c in run(cache, sql) if c.code == "join_unconfirmed"] + assert check.level == "info" + assert "会员" in check.message or "members" in check.message + assert "成倍" in check.message + + async def test_join_level_follows_status(cache): sql = "SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1" assert "join_unconfirmed" not in codes(cache, sql, "confirmed") diff --git a/backend/tests/test_truncated_results.py b/backend/tests/test_truncated_results.py index 2dc1ac2e..59f48d68 100644 --- a/backend/tests/test_truncated_results.py +++ b/backend/tests/test_truncated_results.py @@ -34,6 +34,7 @@ from app.engine.expressions import TruncatedUse, eval_expression from app.engine.issuance import decide_tier, incomplete_gaps from app.engine.runner import run_manager from app.main import app +from tests.fixtures.sources import drop_source # -------------------------------------------------------------------------- # 求值器：整组用到截断结果时记一笔，单格不记 @@ -345,7 +346,7 @@ async def park(ticket_db, engine_up): # 连接池按数据源 id 缓存：换了库文件不作废，查的还是上一个测试的库 await engines.invalidate(source_id) yield - await engines.invalidate(source_id) + await drop_source(source_id) def node(nid, ntype, **config): diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 572ed88e..73fe62c3 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,6 +1,7 @@ import type { ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogDetail, CatalogDraftOut, CatalogImpact, CatalogList, CatalogNotes, - CatalogPatchPreview, CatalogPatchSubmit, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview, EditRequest, ImportRecord, + CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview, + EditRequest, ImportRecord, ManifestOut, PurgeRawBody, PurgeRawOut, QuestionAnswer, Recipe, RedraftRulesOut, RemovePeriodBody, RemovePeriodOut, RevokeAcceptanceBody, RevokeAcceptanceOut, SnapshotOut, Staging, } from '../types' @@ -687,6 +688,13 @@ export const api = { draft: (sourceId: string, body: { tables?: string[]; use_model: boolean }, opts?: RequestOptions) => request(`/datasources/${encodeURIComponent(sourceId)}/catalog/draft`, { method: 'POST', body: JSON.stringify(body), ...opts }), + /** + * 同步剖析：对业务库发少量只读查询，核对关系、取码值候选，结论写进目录。不给 tables 时按使用次数取前 10 张 + * 有待核实关系的表。没开启、数据源停用、没有表结构、同一个数据源正在剖析、快照被改动时回 409（detail 是一句中文） + */ + profile: (sourceId: string, body: { tables?: string[] }, opts?: RequestOptions) => + request(`/datasources/${encodeURIComponent(sourceId)}/catalog/profile`, + { method: 'POST', body: JSON.stringify(body), ...opts }), }, runs: { diff --git a/frontend/src/canvas/InspectorSheet.tsx b/frontend/src/canvas/InspectorSheet.tsx index 4dce4062..98746721 100644 --- a/frontend/src/canvas/InspectorSheet.tsx +++ b/frontend/src/canvas/InspectorSheet.tsx @@ -77,9 +77,28 @@ export function revealField(nodeId: string, field?: FieldRef | string | null, op const to = target.querySelector(FILLABLE) ?? target.querySelector(PRESSABLE) to?.focus({ preventScroll: true }) } + // 子键没有自己的一行、字段是一段 JSON（调用工具的「参数」，问题落在 args.sql 上）：在文字里找到那个键， + // 框内滚到那一行，聚焦时光标放在它的值开头 + if (!part && at.sub) revealJsonKey(target, at.sub, !!opts.focus) })) } +/** JSON 文字框里的某个键：滚到它那一行；focus 时把光标放在值的开头（不选中，免得一敲键就把整段 SQL 换掉） */ +function revealJsonKey(field: HTMLElement, key: string, focus: boolean): void { + const box = field.querySelector('textarea') + if (!box) return + const m = new RegExp(`"${key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}"\\s*:\\s*"?`).exec(box.value) + if (!m) return + const line = box.value.slice(0, m.index).split('\n').length - 1 + const height = parseFloat(getComputedStyle(box).lineHeight) || 16 + box.scrollTop = Math.max(0, (line - 1) * height) + if (focus) { + const at = m.index + m[0].length + box.focus({ preventScroll: true }) + box.setSelectionRange(at, at) + } +} + // 选中 → 面板开不开。放在模块级：选中可能来自别处（问题面板定位、发布弹窗、快捷键）， // 规则只写这一份 useStudio.subscribe((s, prev) => { diff --git a/frontend/src/canvas/PublishDialog.tsx b/frontend/src/canvas/PublishDialog.tsx index 469070c5..a83efba5 100644 --- a/frontend/src/canvas/PublishDialog.tsx +++ b/frontend/src/canvas/PublishDialog.tsx @@ -13,6 +13,7 @@ import { useCatalog } from '../store/catalog' import { Modal, Spinner, toast, useRadioGroup } from '../components/ui' import { humanizeError } from '../lib/errors' import { PUBLISH_FIX_TEXT as T, WORKFLOW_STATUS_HINT, WORKFLOW_STATUS_LABEL } from '../lib/terms' +import { isSqlCheckCode, sqlRuleLabel } from '../lib/sqlcheck' import { setLocalActor, useLocalActor } from '../lib/actor' import { autoFixIds, choiceReady, contentSig, fixFieldLabel, fixFor, fixValueLines, fixValueText, isMissingEndpoint, mergeIssues, @@ -42,7 +43,8 @@ export function PublishDialog({ workflow, onClose, onDone, onLocate }: { workflow: Workflow onClose: () => void onDone: () => void - onLocate: (nodeId: string) => void + /** 定位一条问题：选中节点，能落到具体配置项（调用工具的 SQL）就落到那一项 */ + onLocate: (issue: ValidationIssue) => void }) { const load = useStudio((s) => s.load) const dirty = useStudio((s) => s.dirty) @@ -208,7 +210,7 @@ export function PublishDialog({ workflow, onClose, onDone, onLocate }: {
- { if (issue.node_id) { onLocate(issue.node_id); onClose() } }} /> + { if (issue.node_id) { onLocate(issue); onClose() } }} />
) @@ -564,6 +566,8 @@ function PreflightRow({ issue, others, fix, where, pf, onLocate }: { aria-label={err ? '错误' : '提示'} /> {where && !String(issue.message).includes(`「${where}」`) && 「{where}」} + {/* 对照数据目录的 SQL 检查：先写中文规则名，不露规则编号 */} + {isSqlCheckCode(issue.code) && {sqlRuleLabel(issue.code)}：} {issue.message} {issue.node_id && } diff --git a/frontend/src/components/ui.tsx b/frontend/src/components/ui.tsx index 53c6b3e8..18ccafca 100644 --- a/frontend/src/components/ui.tsx +++ b/frontend/src/components/ui.tsx @@ -310,6 +310,7 @@ function ToastCard({ item }: { item: ToastItem }) {
setPaused(true)} onMouseLeave={() => setPaused(false)} onFocus={() => setPaused(true)} diff --git a/frontend/src/lib/sqlcheck.ts b/frontend/src/lib/sqlcheck.ts new file mode 100644 index 00000000..bc1b9a1d --- /dev/null +++ b/frontend/src/lib/sqlcheck.ts @@ -0,0 +1,65 @@ +import type { SqlCheckCode, SqlCheckItem, SqlCheckLevel } from '../types' +import { SQL_CHECK_LEVEL_LABEL, SQL_CHECK_RULE_LABEL, SQL_CHECK_TEXT } from './terms' + +// =========================================================================== +// 基于数据目录的 SQL 检查（后端 data/sqlcheck.py）在界面上的读法：规则编号 → 中文规则名、级别 → 语义色， +// 以及从证据接口、助手自查、发布前检查的问题里读出一条检查。证据面板、运行时间线、助手面板、发布前检查共用。 +// =========================================================================== + +/** 七条规则的编号（服务端 CHECK_CODES），按严重程度大致排序 */ +export const SQL_CHECK_CODES: readonly SqlCheckCode[] = [ + 'fanout_sum', 'stock_summed', 'join_unconfirmed', 'ratio_aggregated', 'missing_valid_filter', 'unknown_code', 'wrong_date_column', +] + +export const isSqlCheckCode = (code: unknown): code is SqlCheckCode => + typeof code === 'string' && (SQL_CHECK_CODES as readonly string[]).includes(code) + +/** 级别：认不出的按「提示」（最轻的一档），不把拿不准的说重 */ +export const sqlCheckLevelOf = (v: unknown): SqlCheckLevel => (v === 'error' || v === 'warning' ? v : 'info') + +/** 规则的中文名；服务端新加、这里还不认识的规则统称「SQL 检查」，不露编号 */ +export const sqlRuleLabel = (code: unknown): string => (isSqlCheckCode(code) ? SQL_CHECK_RULE_LABEL[code] : SQL_CHECK_TEXT.unknownRule) + +/** 各级别的颜色（语义令牌） */ +export const SQL_CHECK_TONE: Record = { + error: 'var(--st-failed)', + warning: 'var(--st-waiting)', + info: 'var(--accent)', +} + +const LEVEL_RANK: Record = { error: 0, warning: 1, info: 2 } + +/** 错误在前，同级保持原来的顺序 */ +export function sortSqlChecks(list: T[]): T[] { + return list.map((c, i) => ({ c, i })) + .sort((a, b) => LEVEL_RANK[sqlCheckLevelOf(a.c.level)] - LEVEL_RANK[sqlCheckLevelOf(b.c.level)] || a.i - b.i) + .map((x) => x.c) +} + +/** 涉及的表和列：「orders.amount」「orders」 */ +export const sqlCheckWhere = (c: Pick): string => + (c.table ? (c.column ? `${c.table}.${c.column}` : c.table) : (c.column ?? '')) + +/** 从任意对象读一条检查（证据接口的 checks、助手自查和发布前检查的问题）；不是 SQL 检查的返回 null */ +export function sqlCheckOf(v: unknown): SqlCheckItem | null { + if (!v || typeof v !== 'object') return null + const o = v as Record + if (!isSqlCheckCode(o.code)) return null + const str = (x: unknown) => (typeof x === 'string' && x ? x : undefined) + return { + code: o.code, + level: sqlCheckLevelOf(o.level), + message: String(o.message ?? ''), + table: str(o.table) ?? '', + ...(str(o.column) ? { column: str(o.column) } : {}), + ...(str(o.relation_id) ? { relation_id: str(o.relation_id) } : {}), + ...(str(o.sql_excerpt) ? { sql_excerpt: str(o.sql_excerpt) } : {}), + } +} + +/** 一条检查写成一行（时间线的展开区、交给助手再修的清单）：「错误 · 一对多关联后重复计算：……（涉及 orders.amount）」 */ +export function sqlCheckLine(c: SqlCheckItem): string { + const where = sqlCheckWhere(c) + return `${SQL_CHECK_LEVEL_LABEL[sqlCheckLevelOf(c.level)]} · ${sqlRuleLabel(c.code)}：${c.message}` + + (where ? `（${SQL_CHECK_TEXT.where} ${where}）` : '') +} diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 5f1a9e89..3bc567b2 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -43,7 +43,8 @@ */ import type { - CatalogCardinality, CatalogMeasure, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode, ToolTrust, + CatalogCardinality, CatalogMeasure, CatalogProfileNumberKey, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode, + SqlCheckCode, SqlCheckLevel, ToolTrust, } from '../types' import { formatNumber } from './format' @@ -247,6 +248,9 @@ export const EVIDENCE_TEXT = { /** 指标拿截断的查询结果整组算出来（计数、求和……）：值只算到了取回的那部分，出具按缺口降档 */ incomplete: '结果不完整', incompleteFallback: '基于被截断的查询结果计算，结果不完整', + /** 指标所依据的查询对照数据目录查出了错误级的问题：值照算，出具按缺口降档 */ + sqlCheckFailed: 'SQL 检查未通过', + sqlCheckFallback: '所依据的查询未通过 SQL 检查，结果不可靠', /** 输入是 Agent 交来、被截断切开的数组字段 */ inputTruncated: '查询结果已截断，该字段只含取回的部分行', unshowable: (value: string) => `有值（${value}），但无法按口径卡的格式显示`, @@ -2127,7 +2131,7 @@ export const CATALOG_TEXT = { keysPlaceholder: '列名，用顿号或逗号分隔', keysUnknown: (cols: string) => `表结构中没有这些列：${cols}`, codesPlaceholder: '1=已支付', - codesHint: '每行一个，写成「码值=含义」', + codesHint: '每行一个，写成「码值=含义」，含义可暂不填写', codesInvalid: (line: number) => `第 ${formatNumber(line)} 行应写成「码值=含义」`, codesDuplicate: (code: string) => `码值 ${code} 重复`, dateColumnRequired: '填写规则或时区时，日期列必填', @@ -2241,3 +2245,284 @@ export const CATALOG_DRIFT_TEXT = { unchanged: '发布之后这些表的目录没有变化', empty: '这一版的 SQL 没有用到数据库表', } + +// =========================================================================== +// 数据剖析（数据源设置里的开关和预算；数据目录页的「数据剖析」） +// +// 术语： +// - 「数据剖析」：对业务库发少量只读查询，核实推断的关联关系（覆盖率、基数），取状态类列的码值候选，提议业务日期。 +// 动作写「剖析」，不写扫描、探测。默认关闭，按数据源开启。 +// - 预算五项：查询次数上限、单条查询时限、抽样键值数、整表统计行数上限、总时长上限。标签和服务端校验报错里的叫法 +// 一致（catalog_profile._NUMBER_FIELDS），保存被拒时按它认出是哪一项。 +// =========================================================================== + +export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, string> = { + enabled: '开启数据剖析', + max_queries: '查询次数上限', + query_timeout_s: '单条查询时限', + sample_size: '抽样键值数', + max_scan_rows: '整表统计行数上限', + max_total_s: '总时长上限', +} + +export const PROFILE_FIELD_UNIT: Record = { + max_queries: '条', + query_timeout_s: '秒', + sample_size: '个', + max_scan_rows: '行', + max_total_s: '秒', +} + +export const PROFILE_FIELD_HINT: Record = { + max_queries: '一次剖析最多发出的查询条数，用完即停止', + query_timeout_s: '超过即中断这条查询；不超过数据源自身的查询时限', + sample_size: '核对一条关联关系时，从本表抽取的不同键值个数', + max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计', + max_total_s: '一次剖析的总时长，到时即停止', +} + +export const PROFILE_TEXT = { + // ---- 设置 + section: '数据剖析', + enableHint: '默认关闭。开启后可在数据目录中核实推断的关联关系、取状态类列的码值候选', + risk: (queries: number, timeout: number, total: number) => + `开启后，每次剖析会对这个数据源发出只读查询：最多 ${formatNumber(queries)} 条，单条不超过 ${formatNumber(timeout)} 秒，` + + `总时长不超过 ${formatNumber(total)} 秒。不读取明细行，遮罩的列不取样。`, + range: (min: number, max: number, unit: string, def: number) => + `范围 ${formatNumber(min)}–${formatNumber(max)} ${unit}，默认 ${formatNumber(def)}`, + scanZero: '填 0 表示一律不做整表统计', + placeholder: (def: number) => `默认 ${formatNumber(def)}`, + rangeError: (min: number, max: number, integer: boolean) => + `请填写 ${formatNumber(min)} 到 ${formatNumber(max)} 之间的${integer ? '整数' : '数'}`, + invalid: (n: number) => `数据剖析有 ${formatNumber(n)} 项设置不正确`, + // ---- 数据源卡片 + chip: '数据剖析已开启', + chipHint: (queries: number, timeout: number) => + `数据剖析已开启：每次最多 ${formatNumber(queries)} 条只读查询，单条不超过 ${formatNumber(timeout)} 秒`, + open: '设置数据剖析', + dialogTitle: (name: string) => `「${name}」的数据剖析设置`, + dialogBody: '设置保存在数据源上，对数据目录中的每次剖析生效。', + savedOn: (name: string) => `「${name}」已开启数据剖析`, + savedOff: (name: string) => `「${name}」已关闭数据剖析`, + saved: (name: string) => `已保存「${name}」的数据剖析设置`, + // ---- 数据目录页：范围与确认 + action: '数据剖析', + actionHint: '对业务库发少量只读查询：核实推断的关联关系、取码值候选、提议业务日期', + profileSelected: (n: number) => `剖析所选（${formatNumber(n)}）`, + title: '数据剖析', + intro: '对业务库发少量只读查询，核实推断的关联关系（覆盖率、基数），取状态类列的码值候选，提议业务日期。' + + '结论写入数据目录；已确认、已驳回的项不会被改动。', + scope: '剖析范围', + scopeSelected: (n: number) => `已选的 ${formatNumber(n)} 张表`, + scopeFiltered: (n: number) => `当前筛选结果中的 ${formatNumber(n)} 张表`, + scopeDefault: '使用次数最多、有待核实关联关系的表', + scopeDefaultHint: (n: number) => `由服务端挑选，至多 ${formatNumber(n)} 张`, + scopeTooMany: (max: number) => `超过 ${formatNumber(max)} 张，请缩小范围`, + budget: '本次预算', + budgetQueries: (queries: number, timeout: number) => + `最多 ${formatNumber(queries)} 条只读查询，单条不超过 ${formatNumber(timeout)} 秒`, + budgetTotal: (total: number) => `总时长不超过 ${formatNumber(total)} 秒，到时即停止`, + budgetSample: (n: number) => `每条关联关系抽样 ${formatNumber(n)} 个键值`, + budgetScan: (n: number) => (n > 0 ? `行数超过 ${formatNumber(n)} 的表不做整表统计` : '不做整表统计'), + budgetFrom: (name: string) => `按「${name}」的数据剖析设置`, + editSettings: '修改设置', + noCancel: '剖析在服务端一次完成，开始后无法中途停止。', + start: '开始剖析', + again: '重新剖析', + disabledTitle: '这个数据源未开启数据剖析', + disabledBody: '剖析会对业务库发出只读查询，需要先在数据源设置中开启。', + enable: '开启数据剖析', + // ---- 进行中 + running: '正在剖析…', + elapsed: (used: string, total: string) => `已用时 ${used}（总时长上限 ${total}）`, + runningHint: '剖析在服务端一次完成，无法中途停止。关闭此窗口不影响剖析，完成后会提示结果。', + background: '在后台继续', + headerRunning: (used: string) => `正在剖析 ${used}`, + headerRunningHint: '数据剖析进行中，点击查看', + leavePage: '离开本页不会中止剖析，结论仍会写入数据目录，但不再显示本次的报告。', + // ---- 报告 + done: '剖析完成', + stoppedTitle: (reason: string) => `剖析已停止：${reason}`, + summary: (used: number, max: number, tables: number) => + `用了 ${formatNumber(used)} / ${formatNumber(max)} 条查询，剖析 ${formatNumber(tables)} 张表`, + changes: (added: number, updated: number, removed: number) => + `新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`, + noChange: '目录没有变化', + toastDone: (summary: string) => `数据剖析完成：${summary}`, + toastStopped: (reason: string) => `数据剖析已停止：${reason}`, + toastFailed: '数据剖析未能完成', + viewReport: '查看报告', + close: '完成', + tableQueries: (n: number) => `${formatNumber(n)} 条查询`, + rowsStats: (n: number) => `约 ${formatNumber(n)} 行`, + rowsCount: (n: number) => `${formatNumber(n)} 行`, + rowsAtLeast: (n: number) => `超过 ${formatNumber(n - 1)} 行`, + rowsHint: { stats: '数据库统计信息中的估算值', count: '计数到整表统计行数上限为止' } as Record, + openTable: '打开', + openTableLabel: (name: string) => `打开「${name}」`, + noFindings: '没有新的发现', + findingRelations: '关联关系', + findingCodes: '码值候选', + findingDate: '业务日期', + relationVerified: '升为已验证', + relationKept: '保持推断', + relationConfirmed: '已人工确认，只补充覆盖率和基数', + relationLowCoverage: '覆盖率不足，可能不是这条关系', + relationNotUnique: '被指向列未核实唯一', + coverage: (pct: string) => `覆盖率 ${pct}`, + sampled: (sample: number, matched: number) => `抽样 ${formatNumber(sample)} 个键值，对上 ${formatNumber(matched)} 个`, + cardinalityUnknown: '基数未核实', + codeRows: (n: number) => `${formatNumber(n)} 行`, + codesSummary: (values: number, rows: number) => `${formatNumber(values)} 个取值，共 ${formatNumber(rows)} 行`, + codesPendingN: (n: number) => `${formatNumber(n)} 个含义待填写`, + fillMeanings: '填写含义', + fillMeaningsLabel: (col: string) => `填写含义：列 ${col} 的码值`, + dateProposal: (col: string) => `提议按 ${col} 作为业务日期`, + dateSpan: (min: string, max: string) => `取值从 ${min} 到 ${max}`, + dateRanges: '日期列的取值范围', + skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`, + moreSkipped: (n: number) => `…另有 ${formatNumber(n)} 项`, + tableError: '这张表未能剖析', + // ---- 无法开始（409） + blocked: { + disabled: '未开启数据剖析', + inactive: '数据源已停用', + noSchema: '还没有表结构', + busy: '这个数据源正在剖析', + tampered: '数据文件核对未通过', + other: '无法开始剖析', + } as Record, + blockedNext: { + disabled: '开启后即可在这里剖析。', + inactive: '启用后回到这里重试。', + noSchema: '探查完成后回到这里重试。', + busy: '同一个数据源同一时间只能进行一次剖析。', + tampered: '请重新上传表格，或联系管理员核对数据文件。', + other: '请稍后重试。', + } as Record, + goSource: '前往数据源', + retry: '重试', + failedTitle: '剖析未能完成', + backToSetup: '返回', +} + +/** 整次剖析中途停下的原因 */ +export const PROFILE_STOP_LABEL: Record = { + budget: '查询次数已用完', + deadline: '总时长已用完', + failed: '连续多条查询失败', +} + +/** 停下之后怎么办 */ +export const PROFILE_STOP_NEXT: Record = { + budget: '其余检查未执行。可在数据剖析设置中调高查询次数上限，或缩小剖析范围后重新剖析。', + deadline: '其余检查未执行。可在数据剖析设置中调高总时长上限，或缩小剖析范围后重新剖析。', + failed: '其余检查未执行。请先在数据源卡片上测试连接，确认可用后重新剖析。', +} + +/** 跳过一项的原因（skipped.reason）。整句说明在 skipped.detail 里 */ +export const PROFILE_SKIP_REASON_LABEL: Record = { + budget: '查询次数用完', + deadline: '总时长用完', + failed: '连续失败', + timeout: '查询超时', + error: '查询失败', + rejected: '未通过安全守卫', + masked: '列已遮罩', + too_large: '表太大', + view: '视图', + unsupported: '暂不支持', + no_data: '没有数据', + missing: '表结构中没有', + high_cardinality: '取值太多', +} + +/** 跳过的是哪一类检查（skipped.kind） */ +export const PROFILE_SKIP_KIND_LABEL: Record = { + relation: '关联关系', + codes: '码值', + date: '日期列', + row_estimate: '行数估算', + table: '整张表', +} + +/** 码值候选：剖析只知道出现过哪些取值，含义等人填 */ +export const CODES_TEXT = { + pending: '含义待填写', + pendingN: (n: number) => `${formatNumber(n)} 个含义待填写`, + fill: '填写含义', + fillLabel: (col: string) => `填写含义：列 ${col} 的码值`, + title: (col: string) => `列 ${col} 的码值含义`, + hint: '保存后这一项记为人工填写、已确认。暂不清楚的含义可以留空，之后再填。', + head: { code: '码值', meaning: '含义' }, + note: '来源说明', + placeholder: '含义待填写', + saved: (n: number) => (n ? `已填写 ${formatNumber(n)} 个码值的含义` : '已保存'), + save: '保存', + cancel: '取消', +} + +// =========================================================================== +// 基于数据目录的 SQL 检查（后端 data/sqlcheck.py） +// +// 术语： +// - 「SQL 检查」：对照数据目录找出能执行、数却可能不对的写法。七条规则，界面上写中文规则名，不露规则编号。 +// - 级别：错误（依据已确认，结果必然有误）、提醒（依据已确认，结果很可能有误）、提示（依据只是推断，仅供参考）。 +// 颜色走语义令牌：错误 --st-failed，提醒 --st-waiting，提示 --accent。 +// =========================================================================== + +export const SQL_CHECK_RULE_LABEL: Record = { + fanout_sum: '一对多关联后重复计算', + stock_summed: '存量跨期加总', + join_unconfirmed: '关联条件未经确认', + ratio_aggregated: '比率直接加总或平均', + missing_valid_filter: '未筛选有效记录', + unknown_code: '码值不在码值表中', + wrong_date_column: '未按业务日期统计', +} + +export const SQL_CHECK_RULE_HINT: Record = { + fanout_sum: '沿一对多方向关联后，对「一」那一侧的度量求和或计数，会按明细行数重复计算', + stock_summed: '对存量列求和却没有按业务日期分组，不同日期的存量被加在一起', + join_unconfirmed: '关联条件对不上数据目录中有确证的关系，可能连错了列', + ratio_aggregated: '对比率列直接求和或求平均，得不到正确的比率', + missing_valid_filter: '表定义了有效记录条件，查询没有按它筛选', + unknown_code: '按码值筛选时用了码值表中没有的值', + wrong_date_column: '表定义了业务日期，查询却按另一个时间列分组或筛选', +} + +export const SQL_CHECK_LEVEL_LABEL: Record = { + error: '错误', + warning: '提醒', + info: '提示', +} + +export const SQL_CHECK_LEVEL_HINT: Record = { + error: '依据已确认，结果必然有误', + warning: '依据已确认，结果很可能有误', + info: '依据尚未确认（推断），仅供参考', +} + +export const SQL_CHECK_TEXT = { + title: 'SQL 检查', + /** 规则编号不认识（服务端新加了规则）时的叫法 */ + unknownRule: 'SQL 检查', + listLabel: (n: number) => `SQL 检查发现 ${formatNumber(n)} 处问题`, + where: '涉及', + excerpt: 'SQL 片段', + catalogHint: '检查依据来自数据目录。依据有误时，请在数据目录中修正对应项。', + // ---- 运行时间线：指标所依据的查询没通过 SQL 检查（日志 metric_sql_check） + metricTitle: (name: string, n = 1) => + (n > 1 ? `指标「${name}」所依据的查询有 ${formatNumber(n)} 处未通过 SQL 检查` : `指标「${name}」所依据的查询未通过 SQL 检查`), + metricTitleAnon: '指标所依据的查询未通过 SQL 检查', + metricNext: '指标照常计算，但结果不可靠，出具按缺口降档。请在证据面板的查询步骤中查看问题，修改 SQL 后重新运行；' + + '检查依据有误时，请在数据目录中修正对应项', + // ---- 助手：搭图时查出的问题 + counts: (errors: number, warnings: number, infos: number) => + [errors ? `${formatNumber(errors)} 处错误` : '', warnings ? `${formatNumber(warnings)} 处提醒` : '', + infos ? `${formatNumber(infos)} 处提示` : ''].filter(Boolean).join('、'), + assistantTitle: (counts: string) => `SQL 检查：${counts}`, + assistantNext: '请打开对应节点，在「参数」中修改 SQL；检查依据有误时，请在数据目录中修正对应项', + assistantNextChat: '可在画布中打开此工作流，修改对应节点「参数」中的 SQL；检查依据有误时，请在数据目录中修正对应项', + assistantInfoNext: '这些检查的依据尚未确认，仅供参考。在数据目录中确认相关项后，检查结论更可靠', +} diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index de2cd9b1..3720cafc 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -2,7 +2,7 @@ import { useEffect, useId, useMemo, useRef, useState } from 'react' import { useNavigate } from 'react-router-dom' import { AlertTriangle, BookMarked, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, Lock, Plug, Plus, - RefreshCw, Search, Table2, Upload, X, + RefreshCw, ScanSearch, Search, Table2, Upload, X, } from 'lucide-react' import clsx from 'clsx' import { ApiError, api, uploadDecision } from '../api/client' @@ -21,9 +21,16 @@ import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } fro import type { HealthRecord } from '../lib/health' import { workflowList, workflowsMentioning } from '../lib/mentions' import { - CATALOG_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, VERSIONS_TEXT, + CATALOG_TEXT, PROFILE_TEXT, RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, + VERSIONS_TEXT, } from '../lib/terms' import { useRunClock } from '../run/useRunClock' +import { focusProfileField, ProfileSettingsDialog, ProfileSettingsSection } from './catalog/ProfileSettings' +import type { ProfileErrorKey } from './catalog/ProfileSettings' +import { + hasProfileOption, PROFILE_OPTION, profileFieldOfRejection, profileFormOf, profileOptionOf, profileProblems, profileSettingsOf, +} from './catalog/profile' +import type { ProfileForm } from './catalog/profile' import { ImportWizard } from './import/ImportWizard' import type { WizardEntry } from './import/ImportWizard' import { VersionsDialog } from './import/VersionsDialog' @@ -285,9 +292,11 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci // 传上来的表通常只有一两张，直接摊开；库动辄几十上百个对象，默认收着 const [open, setOpen] = useState(() => uploaded && row.table_count > 0 && row.table_count <= 3) const [masking, setMasking] = useState(false) + const [profiling, setProfiling] = useState(false) /** 版本页（当前版本的各期、历史版本、导入记录） */ const [versions, setVersions] = useState(false) const masks = maskList(row.options?.[MASK_KEY]) + const profile = profileSettingsOf(row.options) const clock = useRunClock(!!busy) const busySince = useRef(0) useTicker(60_000) @@ -525,12 +534,28 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci {masks.length ? '修改遮罩列' : '设置遮罩列'} )} + {profile.enabled && ( + + {PROFILE_TEXT.chip} + + )} + {uploaded && ( + // 数据剖析的设置同理：手工登记的库在编辑框里改 + + )} {!!row.tools?.length && {row.tools.join(' · ')}}
{masking && ( setMasking(false)} onSaved={(next) => { setMasking(false); onChange(next); syncCatalog() }} /> )} + {profiling && ( + setProfiling(false)} + onSaved={(next) => { setProfiling(false); onChange(next); syncCatalog() }} /> + )} {versions && ( // onChange 是父级的「更新这一行并刷新全局目录」：回滚之后表结构可能变了，检查器、问数据读的目录要跟上 setVersions(false)} onChange={onChange} onReload={onReload} /> @@ -940,11 +965,16 @@ interface SourceForm { sid: string /** options.mask_columns：证据面板展示查询原始行时遮掉的列 */ maskColumns: string[] - /** options 里除 schema / service_name / sid / mask_columns 之外的键，全部摊开可编辑 */ + /** options.catalog_profile：数据剖析的开关和预算 */ + profile: ProfileForm + /** 打开时 options 里已经有剖析设置：没开、也没填时也要照写（关掉之后写回 enabled: false） */ + profileStored: boolean + /** options 里除 schema / service_name / sid / mask_columns / catalog_profile 之外的键，全部摊开可编辑 */ advanced: [string, string][] } -const OWN_OPTION_KEYS = new Set(['schema', 'service_name', 'sid', MASK_KEY]) +// 剖析设置是一个对象，不能摊进「高级连接参数」：那里按文字回显，保存时会写成「[object Object]」被服务端拒收 +const OWN_OPTION_KEYS = new Set(['schema', 'service_name', 'sid', MASK_KEY, PROFILE_OPTION]) function toForm(source: any): SourceForm { const o = source.options ?? {} @@ -966,6 +996,8 @@ function toForm(source: any): SourceForm { serviceName: o.service_name ?? (oracle ? source.database ?? '' : ''), sid: o.sid ?? '', maskColumns: maskList(o[MASK_KEY]), + profile: profileFormOf(o), + profileStored: hasProfileOption(o), advanced: Object.entries(o) .filter(([k]) => !OWN_OPTION_KEYS.has(k)) .map(([k, v]) => [k, String(v ?? '')]), @@ -973,10 +1005,12 @@ function toForm(source: any): SourceForm { } function toBody(form: SourceForm, isNew: boolean, wasOracle = true): any { - const options: Record = {} + const options: Record = {} for (const [k, v] of form.advanced) if (k.trim() && v.trim()) options[k.trim()] = v.trim() if (form.schema.trim()) options.schema = form.schema.trim() if (form.maskColumns.length) options[MASK_KEY] = form.maskColumns + const profile = profileOptionOf(form.profile, form.profileStored) + if (profile) options[PROFILE_OPTION] = profile const oracle = form.kind === 'oracle' if (oracle) { // 只写选中的那一个，另一个不带：两处都有值时引擎听 service_name， @@ -1005,10 +1039,12 @@ function toBody(form: SourceForm, isNew: boolean, wasOracle = true): any { } /** - * 422 的中文 detail 说的是哪一项。眼下只有查询时限按字段拒（后端 query_timeout_problem）， - * 认不出的交回 toast + * 422 的中文 detail 说的是哪一项：查询时限（后端 query_timeout_problem）；数据剖析的某一项（profile_settings_problem， + * 记为 profile.<键>，它的「单条查询时限」也带「查询时限」四个字，要先认）。认不出的交回 toast */ function fieldOfRejection(message: string): string | null { + const profile = profileFieldOfRejection(message) + if (profile) return `profile.${profile}` if (/查询时限/.test(message)) return 'query_timeout_s' return null } @@ -1019,10 +1055,11 @@ const FIELD_LABEL: Record = { /** * 决定连到哪个库的那几项（提交体里的）。schema 不在内：它只影响探查哪一片，不影响 - * 连不连得上，后端判断测连接结果还作不作数时同样不看它；遮罩列只管面板上怎么显示，也不在内 + * 连不连得上，后端判断测连接结果还作不作数时同样不看它；遮罩列只管面板上怎么显示，数据剖析的设置只管剖析， + * 都不在内（服务端拼连接串时同样摘掉这几项） */ function connectionOf(body: any): string { - const { schema: _schema, [MASK_KEY]: _mask, ...options } = body.options ?? {} + const { schema: _schema, [MASK_KEY]: _mask, [PROFILE_OPTION]: _profile, ...options } = body.options ?? {} return JSON.stringify([ body.kind, body.host ?? null, body.port ?? null, body.database ?? null, body.username ?? null, options, !!body.password, @@ -1076,6 +1113,12 @@ function SourceEditor({ source, kinds, onClose, onSaved }: { } const required = (key: string) => needs.includes(key) const blocked = missing.length > 0 || !!nameError + // 剖析设置填错只拦保存，不拦测试连接（和连接无关） + const profileBad = Object.keys(profileProblems(form.profile)).length + const saveBlocked = blocked || profileBad > 0 + const saveHint = blocked ? `缺少必填项：${missing.join('、') || '标识格式'}` : profileBad ? PROFILE_TEXT.invalid(profileBad) : undefined + const profileError = fieldError?.key.startsWith('profile.') + ? { key: fieldError.key.slice('profile.'.length) as ProfileErrorKey, message: fieldError.message } : null const body = toBody(form, isNew, source.kind === 'oracle') const sig = JSON.stringify({ ...body, password: form.password }) const dirty = JSON.stringify(form) !== JSON.stringify(initial) @@ -1097,7 +1140,7 @@ function SourceEditor({ source, kinds, onClose, onSaved }: { } const save = async () => { - if (blocked) return + if (saveBlocked) return setSaving(true) setFieldError(null) try { @@ -1109,7 +1152,10 @@ function SourceEditor({ source, kinds, onClose, onSaved }: { } catch (e) { const rejected = e instanceof ApiError && e.status === 422 ? e.message : '' const key = rejected ? fieldOfRejection(rejected) : null - if (key) { + if (key?.startsWith('profile.')) { + setFieldError({ key, message: rejected }) + focusProfileField('ds-profile', key.slice('profile.'.length) as ProfileErrorKey) + } else if (key) { setFieldError({ key, message: rejected }) setAdvOpen(true) requestAnimationFrame(() => document.getElementById(`ds-adv-${key}`)?.focus()) @@ -1200,8 +1246,8 @@ function SourceEditor({ source, kinds, onClose, onSaved }: { title={blocked ? `缺少必填项：${missing.join('、') || '标识格式'}` : '使用当前配置测试连接，不保存'}> {test.since ? : } 测试连接 - @@ -1318,6 +1364,12 @@ function SourceEditor({ source, kinds, onClose, onSaved }: { set({ maskColumns })} /> + { + if (profileError) setFieldError(null) + set({ profile }) + }} /> +