数据剖析的关系验证:抽样键值去父表核对覆盖率,父键唯一且覆盖率够的升为有确证(来源记为数据剖析),覆盖率低的保持推断并写明原因,人工确认过的只补覆盖率和基数;撤销审阅回到剖析的结论

This commit is contained in:
ThearchyHelios
2026-10-03 10:56:08 +08:00
parent 32c3f4a180
commit fefb708757
3 changed files with 621 additions and 13 deletions
+35 -6
View File
@@ -19,9 +19,10 @@ notes 结构(每个「项」都是 {value, source, status},可选 note、upd
"relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage",
"source", "status", "note"?}]}
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(阶段 2)、llm 模型起草、
human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证(外键
约束;阶段 2 的数据剖析也写这个状态)、confirmed 人工确认、rejected 人工驳回。只有 confirmed 和 verified
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(data/catalog_profile.py)、
llm 模型起草、human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证
(外键约束;数据剖析核实了覆盖率和父键唯一的关系也是,见 profile_relation_holds)、confirmed 人工确认、
rejected 人工驳回。只有 confirmed 和 verified
将来会触发错误级检查;给模型看时 proposed 的项标「推断,未确认」;rejected 的项留着(防止下次起草又提出
来),但模型和检查都看不到。
@@ -98,6 +99,33 @@ def initial_status(source: str) -> str:
return _INITIAL_STATUS.get(source, "proposed")
#: 数据剖析把关系升为有确证的抽样覆盖率下限(data/catalog_profile.py 判定,撤销审阅时按它判回剖析的结论)
PROFILE_VERIFY_COVERAGE = 0.95
def profile_relation_holds(rel: Mapping[str, Any]) -> bool:
"""一条数据剖析得出的关系站不站得住:抽样覆盖率不低于 PROFILE_VERIFY_COVERAGE,而且被指向列核实唯一。
剖析只在被指向列核实唯一时才写基数(多对一或一对一),所以「基数有值」就是「父键唯一」。结论全在关系
自己的字段里,撤销审阅时照原样判得回来,不必另存一份「审阅前的状态」。
"""
coverage = rel.get("coverage")
return (rel.get("source") == "profile" and isinstance(coverage, (int, float)) and not isinstance(coverage, bool)
and coverage >= PROFILE_VERIFY_COVERAGE and rel.get("cardinality") in ("many_to_one", "one_to_one"))
def unreviewed_status(item: Mapping[str, Any]) -> str:
"""撤销审阅后回到的状态:来源的初始状态。
数据剖析是例外:它的状态随数据而定(覆盖率够、父键唯一的关系是有确证,其余是推断),不能一律回到
initial_status("profile") 的推断——那样人工确认过再撤销,有确证就无故丢了,要等下一次剖析才回来。
剖析提的码值候选、业务日期永远是推断。
"""
if item.get("source") == "profile" and "to_table" in item:
return "verified" if profile_relation_holds(item) else "proposed"
return initial_status(item.get("source"))
def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None,
at: str | None = None) -> dict[str, Any]:
"""构造一个项。status 不给时按来源取初始状态(initial_status)。"""
@@ -574,7 +602,7 @@ def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at:
# 人工填写的项没有「推断时的样子」可回:撤销就是删掉,下次起草可以重新提出
del slots[key]
return _assemble(slots)
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or initial_status(item.get("source"))
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or unreviewed_status(item)
item["updated_at"] = at or now_iso()
return _assemble(slots)
@@ -639,7 +667,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
continue
kept = dict(prev)
status = item.get("status")
if status != prev.get("status") and status in ("confirmed", "rejected", initial_status(prev.get("source"))):
if status != prev.get("status") and status in ("confirmed", "rejected", unreviewed_status(prev)):
kept["status"] = status
kept["updated_at"] = at
if item.get("note") != prev.get("note"):
@@ -1690,7 +1718,8 @@ __all__ = [
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
"MergeStats", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS",
"MergeStats", "PROFILE_VERIFY_COVERAGE", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS",
"TABLE_FIELD_LABEL", "TABLE_KINDS", "profile_relation_holds", "unreviewed_status",
"TableDraft", "TableDraftResult", "apply_human_edit", "describe_slot", "draft_catalog", "draft_structure",
"draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths",
"make_item", "merge_notes", "now_iso", "parse_path", "query_snapshot_meta", "read_catalog", "read_entry",
+535 -6
View File
@@ -18,19 +18,24 @@
"""
from __future__ import annotations
import copy
import logging
import re
import time
from dataclasses import asdict, dataclass
from dataclasses import asdict, dataclass, field
from decimal import Decimal
from typing import Any
from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite
from sqlalchemy.ext.asyncio import AsyncSession
from app.data import catalog, guard
from app.data import engine as data_engine
from app.data import guard
from app.data.engine import CATALOG_PROFILE_OPTION, QueryResult, SnapshotTampered
from app.data.guard import QueryLimits, SqlRejected
logger = logging.getLogger(__name__)
#: 数据源 options 里剖析设置的键
PROFILE_OPTION = CATALOG_PROFILE_OPTION
@@ -283,12 +288,17 @@ class _Stop(Exception):
class _Skip(Exception):
"""这一项没查成:timeout 超时、error 查询失败、rejected 未通过守卫。记下原因,接着查别的。"""
"""这一项没查成或不查:timeout 超时、error 查询失败、rejected 未通过守卫、masked 被遮罩……记下原因,接着查别的。
def __init__(self, reason: str, detail: str) -> None:
carry:目录里剖析以前对这一项写的结论要不要原样留着。没查成的(超时、表太大……)留着——这次没查不等于
结论错了;被遮罩的列上取过值的结论(码值、日期范围)不留:遮罩之后,带着原值的结论也不该再给人看。
"""
def __init__(self, reason: str, detail: str, *, carry: bool = True) -> None:
super().__init__(detail)
self.reason = reason
self.detail = detail
self.carry = carry
def _brief(e: BaseException) -> str:
@@ -424,5 +434,524 @@ async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any],
return TableSize(None, "count", at_least=cap), skips
__all__ = ["MAX_LITERAL_LEN", "PROFILE_OPTION", "ProfileSettings", "SqlDialect", "TableSize", "estimate_size",
"profile_settings", "profile_settings_problem", "sql_literal"]
# ==========================================================================
# 一次剖析:上下文、结果
# ==========================================================================
#: 不指定表时剖析几张:按使用次数取目录里有待核实关系的表
PROFILE_DEFAULT_TABLES = 10
#: IN 列表每批最多几个值:Oracle 的 IN 列表不能超过 1000 项(ORA-01795),各家统一按它分批
IN_CHUNK = 1000
#: 剖析写的备注都以它开头:人工确认过的关系,只覆盖剖析自己写的备注,不碰人写的
NOTE_PREFIX = "数据剖析("
#: 整次剖析停下的原因 → 剩下没查的项的说明
_STOP_DETAIL = {
"budget": "本次剖析的查询次数已用完(上限 {max_queries} 条),未检查",
"deadline": "本次剖析的总时长已用完(上限 {max_total_s} 秒),未检查",
"failed": "连续多条查询失败,已停止剖析,未检查",
}
class ProfileDisabled(Exception):
"""这个数据源没有开启数据剖析。message 给人看。"""
class ProfileBusy(Exception):
"""这个数据源正在剖析。message 给人看。"""
@dataclass
class ProfileSkip:
"""没做的一项和原因。
kind:relation 关系、codes 码值、date 日期列、row_estimate 行数估算、table 整张表。
reason:budget 查询次数用完、deadline 总时长用完、failed 连续失败、timeout 超时、error 查询失败、
rejected 未通过守卫、masked 字段被遮罩、too_large 表太大、view 视图、unsupported 暂不支持、
no_data 没有数据、missing 表结构里没有、high_cardinality 取值太多(不像码值)。detail 是给人看的整句。
"""
kind: str
target: str
reason: str
detail: str
path: str | None = None
def to_dict(self) -> dict[str, Any]:
return {"kind": self.kind, "target": self.target, "path": self.path, "reason": self.reason,
"detail": self.detail}
@dataclass
class TableProfile:
"""一张表这次剖析的结果。findings 是写进目录的发现(给界面看,形状见 _finding_*)。"""
table: str
queries: int = 0
row_estimate: TableSize | None = None
findings: list[dict[str, Any]] = field(default_factory=list)
skipped: list[ProfileSkip] = field(default_factory=list)
#: 日期类列的取值范围(MIN、MAX)。只有一个日期列时它还写进了业务日期的备注;多个时只在这里
date_ranges: list[dict[str, Any]] = field(default_factory=list)
added: int = 0
updated: int = 0
removed: int = 0
version: int = 0
#: 这张表没剖析(表结构里没有)或结果没写进去(写入一直冲突)
error: str | None = None
@dataclass
class ProfileReport:
"""一次剖析的结果。stopped:整次剖析中途停下的原因(budget / deadline / failed),做完了为 None。"""
profiled_at: str
settings: ProfileSettings
tables: list[TableProfile] = field(default_factory=list)
queries_used: int = 0
stopped: str | None = None
@dataclass
class _RelationOutcome:
"""一条关系核对完的结论。写不写、怎么写由 _apply 按写入时目录里那条关系的状态决定。"""
rid: str
target: str
coverage: float
cardinality: str | None
holds: bool
sample: int
matched: int
note: str
@dataclass
class _TableOutcome:
"""一张表所有检查的结论,以及没查成、要原样保留旧结论的槽位。"""
relations: list[_RelationOutcome] = field(default_factory=list)
#: 没查成的关系:目录里剖析以前写的结论原样留着(不然会被 covered={"profile"} 当成「这次没有」删掉)
keep_relations: set[str] = field(default_factory=set)
@dataclass
class _Context:
source: Any
settings: ProfileSettings
dialect: SqlDialect
run: _Runner
tables: dict[str, Any]
#: 遮罩的列(小写)。遮罩按列名生效、不分大小写、不分表,和证据面板的比法一致
masked: set[str]
#: 写进备注的剖析日期(UTC)
day: str
sizes: dict[str, TableSize] = field(default_factory=dict)
async def size(self, table: str, result: TableProfile) -> TableSize:
"""表有多大,一次剖析里每张表只估一次。途中没查成的记进 result(行数估算这一项)。"""
if table not in self.sizes:
size, skips = await estimate_size(self.run, self.dialect, self.tables[table], table,
max_scan_rows=self.settings.max_scan_rows)
self.sizes[table] = size
result.skipped += [ProfileSkip("row_estimate", table, s.reason, s.detail) for s in skips]
return self.sizes[table]
def is_masked(self, column: str) -> bool:
return column.lower() in self.masked
def scan_cap(self, size: TableSize) -> int | None:
"""大表(或不知道多大)抽样、取值分布只看前多少行;小表整表看,返回 None。"""
if size.within(self.settings.max_scan_rows):
return None
return max(self.settings.sample_size, self.settings.max_scan_rows)
def _column(meta: dict[str, Any], name: str) -> str | None:
"""表结构里这一列的写法(目录里的列名大小写可能和表结构不同);没有返回 None。"""
lowered = name.lower()
return next((str(c["name"]) for c in meta.get("columns") or []
if isinstance(c, dict) and str(c.get("name", "")).lower() == lowered), None)
def _constraint(meta: dict[str, Any], column: str) -> str | None:
"""这一列单独是主键或某个唯一约束时返回「主键」/「唯一约束」,否则 None。"""
wanted = [column.lower()]
if [c.lower() for c in meta.get("primary_key") or []] == wanted:
return "主键"
for group in meta.get("unique") or []:
if isinstance(group, list) and [str(c).lower() for c in group] == wanted:
return "唯一约束"
return None
def _percent(ratio: float) -> str:
"""覆盖率写成百分数:整数不带小数,其余留一位;不到 100% 的不四舍五入成 100%。"""
text = f"{ratio * 100:.1f}".rstrip("0").rstrip(".")
return "99.9%" if text == "100" and ratio < 1 else f"{text}%"
def _relation_target(rel: dict[str, Any]) -> str:
left = ", ".join(rel.get("columns") or [])
right = ", ".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or [])
return f"{left} → {right}"
# ==========================================================================
# 检查:关联关系
# ==========================================================================
def _relation_candidates(notes: dict[str, Any] | None) -> list[dict[str, Any]]:
"""要核对的关系:没被驳回、不是外键约束来的(外键由数据库保证,剖析也顶不掉它)。人工确认过的也核对,
只补覆盖率和基数。"""
return [r for r in (notes or {}).get("relations") or []
if isinstance(r, dict) and r.get("id") and r.get("status") != "rejected" and r.get("source") != "fk"]
async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str,
result: TableProfile) -> tuple[bool | None, str]:
"""这一列(非空部分)唯一吗:(True / False / None 不知道, 依据)。有主键或唯一约束直接认;否则只对行数不超过
整表统计上限的表数一次 COUNT 和 COUNT(DISTINCT),大表不扫。"""
how = _constraint(meta, column)
if how:
return True, how
size = await cx.size(table, result)
if not size.within(cx.settings.max_scan_rows):
return None, "too_large"
d = cx.dialect
res = await cx.run(d.unique_check_sql(d.table(meta, table), d.quote(column)))
row = res.rows[0] if res.rows else [None, None]
total, distinct = _as_int(row[0]), _as_int(row[1])
if total is None or distinct is None:
return None, "count"
return total == distinct, "count"
def _relation_note(day: str, *, sample: int, matched: int, coverage: float, scan_cap: int | None,
skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None,
holds: bool) -> str:
"""关系的剖析备注:日期、抽样规模、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。"""
parts: list[str] = []
low = coverage < catalog.PROFILE_VERIFY_COVERAGE
if low:
parts.append(f"抽样覆盖率 {_percent(coverage)},可能不是这条关系")
scope = f"(只看前 {scan_cap} 行)" if scan_cap else ""
sampled = f"子表抽样 {sample} 个不同键值{scope},父表对上 {matched} 个"
parts.append(sampled if low else f"{sampled},覆盖率 {_percent(coverage)}")
if skipped_values:
parts.append(f"另有 {skipped_values} 个键值的类型不支持核对,未计入")
unique, how = parent
if unique is True:
parts.append(f"被指向列是{how}" if how in ("主键", "唯一约束") else "被指向列经计数核对唯一")
elif unique is False:
parts.append("被指向列有重复值,不能作为关联的被指向键")
else:
parts.append("被指向的表超过整表统计的行数上限,未核对被指向列是否唯一")
if child is not None:
child_unique, child_how = child
if child_unique is True:
basis = child_how if child_how in ("主键", "唯一约束") else "计数核对"
parts.append(f"子表一侧也唯一({basis}),一对一")
elif child_unique is False:
parts.append("子表一侧有重复值,多对一")
else:
parts.append("子表超过整表统计的行数上限,未核对子表一侧是否唯一,按多对一记")
if not holds and not low:
parts.append("暂不升为有确证")
return f"{NOTE_PREFIX}{day}):" + ";".join(parts) + "。"
async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _TableOutcome,
result: TableProfile) -> None:
"""从子表取至多 sample_size 个不同的非空键值,去父表里数对上几个,得到覆盖率;再看父键、子键唯不唯一。"""
cols, to_table, to_cols = list(rel.get("columns") or []), str(rel.get("to_table") or ""), \
list(rel.get("to_columns") or [])
if len(cols) != 1 or len(to_cols) != 1:
raise _Skip("unsupported", "多列组成的关联关系暂不剖析")
meta = cx.tables[table]
parent = to_table if to_table in cx.tables else catalog.resolve_table_name({"tables": cx.tables}, to_table)
if parent is None:
raise _Skip("missing", f"表结构里没有被指向的表 {to_table},请先重新探查结构")
pmeta = cx.tables[parent]
col, pcol = _column(meta, cols[0]), _column(pmeta, to_cols[0])
if col is None or pcol is None:
missing = f"{table}.{cols[0]}" if col is None else f"{parent}.{to_cols[0]}"
raise _Skip("missing", f"表结构里没有列 {missing},请先重新探查结构")
if pmeta.get("is_view"):
raise _Skip("view", f"被指向的 {parent} 是视图,不剖析")
for name, owner in ((col, table), (pcol, parent)):
if cx.is_masked(name):
raise _Skip("masked", f"{owner}.{name} 在数据源设置中被遮罩,不取样", carry=True)
d, s = cx.dialect, cx.settings
size = await cx.size(table, result)
scan_cap = cx.scan_cap(size)
child_table, parent_table = d.table(meta, table), d.table(pmeta, parent)
res = await cx.run(d.distinct_sample_sql(child_table, d.quote(col), s.sample_size, scan_cap=scan_cap),
max_rows=s.sample_size)
kind = (res.column_types or {}).get(res.columns[0]) if res.columns else None
values = [row[0] for row in res.rows if row]
literals = list(dict.fromkeys(lit for v in values if (lit := sql_literal(v, kind)) is not None))
if not values:
raise _Skip("no_data", f"{table}.{col} 没有非空值,无法核对")
if not literals:
raise _Skip("unsupported", f"{table}.{col} 的取值类型不支持核对(只核对整数和文字)")
matched = 0
for start in range(0, len(literals), IN_CHUNK):
sql = d.match_count_sql(parent_table, d.quote(pcol), literals[start:start + IN_CHUNK])
matched += _as_int(_first(await cx.run(sql))) or 0
matched = min(matched, len(literals))
coverage = round(matched / len(literals), 4)
parent_unique = await _unique(cx, parent, pmeta, pcol, result)
child_unique = await _unique(cx, table, meta, col, result) if parent_unique[0] else None
cardinality = None
if child_unique is not None:
cardinality = "one_to_one" if child_unique[0] is True else "many_to_one"
holds = coverage >= catalog.PROFILE_VERIFY_COVERAGE and cardinality is not None
note = _relation_note(cx.day, sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap,
skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique,
holds=holds)
out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=coverage,
cardinality=cardinality, holds=holds, sample=len(literals),
matched=matched, note=note))
# ==========================================================================
# 结论进目录
# ==========================================================================
def _relation_finding(o: _RelationOutcome, rel: dict[str, Any], *, status: str, confirmed: bool) -> dict[str, Any]:
if confirmed:
summary = "已人工确认,只补充覆盖率和基数"
elif status == "verified":
summary = f"覆盖率 {_percent(o.coverage)},升为有确证"
elif o.coverage < catalog.PROFILE_VERIFY_COVERAGE:
summary = f"抽样覆盖率 {_percent(o.coverage)},保持推断"
else:
summary = "被指向列未核实唯一,保持推断"
return {"kind": "relation", "path": f"relations.{o.rid}", "target": o.target,
"columns": list(rel.get("columns") or []), "to_table": rel.get("to_table"),
"to_columns": list(rel.get("to_columns") or []), "status": status, "confirmed": confirmed,
"coverage": o.coverage, "cardinality": o.cardinality, "sample": o.sample, "matched": o.matched,
"summary": summary}
def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool:
"""人工确认过的关系只补覆盖率和基数:覆盖率是测量值,照新的写;基数只在原来没有时补,不改人定的;
备注只在原来为空、或者是剖析自己写的时候换成新的。状态、来源不动。返回改没改。"""
changed = False
if rel.get("coverage") != o.coverage:
rel["coverage"] = o.coverage
changed = True
if rel.get("cardinality") is None and o.cardinality is not None:
rel["cardinality"] = o.cardinality
changed = True
note = rel.get("note")
if (not note or str(note).startswith(NOTE_PREFIX)) and note != o.note:
rel["note"] = o.note
changed = True
if changed:
rel["updated_at"] = at
return changed
def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *,
at: str) -> tuple[dict[str, Any], list[dict[str, Any]], catalog.MergeStats]:
"""把一张表的结论并进写入时的目录:(新目录, 写进去的发现, 计数)。纯函数,写入冲突重来时再调一次。
- 没被确认、驳回的关系:记为 source=profile,覆盖率够、父键唯一的 verified,否则 proposed;按 merge_notes
的规则并入(剖析顶得掉命名推断、模型起草,顶不掉外键和人工)。
- 人工确认过的关系:只补覆盖率和基数(_patch_confirmed)。
- covered={"profile"}:这次完整检查过、却没再得出的剖析结论删掉(比如取值变多、不再像码值的列);
没查成的检查(超时、预算用完……)把剖析以前写的结论原样带上,不因为这次没查成就删。
"""
existing = existing or {}
current = {str(r["id"]): r for r in existing.get("relations") or [] if isinstance(r, dict) and r.get("id")}
relations: list[dict[str, Any]] = []
confirmed: dict[str, _RelationOutcome] = {}
findings: list[dict[str, Any]] = []
for o in out.relations:
rel = current.get(o.rid)
if rel is None or rel.get("status") == "rejected" or rel.get("source") == "fk":
continue # 核对期间被删、被驳回了:不写
if rel.get("status") == "confirmed":
confirmed[o.rid] = o
findings.append(_relation_finding(o, rel, status="confirmed", confirmed=True))
continue
status = "verified" if o.holds else "proposed"
relations.append({"id": o.rid, "columns": list(rel["columns"]), "to_table": rel["to_table"],
"to_columns": list(rel["to_columns"]), "cardinality": o.cardinality,
"coverage": o.coverage, "source": "profile", "status": status, "note": o.note})
findings.append(_relation_finding(o, rel, status=status, confirmed=False))
for rid in out.keep_relations:
rel = current.get(rid)
if rel is not None and rel.get("source") == "profile":
relations.append(copy.deepcopy(rel))
draft: dict[str, Any] = {"relations": relations} if relations else {}
merged, stats = catalog.merge_notes(existing, draft, covered={"profile"}, at=at)
for rel in merged.get("relations") or []:
o = confirmed.get(str(rel.get("id")))
if o is not None and rel.get("status") == "confirmed" and _patch_confirmed(rel, o, at):
stats += catalog.MergeStats(updated=1)
return merged, findings, stats
# ==========================================================================
# 入口
# ==========================================================================
#: 正在剖析的数据源。检查和登记之间没有 await,在一个事件循环里是原子的;服务端是单进程部署,
#: 多进程部署时要换成库里的锁
_RUNNING: set[str] = set()
@dataclass
class _Check:
kind: str
target: str
path: str | None
run: Any
#: 没查成时要原样保留的旧结论(关系编号)
keep_relation: str | None = None
def _plan(cx: _Context, table: str, notes: dict[str, Any], out: _TableOutcome,
result: TableProfile) -> list[_Check]:
"""这张表要做的检查,按价值排:关系、码值、日期。先列全再执行:中途停下时,没做的每一项都要记下原因。"""
checks: list[_Check] = []
for rel in _relation_candidates(notes):
checks.append(_Check("relation", _relation_target(rel), f"relations.{rel['id']}",
lambda rel=rel: _check_relation(cx, table, rel, out, result),
keep_relation=str(rel["id"])))
return checks
async def _profile_table(cx: _Context, table: str, notes: dict[str, Any], result: TableProfile) -> _TableOutcome:
out = _TableOutcome()
if cx.tables[table].get("is_view"):
result.skipped.append(ProfileSkip("table", table, "view", "视图没有统计信息,剖析可能触发整个视图的计算,已跳过"))
return out
checks = _plan(cx, table, notes, out, result)
for i, check in enumerate(checks):
try:
await check.run()
except _Skip as e:
result.skipped.append(ProfileSkip(check.kind, check.target, e.reason, e.detail, check.path))
if check.keep_relation and e.carry:
out.keep_relations.add(check.keep_relation)
except _Stop as e:
detail = _STOP_DETAIL[e.reason].format(max_queries=cx.settings.max_queries,
max_total_s=_fmt(cx.settings.max_total_s))
for rest in checks[i:]:
result.skipped.append(ProfileSkip(rest.kind, rest.target, e.reason, detail, rest.path))
if rest.keep_relation:
out.keep_relations.add(rest.keep_relation)
break
if table in cx.sizes:
result.row_estimate = cx.sizes[table]
return out
async def _write(session: AsyncSession, source_id: str, table: str, out: _TableOutcome, result: TableProfile, *,
at: str, actor: str | None) -> None:
"""并入并写入(乐观锁)。撞上别人刚改过就重读重并,最多三次。"""
for _attempt in range(3):
entry = await catalog.read_entry(session, source_id, table)
version = entry.version if entry else 0
notes, findings, stats = _apply(entry.notes if entry else {}, out, at=at)
result.findings = findings
result.added, result.updated, result.removed = stats.added, stats.updated, stats.removed
result.version = version
if entry is None and not notes:
return # 什么也没得出,不建空目录
try:
written = await catalog.write_entry(session, source_id, table, notes, if_version=version, actor=actor)
except catalog.CatalogConflict:
continue
result.version = written.version
return
result.error = "这张表的数据目录正被其他人修改,剖析结果未能写入,请稍后重新剖析"
result.findings = []
result.added = result.updated = result.removed = 0
async def _pick_tables(session: AsyncSession, source: Any, tables: list[str] | None,
entries: dict[str, catalog.CatalogEntry], report: ProfileReport) -> list[str]:
cache = getattr(source, "schema_cache", None) or {}
all_tables = cache.get("tables") or {}
if tables:
picked: list[str] = []
for name in dict.fromkeys(str(t) for t in tables):
key = catalog.resolve_table_name(cache, name)
if key is None:
report.tables.append(TableProfile(table=name, error=f"表结构里没有 {name},请先重新探查结构"))
elif key not in picked:
picked.append(key)
return picked
usage = await catalog.table_usage(session, source)
order = {name: i for i, name in enumerate(all_tables)}
candidates = [name for name, meta in all_tables.items()
if not (meta or {}).get("is_view") and name in entries
and _relation_candidates(entries[name].notes)]
return sorted(candidates, key=lambda n: (-usage.get(n, 0), order[n]))[:PROFILE_DEFAULT_TABLES]
async def profile_catalog(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None = None,
actor: str | None = None, settings: ProfileSettings | None = None) -> ProfileReport:
"""剖析这些表并把结论写进数据目录,同步完成。
- row:数据源记录(开关、遮罩按它的 options);source:用来查询和读表结构的源(上传源是绑定当前快照的视图)。
- settings 不给就按 row.options 读;没开启抛 ProfileDisabled,这个源正在剖析抛 ProfileBusy。
- tables 为空时按使用次数取前 PROFILE_DEFAULT_TABLES 张目录里有待核实关系的表。
- 每张表做完就写入,后面的表停下(预算、总时长)不影响前面已经写进去的。
"""
settings = settings or profile_settings(getattr(row, "options", None))
name = getattr(row, "name", "") or ""
if not settings.enabled:
raise ProfileDisabled(f"数据源「{name}」未开启数据剖析。剖析会对业务库发查询,请先在数据源设置中开启数据剖析")
if source.id in _RUNNING:
raise ProfileBusy(f"数据源「{name}」正在进行数据剖析,请等待完成后再试")
_RUNNING.add(source.id)
try:
return await _profile(session, row, source, tables, actor, settings)
finally:
_RUNNING.discard(source.id)
async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[str] | None, actor: str | None,
settings: ProfileSettings) -> ProfileReport:
at = catalog.now_iso()
report = ProfileReport(profiled_at=at, settings=settings)
dialect = SqlDialect(source.kind)
run = _Runner(source, settings, dialect)
cx = _Context(source=source, settings=settings, dialect=dialect, run=run,
tables=(getattr(source, "schema_cache", None) or {}).get("tables") or {},
masked={c.lower() for c in data_engine.masked_columns(getattr(row, "options", None))},
day=at[:10])
entries = await catalog.read_catalog(session, source.id)
for table in await _pick_tables(session, source, tables, entries, report):
result = TableProfile(table=table)
report.tables.append(result)
if run.stopped:
detail = _STOP_DETAIL[run.stopped].format(max_queries=settings.max_queries,
max_total_s=_fmt(settings.max_total_s))
result.skipped.append(ProfileSkip("table", table, run.stopped, detail))
continue
before = run.used
entry = entries.get(table)
out = await _profile_table(cx, table, entry.notes if entry else {}, result)
result.queries = run.used - before
await _write(session, source.id, table, out, result, at=at, actor=actor)
report.queries_used = run.used
report.stopped = run.stopped
logger.info("数据剖析 源=%s 表=%s 查询=%d 停止=%s 署名=%s", source.id,
",".join(t.table for t in report.tables), run.used, run.stopped, actor)
return report
__all__ = ["IN_CHUNK", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", "PROFILE_OPTION", "ProfileBusy",
"ProfileDisabled", "ProfileReport", "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile",
"TableSize", "estimate_size", "profile_catalog", "profile_settings", "profile_settings_problem",
"sql_literal"]
+51 -1
View File
@@ -6,10 +6,14 @@ from types import SimpleNamespace
import pytest
from app.data import catalog_profile, guard
from app.data import catalog, catalog_profile, guard
from app.data.catalog import make_item
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
from app.data.engine import build_url
AT = "2026-10-03T08:00:00+00:00"
# ---------------------------------------------------------------- 设置
@@ -143,3 +147,49 @@ def test_dialect_quoting_and_row_limits():
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
stats = ora.stats_sql(_META["orders"], "orders")
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
# ---------------------------------------------------------------- 目录对剖析结果的容纳
def _profile_rel(coverage, cardinality, status):
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
"source": "profile", "status": status}
def test_reset_returns_profile_relation_to_what_profiling_concluded():
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定:覆盖率够、父键唯一(基数有值)就是有确证。"""
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
rid = verified["relations"][0]["id"]
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
assert out["relations"][0]["status"] == "confirmed"
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "verified"
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "verified"
# 覆盖率低、父键没核实唯一的:撤销后仍是推断
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "proposed"
# 剖析提的码值候选、业务日期永远是推断
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
"columns.status.codes", "reset", at=AT)
assert out["columns"]["status"]["codes"]["status"] == "proposed"
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
rel = _profile_rel(1.0, "many_to_one", "confirmed")
existing = {"relations": [rel]}
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
low = _profile_rel(0.4, "many_to_one", "proposed")
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
table="visits", at=AT)
assert kept["relations"][0]["status"] == "proposed"