mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 06:02:01 +00:00
合入集成分支（阶段 2A 数据剖析、4B 剖析与 SQL 检查界面及修正）到 4A：两边功能都保留
冲突的解法： - 后端 api/catalog.py：同时导入 catalog_impact 和 catalog_profile、SnapshotTampered；data/catalog.py：__all__ 取两边的并集重排。 - 2A 放宽了码值校验（含义可以为空，剖析写进来的是「含义待填写」）：目录修改提案的码值补充改为含义为空的码不覆盖已有含义、 只在原来没有这个码时加进去，补了测试；卡片和过程里的码值把空含义写成「含义待填写」，和目录页同一个说法。 - 前端 types.ts、terms.ts：两边各在同一处追加了新段落，按顺序都留下（提案、影响面、目录版本在前，剖析、码值、SQL 检查在后）， 没有重复的键；client.ts、decode.ts、CatalogPage.tsx、TableDetail.tsx 的导入取并集。 - check-catalog.mjs：影响面和剖析、保存数据源三条假路由都留下，各节和最后核对真库数量的逻辑不变。 - 4A 的测试数据源改用 tests/fixtures/sources.py 的 drop_source 删除，顺带作废连接池。
This commit is contained in:
ThearchyHelios
@@ -8,9 +8,10 @@
|
||||
- POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认
|
||||
- GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py）
|
||||
- POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）
|
||||
- POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py）
|
||||
|
||||
规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径
|
||||
不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。
|
||||
不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。
|
||||
|
||||
上传表格的源按当前快照的表结构回答（table_versions.resolve_source），和查询实际用的库是同一个版本。
|
||||
"""
|
||||
@@ -23,7 +24,8 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.runs import actor_of
|
||||
from app.data import catalog, catalog_impact, introspect, table_versions
|
||||
from app.data import catalog, catalog_impact, catalog_profile, introspect, table_versions
|
||||
from app.data.engine import SnapshotTampered
|
||||
from app.db.base import get_session
|
||||
from app.db.models import DataSource
|
||||
|
||||
@@ -65,6 +67,11 @@ class CatalogDraftIn(BaseModel):
|
||||
use_model: bool = False
|
||||
|
||||
|
||||
class CatalogProfileIn(BaseModel):
|
||||
#: 要剖析的表；不给或为空时按使用次数取前 10 张目录里有待核实关系的表
|
||||
tables: list[str] | None = Field(default=None, max_length=200)
|
||||
|
||||
|
||||
async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]:
|
||||
"""(数据源记录, 用来读表结构的源)：上传源解析成绑定当前快照的视图。"""
|
||||
row = await session.get(DataSource, source_id)
|
||||
@@ -227,6 +234,62 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
|
||||
}
|
||||
|
||||
|
||||
def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]:
|
||||
return {
|
||||
"table_name": t.table,
|
||||
"queries": t.queries,
|
||||
"row_estimate": t.row_estimate.to_dict() if t.row_estimate else None,
|
||||
"findings": t.findings,
|
||||
"skipped": [s.to_dict() for s in t.skipped],
|
||||
"date_ranges": t.date_ranges,
|
||||
"added": t.added,
|
||||
"updated": t.updated,
|
||||
"removed": t.removed,
|
||||
"version": t.version,
|
||||
"error": t.error,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/profile")
|
||||
async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。
|
||||
|
||||
默认关闭：数据源设置里没开启数据剖析、数据源停用，409；同一个数据源已经在剖析，409。
|
||||
"""
|
||||
row, source = await _resolved(session, source_id)
|
||||
try:
|
||||
catalog_profile.ensure_enabled(row)
|
||||
except catalog_profile.ProfileDisabled as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
cache = source.schema_cache or {}
|
||||
if not cache.get("tables"):
|
||||
raise HTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」")
|
||||
actor = actor_of(x_actor)
|
||||
try:
|
||||
report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None,
|
||||
actor=actor)
|
||||
except (catalog_profile.ProfileDisabled, catalog_profile.ProfileBusy) as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
except SnapshotTampered as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
rows = [_profile_table_out(t) for t in report.tables]
|
||||
note = None
|
||||
if not rows:
|
||||
note = "目录里还没有待核实的关联关系，请先起草数据目录，或指定要剖析的表"
|
||||
return {
|
||||
"profiled_at": report.profiled_at,
|
||||
"actor": actor,
|
||||
"settings": report.settings.to_dict(),
|
||||
"queries_used": report.queries_used,
|
||||
"stopped": report.stopped,
|
||||
"tables": rows,
|
||||
"total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")},
|
||||
"note": note,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/{table}/review")
|
||||
async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw
|
||||
from app.core.config import settings
|
||||
from app.core.crypto import encrypt, mask
|
||||
from app.data import catalog as data_catalog
|
||||
from app.data.catalog_profile import profile_settings_problem
|
||||
from app.data import introspect as introspect_mod
|
||||
from app.data import table_versions
|
||||
from app.data.engine import (
|
||||
@@ -336,7 +337,8 @@ _MASK_FIELD = {
|
||||
|
||||
|
||||
def _refuse_bad_options(options: dict[str, Any] | None) -> None:
|
||||
problem = query_timeout_problem(options) or mask_columns_problem(options)
|
||||
problem = (query_timeout_problem(options) or mask_columns_problem(options)
|
||||
or profile_settings_problem(options))
|
||||
if problem:
|
||||
raise HTTPException(422, problem)
|
||||
|
||||
|
||||
+80
-18
@@ -15,13 +15,14 @@ notes 结构（每个「项」都是 {value, source, status}，可选 note、upd
|
||||
"valid_filter": 项(str，SQL 条件片段), "dedup": 项(str),
|
||||
"columns": {列名: {"label": 项, "meaning": 项, "unit": 项,
|
||||
"measure": 项("flow"|"stock"|"ratio"|"identifier"|"status"|"attribute"),
|
||||
"codes": 项(dict[str, str])}},
|
||||
"codes": 项(dict[码值, 含义]；含义可为空串，表示待填写)}},
|
||||
"relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage",
|
||||
"source", "status", "note"?}]}
|
||||
|
||||
来源（source）：comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析（阶段 2）、llm 模型起草、
|
||||
human 人工填写。状态（status）：proposed 推断（命名推断、模型起草、数据库注释）、verified 有确证（外键
|
||||
约束；阶段 2 的数据剖析也写这个状态）、confirmed 人工确认、rejected 人工驳回。只有 confirmed 和 verified
|
||||
来源（source）：comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析（data/catalog_profile.py）、
|
||||
llm 模型起草、human 人工填写。状态（status）：proposed 推断（命名推断、模型起草、数据库注释）、verified 有确证
|
||||
（外键约束；数据剖析核实了覆盖率和父键唯一的关系也是，见 profile_relation_holds）、confirmed 人工确认、
|
||||
rejected 人工驳回。只有 confirmed 和 verified
|
||||
将来会触发错误级检查；给模型看时 proposed 的项标「推断，未确认」；rejected 的项留着（防止下次起草又提出
|
||||
来），但模型和检查都看不到。
|
||||
|
||||
@@ -98,6 +99,33 @@ def initial_status(source: str) -> str:
|
||||
return _INITIAL_STATUS.get(source, "proposed")
|
||||
|
||||
|
||||
#: 数据剖析把关系升为有确证的抽样覆盖率下限（data/catalog_profile.py 判定，撤销审阅时按它判回剖析的结论）
|
||||
PROFILE_VERIFY_COVERAGE = 0.95
|
||||
|
||||
|
||||
def profile_relation_holds(rel: Mapping[str, Any]) -> bool:
|
||||
"""一条数据剖析得出的关系站不站得住：抽样覆盖率不低于 PROFILE_VERIFY_COVERAGE，而且被指向列核实唯一。
|
||||
|
||||
剖析只在被指向列核实唯一时才写基数（多对一或一对一），所以「基数有值」就是「父键唯一」。结论全在关系
|
||||
自己的字段里，撤销审阅时照原样判得回来，不必另存一份「审阅前的状态」。
|
||||
"""
|
||||
coverage = rel.get("coverage")
|
||||
return (rel.get("source") == "profile" and isinstance(coverage, (int, float)) and not isinstance(coverage, bool)
|
||||
and coverage >= PROFILE_VERIFY_COVERAGE and rel.get("cardinality") in ("many_to_one", "one_to_one"))
|
||||
|
||||
|
||||
def unreviewed_status(item: Mapping[str, Any]) -> str:
|
||||
"""撤销审阅后回到的状态：来源的初始状态。
|
||||
|
||||
数据剖析是例外：它的状态随数据而定（覆盖率够、父键唯一的关系是有确证，其余是推断），不能一律回到
|
||||
initial_status("profile") 的推断——那样人工确认过再撤销，有确证就无故丢了，要等下一次剖析才回来。
|
||||
剖析提的码值候选、业务日期永远是推断。
|
||||
"""
|
||||
if item.get("source") == "profile" and "to_table" in item:
|
||||
return "verified" if profile_relation_holds(item) else "proposed"
|
||||
return initial_status(item.get("source"))
|
||||
|
||||
|
||||
def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None,
|
||||
at: str | None = None) -> dict[str, Any]:
|
||||
"""构造一个项。status 不给时按来源取初始状态（initial_status）。"""
|
||||
@@ -153,8 +181,10 @@ def _business_date_ok(value: Any) -> bool:
|
||||
|
||||
|
||||
def _codes_ok(value: Any) -> bool:
|
||||
"""码值对照：{码值: 含义}，都是文字。含义可以是空串——数据剖析只知道列里出现过哪些取值，含义要等人填；
|
||||
以前要求含义非空，剖析得到的码值候选就写不进来。"""
|
||||
return (isinstance(value, dict) and bool(value)
|
||||
and all(isinstance(k, str) and isinstance(v, str) and v.strip() for k, v in value.items()))
|
||||
and all(isinstance(k, str) and isinstance(v, str) for k, v in value.items()))
|
||||
|
||||
|
||||
#: 表类型、度量类型在界面上的叫法（和 frontend/src/lib/terms.ts 的 CATALOG_KIND_LABEL / CATALOG_MEASURE_LABEL 一致）。
|
||||
@@ -587,7 +617,7 @@ def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at:
|
||||
# 人工填写的项没有「推断时的样子」可回：撤销就是删掉，下次起草可以重新提出
|
||||
del slots[key]
|
||||
return _assemble(slots)
|
||||
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or initial_status(item.get("source"))
|
||||
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or unreviewed_status(item)
|
||||
item["updated_at"] = at or now_iso()
|
||||
return _assemble(slots)
|
||||
|
||||
@@ -652,7 +682,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
|
||||
continue
|
||||
kept = dict(prev)
|
||||
status = item.get("status")
|
||||
if status != prev.get("status") and status in ("confirmed", "rejected", initial_status(prev.get("source"))):
|
||||
if status != prev.get("status") and status in ("confirmed", "rejected", unreviewed_status(prev)):
|
||||
kept["status"] = status
|
||||
kept["updated_at"] = at
|
||||
if item.get("note") != prev.get("note"):
|
||||
@@ -779,6 +809,17 @@ def _patch_relation(table: str, raw: Any, path_id: str | None, tables: Mapping[s
|
||||
return value, None
|
||||
|
||||
|
||||
def _merge_codes(before: Mapping[str, str], value: Mapping[str, str]) -> dict[str, str]:
|
||||
"""码值的补充：给出的码加进去或改掉原来的含义。含义为空表示「只知道有这个码、含义待填写」（数据剖析的码值
|
||||
候选就是这样），所以提案里含义为空的码不覆盖已有的含义，只在原来没有这个码时加进去。"""
|
||||
out = dict(before)
|
||||
for code, meaning in value.items():
|
||||
if not str(meaning).strip() and str(out.get(code) or "").strip():
|
||||
continue
|
||||
out[code] = meaning
|
||||
return out
|
||||
|
||||
|
||||
def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
|
||||
tables: Mapping[str, Any] | None = None) -> PatchPlan:
|
||||
"""逐项核对一条目录修改提案，算出改前、改后。不改入参、不抛异常：不合法的项写进 problems。
|
||||
@@ -790,7 +831,8 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
|
||||
规则：
|
||||
- 取值格式和 validate_notes 同一套（_TABLE_VALUE_RULES / _COLUMN_VALUE_RULES / _relation_problems）。
|
||||
- 码值是补充：在现有（没被驳回的）码值上加或改给出的码，不整份替换——「status=9 表示作废」不该把已确认的
|
||||
其余码值删掉。
|
||||
其余码值删掉。含义为空的码（数据剖析写进来的「含义待填写」）可以被补上含义；提案里含义为空的码只在原来
|
||||
没有这个码时加进去，不会把已有的含义抹掉（_merge_codes）。
|
||||
- 看起来像计算公式的文字不收（_FORMULA）：公式进口径卡。
|
||||
- 同一个路径出现两次只认第一次。
|
||||
"""
|
||||
@@ -864,7 +906,7 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
|
||||
before = _value_of(key, live) if live is not None else None
|
||||
after = value
|
||||
if key[0] == "c" and key[2] == "codes" and isinstance(before, dict):
|
||||
after = {**before, **value}
|
||||
after = _merge_codes(before, value)
|
||||
if before is not None and before == after:
|
||||
state = "same" if live.get("status") == "confirmed" else "confirm"
|
||||
else:
|
||||
@@ -1094,6 +1136,15 @@ def infer_name_relations(schema_cache: Mapping[str, Any] | None, table: str, *,
|
||||
candidates: set[str] = set()
|
||||
for form in _singulars(words[-1]):
|
||||
candidates |= index.get((prefix, form), set())
|
||||
if not candidates:
|
||||
# 直接对不上：库里的表常带业务前缀（store_tags），列名却只写词根（tagId）。
|
||||
# 只认和引用它的表共用前缀的那张，从最长的共用前缀试起；仍然不止一张就不推
|
||||
own = _words(table)
|
||||
for k in range(len(own) - 1, 0, -1):
|
||||
for form in _singulars(words[-1]):
|
||||
candidates |= index.get(("".join(own[:k]) + prefix, form), set())
|
||||
if candidates:
|
||||
break
|
||||
if len(candidates) != 1:
|
||||
continue
|
||||
target = next(iter(candidates))
|
||||
@@ -1625,6 +1676,17 @@ def _date_text(value: Mapping[str, Any]) -> str:
|
||||
return "；".join(parts)
|
||||
|
||||
|
||||
def _codes_text(codes: Mapping[str, Any]) -> str:
|
||||
"""码值对照的文字：有含义的写「1=有效」，含义待填写的单独说明——不能写成「1=」，模型会当成含义就是空。"""
|
||||
filled = [f"{k}={v}" for k, v in codes.items() if str(v).strip()]
|
||||
pending = [str(k) for k, v in codes.items() if not str(v).strip()]
|
||||
if not pending:
|
||||
return "、".join(filled)
|
||||
if not filled:
|
||||
return f"{'、'.join(pending)}（含义待填写）"
|
||||
return f"{'、'.join(filled)}，{'、'.join(pending)} 的含义待填写"
|
||||
|
||||
|
||||
def _relation_text(rel: Mapping[str, Any]) -> str:
|
||||
left = ", ".join(rel.get("columns") or [])
|
||||
right = ", ".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or [])
|
||||
@@ -1675,7 +1737,7 @@ def _notes_lines(notes: Mapping[str, Any] | None, *, meta: Mapping[str, Any] | N
|
||||
if name == "measure":
|
||||
text = MEASURE_LABEL.get(value, str(value))
|
||||
elif name == "codes":
|
||||
text = "、".join(f"{k}={v}" for k, v in value.items())
|
||||
text = _codes_text(value)
|
||||
else:
|
||||
text = str(value)
|
||||
# 第一段是中文名（没有中文名时是含义），不加前缀；其余写明是什么
|
||||
@@ -1932,13 +1994,13 @@ __all__ = [
|
||||
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
|
||||
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
|
||||
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
|
||||
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND",
|
||||
"REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult",
|
||||
"UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot", "draft_catalog",
|
||||
"draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations",
|
||||
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_VERIFY_COVERAGE", "PatchChange", "PatchPlan",
|
||||
"QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft",
|
||||
"TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot",
|
||||
"draft_catalog", "draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations",
|
||||
"initial_status", "join_paths", "make_item", "merge_notes", "now_iso", "parse_path", "plan_patch",
|
||||
"query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index",
|
||||
"render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes",
|
||||
"save_human_edit", "save_patch", "status_counts", "system_notes_source", "table_usage", "validate_notes",
|
||||
"visible_notes", "write_entry",
|
||||
"profile_relation_holds", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id",
|
||||
"render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry",
|
||||
"review_item", "same_notes", "save_human_edit", "save_patch", "status_counts", "system_notes_source",
|
||||
"table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry",
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -58,11 +58,16 @@ MAX_QUERY_TIMEOUT_S = 600
|
||||
#: 完整快照仍能按工件 id 取到，SQL 里给列起个别名也能绕开
|
||||
MASK_COLUMNS_OPTION = "mask_columns"
|
||||
|
||||
#: 数据源 options 里的数据剖析设置（一个对象：开关、查询次数上限、单条时限……）。取值和校验见
|
||||
#: data/catalog_profile.py；放在这里只为让拼连接串时摘掉它
|
||||
CATALOG_PROFILE_OPTION = "catalog_profile"
|
||||
|
||||
# options 里这些 key 是 AgentLab 自己的配置，不是驱动参数，拼 URL 时要摘掉。
|
||||
# schema：探查哪个 schema（企业库里只读账号名下常常什么都没有，数据在别处）
|
||||
# query_timeout_s：查询时限，由数据层按语句下发给数据库，见 _server_deadline
|
||||
# mask_columns：证据面板遮罩的列，驱动不认识它，拼进连接串会被当成未知参数拒掉
|
||||
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION})
|
||||
# catalog_profile：数据剖析设置，同上；它还是个对象，拼进去连 URL 都不成形
|
||||
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION, CATALOG_PROFILE_OPTION})
|
||||
|
||||
_MASK_SPLIT = re.compile(r"[,，、;；\n]+")
|
||||
|
||||
|
||||
@@ -169,13 +169,17 @@ class SqlChecker:
|
||||
和助手挑表看到的是同一张图（catalog.relation_graph）：目录里的关系，加上没起草过的表按外键、命名现推的；
|
||||
被驳回的关系不进图，也不会被现推回来。
|
||||
"""
|
||||
return [e for e in self.edges_from(a) if e.to_table.lower() == b.lower()]
|
||||
|
||||
def edges_from(self, a: str) -> list[catalog.JoinEdge]:
|
||||
"""从 a 出发的全部边。关系图第一次用到时才建，之后复用。"""
|
||||
if self._edges is None:
|
||||
graph = catalog.relation_graph(self._raw, schema_cache=self.schema_cache)
|
||||
index: dict[str, list[catalog.JoinEdge]] = {}
|
||||
for table, edges in graph.items():
|
||||
index.setdefault(table.lower(), []).extend(edges)
|
||||
self._edges = index
|
||||
return [e for e in self._edges.get(a.lower(), []) if e.to_table.lower() == b.lower()]
|
||||
return list(self._edges.get(a.lower(), []))
|
||||
|
||||
# ---- 检查
|
||||
|
||||
@@ -557,10 +561,46 @@ class _ScopeCheck:
|
||||
continue # 两张表都没写目录：没有可对照的，不判
|
||||
condition = " AND ".join(filter(None, (self.excerpt(c) for c in group["conds"])))
|
||||
if best is None:
|
||||
self._join_checks.append(self._join_unknown(a, b, candidates, condition))
|
||||
parent = self._shared_parent(a, b, pairs)
|
||||
self._join_checks.append(self._join_sibling(a, b, parent, condition) if parent
|
||||
else self._join_unknown(a, b, candidates, condition))
|
||||
elif best.status not in _STRONG:
|
||||
self._join_checks.append(self._join_proposed(a, b, best, condition))
|
||||
|
||||
def _shared_parent(self, a: _Table, b: _Table, pairs: set[tuple[str, str]]) -> str | None:
|
||||
"""兄弟关联：每一对连接列都是两边各自指向同一张父表的同一组列（a.member_id → members.id ← b.member_id）。
|
||||
返回父表名；不是这种连法返回 None。
|
||||
|
||||
目录里不会记 a 和 b 之间的关系，但这不是「连错了列」：比如评价表和标签关联表各自按门店编号指向门店、
|
||||
彼此按门店编号相连。照「对不上任何关系」报警告是误报。
|
||||
"""
|
||||
def parents(table: _Table, column: str) -> set[tuple[str, tuple[str, ...]]]:
|
||||
return {(e.to_table.lower(), tuple(c.lower() for c in e.to_columns))
|
||||
for e in self.checker.edges_from(table.name)
|
||||
if tuple(c.lower() for c in e.from_columns) == (column,)
|
||||
and e.cardinality in (None, "many_to_one", "one_to_one")}
|
||||
|
||||
shared: set[tuple[str, tuple[str, ...]]] | None = None
|
||||
for left, right in pairs:
|
||||
common = parents(a, left) & parents(b, right)
|
||||
shared = common if shared is None else shared & common
|
||||
if not shared:
|
||||
return None
|
||||
names = {table for table, _ in shared or ()}
|
||||
return next(iter(names)) if len(names) == 1 else None
|
||||
|
||||
def _join_sibling(self, a: _Table, b: _Table, parent: str, condition: str) -> SqlCheck:
|
||||
item = (self.checker.notes_of(parent) or {}).get("label")
|
||||
label = str(item["value"]) if isinstance(item, dict) and isinstance(item.get("value"), str) \
|
||||
and item["value"].strip() else parent
|
||||
return SqlCheck(
|
||||
code="join_unconfirmed", level="info", table=b.name, sql_excerpt=condition or None,
|
||||
message=(f"「{a.label()}」与「{b.label()}」之间没有直接的关系，这里经共同的「{label}」相连。"
|
||||
"关联后行数可能成倍增加：确认其中一侧每个键只有一行，或先各自汇总再关联"),
|
||||
for_model=(f"{a.name} 和 {b.name} 按 {condition or '这组条件'} 关联：两边都指向 {parent}，彼此是一对多对多。"
|
||||
"如果其中一侧每个键不止一行，关联后另一侧的行会被重复，求和、计数会放大；"
|
||||
"先把一侧筛到每个键一行（例如只取主记录），或分别汇总到这个键再关联"))
|
||||
|
||||
def _join_unknown(self, a: _Table, b: _Table, candidates: list[catalog.JoinEdge], condition: str) -> SqlCheck:
|
||||
known = "；".join(dict.fromkeys(e.condition() for e in candidates))
|
||||
hint = f"目录里这两张表的关系是：{known}。照它改关联条件" if known else \
|
||||
|
||||
+5
-3
@@ -9,8 +9,9 @@
|
||||
从 visits 到 channels 要经过它，正好是两跳的连接路径。group_visits 同理（旅行团与入园记录）。
|
||||
- **命名推断必须放过的列**（NOT_INFERRED）：自引用（employees.employee_id 是工号，不是指向自己）、
|
||||
多个候选（guide 和 guides 两张表都在，tour_assignments.guide_id 不知道指哪张）、没有对应的表
|
||||
（parking_records.lot_id、operation_logs.operator_id、categories.parentId）、类型对不上
|
||||
（feedback_forms.store_id 是文字，stores.id 是整数）。
|
||||
（operation_logs.operator_id、categories.parentId）、类型对不上（feedback_forms.store_id 是文字，stores.id 是整数）。
|
||||
parking_records.lot_id 没有直接对上的 lot / lots，但和引用它的表共用 parking 前缀的 parking_lots 只有一张，
|
||||
推得出来（在 NAME_RELATIONS 里）。
|
||||
- **唯一约束 / 唯一索引。** orders.order_no、channels.channel_code 是列上的 UNIQUE；members.member_no、
|
||||
visits.ticket_no 是单独建的唯一索引。
|
||||
|
||||
@@ -188,6 +189,8 @@ NAME_RELATIONS = frozenset({
|
||||
("tour_groups", ("agency_id",), "agencies", ("id",)),
|
||||
("tour_assignments", ("tour_group_id",), "tour_groups", ("id",)),
|
||||
("lost_and_found", ("park_id",), "parks", ("id",)),
|
||||
# 没有 lot / lots，按引用表共用的 parking 前缀对上 parking_lots
|
||||
("parking_records", ("lot_id",), "parking_lots", ("id",)),
|
||||
# 视图没有约束，按列名同样推得出来：按日汇总的入园人数照样能按 park_id 连到景区
|
||||
("v_daily_visits", ("park_id",), "parks", ("id",)),
|
||||
})
|
||||
@@ -196,7 +199,6 @@ NAME_RELATIONS = frozenset({
|
||||
NOT_INFERRED = {
|
||||
("employees", "employee_id"): "自引用：工号，不是指向本表",
|
||||
("tour_assignments", "guide_id"): "多个候选：guide 与 guides",
|
||||
("parking_records", "lot_id"): "没有名为 lot / lots 的表",
|
||||
("operation_logs", "operator_id"): "没有名为 operator / operators 的表",
|
||||
("categories", "parentId"): "没有名为 parent / parents 的表",
|
||||
("feedback_forms", "store_id"): "类型对不上：文字列指向整数主键",
|
||||
|
||||
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
"""测试里建的数据源，用完就删。
|
||||
|
||||
数据源表在每个测试进程的测试库里是共用的：留下来的源会出现在助手的数据源清单里（50 张表的合成库还会
|
||||
触发挑表、多一次模型调用），别的用例的结果就跟着执行顺序变。阶段 1B 就是这样被绊过一次。
|
||||
删数据源时它的数据目录随外键级联删除（测试库开着 PRAGMA foreign_keys=ON）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
|
||||
|
||||
async def drop_source(source_id: str | None = None, *, name: str | None = None) -> None:
|
||||
"""按 id 或名字删一个数据源，并作废它的连接池。不存在就什么都不做。"""
|
||||
async with SessionLocal() as session:
|
||||
if source_id is None:
|
||||
source_id = (await session.execute(
|
||||
select(DataSource.id).where(DataSource.name == name))).scalar_one_or_none()
|
||||
if source_id is None:
|
||||
return
|
||||
await session.execute(delete(DataSource).where(DataSource.id == source_id))
|
||||
await session.commit()
|
||||
await engines.invalidate(source_id)
|
||||
@@ -11,12 +11,12 @@ import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.data import catalog
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.catalog import scenic
|
||||
from tests.fixtures.catalog.fake_model import FakeModel
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
ACTOR = {"X-Actor": quote("王敏")}
|
||||
|
||||
@@ -38,7 +38,7 @@ async def source_id(client, scenic_db):
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
yield sid
|
||||
await engines.invalidate(sid)
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
def _base(sid: str) -> str:
|
||||
@@ -184,9 +184,12 @@ async def test_draft_without_schema_is_refused(client):
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
r = await client.post(f"{_base(sid)}/draft", json={})
|
||||
assert r.status_code == 409 and "探查结构" in r.json()["detail"]
|
||||
assert (await client.get(_base(sid))).json()["tables"] == []
|
||||
try:
|
||||
r = await client.post(f"{_base(sid)}/draft", json={})
|
||||
assert r.status_code == 409 and "探查结构" in r.json()["detail"]
|
||||
assert (await client.get(_base(sid))).json()["tables"] == []
|
||||
finally:
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
async def test_schema_endpoint_detail_carries_catalog(client, source_id):
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.data.catalog import make_item
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from tests.fixtures.catalog.fake_model import FakeModel
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
|
||||
def _col(name: str, type_: str = "INTEGER", comment: str | None = None, nullable: bool = True) -> dict:
|
||||
@@ -156,6 +157,31 @@ def test_name_inference_negative(tables, why):
|
||||
assert _infer(tables, next(iter(tables))) == set(), why
|
||||
|
||||
|
||||
def test_name_inference_falls_back_to_a_table_sharing_the_prefix():
|
||||
"""库里没有 tags，只有带前缀的 store_tags 和 menu_tag：和引用它的表共用 store 前缀的那张才是它要指的。
|
||||
以前这种命名推不出关系，SQL 检查跟着误报「对不上任何关系」。"""
|
||||
tables = {
|
||||
"store_tag_links": _table(_col("id"), _col("tagId"), _col("storeID")),
|
||||
"store_tags": _table(_col("id")),
|
||||
"menu_tag": _table(_col("id")),
|
||||
"store": _table(_col("id")),
|
||||
}
|
||||
assert _infer(tables, "store_tag_links") == {
|
||||
(("tagId",), "store_tags", ("id",)), (("storeID",), "store", ("id",))}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tables, why", [
|
||||
# 直接对得上就用直接的，不去找带前缀的
|
||||
({"shop_category_links": _table(_col("id"), _col("category_id")), "categories": _table(_col("id")),
|
||||
"shop_categories": _table(_col("id"))}, {(("category_id",), "categories", ("id",))}),
|
||||
# 引用它的表没有可共用的前缀：带前缀的表不止一张时照旧不推
|
||||
({"links": _table(_col("id"), _col("tag_id")), "store_tags": _table(_col("id")),
|
||||
"menu_tags": _table(_col("id"))}, set()),
|
||||
])
|
||||
def test_prefix_fallback_only_when_direct_match_is_missing(tables, why):
|
||||
assert _infer(tables, next(iter(tables))) == why
|
||||
|
||||
|
||||
def test_name_inference_one_to_one_when_column_is_own_key():
|
||||
tables = {"member_profiles": _table(_col("member_id"), pk=("member_id",)), "members": _table(_col("id"))}
|
||||
rel = catalog.infer_name_relations(_source(tables).schema_cache, "member_profiles")[0]
|
||||
@@ -255,6 +281,16 @@ async def test_model_draft_respects_system_notes_of_imported_tables():
|
||||
# ---------------------------------------------------------------- 起草并写入
|
||||
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _drop_created_sources():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
async def _db_source(tables: dict, **kw) -> DataSource:
|
||||
for name, meta in tables.items():
|
||||
meta["qualified"] = name
|
||||
@@ -263,6 +299,7 @@ async def _db_source(tables: dict, **kw) -> DataSource:
|
||||
schema_cache={"tables": tables}, **kw)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
_CREATED.append(row.id)
|
||||
return row
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
# 模型要在建表之前注册进 metadata：conftest 的建表只导入了 app.db.base，这个文件单独跑时没有别的模块先导入它们
|
||||
from app.db import models # noqa: F401
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
|
||||
@@ -106,10 +107,9 @@ async def world(scenic_db):
|
||||
for wid in created:
|
||||
if (wf := await session.get(Workflow, wid)) is not None:
|
||||
await session.delete(wf)
|
||||
for sid in (ids.source, ids.other):
|
||||
if (row := await session.get(DataSource, sid)) is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
for sid in (ids.source, ids.other):
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
async def _impact(source_id: str, table: str) -> dict:
|
||||
|
||||
@@ -25,6 +25,7 @@ from app.data import catalog
|
||||
from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
|
||||
@@ -76,6 +77,32 @@ async def test_codes_are_supplemented_not_replaced(scenic_db):
|
||||
assert change.value == {"9": "作废"}
|
||||
|
||||
|
||||
async def test_codes_with_pending_meanings(scenic_db):
|
||||
"""数据剖析写进来的码值候选含义为空（界面上写「含义待填写」）：提案可以补上含义；提案里含义为空的码只在原来
|
||||
没有时加进去，不抹掉已有的含义；含义为空的码合法，保存不报错。"""
|
||||
tables = (await _schema(scenic_db))["tables"]
|
||||
profiled = _visits()
|
||||
profiled["columns"]["status"]["codes"] = catalog.make_item({"1": "有效", "0": "", "9": ""}, "profile", "proposed")
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废", "1": ""}}],
|
||||
table="visits", tables=tables)
|
||||
assert plan.problems == []
|
||||
[change] = plan.changes
|
||||
assert change.after == {"1": "有效", "0": "", "9": "作废"} and change.state == "change"
|
||||
# 只给了含义为空的新码：照样加进去，含义待人填
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"7": ""}}], table="visits",
|
||||
tables=tables)
|
||||
assert plan.changes[0].after == {"1": "有效", "0": "", "9": "", "7": ""}
|
||||
# 只给了含义为空的已有码：什么都没变，这一项是「保存即确认」（原来是推断）
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"1": ""}}], table="visits",
|
||||
tables=tables)
|
||||
assert plan.changes[0].state == "confirm"
|
||||
out = catalog.apply_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废"}}], table="visits",
|
||||
tables=tables)
|
||||
codes = out["columns"]["status"]["codes"]
|
||||
assert codes["value"] == {"1": "有效", "0": "", "9": "作废"} and (codes["source"], codes["status"]) == (
|
||||
"human", "confirmed")
|
||||
|
||||
|
||||
async def test_new_relation_gets_its_id_from_both_ends(scenic_db):
|
||||
tables = (await _schema(scenic_db))["tables"]
|
||||
value = {"columns": ["visit_id"], "to_table": "visits", "to_columns": ["id"], "cardinality": "many_to_one"}
|
||||
@@ -190,11 +217,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield SimpleNamespace(id=source_id, name=name)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
async def _stream(monkeypatch, model, source, base_graph=None):
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
"""数据剖析接口：POST /api/datasources/{id}/catalog/profile（阶段 2A）。
|
||||
|
||||
剖析是数据目录里唯一对业务库发查询的地方，这里逐条钉住方案定的约束：默认关闭、只读、每条查询都过守卫、
|
||||
查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析；
|
||||
以及剖析的结论怎么进目录：命名推断对上数据的升为有确证（来源记为数据剖析，和外键分开）、对不上的写明
|
||||
覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。
|
||||
|
||||
所有测试都在合成的景区库上做（tests/fixtures/catalog/scenic.py），数据和名称全部虚构。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from app.data import catalog, catalog_profile
|
||||
from app.data import engine as data_engine
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.catalog import scenic
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
ACTOR = {"X-Actor": quote("周宁")}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def _on(**settings) -> dict:
|
||||
return {PROFILE_OPTION: {"enabled": True, **settings}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def make_source(client):
|
||||
"""登记一个接上 path 的数据源并走真实的探查接口；draft 给了就先起草这些表的目录。"""
|
||||
made: list[str] = []
|
||||
|
||||
async def _make(path: str, *, options: dict | None = None, draft: list[str] | None = None) -> str:
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=f"prof_{uuid.uuid4().hex[:10]}", kind="sqlite", database=str(path),
|
||||
options=options or {})
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
made.append(sid)
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
if draft is not None:
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": draft})
|
||||
assert r.status_code == 200, r.text
|
||||
return sid
|
||||
|
||||
yield _make
|
||||
for sid in made:
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
class _Spy:
|
||||
"""记下两样东西：守卫放行（guard.check 返回）的语句，和真正发到这个业务库的语句。
|
||||
|
||||
后者挂在 SQLAlchemy 的 before_cursor_execute 上，任何一条到达驱动的语句都逃不掉——包括绕开查询层
|
||||
另开连接发的。断言「发到库的 ⊆ 守卫放行的」就是「所有查询都经过守卫」。
|
||||
"""
|
||||
|
||||
def __init__(self, db_name: str) -> None:
|
||||
self.db_name = db_name
|
||||
self.checked: list[str] = []
|
||||
self.executed: list[str] = []
|
||||
|
||||
def clear(self) -> None:
|
||||
self.checked.clear()
|
||||
self.executed.clear()
|
||||
|
||||
def all_guarded(self) -> bool:
|
||||
return bool(self.executed) and all(s in self.checked for s in self.executed)
|
||||
|
||||
def mentions(self, word: str) -> list[str]:
|
||||
return [s for s in self.executed if word.lower() in s.lower()]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def spy(monkeypatch, scenic_db):
|
||||
holder = SimpleNamespace(spy=None)
|
||||
|
||||
def watch(path: str) -> _Spy:
|
||||
holder.spy = _Spy(Path(path).name)
|
||||
return holder.spy
|
||||
|
||||
real_check = data_engine.check
|
||||
|
||||
def checking(sql, **kw):
|
||||
out = real_check(sql, **kw)
|
||||
if holder.spy is not None:
|
||||
holder.spy.checked.append(out)
|
||||
return out
|
||||
|
||||
def on_execute(conn, cursor, statement, parameters, context, executemany):
|
||||
if holder.spy is not None and holder.spy.db_name in str(conn.engine.url):
|
||||
holder.spy.executed.append(statement)
|
||||
|
||||
monkeypatch.setattr(data_engine, "check", checking)
|
||||
event.listen(Engine, "before_cursor_execute", on_execute)
|
||||
yield watch
|
||||
event.remove(Engine, "before_cursor_execute", on_execute)
|
||||
|
||||
|
||||
def _url(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog/profile"
|
||||
|
||||
|
||||
async def _notes(client, sid: str, table: str) -> dict:
|
||||
r = await client.get(f"/api/datasources/{sid}/catalog/{table}")
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
|
||||
def _relation(notes: dict, column: str) -> dict:
|
||||
return next(r for r in notes.get("relations") or [] if r["columns"] == [column])
|
||||
|
||||
|
||||
def _table(body: dict, name: str) -> dict:
|
||||
return next(t for t in body["tables"] if t["table_name"] == name)
|
||||
|
||||
|
||||
def _reasons(table: dict) -> set[str]:
|
||||
return {s["reason"] for s in table["skipped"]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 开关与设置
|
||||
|
||||
|
||||
async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 409
|
||||
detail = r.json()["detail"]
|
||||
assert "数据源设置" in detail and "开启数据剖析" in detail
|
||||
assert watch.executed == [] # 一条查询都没发
|
||||
|
||||
|
||||
async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db)
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)})
|
||||
assert r.status_code == 422 and "查询次数上限" in r.json()["detail"]
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500}
|
||||
# 设好以后就能剖析；返回里带着这次实际用的设置
|
||||
r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 剖析的结论
|
||||
|
||||
|
||||
async def test_profile_verifies_name_relations_and_proposes_codes_and_business_date(client, make_source, scenic_db,
|
||||
spy):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
|
||||
# 所有发到业务库的语句都经过了守卫
|
||||
assert watch.all_guarded(), watch.executed
|
||||
assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"]
|
||||
assert body["queries_used"] == sum(t["queries"] for t in body["tables"])
|
||||
assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"]
|
||||
|
||||
notes = (await _notes(client, sid, "visits"))
|
||||
assert notes["updated_by"] == "周宁"
|
||||
notes = notes["notes"]
|
||||
# 命名推断出来的两条关系：数据对得上、父表主键唯一 → 有确证，来源记为数据剖析
|
||||
for column, parent in (("gate_id", "gates"), ("member_id", "members")):
|
||||
rel = _relation(notes, column)
|
||||
assert rel["to_table"] == parent
|
||||
assert (rel["source"], rel["status"]) == ("profile", "verified")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
assert "数据剖析" in rel["note"] and "抽样" in rel["note"]
|
||||
# 外键约束来的关系不动：外键是外键，剖析是剖析
|
||||
park = _relation(notes, "park_id")
|
||||
assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None)
|
||||
|
||||
# 状态列：码值候选，值有了、含义留空，推断
|
||||
codes = notes["columns"]["status"]["codes"]
|
||||
assert codes["value"] == {"1": "", "0": ""}
|
||||
assert (codes["source"], codes["status"]) == ("profile", "proposed") and "1500" in codes["note"]
|
||||
# 只有一个日期类列：提议为业务日期，备注里写明取值范围
|
||||
bdate = notes["business_date"]
|
||||
assert bdate["value"] == {"column": "visit_time"} and (bdate["source"], bdate["status"]) == ("profile", "proposed")
|
||||
assert "2026-07" in bdate["note"] and "2026-09" in bdate["note"]
|
||||
|
||||
channels = (await _notes(client, sid, "channels"))["notes"]
|
||||
assert set(channels["columns"]["channel_type"]["codes"]["value"]) == {"线上直销", "线下", "分销"}
|
||||
assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值
|
||||
|
||||
visits = _table(body, "visits")
|
||||
kinds = sorted(f["kind"] for f in visits["findings"])
|
||||
assert kinds == ["business_date", "codes", "relation", "relation"]
|
||||
gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"])
|
||||
assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}"
|
||||
assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"]
|
||||
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
|
||||
assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None
|
||||
|
||||
|
||||
async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path):
|
||||
"""推错的命名关系：parking_records.lot_id 按名字对上了新建的 lots 表，但停车记录里的区号大多不在里面。"""
|
||||
path = scenic.build(tmp_path / "lots.db")
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)")
|
||||
db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)])
|
||||
db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)",
|
||||
[(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)])
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
sid = await make_source(path, options=_on(), draft=["parking_records"])
|
||||
before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed")
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert rel["status"] == "proposed" and rel["source"] == "profile"
|
||||
assert rel["coverage"] == 0.15
|
||||
assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"]
|
||||
finding = _table(r.json(), "parking_records")["findings"][0]
|
||||
assert finding["status"] == "proposed" and finding["coverage"] == 0.15
|
||||
# 两个日期类列：不提议业务日期，取值范围只在返回里
|
||||
ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]}
|
||||
assert ranges == {"entered_at"} # exited_at 全是空值
|
||||
assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"]
|
||||
|
||||
|
||||
async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
detail = await _notes(client, sid, "visits")
|
||||
gate = _relation(detail["notes"], "gate_id")
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/visits/review",
|
||||
json={"path": f"relations.{gate['id']}", "action": "confirm",
|
||||
"if_version": detail["version"]})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "confirmed")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"])
|
||||
assert finding["status"] == "confirmed" and finding["confirmed"] is True
|
||||
|
||||
|
||||
async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db):
|
||||
"""同一天、数据没变：第二次剖析不改目录、不升版本（冻结进快照的目录不会无故变）。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
first = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
second = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
a, b = _table(first, "visits"), _table(second, "visits")
|
||||
assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0)
|
||||
assert b["version"] == a["version"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 安全约束
|
||||
|
||||
|
||||
async def test_masked_columns_are_never_queried(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options={**_on(), "mask_columns": "member_id, Channel_Type"},
|
||||
draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert watch.all_guarded()
|
||||
assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == []
|
||||
visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels")
|
||||
masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"]
|
||||
assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"}
|
||||
assert all("遮罩" in s["detail"] for s in masked)
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "member_id")["source"] == "name" # 没动
|
||||
assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常
|
||||
assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {})
|
||||
|
||||
|
||||
async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget"
|
||||
visits, channels = _table(body, "visits"), _table(body, "channels")
|
||||
assert "budget" in _reasons(visits)
|
||||
assert channels["queries"] == 0 and _reasons(channels) == {"budget"}
|
||||
assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget")
|
||||
# 没查完的关系不改：原来的命名推断原样留着
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "proposed")
|
||||
|
||||
|
||||
async def test_single_query_timeout_is_recorded_and_profiling_continues(client, make_source, scenic_db, monkeypatch):
|
||||
"""闸口的抽样查询换成一条停不下来的递归查询：1 秒时限到点由数据库掐断，记为超时，其余照常。"""
|
||||
sid = await make_source(scenic_db, options=_on(query_timeout_s=1), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
endless = "WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM r) SELECT COUNT(*) FROM r"
|
||||
|
||||
async def slow_gate(source, sql, *, limits=None):
|
||||
if "gate_id" in sql and "DISTINCT" in sql and "COUNT" not in sql:
|
||||
assert limits is not None and limits.timeout_seconds == 1
|
||||
return await real(source, endless, limits=limits)
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow_gate)
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"]
|
||||
assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"]
|
||||
assert r.json()["stopped"] is None
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改
|
||||
assert _relation(notes, "member_id")["status"] == "verified" # 其余照常
|
||||
|
||||
|
||||
async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
|
||||
async def slow(source, sql, *, limits=None):
|
||||
await asyncio.sleep(0.5)
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
settings = ProfileSettings(enabled=True, max_total_s=1.4)
|
||||
report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None,
|
||||
settings=settings)
|
||||
assert report.stopped == "deadline"
|
||||
assert 1 <= report.queries_used <= 3
|
||||
reasons = {s.reason for t in report.tables for s in t.skipped}
|
||||
assert "deadline" in reasons
|
||||
|
||||
|
||||
async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy):
|
||||
"""行数超过整表统计上限（这里设成 100 行）：不做 COUNT(DISTINCT)、MIN/MAX，抽样和取值分布都带上限。"""
|
||||
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101}
|
||||
on_visits = [s for s in watch.executed if "from visits" in s.lower()]
|
||||
assert on_visits and all("LIMIT" in s for s in on_visits), on_visits
|
||||
assert not any("MIN(" in s for s in on_visits)
|
||||
assert "too_large" in _reasons(visits)
|
||||
assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large")
|
||||
# 父表有主键，覆盖率又够：照样有确证；子表一侧没核对唯一，按多对一记
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 接口
|
||||
|
||||
|
||||
async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def held(source, sql, *, limits=None):
|
||||
started.set()
|
||||
await release.wait()
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", held)
|
||||
first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR))
|
||||
await asyncio.wait_for(started.wait(), 10)
|
||||
second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"]
|
||||
release.set()
|
||||
done = await asyncio.wait_for(first, 30)
|
||||
assert done.status_code == 200, done.text
|
||||
# 做完以后可以再来
|
||||
again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert again.status_code == 200, again.text
|
||||
|
||||
|
||||
async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=[])
|
||||
r = await client.post(_url(sid), json={}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
names = [t["table_name"] for t in r.json()["tables"]]
|
||||
assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES
|
||||
async with SessionLocal() as session:
|
||||
entries = await catalog.read_catalog(session, sid)
|
||||
for name in names:
|
||||
rels = entries[name].notes.get("relations") or []
|
||||
assert any(rel["source"] not in ("fk",) for rel in rels), name
|
||||
assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选
|
||||
|
||||
|
||||
async def test_unknown_table_and_missing_source(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR)
|
||||
assert r.status_code == 200
|
||||
assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0
|
||||
r = await client.post(_url("0" * 32), json={}, headers=ACTOR)
|
||||
assert r.status_code == 404
|
||||
@@ -0,0 +1,205 @@
|
||||
"""数据剖析（阶段 2A）里不连库的部分：设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、
|
||||
目录对剖析结果的容纳（待填含义的码值、撤销审阅回到剖析的结论）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data import catalog, catalog_profile, guard
|
||||
from app.data.catalog import make_item
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
|
||||
from app.data.engine import build_url
|
||||
|
||||
AT = "2026-10-03T08:00:00+00:00"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 设置
|
||||
|
||||
|
||||
def test_settings_default_off_and_parse_leniently():
|
||||
s = profile_settings({})
|
||||
assert s == ProfileSettings()
|
||||
assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000
|
||||
assert s.max_scan_rows == 100_000 and s.max_total_s == 120
|
||||
on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}})
|
||||
assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500
|
||||
# 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错
|
||||
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}})
|
||||
assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10
|
||||
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, label", [
|
||||
({"max_queries": 0}, "查询次数上限"),
|
||||
({"max_queries": 501}, "查询次数上限"),
|
||||
({"max_queries": 2.5}, "查询次数上限"),
|
||||
({"query_timeout_s": 61}, "单条查询时限"),
|
||||
({"sample_size": 5}, "抽样键值数"),
|
||||
({"max_scan_rows": -1}, "整表统计行数上限"),
|
||||
({"max_total_s": 5}, "总时长上限"),
|
||||
({"enabled": "true"}, "开启数据剖析"),
|
||||
({"max_queries": True}, "查询次数上限"),
|
||||
])
|
||||
def test_settings_problem_names_the_field_in_chinese(value, label):
|
||||
problem = profile_settings_problem({PROFILE_OPTION: value})
|
||||
assert problem and label in problem and "数据剖析" in problem
|
||||
|
||||
|
||||
def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
|
||||
assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}})
|
||||
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
|
||||
assert profile_settings_problem({}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: None}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5,
|
||||
"sample_size": 10, "max_scan_rows": 0,
|
||||
"max_total_s": 600}}) is None
|
||||
|
||||
|
||||
def test_profile_option_never_reaches_the_connection_url():
|
||||
"""剖析设置是 AgentLab 自己的配置：拼进连接串，驱动会把它当未知参数拒掉。"""
|
||||
source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro",
|
||||
password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}})
|
||||
url = build_url(source)
|
||||
assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 字面量
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind, expected", [
|
||||
(42, "number", "42"),
|
||||
(-7, None, "-7"),
|
||||
(3.0, "number", "3"),
|
||||
("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串
|
||||
("A-01", "text", "'A-01'"),
|
||||
("O'Neil", "text", "'O''Neil'"),
|
||||
("12:30", "text", "'12:30'"),
|
||||
])
|
||||
def test_sql_literal_quotes_safe_values(value, kind, expected):
|
||||
assert catalog_profile.sql_literal(value, kind) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind", [
|
||||
(True, "boolean"), # 布尔不当键
|
||||
(2.5, "number"), # 非整数的数不当键：浮点相等比较靠不住
|
||||
("12.50", "number"),
|
||||
("1e3", "number"),
|
||||
("2026-07-01", "date"), # 日期的字面量各家写法不同，不猜
|
||||
("a\\b", "text"), # 反斜杠：MySQL 不同设置下读法不同，守卫会拒
|
||||
("a :b", "text"), # 「:名字」会被查询层当成绑定参数
|
||||
(":b", "text"),
|
||||
("a\nb", "text"), # 控制字符
|
||||
("x" * 201, "text"),
|
||||
(None, "text"),
|
||||
([1], None),
|
||||
])
|
||||
def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind):
|
||||
assert catalog_profile.sql_literal(value, kind) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 各方言的剖析 SQL
|
||||
|
||||
|
||||
_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]},
|
||||
"orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"])
|
||||
def test_profile_sql_passes_readonly_guard_on_every_dialect(kind):
|
||||
"""剖析的每一种查询（含读统计信息的系统视图）都得过只读守卫——守卫不为剖析放宽。"""
|
||||
d = catalog_profile.SqlDialect(kind)
|
||||
sqls = []
|
||||
for table in ("visits", "orders"):
|
||||
stats = d.stats_sql(_META[table], table)
|
||||
if kind == "sqlite":
|
||||
assert stats is None # SQLite 没有可读的统计信息，改为数到上限
|
||||
else:
|
||||
assert stats
|
||||
sqls.append(stats)
|
||||
tbl = d.table(_META[table], table)
|
||||
col = d.quote(_META[table]["columns"][0]["name"])
|
||||
sqls += [
|
||||
d.bounded_count_sql(tbl, 100_001),
|
||||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=None),
|
||||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000),
|
||||
d.match_count_sql(tbl, col, ["1", "'A'"]),
|
||||
d.unique_check_sql(tbl, col),
|
||||
d.value_counts_sql(tbl, col, 21, scan_cap=None),
|
||||
d.value_counts_sql(tbl, col, 21, scan_cap=100_000),
|
||||
d.min_max_sql(tbl, [col]),
|
||||
]
|
||||
for sql in sqls:
|
||||
assert guard.check(sql, readonly=True, dialect=kind) == sql, sql
|
||||
|
||||
|
||||
def test_dialect_quoting_and_row_limits():
|
||||
my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql"))
|
||||
assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"'
|
||||
assert my.table(_META["orders"], "orders") == "sales.orders"
|
||||
assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"'
|
||||
# Oracle 没有 LIMIT（11g 也没有 FETCH FIRST）：一律用 ROWNUM
|
||||
sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None)
|
||||
assert "LIMIT" not in sample and "ROWNUM <= 50" in sample
|
||||
assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None)
|
||||
# 统计信息按方言读各自的系统视图
|
||||
assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits")
|
||||
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
|
||||
stats = ora.stats_sql(_META["orders"], "orders")
|
||||
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 目录对剖析结果的容纳
|
||||
|
||||
|
||||
def test_codes_with_pending_meanings_are_valid_and_render_as_pending():
|
||||
"""剖析得到的码值只有取值、含义待人填写：目录要收得下，给模型看时写明含义待填写。"""
|
||||
notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")},
|
||||
"channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}}
|
||||
assert catalog.validate_notes(notes) == []
|
||||
text = catalog.render_table_notes(notes)
|
||||
assert "1、0（含义待填写）" in text
|
||||
assert "WEB=官网，BOX 的含义待填写" in text
|
||||
# 码值仍得是对照表：值不是文字的照样拒收
|
||||
assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}})
|
||||
|
||||
|
||||
def _profile_rel(coverage, cardinality, status):
|
||||
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
|
||||
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
|
||||
"source": "profile", "status": status}
|
||||
|
||||
|
||||
def test_reset_returns_profile_relation_to_what_profiling_concluded():
|
||||
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定：覆盖率够、父键唯一（基数有值）就是有确证。"""
|
||||
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
|
||||
rid = verified["relations"][0]["id"]
|
||||
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
|
||||
assert out["relations"][0]["status"] == "confirmed"
|
||||
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "verified"
|
||||
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
|
||||
f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "verified"
|
||||
# 覆盖率低、父键没核实唯一的：撤销后仍是推断
|
||||
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
|
||||
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
|
||||
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
|
||||
f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "proposed"
|
||||
# 剖析提的码值候选、业务日期永远是推断
|
||||
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
|
||||
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
|
||||
"columns.status.codes", "reset", at=AT)
|
||||
assert out["columns"]["status"]["codes"]["status"] == "proposed"
|
||||
|
||||
|
||||
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
|
||||
rel = _profile_rel(1.0, "many_to_one", "confirmed")
|
||||
existing = {"relations": [rel]}
|
||||
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
|
||||
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
|
||||
low = _profile_rel(0.4, "many_to_one", "proposed")
|
||||
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
|
||||
table="visits", at=AT)
|
||||
assert kept["relations"][0]["status"] == "proposed"
|
||||
@@ -24,6 +24,7 @@ from app.db.models import DataSource, Run, RunEvent, Workflow, WorkflowVersion
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
SQL = "SELECT COUNT(*) AS n FROM visits v JOIN parks p ON p.id = v.park_id WHERE v.status = 1"
|
||||
@@ -72,9 +73,8 @@ async def source(scenic_db):
|
||||
for wid in made:
|
||||
if (wf := await session.get(Workflow, wid)) is not None:
|
||||
await session.delete(wf)
|
||||
if (row := await session.get(DataSource, sid)) is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
|
||||
@@ -7,6 +7,7 @@ from app.data import catalog
|
||||
from app.data.catalog import make_item
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource, Run
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
MARK = "（推断，未确认）"
|
||||
|
||||
@@ -184,4 +185,5 @@ async def test_table_usage_counts_run_queries_per_table(monkeypatch):
|
||||
await session.commit()
|
||||
source = await session.get(DataSource, sid)
|
||||
usage = await catalog.table_usage(session, source)
|
||||
await drop_source(sid)
|
||||
assert usage == {"visits": 4, "明细": 2, "parks": 1}
|
||||
|
||||
@@ -14,12 +14,12 @@ from types import SimpleNamespace
|
||||
from app.core.artifact_store import canonical_json, content_hash, load
|
||||
from app.data import catalog
|
||||
from app.data.catalog import make_item
|
||||
from app.data.engine import engines
|
||||
from app.data.introspect import describe_table, introspect
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource
|
||||
from app.tools.datasource import _store_schema, build_datasource_tools
|
||||
from app.tools.registry import ToolContext
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
MARK = "（推断，未确认）"
|
||||
|
||||
@@ -182,4 +182,4 @@ async def test_tools_show_and_freeze_catalog_and_record_query_tables(scenic_db):
|
||||
ref = await session.get(Artifact, (result["artifact"], run_id))
|
||||
assert ref.meta == {"source_id": row.id, "source": row.name, "tables": ["visits", "parks"]}
|
||||
finally:
|
||||
await engines.invalidate(row.id)
|
||||
await drop_source(row.id)
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
from app.data.sqlcheck import SqlChecker
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -177,10 +178,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield name
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
row.enabled = False
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
async def _stream(monkeypatch, model, source):
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.engine.governance import lint_for_publish, publish_issues
|
||||
from app.engine.schema import GraphSpec
|
||||
from app.main import app
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -133,10 +134,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield name
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
row.enabled = False
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -32,6 +32,7 @@ from app.main import app
|
||||
from app.tools.datasource import build_datasource_tools
|
||||
from app.tools.registry import ToolContext
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -54,6 +55,16 @@ async def _schema(path: str) -> dict:
|
||||
return _CACHE[path]
|
||||
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _drop_created_sources():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
name = f"scenic_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
@@ -61,6 +72,7 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
row.schema_cache = await _schema(path)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
_CREATED.append(row.id)
|
||||
if with_catalog:
|
||||
for table, notes in scenic_notes.notes().items():
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
@@ -69,9 +81,8 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
|
||||
@pytest.fixture
|
||||
async def scenic_source(scenic_db):
|
||||
source_id, name = await _source(scenic_db)
|
||||
_, name = await _source(scenic_db)
|
||||
yield name
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
async def _query_tool(name: str):
|
||||
|
||||
@@ -254,6 +254,16 @@ async def test_join_unconfirmed_negative(cache, sql):
|
||||
assert "join_unconfirmed" not in codes(cache, sql)
|
||||
|
||||
|
||||
async def test_sibling_join_through_a_shared_parent_is_only_a_note(cache):
|
||||
"""两张表各自按 member_id 指向会员（都是外键），彼此按 member_id 相连：目录里没有这两张表之间的关系，
|
||||
但这是经共同父表推得出来的连法，不是「连错了列」。只提示一句：关联后行数可能成倍增加。"""
|
||||
sql = "SELECT COUNT(*) FROM orders o JOIN member_tag_links t ON t.member_id = o.member_id WHERE o.status = 1"
|
||||
[check] = [c for c in run(cache, sql) if c.code == "join_unconfirmed"]
|
||||
assert check.level == "info"
|
||||
assert "会员" in check.message or "members" in check.message
|
||||
assert "成倍" in check.message
|
||||
|
||||
|
||||
async def test_join_level_follows_status(cache):
|
||||
sql = "SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1"
|
||||
assert "join_unconfirmed" not in codes(cache, sql, "confirmed")
|
||||
|
||||
@@ -34,6 +34,7 @@ from app.engine.expressions import TruncatedUse, eval_expression
|
||||
from app.engine.issuance import decide_tier, incomplete_gaps
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 求值器：整组用到截断结果时记一笔，单格不记
|
||||
@@ -345,7 +346,7 @@ async def park(ticket_db, engine_up):
|
||||
# 连接池按数据源 id 缓存：换了库文件不作废，查的还是上一个测试的库
|
||||
await engines.invalidate(source_id)
|
||||
yield
|
||||
await engines.invalidate(source_id)
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
|
||||
Reference in New Issue
Block a user