合入数据目录阶段 2A:数据剖析——默认关闭、按源开启,查询次数和时限有上限,抽样核实推断的关联关系并补基数与覆盖率,状态列给出码值候选,遮罩列不取样

This commit is contained in:
ThearchyHelios
2026-10-03 11:09:20 +08:00
7 changed files with 1988 additions and 13 deletions
+65 -2
View File
@@ -5,9 +5,10 @@
- PUT /catalog/{table} 整份提交(乐观锁),改动过的项记为人工确认
- POST /catalog/{table}/review 单项审阅:确认、驳回、撤销审阅
- POST /catalog/draft 同步起草(注释、外键、命名推断,可选模型)
- POST /catalog/profile 同步剖析:对业务库发少量只读查询,核对关系、取码值候选(data/catalog_profile.py)
规则都在 data/catalog.py,这里只做取源、转换形状和把异常翻成状态码:版本不符 409,结构不合规、审阅路径
不对 422,源或表不存在 404。所有写操作记录请求头 X-Actor 的署名(自报、未认证)。
不对 422,源或表不存在 404。所有写操作记录请求头 X-Actor 的署名(自报、未认证)。剖析没开启、正在剖析 409。
上传表格的源按当前快照的表结构回答(table_versions.resolve_source),和查询实际用的库是同一个版本。
"""
@@ -20,7 +21,8 @@ from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.runs import actor_of
from app.data import catalog, introspect, table_versions
from app.data import catalog, catalog_profile, introspect, table_versions
from app.data.engine import SnapshotTampered
from app.db.base import get_session
from app.db.models import DataSource
@@ -47,6 +49,11 @@ class CatalogDraftIn(BaseModel):
use_model: bool = False
class CatalogProfileIn(BaseModel):
#: 要剖析的表;不给或为空时按使用次数取前 10 张目录里有待核实关系的表
tables: list[str] | None = Field(default=None, max_length=200)
async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]:
"""(数据源记录, 用来读表结构的源):上传源解析成绑定当前快照的视图。"""
row = await session.get(DataSource, source_id)
@@ -209,6 +216,62 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
}
def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]:
return {
"table_name": t.table,
"queries": t.queries,
"row_estimate": t.row_estimate.to_dict() if t.row_estimate else None,
"findings": t.findings,
"skipped": [s.to_dict() for s in t.skipped],
"date_ranges": t.date_ranges,
"added": t.added,
"updated": t.updated,
"removed": t.removed,
"version": t.version,
"error": t.error,
}
@router.post("/{source_id}/catalog/profile")
async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn,
x_actor: str | None = Header(default=None),
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。
默认关闭:数据源设置里没开启数据剖析、数据源停用,409;同一个数据源已经在剖析,409。
"""
row, source = await _resolved(session, source_id)
try:
catalog_profile.ensure_enabled(row)
except catalog_profile.ProfileDisabled as e:
raise HTTPException(409, str(e)) from e
cache = source.schema_cache or {}
if not cache.get("tables"):
raise HTTPException(409, f"{introspect.why_empty(cache)},无法进行数据剖析。请先在数据源卡片上点「探查结构」")
actor = actor_of(x_actor)
try:
report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None,
actor=actor)
except (catalog_profile.ProfileDisabled, catalog_profile.ProfileBusy) as e:
raise HTTPException(409, str(e)) from e
except SnapshotTampered as e:
raise HTTPException(409, str(e)) from e
rows = [_profile_table_out(t) for t in report.tables]
note = None
if not rows:
note = "目录里还没有待核实的关联关系,请先起草数据目录,或指定要剖析的表"
return {
"profiled_at": report.profiled_at,
"actor": actor,
"settings": report.settings.to_dict(),
"queries_used": report.queries_used,
"stopped": report.stopped,
"tables": rows,
"total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")},
"note": note,
}
@router.post("/{source_id}/catalog/{table}/review")
async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn,
x_actor: str | None = Header(default=None),
+3 -1
View File
@@ -23,6 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw
from app.core.config import settings
from app.core.crypto import encrypt, mask
from app.data import catalog as data_catalog
from app.data.catalog_profile import profile_settings_problem
from app.data import introspect as introspect_mod
from app.data import table_versions
from app.data.engine import (
@@ -336,7 +337,8 @@ _MASK_FIELD = {
def _refuse_bad_options(options: dict[str, Any] | None) -> None:
problem = query_timeout_problem(options) or mask_columns_problem(options)
problem = (query_timeout_problem(options) or mask_columns_problem(options)
or profile_settings_problem(options))
if problem:
raise HTTPException(422, problem)
+51 -9
View File
@@ -15,13 +15,14 @@ notes 结构(每个「项」都是 {value, source, status},可选 note、upd
"valid_filter": 项(str,SQL 条件片段), "dedup": 项(str),
"columns": {列名: {"label": 项, "meaning": 项, "unit": 项,
"measure": 项("flow"|"stock"|"ratio"|"identifier"|"status"|"attribute"),
"codes": 项(dict[str, str])}},
"codes": 项(dict[码值, 含义];含义可为空串,表示待填写)}},
"relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage",
"source", "status", "note"?}]}
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(阶段 2)、llm 模型起草、
human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证(外键
约束;阶段 2 的数据剖析也写这个状态)、confirmed 人工确认、rejected 人工驳回。只有 confirmed 和 verified
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(data/catalog_profile.py)、
llm 模型起草、human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证
(外键约束;数据剖析核实了覆盖率和父键唯一的关系也是,见 profile_relation_holds)、confirmed 人工确认、
rejected 人工驳回。只有 confirmed 和 verified
将来会触发错误级检查;给模型看时 proposed 的项标「推断,未确认」;rejected 的项留着(防止下次起草又提出
来),但模型和检查都看不到。
@@ -98,6 +99,33 @@ def initial_status(source: str) -> str:
return _INITIAL_STATUS.get(source, "proposed")
#: 数据剖析把关系升为有确证的抽样覆盖率下限(data/catalog_profile.py 判定,撤销审阅时按它判回剖析的结论)
PROFILE_VERIFY_COVERAGE = 0.95
def profile_relation_holds(rel: Mapping[str, Any]) -> bool:
"""一条数据剖析得出的关系站不站得住:抽样覆盖率不低于 PROFILE_VERIFY_COVERAGE,而且被指向列核实唯一。
剖析只在被指向列核实唯一时才写基数(多对一或一对一),所以「基数有值」就是「父键唯一」。结论全在关系
自己的字段里,撤销审阅时照原样判得回来,不必另存一份「审阅前的状态」。
"""
coverage = rel.get("coverage")
return (rel.get("source") == "profile" and isinstance(coverage, (int, float)) and not isinstance(coverage, bool)
and coverage >= PROFILE_VERIFY_COVERAGE and rel.get("cardinality") in ("many_to_one", "one_to_one"))
def unreviewed_status(item: Mapping[str, Any]) -> str:
"""撤销审阅后回到的状态:来源的初始状态。
数据剖析是例外:它的状态随数据而定(覆盖率够、父键唯一的关系是有确证,其余是推断),不能一律回到
initial_status("profile") 的推断——那样人工确认过再撤销,有确证就无故丢了,要等下一次剖析才回来。
剖析提的码值候选、业务日期永远是推断。
"""
if item.get("source") == "profile" and "to_table" in item:
return "verified" if profile_relation_holds(item) else "proposed"
return initial_status(item.get("source"))
def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None,
at: str | None = None) -> dict[str, Any]:
"""构造一个项。status 不给时按来源取初始状态(initial_status)。"""
@@ -153,8 +181,10 @@ def _business_date_ok(value: Any) -> bool:
def _codes_ok(value: Any) -> bool:
"""码值对照:{码值: 含义},都是文字。含义可以是空串——数据剖析只知道列里出现过哪些取值,含义要等人填;
以前要求含义非空,剖析得到的码值候选就写不进来。"""
return (isinstance(value, dict) and bool(value)
and all(isinstance(k, str) and isinstance(v, str) and v.strip() for k, v in value.items()))
and all(isinstance(k, str) and isinstance(v, str) for k, v in value.items()))
#: 每个字段的值怎么查、不合规时怎么说
@@ -574,7 +604,7 @@ def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at:
# 人工填写的项没有「推断时的样子」可回:撤销就是删掉,下次起草可以重新提出
del slots[key]
return _assemble(slots)
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or initial_status(item.get("source"))
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or unreviewed_status(item)
item["updated_at"] = at or now_iso()
return _assemble(slots)
@@ -639,7 +669,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
continue
kept = dict(prev)
status = item.get("status")
if status != prev.get("status") and status in ("confirmed", "rejected", initial_status(prev.get("source"))):
if status != prev.get("status") and status in ("confirmed", "rejected", unreviewed_status(prev)):
kept["status"] = status
kept["updated_at"] = at
if item.get("note") != prev.get("note"):
@@ -1383,6 +1413,17 @@ def _date_text(value: Mapping[str, Any]) -> str:
return ";".join(parts)
def _codes_text(codes: Mapping[str, Any]) -> str:
"""码值对照的文字:有含义的写「1=有效」,含义待填写的单独说明——不能写成「1=」,模型会当成含义就是空。"""
filled = [f"{k}={v}" for k, v in codes.items() if str(v).strip()]
pending = [str(k) for k, v in codes.items() if not str(v).strip()]
if not pending:
return "、".join(filled)
if not filled:
return f"{'、'.join(pending)}(含义待填写)"
return f"{'、'.join(filled)},{'、'.join(pending)} 的含义待填写"
def _relation_text(rel: Mapping[str, Any]) -> str:
left = ", ".join(rel.get("columns") or [])
right = ", ".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or [])
@@ -1433,7 +1474,7 @@ def _notes_lines(notes: Mapping[str, Any] | None, *, meta: Mapping[str, Any] | N
if name == "measure":
text = MEASURE_LABEL.get(value, str(value))
elif name == "codes":
text = "、".join(f"{k}={v}" for k, v in value.items())
text = _codes_text(value)
else:
text = str(value)
# 第一段是中文名(没有中文名时是含义),不加前缀;其余写明是什么
@@ -1690,7 +1731,8 @@ __all__ = [
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
"MergeStats", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS",
"MergeStats", "PROFILE_VERIFY_COVERAGE", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS",
"TABLE_FIELD_LABEL", "TABLE_KINDS", "profile_relation_holds", "unreviewed_status",
"TableDraft", "TableDraftResult", "apply_human_edit", "describe_slot", "draft_catalog", "draft_structure",
"draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths",
"make_item", "merge_notes", "now_iso", "parse_path", "query_snapshot_meta", "read_catalog", "read_entry",
File diff suppressed because it is too large Load Diff
+6 -1
View File
@@ -58,11 +58,16 @@ MAX_QUERY_TIMEOUT_S = 600
#: 完整快照仍能按工件 id 取到,SQL 里给列起个别名也能绕开
MASK_COLUMNS_OPTION = "mask_columns"
#: 数据源 options 里的数据剖析设置(一个对象:开关、查询次数上限、单条时限……)。取值和校验见
#: data/catalog_profile.py;放在这里只为让拼连接串时摘掉它
CATALOG_PROFILE_OPTION = "catalog_profile"
# options 里这些 key 是 AgentLab 自己的配置,不是驱动参数,拼 URL 时要摘掉。
# schema:探查哪个 schema(企业库里只读账号名下常常什么都没有,数据在别处)
# query_timeout_s:查询时限,由数据层按语句下发给数据库,见 _server_deadline
# mask_columns:证据面板遮罩的列,驱动不认识它,拼进连接串会被当成未知参数拒掉
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION})
# catalog_profile:数据剖析设置,同上;它还是个对象,拼进去连 URL 都不成形
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION, CATALOG_PROFILE_OPTION})
_MASK_SPLIT = re.compile(r"[,,、;;\n]+")
+430
View File
@@ -0,0 +1,430 @@
"""数据剖析接口:POST /api/datasources/{id}/catalog/profile(阶段 2A)。
剖析是数据目录里唯一对业务库发查询的地方,这里逐条钉住方案定的约束:默认关闭、只读、每条查询都过守卫、
查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析;
以及剖析的结论怎么进目录:命名推断对上数据的升为有确证(来源记为数据剖析,和外键分开)、对不上的写明
覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。
所有测试都在合成的景区库上做(tests/fixtures/catalog/scenic.py),数据和名称全部虚构。
"""
from __future__ import annotations
import asyncio
import sqlite3
import uuid
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import quote
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import event
from sqlalchemy.engine import Engine
from app.data import catalog, catalog_profile
from app.data import engine as data_engine
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.main import app as _fastapi_app
from tests.fixtures.catalog import scenic
ACTOR = {"X-Actor": quote("周宁")}
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
yield c
def _on(**settings) -> dict:
return {PROFILE_OPTION: {"enabled": True, **settings}}
@pytest.fixture
async def make_source(client):
"""登记一个接上 path 的数据源并走真实的探查接口;draft 给了就先起草这些表的目录。"""
made: list[str] = []
async def _make(path: str, *, options: dict | None = None, draft: list[str] | None = None) -> str:
async with SessionLocal() as session:
row = DataSource(name=f"prof_{uuid.uuid4().hex[:10]}", kind="sqlite", database=str(path),
options=options or {})
session.add(row)
await session.commit()
sid = row.id
made.append(sid)
r = await client.post(f"/api/datasources/{sid}/introspect")
assert r.status_code == 200, r.text
if draft is not None:
r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": draft})
assert r.status_code == 200, r.text
return sid
yield _make
for sid in made:
await engines.invalidate(sid)
class _Spy:
"""记下两样东西:守卫放行(guard.check 返回)的语句,和真正发到这个业务库的语句。
后者挂在 SQLAlchemy 的 before_cursor_execute 上,任何一条到达驱动的语句都逃不掉——包括绕开查询层
另开连接发的。断言「发到库的 ⊆ 守卫放行的」就是「所有查询都经过守卫」。
"""
def __init__(self, db_name: str) -> None:
self.db_name = db_name
self.checked: list[str] = []
self.executed: list[str] = []
def clear(self) -> None:
self.checked.clear()
self.executed.clear()
def all_guarded(self) -> bool:
return bool(self.executed) and all(s in self.checked for s in self.executed)
def mentions(self, word: str) -> list[str]:
return [s for s in self.executed if word.lower() in s.lower()]
@pytest.fixture
def spy(monkeypatch, scenic_db):
holder = SimpleNamespace(spy=None)
def watch(path: str) -> _Spy:
holder.spy = _Spy(Path(path).name)
return holder.spy
real_check = data_engine.check
def checking(sql, **kw):
out = real_check(sql, **kw)
if holder.spy is not None:
holder.spy.checked.append(out)
return out
def on_execute(conn, cursor, statement, parameters, context, executemany):
if holder.spy is not None and holder.spy.db_name in str(conn.engine.url):
holder.spy.executed.append(statement)
monkeypatch.setattr(data_engine, "check", checking)
event.listen(Engine, "before_cursor_execute", on_execute)
yield watch
event.remove(Engine, "before_cursor_execute", on_execute)
def _url(sid: str) -> str:
return f"/api/datasources/{sid}/catalog/profile"
async def _notes(client, sid: str, table: str) -> dict:
r = await client.get(f"/api/datasources/{sid}/catalog/{table}")
assert r.status_code == 200, r.text
return r.json()
def _relation(notes: dict, column: str) -> dict:
return next(r for r in notes.get("relations") or [] if r["columns"] == [column])
def _table(body: dict, name: str) -> dict:
return next(t for t in body["tables"] if t["table_name"] == name)
def _reasons(table: dict) -> set[str]:
return {s["reason"] for s in table["skipped"]}
# ---------------------------------------------------------------- 开关与设置
async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, make_source, scenic_db, spy):
sid = await make_source(scenic_db, draft=["visits"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert r.status_code == 409
detail = r.json()["detail"]
assert "数据源设置" in detail and "开启数据剖析" in detail
assert watch.executed == [] # 一条查询都没发
async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db):
sid = await make_source(scenic_db)
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)})
assert r.status_code == 422 and "查询次数上限" in r.json()["detail"]
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)})
assert r.status_code == 200, r.text
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500}
# 设好以后就能剖析;返回里带着这次实际用的设置
r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
assert r.status_code == 200, r.text
assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500
# ---------------------------------------------------------------- 剖析的结论
async def test_profile_verifies_name_relations_and_proposes_codes_and_business_date(client, make_source, scenic_db,
spy):
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
assert r.status_code == 200, r.text
body = r.json()
# 所有发到业务库的语句都经过了守卫
assert watch.all_guarded(), watch.executed
assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"]
assert body["queries_used"] == sum(t["queries"] for t in body["tables"])
assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"]
notes = (await _notes(client, sid, "visits"))
assert notes["updated_by"] == "周宁"
notes = notes["notes"]
# 命名推断出来的两条关系:数据对得上、父表主键唯一 → 有确证,来源记为数据剖析
for column, parent in (("gate_id", "gates"), ("member_id", "members")):
rel = _relation(notes, column)
assert rel["to_table"] == parent
assert (rel["source"], rel["status"]) == ("profile", "verified")
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
assert "数据剖析" in rel["note"] and "抽样" in rel["note"]
# 外键约束来的关系不动:外键是外键,剖析是剖析
park = _relation(notes, "park_id")
assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None)
# 状态列:码值候选,值有了、含义留空,推断
codes = notes["columns"]["status"]["codes"]
assert codes["value"] == {"1": "", "0": ""}
assert (codes["source"], codes["status"]) == ("profile", "proposed") and "1500" in codes["note"]
# 只有一个日期类列:提议为业务日期,备注里写明取值范围
bdate = notes["business_date"]
assert bdate["value"] == {"column": "visit_time"} and (bdate["source"], bdate["status"]) == ("profile", "proposed")
assert "2026-07" in bdate["note"] and "2026-09" in bdate["note"]
channels = (await _notes(client, sid, "channels"))["notes"]
assert set(channels["columns"]["channel_type"]["codes"]["value"]) == {"线上直销", "线下", "分销"}
assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值
visits = _table(body, "visits")
kinds = sorted(f["kind"] for f in visits["findings"])
assert kinds == ["business_date", "codes", "relation", "relation"]
gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"])
assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}"
assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"]
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None
async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path):
"""推错的命名关系:parking_records.lot_id 按名字对上了新建的 lots 表,但停车记录里的区号大多不在里面。"""
path = scenic.build(tmp_path / "lots.db")
db = sqlite3.connect(path)
try:
db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)")
db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)])
db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)",
[(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)])
db.commit()
finally:
db.close()
sid = await make_source(path, options=_on(), draft=["parking_records"])
before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed")
r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR)
assert r.status_code == 200, r.text
rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
assert rel["status"] == "proposed" and rel["source"] == "profile"
assert rel["coverage"] == 0.15
assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"]
finding = _table(r.json(), "parking_records")["findings"][0]
assert finding["status"] == "proposed" and finding["coverage"] == 0.15
# 两个日期类列:不提议业务日期,取值范围只在返回里
ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]}
assert ranges == {"entered_at"} # exited_at 全是空值
assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"]
async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db):
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
detail = await _notes(client, sid, "visits")
gate = _relation(detail["notes"], "gate_id")
r = await client.post(f"/api/datasources/{sid}/catalog/visits/review",
json={"path": f"relations.{gate['id']}", "action": "confirm",
"if_version": detail["version"]})
assert r.status_code == 200, r.text
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert r.status_code == 200, r.text
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
assert (rel["source"], rel["status"]) == ("name", "confirmed")
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"])
assert finding["status"] == "confirmed" and finding["confirmed"] is True
async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db):
"""同一天、数据没变:第二次剖析不改目录、不升版本(冻结进快照的目录不会无故变)。"""
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
first = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
second = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
a, b = _table(first, "visits"), _table(second, "visits")
assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0)
assert b["version"] == a["version"]
# ---------------------------------------------------------------- 安全约束
async def test_masked_columns_are_never_queried(client, make_source, scenic_db, spy):
sid = await make_source(scenic_db, options={**_on(), "mask_columns": "member_id, Channel_Type"},
draft=["visits", "channels"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
assert r.status_code == 200, r.text
assert watch.all_guarded()
assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == []
visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels")
masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"]
assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"}
assert all("遮罩" in s["detail"] for s in masked)
notes = (await _notes(client, sid, "visits"))["notes"]
assert _relation(notes, "member_id")["source"] == "name" # 没动
assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常
assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {})
async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy):
sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
assert r.status_code == 200, r.text
body = r.json()
assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget"
visits, channels = _table(body, "visits"), _table(body, "channels")
assert "budget" in _reasons(visits)
assert channels["queries"] == 0 and _reasons(channels) == {"budget"}
assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget")
# 没查完的关系不改:原来的命名推断原样留着
rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id")
assert (rel["source"], rel["status"]) == ("name", "proposed")
async def test_single_query_timeout_is_recorded_and_profiling_continues(client, make_source, scenic_db, monkeypatch):
"""闸口的抽样查询换成一条停不下来的递归查询:1 秒时限到点由数据库掐断,记为超时,其余照常。"""
sid = await make_source(scenic_db, options=_on(query_timeout_s=1), draft=["visits"])
real = data_engine.run_query
endless = "WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM r) SELECT COUNT(*) FROM r"
async def slow_gate(source, sql, *, limits=None):
if "gate_id" in sql and "DISTINCT" in sql and "COUNT" not in sql:
assert limits is not None and limits.timeout_seconds == 1
return await real(source, endless, limits=limits)
return await real(source, sql, limits=limits)
monkeypatch.setattr(data_engine, "run_query", slow_gate)
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert r.status_code == 200, r.text
visits = _table(r.json(), "visits")
timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"]
assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"]
assert r.json()["stopped"] is None
notes = (await _notes(client, sid, "visits"))["notes"]
assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改
assert _relation(notes, "member_id")["status"] == "verified" # 其余照常
async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch):
sid = await make_source(scenic_db, draft=["visits"])
real = data_engine.run_query
async def slow(source, sql, *, limits=None):
await asyncio.sleep(0.5)
return await real(source, sql, limits=limits)
monkeypatch.setattr(data_engine, "run_query", slow)
async with SessionLocal() as session:
row = await session.get(DataSource, sid)
settings = ProfileSettings(enabled=True, max_total_s=1.4)
report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None,
settings=settings)
assert report.stopped == "deadline"
assert 1 <= report.queries_used <= 3
reasons = {s.reason for t in report.tables for s in t.skipped}
assert "deadline" in reasons
async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy):
"""行数超过整表统计上限(这里设成 100 行):不做 COUNT(DISTINCT)、MIN/MAX,抽样和取值分布都带上限。"""
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"])
watch = spy(scenic_db)
watch.clear()
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert r.status_code == 200, r.text
visits = _table(r.json(), "visits")
assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101}
on_visits = [s for s in watch.executed if "from visits" in s.lower()]
assert on_visits and all("LIMIT" in s for s in on_visits), on_visits
assert not any("MIN(" in s for s in on_visits)
assert "too_large" in _reasons(visits)
assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large")
# 父表有主键,覆盖率又够:照样有确证;子表一侧没核对唯一,按多对一记
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one"
# ---------------------------------------------------------------- 接口
async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch):
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
real = data_engine.run_query
started, release = asyncio.Event(), asyncio.Event()
async def held(source, sql, *, limits=None):
started.set()
await release.wait()
return await real(source, sql, limits=limits)
monkeypatch.setattr(data_engine, "run_query", held)
first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR))
await asyncio.wait_for(started.wait(), 10)
second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"]
release.set()
done = await asyncio.wait_for(first, 30)
assert done.status_code == 200, done.text
# 做完以后可以再来
again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
assert again.status_code == 200, again.text
async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db):
sid = await make_source(scenic_db, options=_on(), draft=[])
r = await client.post(_url(sid), json={}, headers=ACTOR)
assert r.status_code == 200, r.text
names = [t["table_name"] for t in r.json()["tables"]]
assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES
async with SessionLocal() as session:
entries = await catalog.read_catalog(session, sid)
for name in names:
rels = entries[name].notes.get("relations") or []
assert any(rel["source"] not in ("fk",) for rel in rels), name
assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选
async def test_unknown_table_and_missing_source(client, make_source, scenic_db):
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR)
assert r.status_code == 200
assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0
r = await client.post(_url("0" * 32), json={}, headers=ACTOR)
assert r.status_code == 404
+205
View File
@@ -0,0 +1,205 @@
"""数据剖析(阶段 2A)里不连库的部分:设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、
目录对剖析结果的容纳(待填含义的码值、撤销审阅回到剖析的结论)。"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from app.data import catalog, catalog_profile, guard
from app.data.catalog import make_item
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
from app.data.engine import build_url
AT = "2026-10-03T08:00:00+00:00"
# ---------------------------------------------------------------- 设置
def test_settings_default_off_and_parse_leniently():
s = profile_settings({})
assert s == ProfileSettings()
assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000
assert s.max_scan_rows == 100_000 and s.max_total_s == 120
on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}})
assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500
# 库里存着的值不对(老数据、手改过):那一项用缺省,不因此开启,也不报错
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}})
assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
@pytest.mark.parametrize("value, label", [
({"max_queries": 0}, "查询次数上限"),
({"max_queries": 501}, "查询次数上限"),
({"max_queries": 2.5}, "查询次数上限"),
({"query_timeout_s": 61}, "单条查询时限"),
({"sample_size": 5}, "抽样键值数"),
({"max_scan_rows": -1}, "整表统计行数上限"),
({"max_total_s": 5}, "总时长上限"),
({"enabled": "true"}, "开启数据剖析"),
({"max_queries": True}, "查询次数上限"),
])
def test_settings_problem_names_the_field_in_chinese(value, label):
problem = profile_settings_problem({PROFILE_OPTION: value})
assert problem and label in problem and "数据剖析" in problem
def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}})
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
assert profile_settings_problem({}) is None
assert profile_settings_problem({PROFILE_OPTION: None}) is None
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5,
"sample_size": 10, "max_scan_rows": 0,
"max_total_s": 600}}) is None
def test_profile_option_never_reaches_the_connection_url():
"""剖析设置是 AgentLab 自己的配置:拼进连接串,驱动会把它当未知参数拒掉。"""
source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro",
password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}})
url = build_url(source)
assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url
# ---------------------------------------------------------------- 字面量
@pytest.mark.parametrize("value, kind, expected", [
(42, "number", "42"),
(-7, None, "-7"),
(3.0, "number", "3"),
("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串
("A-01", "text", "'A-01'"),
("O'Neil", "text", "'O''Neil'"),
("12:30", "text", "'12:30'"),
])
def test_sql_literal_quotes_safe_values(value, kind, expected):
assert catalog_profile.sql_literal(value, kind) == expected
@pytest.mark.parametrize("value, kind", [
(True, "boolean"), # 布尔不当键
(2.5, "number"), # 非整数的数不当键:浮点相等比较靠不住
("12.50", "number"),
("1e3", "number"),
("2026-07-01", "date"), # 日期的字面量各家写法不同,不猜
("a\\b", "text"), # 反斜杠:MySQL 不同设置下读法不同,守卫会拒
("a :b", "text"), # 「:名字」会被查询层当成绑定参数
(":b", "text"),
("a\nb", "text"), # 控制字符
("x" * 201, "text"),
(None, "text"),
([1], None),
])
def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind):
assert catalog_profile.sql_literal(value, kind) is None
# ---------------------------------------------------------------- 各方言的剖析 SQL
_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]},
"orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}}
@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"])
def test_profile_sql_passes_readonly_guard_on_every_dialect(kind):
"""剖析的每一种查询(含读统计信息的系统视图)都得过只读守卫——守卫不为剖析放宽。"""
d = catalog_profile.SqlDialect(kind)
sqls = []
for table in ("visits", "orders"):
stats = d.stats_sql(_META[table], table)
if kind == "sqlite":
assert stats is None # SQLite 没有可读的统计信息,改为数到上限
else:
assert stats
sqls.append(stats)
tbl = d.table(_META[table], table)
col = d.quote(_META[table]["columns"][0]["name"])
sqls += [
d.bounded_count_sql(tbl, 100_001),
d.distinct_sample_sql(tbl, col, 2000, scan_cap=None),
d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000),
d.match_count_sql(tbl, col, ["1", "'A'"]),
d.unique_check_sql(tbl, col),
d.value_counts_sql(tbl, col, 21, scan_cap=None),
d.value_counts_sql(tbl, col, 21, scan_cap=100_000),
d.min_max_sql(tbl, [col]),
]
for sql in sqls:
assert guard.check(sql, readonly=True, dialect=kind) == sql, sql
def test_dialect_quoting_and_row_limits():
my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql"))
assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"'
assert my.table(_META["orders"], "orders") == "sales.orders"
assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"'
# Oracle 没有 LIMIT(11g 也没有 FETCH FIRST):一律用 ROWNUM
sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None)
assert "LIMIT" not in sample and "ROWNUM <= 50" in sample
assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None)
# 统计信息按方言读各自的系统视图
assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits")
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
stats = ora.stats_sql(_META["orders"], "orders")
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
# ---------------------------------------------------------------- 目录对剖析结果的容纳
def test_codes_with_pending_meanings_are_valid_and_render_as_pending():
"""剖析得到的码值只有取值、含义待人填写:目录要收得下,给模型看时写明含义待填写。"""
notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")},
"channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}}
assert catalog.validate_notes(notes) == []
text = catalog.render_table_notes(notes)
assert "1、0(含义待填写)" in text
assert "WEB=官网,BOX 的含义待填写" in text
# 码值仍得是对照表:值不是文字的照样拒收
assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}})
def _profile_rel(coverage, cardinality, status):
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
"source": "profile", "status": status}
def test_reset_returns_profile_relation_to_what_profiling_concluded():
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定:覆盖率够、父键唯一(基数有值)就是有确证。"""
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
rid = verified["relations"][0]["id"]
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
assert out["relations"][0]["status"] == "confirmed"
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "verified"
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "verified"
# 覆盖率低、父键没核实唯一的:撤销后仍是推断
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "proposed"
# 剖析提的码值候选、业务日期永远是推断
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
"columns.status.codes", "reset", at=AT)
assert out["columns"]["status"]["codes"]["status"] == "proposed"
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
rel = _profile_rel(1.0, "many_to_one", "confirmed")
existing = {"relations": [rel]}
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
low = _profile_rel(0.4, "many_to_one", "proposed")
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
table="visits", at=AT)
assert kept["relations"][0]["status"] == "proposed"