mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-17 19:01:59 +00:00
合入数据目录阶段 2A:数据剖析——默认关闭、按源开启,查询次数和时限有上限,抽样核实推断的关联关系并补基数与覆盖率,状态列给出码值候选,遮罩列不取样
This commit is contained in:
@@ -5,9 +5,10 @@
|
||||
- PUT /catalog/{table} 整份提交(乐观锁),改动过的项记为人工确认
|
||||
- POST /catalog/{table}/review 单项审阅:确认、驳回、撤销审阅
|
||||
- POST /catalog/draft 同步起草(注释、外键、命名推断,可选模型)
|
||||
- POST /catalog/profile 同步剖析:对业务库发少量只读查询,核对关系、取码值候选(data/catalog_profile.py)
|
||||
|
||||
规则都在 data/catalog.py,这里只做取源、转换形状和把异常翻成状态码:版本不符 409,结构不合规、审阅路径
|
||||
不对 422,源或表不存在 404。所有写操作记录请求头 X-Actor 的署名(自报、未认证)。
|
||||
不对 422,源或表不存在 404。所有写操作记录请求头 X-Actor 的署名(自报、未认证)。剖析没开启、正在剖析 409。
|
||||
|
||||
上传表格的源按当前快照的表结构回答(table_versions.resolve_source),和查询实际用的库是同一个版本。
|
||||
"""
|
||||
@@ -20,7 +21,8 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.runs import actor_of
|
||||
from app.data import catalog, introspect, table_versions
|
||||
from app.data import catalog, catalog_profile, introspect, table_versions
|
||||
from app.data.engine import SnapshotTampered
|
||||
from app.db.base import get_session
|
||||
from app.db.models import DataSource
|
||||
|
||||
@@ -47,6 +49,11 @@ class CatalogDraftIn(BaseModel):
|
||||
use_model: bool = False
|
||||
|
||||
|
||||
class CatalogProfileIn(BaseModel):
|
||||
#: 要剖析的表;不给或为空时按使用次数取前 10 张目录里有待核实关系的表
|
||||
tables: list[str] | None = Field(default=None, max_length=200)
|
||||
|
||||
|
||||
async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]:
|
||||
"""(数据源记录, 用来读表结构的源):上传源解析成绑定当前快照的视图。"""
|
||||
row = await session.get(DataSource, source_id)
|
||||
@@ -209,6 +216,62 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
|
||||
}
|
||||
|
||||
|
||||
def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]:
|
||||
return {
|
||||
"table_name": t.table,
|
||||
"queries": t.queries,
|
||||
"row_estimate": t.row_estimate.to_dict() if t.row_estimate else None,
|
||||
"findings": t.findings,
|
||||
"skipped": [s.to_dict() for s in t.skipped],
|
||||
"date_ranges": t.date_ranges,
|
||||
"added": t.added,
|
||||
"updated": t.updated,
|
||||
"removed": t.removed,
|
||||
"version": t.version,
|
||||
"error": t.error,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/profile")
|
||||
async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。
|
||||
|
||||
默认关闭:数据源设置里没开启数据剖析、数据源停用,409;同一个数据源已经在剖析,409。
|
||||
"""
|
||||
row, source = await _resolved(session, source_id)
|
||||
try:
|
||||
catalog_profile.ensure_enabled(row)
|
||||
except catalog_profile.ProfileDisabled as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
cache = source.schema_cache or {}
|
||||
if not cache.get("tables"):
|
||||
raise HTTPException(409, f"{introspect.why_empty(cache)},无法进行数据剖析。请先在数据源卡片上点「探查结构」")
|
||||
actor = actor_of(x_actor)
|
||||
try:
|
||||
report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None,
|
||||
actor=actor)
|
||||
except (catalog_profile.ProfileDisabled, catalog_profile.ProfileBusy) as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
except SnapshotTampered as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
rows = [_profile_table_out(t) for t in report.tables]
|
||||
note = None
|
||||
if not rows:
|
||||
note = "目录里还没有待核实的关联关系,请先起草数据目录,或指定要剖析的表"
|
||||
return {
|
||||
"profiled_at": report.profiled_at,
|
||||
"actor": actor,
|
||||
"settings": report.settings.to_dict(),
|
||||
"queries_used": report.queries_used,
|
||||
"stopped": report.stopped,
|
||||
"tables": rows,
|
||||
"total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")},
|
||||
"note": note,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/{table}/review")
|
||||
async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw
|
||||
from app.core.config import settings
|
||||
from app.core.crypto import encrypt, mask
|
||||
from app.data import catalog as data_catalog
|
||||
from app.data.catalog_profile import profile_settings_problem
|
||||
from app.data import introspect as introspect_mod
|
||||
from app.data import table_versions
|
||||
from app.data.engine import (
|
||||
@@ -336,7 +337,8 @@ _MASK_FIELD = {
|
||||
|
||||
|
||||
def _refuse_bad_options(options: dict[str, Any] | None) -> None:
|
||||
problem = query_timeout_problem(options) or mask_columns_problem(options)
|
||||
problem = (query_timeout_problem(options) or mask_columns_problem(options)
|
||||
or profile_settings_problem(options))
|
||||
if problem:
|
||||
raise HTTPException(422, problem)
|
||||
|
||||
|
||||
@@ -15,13 +15,14 @@ notes 结构(每个「项」都是 {value, source, status},可选 note、upd
|
||||
"valid_filter": 项(str,SQL 条件片段), "dedup": 项(str),
|
||||
"columns": {列名: {"label": 项, "meaning": 项, "unit": 项,
|
||||
"measure": 项("flow"|"stock"|"ratio"|"identifier"|"status"|"attribute"),
|
||||
"codes": 项(dict[str, str])}},
|
||||
"codes": 项(dict[码值, 含义];含义可为空串,表示待填写)}},
|
||||
"relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage",
|
||||
"source", "status", "note"?}]}
|
||||
|
||||
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(阶段 2)、llm 模型起草、
|
||||
human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证(外键
|
||||
约束;阶段 2 的数据剖析也写这个状态)、confirmed 人工确认、rejected 人工驳回。只有 confirmed 和 verified
|
||||
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(data/catalog_profile.py)、
|
||||
llm 模型起草、human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证
|
||||
(外键约束;数据剖析核实了覆盖率和父键唯一的关系也是,见 profile_relation_holds)、confirmed 人工确认、
|
||||
rejected 人工驳回。只有 confirmed 和 verified
|
||||
将来会触发错误级检查;给模型看时 proposed 的项标「推断,未确认」;rejected 的项留着(防止下次起草又提出
|
||||
来),但模型和检查都看不到。
|
||||
|
||||
@@ -98,6 +99,33 @@ def initial_status(source: str) -> str:
|
||||
return _INITIAL_STATUS.get(source, "proposed")
|
||||
|
||||
|
||||
#: 数据剖析把关系升为有确证的抽样覆盖率下限(data/catalog_profile.py 判定,撤销审阅时按它判回剖析的结论)
|
||||
PROFILE_VERIFY_COVERAGE = 0.95
|
||||
|
||||
|
||||
def profile_relation_holds(rel: Mapping[str, Any]) -> bool:
|
||||
"""一条数据剖析得出的关系站不站得住:抽样覆盖率不低于 PROFILE_VERIFY_COVERAGE,而且被指向列核实唯一。
|
||||
|
||||
剖析只在被指向列核实唯一时才写基数(多对一或一对一),所以「基数有值」就是「父键唯一」。结论全在关系
|
||||
自己的字段里,撤销审阅时照原样判得回来,不必另存一份「审阅前的状态」。
|
||||
"""
|
||||
coverage = rel.get("coverage")
|
||||
return (rel.get("source") == "profile" and isinstance(coverage, (int, float)) and not isinstance(coverage, bool)
|
||||
and coverage >= PROFILE_VERIFY_COVERAGE and rel.get("cardinality") in ("many_to_one", "one_to_one"))
|
||||
|
||||
|
||||
def unreviewed_status(item: Mapping[str, Any]) -> str:
|
||||
"""撤销审阅后回到的状态:来源的初始状态。
|
||||
|
||||
数据剖析是例外:它的状态随数据而定(覆盖率够、父键唯一的关系是有确证,其余是推断),不能一律回到
|
||||
initial_status("profile") 的推断——那样人工确认过再撤销,有确证就无故丢了,要等下一次剖析才回来。
|
||||
剖析提的码值候选、业务日期永远是推断。
|
||||
"""
|
||||
if item.get("source") == "profile" and "to_table" in item:
|
||||
return "verified" if profile_relation_holds(item) else "proposed"
|
||||
return initial_status(item.get("source"))
|
||||
|
||||
|
||||
def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None,
|
||||
at: str | None = None) -> dict[str, Any]:
|
||||
"""构造一个项。status 不给时按来源取初始状态(initial_status)。"""
|
||||
@@ -153,8 +181,10 @@ def _business_date_ok(value: Any) -> bool:
|
||||
|
||||
|
||||
def _codes_ok(value: Any) -> bool:
|
||||
"""码值对照:{码值: 含义},都是文字。含义可以是空串——数据剖析只知道列里出现过哪些取值,含义要等人填;
|
||||
以前要求含义非空,剖析得到的码值候选就写不进来。"""
|
||||
return (isinstance(value, dict) and bool(value)
|
||||
and all(isinstance(k, str) and isinstance(v, str) and v.strip() for k, v in value.items()))
|
||||
and all(isinstance(k, str) and isinstance(v, str) for k, v in value.items()))
|
||||
|
||||
|
||||
#: 每个字段的值怎么查、不合规时怎么说
|
||||
@@ -574,7 +604,7 @@ def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at:
|
||||
# 人工填写的项没有「推断时的样子」可回:撤销就是删掉,下次起草可以重新提出
|
||||
del slots[key]
|
||||
return _assemble(slots)
|
||||
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or initial_status(item.get("source"))
|
||||
item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or unreviewed_status(item)
|
||||
item["updated_at"] = at or now_iso()
|
||||
return _assemble(slots)
|
||||
|
||||
@@ -639,7 +669,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
|
||||
continue
|
||||
kept = dict(prev)
|
||||
status = item.get("status")
|
||||
if status != prev.get("status") and status in ("confirmed", "rejected", initial_status(prev.get("source"))):
|
||||
if status != prev.get("status") and status in ("confirmed", "rejected", unreviewed_status(prev)):
|
||||
kept["status"] = status
|
||||
kept["updated_at"] = at
|
||||
if item.get("note") != prev.get("note"):
|
||||
@@ -1383,6 +1413,17 @@ def _date_text(value: Mapping[str, Any]) -> str:
|
||||
return ";".join(parts)
|
||||
|
||||
|
||||
def _codes_text(codes: Mapping[str, Any]) -> str:
|
||||
"""码值对照的文字:有含义的写「1=有效」,含义待填写的单独说明——不能写成「1=」,模型会当成含义就是空。"""
|
||||
filled = [f"{k}={v}" for k, v in codes.items() if str(v).strip()]
|
||||
pending = [str(k) for k, v in codes.items() if not str(v).strip()]
|
||||
if not pending:
|
||||
return "、".join(filled)
|
||||
if not filled:
|
||||
return f"{'、'.join(pending)}(含义待填写)"
|
||||
return f"{'、'.join(filled)},{'、'.join(pending)} 的含义待填写"
|
||||
|
||||
|
||||
def _relation_text(rel: Mapping[str, Any]) -> str:
|
||||
left = ", ".join(rel.get("columns") or [])
|
||||
right = ", ".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or [])
|
||||
@@ -1433,7 +1474,7 @@ def _notes_lines(notes: Mapping[str, Any] | None, *, meta: Mapping[str, Any] | N
|
||||
if name == "measure":
|
||||
text = MEASURE_LABEL.get(value, str(value))
|
||||
elif name == "codes":
|
||||
text = "、".join(f"{k}={v}" for k, v in value.items())
|
||||
text = _codes_text(value)
|
||||
else:
|
||||
text = str(value)
|
||||
# 第一段是中文名(没有中文名时是含义),不加前缀;其余写明是什么
|
||||
@@ -1690,7 +1731,8 @@ __all__ = [
|
||||
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
|
||||
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
|
||||
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
|
||||
"MergeStats", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS",
|
||||
"MergeStats", "PROFILE_VERIFY_COVERAGE", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS",
|
||||
"TABLE_FIELD_LABEL", "TABLE_KINDS", "profile_relation_holds", "unreviewed_status",
|
||||
"TableDraft", "TableDraftResult", "apply_human_edit", "describe_slot", "draft_catalog", "draft_structure",
|
||||
"draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths",
|
||||
"make_item", "merge_notes", "now_iso", "parse_path", "query_snapshot_meta", "read_catalog", "read_entry",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -58,11 +58,16 @@ MAX_QUERY_TIMEOUT_S = 600
|
||||
#: 完整快照仍能按工件 id 取到,SQL 里给列起个别名也能绕开
|
||||
MASK_COLUMNS_OPTION = "mask_columns"
|
||||
|
||||
#: 数据源 options 里的数据剖析设置(一个对象:开关、查询次数上限、单条时限……)。取值和校验见
|
||||
#: data/catalog_profile.py;放在这里只为让拼连接串时摘掉它
|
||||
CATALOG_PROFILE_OPTION = "catalog_profile"
|
||||
|
||||
# options 里这些 key 是 AgentLab 自己的配置,不是驱动参数,拼 URL 时要摘掉。
|
||||
# schema:探查哪个 schema(企业库里只读账号名下常常什么都没有,数据在别处)
|
||||
# query_timeout_s:查询时限,由数据层按语句下发给数据库,见 _server_deadline
|
||||
# mask_columns:证据面板遮罩的列,驱动不认识它,拼进连接串会被当成未知参数拒掉
|
||||
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION})
|
||||
# catalog_profile:数据剖析设置,同上;它还是个对象,拼进去连 URL 都不成形
|
||||
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION, CATALOG_PROFILE_OPTION})
|
||||
|
||||
_MASK_SPLIT = re.compile(r"[,,、;;\n]+")
|
||||
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
"""数据剖析接口:POST /api/datasources/{id}/catalog/profile(阶段 2A)。
|
||||
|
||||
剖析是数据目录里唯一对业务库发查询的地方,这里逐条钉住方案定的约束:默认关闭、只读、每条查询都过守卫、
|
||||
查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析;
|
||||
以及剖析的结论怎么进目录:命名推断对上数据的升为有确证(来源记为数据剖析,和外键分开)、对不上的写明
|
||||
覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。
|
||||
|
||||
所有测试都在合成的景区库上做(tests/fixtures/catalog/scenic.py),数据和名称全部虚构。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from app.data import catalog, catalog_profile
|
||||
from app.data import engine as data_engine
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.catalog import scenic
|
||||
|
||||
ACTOR = {"X-Actor": quote("周宁")}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def _on(**settings) -> dict:
|
||||
return {PROFILE_OPTION: {"enabled": True, **settings}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def make_source(client):
|
||||
"""登记一个接上 path 的数据源并走真实的探查接口;draft 给了就先起草这些表的目录。"""
|
||||
made: list[str] = []
|
||||
|
||||
async def _make(path: str, *, options: dict | None = None, draft: list[str] | None = None) -> str:
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=f"prof_{uuid.uuid4().hex[:10]}", kind="sqlite", database=str(path),
|
||||
options=options or {})
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
made.append(sid)
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
if draft is not None:
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": draft})
|
||||
assert r.status_code == 200, r.text
|
||||
return sid
|
||||
|
||||
yield _make
|
||||
for sid in made:
|
||||
await engines.invalidate(sid)
|
||||
|
||||
|
||||
class _Spy:
|
||||
"""记下两样东西:守卫放行(guard.check 返回)的语句,和真正发到这个业务库的语句。
|
||||
|
||||
后者挂在 SQLAlchemy 的 before_cursor_execute 上,任何一条到达驱动的语句都逃不掉——包括绕开查询层
|
||||
另开连接发的。断言「发到库的 ⊆ 守卫放行的」就是「所有查询都经过守卫」。
|
||||
"""
|
||||
|
||||
def __init__(self, db_name: str) -> None:
|
||||
self.db_name = db_name
|
||||
self.checked: list[str] = []
|
||||
self.executed: list[str] = []
|
||||
|
||||
def clear(self) -> None:
|
||||
self.checked.clear()
|
||||
self.executed.clear()
|
||||
|
||||
def all_guarded(self) -> bool:
|
||||
return bool(self.executed) and all(s in self.checked for s in self.executed)
|
||||
|
||||
def mentions(self, word: str) -> list[str]:
|
||||
return [s for s in self.executed if word.lower() in s.lower()]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def spy(monkeypatch, scenic_db):
|
||||
holder = SimpleNamespace(spy=None)
|
||||
|
||||
def watch(path: str) -> _Spy:
|
||||
holder.spy = _Spy(Path(path).name)
|
||||
return holder.spy
|
||||
|
||||
real_check = data_engine.check
|
||||
|
||||
def checking(sql, **kw):
|
||||
out = real_check(sql, **kw)
|
||||
if holder.spy is not None:
|
||||
holder.spy.checked.append(out)
|
||||
return out
|
||||
|
||||
def on_execute(conn, cursor, statement, parameters, context, executemany):
|
||||
if holder.spy is not None and holder.spy.db_name in str(conn.engine.url):
|
||||
holder.spy.executed.append(statement)
|
||||
|
||||
monkeypatch.setattr(data_engine, "check", checking)
|
||||
event.listen(Engine, "before_cursor_execute", on_execute)
|
||||
yield watch
|
||||
event.remove(Engine, "before_cursor_execute", on_execute)
|
||||
|
||||
|
||||
def _url(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog/profile"
|
||||
|
||||
|
||||
async def _notes(client, sid: str, table: str) -> dict:
|
||||
r = await client.get(f"/api/datasources/{sid}/catalog/{table}")
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
|
||||
def _relation(notes: dict, column: str) -> dict:
|
||||
return next(r for r in notes.get("relations") or [] if r["columns"] == [column])
|
||||
|
||||
|
||||
def _table(body: dict, name: str) -> dict:
|
||||
return next(t for t in body["tables"] if t["table_name"] == name)
|
||||
|
||||
|
||||
def _reasons(table: dict) -> set[str]:
|
||||
return {s["reason"] for s in table["skipped"]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 开关与设置
|
||||
|
||||
|
||||
async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 409
|
||||
detail = r.json()["detail"]
|
||||
assert "数据源设置" in detail and "开启数据剖析" in detail
|
||||
assert watch.executed == [] # 一条查询都没发
|
||||
|
||||
|
||||
async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db)
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)})
|
||||
assert r.status_code == 422 and "查询次数上限" in r.json()["detail"]
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500}
|
||||
# 设好以后就能剖析;返回里带着这次实际用的设置
|
||||
r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 剖析的结论
|
||||
|
||||
|
||||
async def test_profile_verifies_name_relations_and_proposes_codes_and_business_date(client, make_source, scenic_db,
|
||||
spy):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
|
||||
# 所有发到业务库的语句都经过了守卫
|
||||
assert watch.all_guarded(), watch.executed
|
||||
assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"]
|
||||
assert body["queries_used"] == sum(t["queries"] for t in body["tables"])
|
||||
assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"]
|
||||
|
||||
notes = (await _notes(client, sid, "visits"))
|
||||
assert notes["updated_by"] == "周宁"
|
||||
notes = notes["notes"]
|
||||
# 命名推断出来的两条关系:数据对得上、父表主键唯一 → 有确证,来源记为数据剖析
|
||||
for column, parent in (("gate_id", "gates"), ("member_id", "members")):
|
||||
rel = _relation(notes, column)
|
||||
assert rel["to_table"] == parent
|
||||
assert (rel["source"], rel["status"]) == ("profile", "verified")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
assert "数据剖析" in rel["note"] and "抽样" in rel["note"]
|
||||
# 外键约束来的关系不动:外键是外键,剖析是剖析
|
||||
park = _relation(notes, "park_id")
|
||||
assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None)
|
||||
|
||||
# 状态列:码值候选,值有了、含义留空,推断
|
||||
codes = notes["columns"]["status"]["codes"]
|
||||
assert codes["value"] == {"1": "", "0": ""}
|
||||
assert (codes["source"], codes["status"]) == ("profile", "proposed") and "1500" in codes["note"]
|
||||
# 只有一个日期类列:提议为业务日期,备注里写明取值范围
|
||||
bdate = notes["business_date"]
|
||||
assert bdate["value"] == {"column": "visit_time"} and (bdate["source"], bdate["status"]) == ("profile", "proposed")
|
||||
assert "2026-07" in bdate["note"] and "2026-09" in bdate["note"]
|
||||
|
||||
channels = (await _notes(client, sid, "channels"))["notes"]
|
||||
assert set(channels["columns"]["channel_type"]["codes"]["value"]) == {"线上直销", "线下", "分销"}
|
||||
assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值
|
||||
|
||||
visits = _table(body, "visits")
|
||||
kinds = sorted(f["kind"] for f in visits["findings"])
|
||||
assert kinds == ["business_date", "codes", "relation", "relation"]
|
||||
gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"])
|
||||
assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}"
|
||||
assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"]
|
||||
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
|
||||
assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None
|
||||
|
||||
|
||||
async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path):
|
||||
"""推错的命名关系:parking_records.lot_id 按名字对上了新建的 lots 表,但停车记录里的区号大多不在里面。"""
|
||||
path = scenic.build(tmp_path / "lots.db")
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)")
|
||||
db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)])
|
||||
db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)",
|
||||
[(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)])
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
sid = await make_source(path, options=_on(), draft=["parking_records"])
|
||||
before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed")
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert rel["status"] == "proposed" and rel["source"] == "profile"
|
||||
assert rel["coverage"] == 0.15
|
||||
assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"]
|
||||
finding = _table(r.json(), "parking_records")["findings"][0]
|
||||
assert finding["status"] == "proposed" and finding["coverage"] == 0.15
|
||||
# 两个日期类列:不提议业务日期,取值范围只在返回里
|
||||
ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]}
|
||||
assert ranges == {"entered_at"} # exited_at 全是空值
|
||||
assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"]
|
||||
|
||||
|
||||
async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
detail = await _notes(client, sid, "visits")
|
||||
gate = _relation(detail["notes"], "gate_id")
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/visits/review",
|
||||
json={"path": f"relations.{gate['id']}", "action": "confirm",
|
||||
"if_version": detail["version"]})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "confirmed")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"])
|
||||
assert finding["status"] == "confirmed" and finding["confirmed"] is True
|
||||
|
||||
|
||||
async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db):
|
||||
"""同一天、数据没变:第二次剖析不改目录、不升版本(冻结进快照的目录不会无故变)。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
first = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
second = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
a, b = _table(first, "visits"), _table(second, "visits")
|
||||
assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0)
|
||||
assert b["version"] == a["version"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 安全约束
|
||||
|
||||
|
||||
async def test_masked_columns_are_never_queried(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options={**_on(), "mask_columns": "member_id, Channel_Type"},
|
||||
draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert watch.all_guarded()
|
||||
assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == []
|
||||
visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels")
|
||||
masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"]
|
||||
assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"}
|
||||
assert all("遮罩" in s["detail"] for s in masked)
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "member_id")["source"] == "name" # 没动
|
||||
assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常
|
||||
assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {})
|
||||
|
||||
|
||||
async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget"
|
||||
visits, channels = _table(body, "visits"), _table(body, "channels")
|
||||
assert "budget" in _reasons(visits)
|
||||
assert channels["queries"] == 0 and _reasons(channels) == {"budget"}
|
||||
assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget")
|
||||
# 没查完的关系不改:原来的命名推断原样留着
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "proposed")
|
||||
|
||||
|
||||
async def test_single_query_timeout_is_recorded_and_profiling_continues(client, make_source, scenic_db, monkeypatch):
|
||||
"""闸口的抽样查询换成一条停不下来的递归查询:1 秒时限到点由数据库掐断,记为超时,其余照常。"""
|
||||
sid = await make_source(scenic_db, options=_on(query_timeout_s=1), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
endless = "WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM r) SELECT COUNT(*) FROM r"
|
||||
|
||||
async def slow_gate(source, sql, *, limits=None):
|
||||
if "gate_id" in sql and "DISTINCT" in sql and "COUNT" not in sql:
|
||||
assert limits is not None and limits.timeout_seconds == 1
|
||||
return await real(source, endless, limits=limits)
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow_gate)
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"]
|
||||
assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"]
|
||||
assert r.json()["stopped"] is None
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改
|
||||
assert _relation(notes, "member_id")["status"] == "verified" # 其余照常
|
||||
|
||||
|
||||
async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
|
||||
async def slow(source, sql, *, limits=None):
|
||||
await asyncio.sleep(0.5)
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
settings = ProfileSettings(enabled=True, max_total_s=1.4)
|
||||
report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None,
|
||||
settings=settings)
|
||||
assert report.stopped == "deadline"
|
||||
assert 1 <= report.queries_used <= 3
|
||||
reasons = {s.reason for t in report.tables for s in t.skipped}
|
||||
assert "deadline" in reasons
|
||||
|
||||
|
||||
async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy):
|
||||
"""行数超过整表统计上限(这里设成 100 行):不做 COUNT(DISTINCT)、MIN/MAX,抽样和取值分布都带上限。"""
|
||||
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101}
|
||||
on_visits = [s for s in watch.executed if "from visits" in s.lower()]
|
||||
assert on_visits and all("LIMIT" in s for s in on_visits), on_visits
|
||||
assert not any("MIN(" in s for s in on_visits)
|
||||
assert "too_large" in _reasons(visits)
|
||||
assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large")
|
||||
# 父表有主键,覆盖率又够:照样有确证;子表一侧没核对唯一,按多对一记
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 接口
|
||||
|
||||
|
||||
async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def held(source, sql, *, limits=None):
|
||||
started.set()
|
||||
await release.wait()
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", held)
|
||||
first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR))
|
||||
await asyncio.wait_for(started.wait(), 10)
|
||||
second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"]
|
||||
release.set()
|
||||
done = await asyncio.wait_for(first, 30)
|
||||
assert done.status_code == 200, done.text
|
||||
# 做完以后可以再来
|
||||
again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert again.status_code == 200, again.text
|
||||
|
||||
|
||||
async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=[])
|
||||
r = await client.post(_url(sid), json={}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
names = [t["table_name"] for t in r.json()["tables"]]
|
||||
assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES
|
||||
async with SessionLocal() as session:
|
||||
entries = await catalog.read_catalog(session, sid)
|
||||
for name in names:
|
||||
rels = entries[name].notes.get("relations") or []
|
||||
assert any(rel["source"] not in ("fk",) for rel in rels), name
|
||||
assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选
|
||||
|
||||
|
||||
async def test_unknown_table_and_missing_source(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR)
|
||||
assert r.status_code == 200
|
||||
assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0
|
||||
r = await client.post(_url("0" * 32), json={}, headers=ACTOR)
|
||||
assert r.status_code == 404
|
||||
@@ -0,0 +1,205 @@
|
||||
"""数据剖析(阶段 2A)里不连库的部分:设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、
|
||||
目录对剖析结果的容纳(待填含义的码值、撤销审阅回到剖析的结论)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data import catalog, catalog_profile, guard
|
||||
from app.data.catalog import make_item
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
|
||||
from app.data.engine import build_url
|
||||
|
||||
AT = "2026-10-03T08:00:00+00:00"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 设置
|
||||
|
||||
|
||||
def test_settings_default_off_and_parse_leniently():
|
||||
s = profile_settings({})
|
||||
assert s == ProfileSettings()
|
||||
assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000
|
||||
assert s.max_scan_rows == 100_000 and s.max_total_s == 120
|
||||
on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}})
|
||||
assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500
|
||||
# 库里存着的值不对(老数据、手改过):那一项用缺省,不因此开启,也不报错
|
||||
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}})
|
||||
assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10
|
||||
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, label", [
|
||||
({"max_queries": 0}, "查询次数上限"),
|
||||
({"max_queries": 501}, "查询次数上限"),
|
||||
({"max_queries": 2.5}, "查询次数上限"),
|
||||
({"query_timeout_s": 61}, "单条查询时限"),
|
||||
({"sample_size": 5}, "抽样键值数"),
|
||||
({"max_scan_rows": -1}, "整表统计行数上限"),
|
||||
({"max_total_s": 5}, "总时长上限"),
|
||||
({"enabled": "true"}, "开启数据剖析"),
|
||||
({"max_queries": True}, "查询次数上限"),
|
||||
])
|
||||
def test_settings_problem_names_the_field_in_chinese(value, label):
|
||||
problem = profile_settings_problem({PROFILE_OPTION: value})
|
||||
assert problem and label in problem and "数据剖析" in problem
|
||||
|
||||
|
||||
def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
|
||||
assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}})
|
||||
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
|
||||
assert profile_settings_problem({}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: None}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5,
|
||||
"sample_size": 10, "max_scan_rows": 0,
|
||||
"max_total_s": 600}}) is None
|
||||
|
||||
|
||||
def test_profile_option_never_reaches_the_connection_url():
|
||||
"""剖析设置是 AgentLab 自己的配置:拼进连接串,驱动会把它当未知参数拒掉。"""
|
||||
source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro",
|
||||
password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}})
|
||||
url = build_url(source)
|
||||
assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 字面量
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind, expected", [
|
||||
(42, "number", "42"),
|
||||
(-7, None, "-7"),
|
||||
(3.0, "number", "3"),
|
||||
("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串
|
||||
("A-01", "text", "'A-01'"),
|
||||
("O'Neil", "text", "'O''Neil'"),
|
||||
("12:30", "text", "'12:30'"),
|
||||
])
|
||||
def test_sql_literal_quotes_safe_values(value, kind, expected):
|
||||
assert catalog_profile.sql_literal(value, kind) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind", [
|
||||
(True, "boolean"), # 布尔不当键
|
||||
(2.5, "number"), # 非整数的数不当键:浮点相等比较靠不住
|
||||
("12.50", "number"),
|
||||
("1e3", "number"),
|
||||
("2026-07-01", "date"), # 日期的字面量各家写法不同,不猜
|
||||
("a\\b", "text"), # 反斜杠:MySQL 不同设置下读法不同,守卫会拒
|
||||
("a :b", "text"), # 「:名字」会被查询层当成绑定参数
|
||||
(":b", "text"),
|
||||
("a\nb", "text"), # 控制字符
|
||||
("x" * 201, "text"),
|
||||
(None, "text"),
|
||||
([1], None),
|
||||
])
|
||||
def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind):
|
||||
assert catalog_profile.sql_literal(value, kind) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 各方言的剖析 SQL
|
||||
|
||||
|
||||
_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]},
|
||||
"orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"])
|
||||
def test_profile_sql_passes_readonly_guard_on_every_dialect(kind):
|
||||
"""剖析的每一种查询(含读统计信息的系统视图)都得过只读守卫——守卫不为剖析放宽。"""
|
||||
d = catalog_profile.SqlDialect(kind)
|
||||
sqls = []
|
||||
for table in ("visits", "orders"):
|
||||
stats = d.stats_sql(_META[table], table)
|
||||
if kind == "sqlite":
|
||||
assert stats is None # SQLite 没有可读的统计信息,改为数到上限
|
||||
else:
|
||||
assert stats
|
||||
sqls.append(stats)
|
||||
tbl = d.table(_META[table], table)
|
||||
col = d.quote(_META[table]["columns"][0]["name"])
|
||||
sqls += [
|
||||
d.bounded_count_sql(tbl, 100_001),
|
||||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=None),
|
||||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000),
|
||||
d.match_count_sql(tbl, col, ["1", "'A'"]),
|
||||
d.unique_check_sql(tbl, col),
|
||||
d.value_counts_sql(tbl, col, 21, scan_cap=None),
|
||||
d.value_counts_sql(tbl, col, 21, scan_cap=100_000),
|
||||
d.min_max_sql(tbl, [col]),
|
||||
]
|
||||
for sql in sqls:
|
||||
assert guard.check(sql, readonly=True, dialect=kind) == sql, sql
|
||||
|
||||
|
||||
def test_dialect_quoting_and_row_limits():
|
||||
my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql"))
|
||||
assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"'
|
||||
assert my.table(_META["orders"], "orders") == "sales.orders"
|
||||
assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"'
|
||||
# Oracle 没有 LIMIT(11g 也没有 FETCH FIRST):一律用 ROWNUM
|
||||
sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None)
|
||||
assert "LIMIT" not in sample and "ROWNUM <= 50" in sample
|
||||
assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None)
|
||||
# 统计信息按方言读各自的系统视图
|
||||
assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits")
|
||||
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
|
||||
stats = ora.stats_sql(_META["orders"], "orders")
|
||||
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 目录对剖析结果的容纳
|
||||
|
||||
|
||||
def test_codes_with_pending_meanings_are_valid_and_render_as_pending():
|
||||
"""剖析得到的码值只有取值、含义待人填写:目录要收得下,给模型看时写明含义待填写。"""
|
||||
notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")},
|
||||
"channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}}
|
||||
assert catalog.validate_notes(notes) == []
|
||||
text = catalog.render_table_notes(notes)
|
||||
assert "1、0(含义待填写)" in text
|
||||
assert "WEB=官网,BOX 的含义待填写" in text
|
||||
# 码值仍得是对照表:值不是文字的照样拒收
|
||||
assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}})
|
||||
|
||||
|
||||
def _profile_rel(coverage, cardinality, status):
|
||||
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
|
||||
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
|
||||
"source": "profile", "status": status}
|
||||
|
||||
|
||||
def test_reset_returns_profile_relation_to_what_profiling_concluded():
|
||||
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定:覆盖率够、父键唯一(基数有值)就是有确证。"""
|
||||
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
|
||||
rid = verified["relations"][0]["id"]
|
||||
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
|
||||
assert out["relations"][0]["status"] == "confirmed"
|
||||
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "verified"
|
||||
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
|
||||
f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "verified"
|
||||
# 覆盖率低、父键没核实唯一的:撤销后仍是推断
|
||||
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
|
||||
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
|
||||
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
|
||||
f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "proposed"
|
||||
# 剖析提的码值候选、业务日期永远是推断
|
||||
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
|
||||
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
|
||||
"columns.status.codes", "reset", at=AT)
|
||||
assert out["columns"]["status"]["codes"]["status"] == "proposed"
|
||||
|
||||
|
||||
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
|
||||
rel = _profile_rel(1.0, "many_to_one", "confirmed")
|
||||
existing = {"relations": [rel]}
|
||||
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
|
||||
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
|
||||
low = _profile_rel(0.4, "many_to_one", "proposed")
|
||||
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
|
||||
table="visits", at=AT)
|
||||
assert kept["relations"][0]["status"] == "proposed"
|
||||
Reference in New Issue
Block a user