mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 07:31:58 +00:00
合入集成分支(阶段 2A 数据剖析、4B 剖析与 SQL 检查界面及修正)到 4A:两边功能都保留
冲突的解法: - 后端 api/catalog.py:同时导入 catalog_impact 和 catalog_profile、SnapshotTampered;data/catalog.py:__all__ 取两边的并集重排。 - 2A 放宽了码值校验(含义可以为空,剖析写进来的是「含义待填写」):目录修改提案的码值补充改为含义为空的码不覆盖已有含义、 只在原来没有这个码时加进去,补了测试;卡片和过程里的码值把空含义写成「含义待填写」,和目录页同一个说法。 - 前端 types.ts、terms.ts:两边各在同一处追加了新段落,按顺序都留下(提案、影响面、目录版本在前,剖析、码值、SQL 检查在后), 没有重复的键;client.ts、decode.ts、CatalogPage.tsx、TableDetail.tsx 的导入取并集。 - check-catalog.mjs:影响面和剖析、保存数据源三条假路由都留下,各节和最后核对真库数量的逻辑不变。 - 4A 的测试数据源改用 tests/fixtures/sources.py 的 drop_source 删除,顺带作废连接池。
This commit is contained in:
+5
-3
@@ -9,8 +9,9 @@
|
||||
从 visits 到 channels 要经过它,正好是两跳的连接路径。group_visits 同理(旅行团与入园记录)。
|
||||
- **命名推断必须放过的列**(NOT_INFERRED):自引用(employees.employee_id 是工号,不是指向自己)、
|
||||
多个候选(guide 和 guides 两张表都在,tour_assignments.guide_id 不知道指哪张)、没有对应的表
|
||||
(parking_records.lot_id、operation_logs.operator_id、categories.parentId)、类型对不上
|
||||
(feedback_forms.store_id 是文字,stores.id 是整数)。
|
||||
(operation_logs.operator_id、categories.parentId)、类型对不上(feedback_forms.store_id 是文字,stores.id 是整数)。
|
||||
parking_records.lot_id 没有直接对上的 lot / lots,但和引用它的表共用 parking 前缀的 parking_lots 只有一张,
|
||||
推得出来(在 NAME_RELATIONS 里)。
|
||||
- **唯一约束 / 唯一索引。** orders.order_no、channels.channel_code 是列上的 UNIQUE;members.member_no、
|
||||
visits.ticket_no 是单独建的唯一索引。
|
||||
|
||||
@@ -188,6 +189,8 @@ NAME_RELATIONS = frozenset({
|
||||
("tour_groups", ("agency_id",), "agencies", ("id",)),
|
||||
("tour_assignments", ("tour_group_id",), "tour_groups", ("id",)),
|
||||
("lost_and_found", ("park_id",), "parks", ("id",)),
|
||||
# 没有 lot / lots,按引用表共用的 parking 前缀对上 parking_lots
|
||||
("parking_records", ("lot_id",), "parking_lots", ("id",)),
|
||||
# 视图没有约束,按列名同样推得出来:按日汇总的入园人数照样能按 park_id 连到景区
|
||||
("v_daily_visits", ("park_id",), "parks", ("id",)),
|
||||
})
|
||||
@@ -196,7 +199,6 @@ NAME_RELATIONS = frozenset({
|
||||
NOT_INFERRED = {
|
||||
("employees", "employee_id"): "自引用:工号,不是指向本表",
|
||||
("tour_assignments", "guide_id"): "多个候选:guide 与 guides",
|
||||
("parking_records", "lot_id"): "没有名为 lot / lots 的表",
|
||||
("operation_logs", "operator_id"): "没有名为 operator / operators 的表",
|
||||
("categories", "parentId"): "没有名为 parent / parents 的表",
|
||||
("feedback_forms", "store_id"): "类型对不上:文字列指向整数主键",
|
||||
|
||||
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
"""测试里建的数据源,用完就删。
|
||||
|
||||
数据源表在每个测试进程的测试库里是共用的:留下来的源会出现在助手的数据源清单里(50 张表的合成库还会
|
||||
触发挑表、多一次模型调用),别的用例的结果就跟着执行顺序变。阶段 1B 就是这样被绊过一次。
|
||||
删数据源时它的数据目录随外键级联删除(测试库开着 PRAGMA foreign_keys=ON)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
|
||||
|
||||
async def drop_source(source_id: str | None = None, *, name: str | None = None) -> None:
|
||||
"""按 id 或名字删一个数据源,并作废它的连接池。不存在就什么都不做。"""
|
||||
async with SessionLocal() as session:
|
||||
if source_id is None:
|
||||
source_id = (await session.execute(
|
||||
select(DataSource.id).where(DataSource.name == name))).scalar_one_or_none()
|
||||
if source_id is None:
|
||||
return
|
||||
await session.execute(delete(DataSource).where(DataSource.id == source_id))
|
||||
await session.commit()
|
||||
await engines.invalidate(source_id)
|
||||
@@ -11,12 +11,12 @@ import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.data import catalog
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.catalog import scenic
|
||||
from tests.fixtures.catalog.fake_model import FakeModel
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
ACTOR = {"X-Actor": quote("王敏")}
|
||||
|
||||
@@ -38,7 +38,7 @@ async def source_id(client, scenic_db):
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
yield sid
|
||||
await engines.invalidate(sid)
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
def _base(sid: str) -> str:
|
||||
@@ -184,9 +184,12 @@ async def test_draft_without_schema_is_refused(client):
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
r = await client.post(f"{_base(sid)}/draft", json={})
|
||||
assert r.status_code == 409 and "探查结构" in r.json()["detail"]
|
||||
assert (await client.get(_base(sid))).json()["tables"] == []
|
||||
try:
|
||||
r = await client.post(f"{_base(sid)}/draft", json={})
|
||||
assert r.status_code == 409 and "探查结构" in r.json()["detail"]
|
||||
assert (await client.get(_base(sid))).json()["tables"] == []
|
||||
finally:
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
async def test_schema_endpoint_detail_carries_catalog(client, source_id):
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.data.catalog import make_item
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from tests.fixtures.catalog.fake_model import FakeModel
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
|
||||
def _col(name: str, type_: str = "INTEGER", comment: str | None = None, nullable: bool = True) -> dict:
|
||||
@@ -156,6 +157,31 @@ def test_name_inference_negative(tables, why):
|
||||
assert _infer(tables, next(iter(tables))) == set(), why
|
||||
|
||||
|
||||
def test_name_inference_falls_back_to_a_table_sharing_the_prefix():
|
||||
"""库里没有 tags,只有带前缀的 store_tags 和 menu_tag:和引用它的表共用 store 前缀的那张才是它要指的。
|
||||
以前这种命名推不出关系,SQL 检查跟着误报「对不上任何关系」。"""
|
||||
tables = {
|
||||
"store_tag_links": _table(_col("id"), _col("tagId"), _col("storeID")),
|
||||
"store_tags": _table(_col("id")),
|
||||
"menu_tag": _table(_col("id")),
|
||||
"store": _table(_col("id")),
|
||||
}
|
||||
assert _infer(tables, "store_tag_links") == {
|
||||
(("tagId",), "store_tags", ("id",)), (("storeID",), "store", ("id",))}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tables, why", [
|
||||
# 直接对得上就用直接的,不去找带前缀的
|
||||
({"shop_category_links": _table(_col("id"), _col("category_id")), "categories": _table(_col("id")),
|
||||
"shop_categories": _table(_col("id"))}, {(("category_id",), "categories", ("id",))}),
|
||||
# 引用它的表没有可共用的前缀:带前缀的表不止一张时照旧不推
|
||||
({"links": _table(_col("id"), _col("tag_id")), "store_tags": _table(_col("id")),
|
||||
"menu_tags": _table(_col("id"))}, set()),
|
||||
])
|
||||
def test_prefix_fallback_only_when_direct_match_is_missing(tables, why):
|
||||
assert _infer(tables, next(iter(tables))) == why
|
||||
|
||||
|
||||
def test_name_inference_one_to_one_when_column_is_own_key():
|
||||
tables = {"member_profiles": _table(_col("member_id"), pk=("member_id",)), "members": _table(_col("id"))}
|
||||
rel = catalog.infer_name_relations(_source(tables).schema_cache, "member_profiles")[0]
|
||||
@@ -255,6 +281,16 @@ async def test_model_draft_respects_system_notes_of_imported_tables():
|
||||
# ---------------------------------------------------------------- 起草并写入
|
||||
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _drop_created_sources():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
async def _db_source(tables: dict, **kw) -> DataSource:
|
||||
for name, meta in tables.items():
|
||||
meta["qualified"] = name
|
||||
@@ -263,6 +299,7 @@ async def _db_source(tables: dict, **kw) -> DataSource:
|
||||
schema_cache={"tables": tables}, **kw)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
_CREATED.append(row.id)
|
||||
return row
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
# 模型要在建表之前注册进 metadata:conftest 的建表只导入了 app.db.base,这个文件单独跑时没有别的模块先导入它们
|
||||
from app.db import models # noqa: F401
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
|
||||
@@ -106,10 +107,9 @@ async def world(scenic_db):
|
||||
for wid in created:
|
||||
if (wf := await session.get(Workflow, wid)) is not None:
|
||||
await session.delete(wf)
|
||||
for sid in (ids.source, ids.other):
|
||||
if (row := await session.get(DataSource, sid)) is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
for sid in (ids.source, ids.other):
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
async def _impact(source_id: str, table: str) -> dict:
|
||||
|
||||
@@ -25,6 +25,7 @@ from app.data import catalog
|
||||
from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
|
||||
@@ -76,6 +77,32 @@ async def test_codes_are_supplemented_not_replaced(scenic_db):
|
||||
assert change.value == {"9": "作废"}
|
||||
|
||||
|
||||
async def test_codes_with_pending_meanings(scenic_db):
|
||||
"""数据剖析写进来的码值候选含义为空(界面上写「含义待填写」):提案可以补上含义;提案里含义为空的码只在原来
|
||||
没有时加进去,不抹掉已有的含义;含义为空的码合法,保存不报错。"""
|
||||
tables = (await _schema(scenic_db))["tables"]
|
||||
profiled = _visits()
|
||||
profiled["columns"]["status"]["codes"] = catalog.make_item({"1": "有效", "0": "", "9": ""}, "profile", "proposed")
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废", "1": ""}}],
|
||||
table="visits", tables=tables)
|
||||
assert plan.problems == []
|
||||
[change] = plan.changes
|
||||
assert change.after == {"1": "有效", "0": "", "9": "作废"} and change.state == "change"
|
||||
# 只给了含义为空的新码:照样加进去,含义待人填
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"7": ""}}], table="visits",
|
||||
tables=tables)
|
||||
assert plan.changes[0].after == {"1": "有效", "0": "", "9": "", "7": ""}
|
||||
# 只给了含义为空的已有码:什么都没变,这一项是「保存即确认」(原来是推断)
|
||||
plan = catalog.plan_patch(profiled, [{"path": "columns.status.codes", "value": {"1": ""}}], table="visits",
|
||||
tables=tables)
|
||||
assert plan.changes[0].state == "confirm"
|
||||
out = catalog.apply_patch(profiled, [{"path": "columns.status.codes", "value": {"9": "作废"}}], table="visits",
|
||||
tables=tables)
|
||||
codes = out["columns"]["status"]["codes"]
|
||||
assert codes["value"] == {"1": "有效", "0": "", "9": "作废"} and (codes["source"], codes["status"]) == (
|
||||
"human", "confirmed")
|
||||
|
||||
|
||||
async def test_new_relation_gets_its_id_from_both_ends(scenic_db):
|
||||
tables = (await _schema(scenic_db))["tables"]
|
||||
value = {"columns": ["visit_id"], "to_table": "visits", "to_columns": ["id"], "cardinality": "many_to_one"}
|
||||
@@ -190,11 +217,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield SimpleNamespace(id=source_id, name=name)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
async def _stream(monkeypatch, model, source, base_graph=None):
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
"""数据剖析接口:POST /api/datasources/{id}/catalog/profile(阶段 2A)。
|
||||
|
||||
剖析是数据目录里唯一对业务库发查询的地方,这里逐条钉住方案定的约束:默认关闭、只读、每条查询都过守卫、
|
||||
查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析;
|
||||
以及剖析的结论怎么进目录:命名推断对上数据的升为有确证(来源记为数据剖析,和外键分开)、对不上的写明
|
||||
覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。
|
||||
|
||||
所有测试都在合成的景区库上做(tests/fixtures/catalog/scenic.py),数据和名称全部虚构。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from app.data import catalog, catalog_profile
|
||||
from app.data import engine as data_engine
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.catalog import scenic
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
ACTOR = {"X-Actor": quote("周宁")}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def _on(**settings) -> dict:
|
||||
return {PROFILE_OPTION: {"enabled": True, **settings}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def make_source(client):
|
||||
"""登记一个接上 path 的数据源并走真实的探查接口;draft 给了就先起草这些表的目录。"""
|
||||
made: list[str] = []
|
||||
|
||||
async def _make(path: str, *, options: dict | None = None, draft: list[str] | None = None) -> str:
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=f"prof_{uuid.uuid4().hex[:10]}", kind="sqlite", database=str(path),
|
||||
options=options or {})
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
made.append(sid)
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
if draft is not None:
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": draft})
|
||||
assert r.status_code == 200, r.text
|
||||
return sid
|
||||
|
||||
yield _make
|
||||
for sid in made:
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
class _Spy:
|
||||
"""记下两样东西:守卫放行(guard.check 返回)的语句,和真正发到这个业务库的语句。
|
||||
|
||||
后者挂在 SQLAlchemy 的 before_cursor_execute 上,任何一条到达驱动的语句都逃不掉——包括绕开查询层
|
||||
另开连接发的。断言「发到库的 ⊆ 守卫放行的」就是「所有查询都经过守卫」。
|
||||
"""
|
||||
|
||||
def __init__(self, db_name: str) -> None:
|
||||
self.db_name = db_name
|
||||
self.checked: list[str] = []
|
||||
self.executed: list[str] = []
|
||||
|
||||
def clear(self) -> None:
|
||||
self.checked.clear()
|
||||
self.executed.clear()
|
||||
|
||||
def all_guarded(self) -> bool:
|
||||
return bool(self.executed) and all(s in self.checked for s in self.executed)
|
||||
|
||||
def mentions(self, word: str) -> list[str]:
|
||||
return [s for s in self.executed if word.lower() in s.lower()]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def spy(monkeypatch, scenic_db):
|
||||
holder = SimpleNamespace(spy=None)
|
||||
|
||||
def watch(path: str) -> _Spy:
|
||||
holder.spy = _Spy(Path(path).name)
|
||||
return holder.spy
|
||||
|
||||
real_check = data_engine.check
|
||||
|
||||
def checking(sql, **kw):
|
||||
out = real_check(sql, **kw)
|
||||
if holder.spy is not None:
|
||||
holder.spy.checked.append(out)
|
||||
return out
|
||||
|
||||
def on_execute(conn, cursor, statement, parameters, context, executemany):
|
||||
if holder.spy is not None and holder.spy.db_name in str(conn.engine.url):
|
||||
holder.spy.executed.append(statement)
|
||||
|
||||
monkeypatch.setattr(data_engine, "check", checking)
|
||||
event.listen(Engine, "before_cursor_execute", on_execute)
|
||||
yield watch
|
||||
event.remove(Engine, "before_cursor_execute", on_execute)
|
||||
|
||||
|
||||
def _url(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog/profile"
|
||||
|
||||
|
||||
async def _notes(client, sid: str, table: str) -> dict:
|
||||
r = await client.get(f"/api/datasources/{sid}/catalog/{table}")
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
|
||||
def _relation(notes: dict, column: str) -> dict:
|
||||
return next(r for r in notes.get("relations") or [] if r["columns"] == [column])
|
||||
|
||||
|
||||
def _table(body: dict, name: str) -> dict:
|
||||
return next(t for t in body["tables"] if t["table_name"] == name)
|
||||
|
||||
|
||||
def _reasons(table: dict) -> set[str]:
|
||||
return {s["reason"] for s in table["skipped"]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 开关与设置
|
||||
|
||||
|
||||
async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 409
|
||||
detail = r.json()["detail"]
|
||||
assert "数据源设置" in detail and "开启数据剖析" in detail
|
||||
assert watch.executed == [] # 一条查询都没发
|
||||
|
||||
|
||||
async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db)
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)})
|
||||
assert r.status_code == 422 and "查询次数上限" in r.json()["detail"]
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500}
|
||||
# 设好以后就能剖析;返回里带着这次实际用的设置
|
||||
r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 剖析的结论
|
||||
|
||||
|
||||
async def test_profile_verifies_name_relations_and_proposes_codes_and_business_date(client, make_source, scenic_db,
|
||||
spy):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
|
||||
# 所有发到业务库的语句都经过了守卫
|
||||
assert watch.all_guarded(), watch.executed
|
||||
assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"]
|
||||
assert body["queries_used"] == sum(t["queries"] for t in body["tables"])
|
||||
assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"]
|
||||
|
||||
notes = (await _notes(client, sid, "visits"))
|
||||
assert notes["updated_by"] == "周宁"
|
||||
notes = notes["notes"]
|
||||
# 命名推断出来的两条关系:数据对得上、父表主键唯一 → 有确证,来源记为数据剖析
|
||||
for column, parent in (("gate_id", "gates"), ("member_id", "members")):
|
||||
rel = _relation(notes, column)
|
||||
assert rel["to_table"] == parent
|
||||
assert (rel["source"], rel["status"]) == ("profile", "verified")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
assert "数据剖析" in rel["note"] and "抽样" in rel["note"]
|
||||
# 外键约束来的关系不动:外键是外键,剖析是剖析
|
||||
park = _relation(notes, "park_id")
|
||||
assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None)
|
||||
|
||||
# 状态列:码值候选,值有了、含义留空,推断
|
||||
codes = notes["columns"]["status"]["codes"]
|
||||
assert codes["value"] == {"1": "", "0": ""}
|
||||
assert (codes["source"], codes["status"]) == ("profile", "proposed") and "1500" in codes["note"]
|
||||
# 只有一个日期类列:提议为业务日期,备注里写明取值范围
|
||||
bdate = notes["business_date"]
|
||||
assert bdate["value"] == {"column": "visit_time"} and (bdate["source"], bdate["status"]) == ("profile", "proposed")
|
||||
assert "2026-07" in bdate["note"] and "2026-09" in bdate["note"]
|
||||
|
||||
channels = (await _notes(client, sid, "channels"))["notes"]
|
||||
assert set(channels["columns"]["channel_type"]["codes"]["value"]) == {"线上直销", "线下", "分销"}
|
||||
assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值
|
||||
|
||||
visits = _table(body, "visits")
|
||||
kinds = sorted(f["kind"] for f in visits["findings"])
|
||||
assert kinds == ["business_date", "codes", "relation", "relation"]
|
||||
gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"])
|
||||
assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}"
|
||||
assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"]
|
||||
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
|
||||
assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None
|
||||
|
||||
|
||||
async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path):
|
||||
"""推错的命名关系:parking_records.lot_id 按名字对上了新建的 lots 表,但停车记录里的区号大多不在里面。"""
|
||||
path = scenic.build(tmp_path / "lots.db")
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)")
|
||||
db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)])
|
||||
db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)",
|
||||
[(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)])
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
sid = await make_source(path, options=_on(), draft=["parking_records"])
|
||||
before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed")
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert rel["status"] == "proposed" and rel["source"] == "profile"
|
||||
assert rel["coverage"] == 0.15
|
||||
assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"]
|
||||
finding = _table(r.json(), "parking_records")["findings"][0]
|
||||
assert finding["status"] == "proposed" and finding["coverage"] == 0.15
|
||||
# 两个日期类列:不提议业务日期,取值范围只在返回里
|
||||
ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]}
|
||||
assert ranges == {"entered_at"} # exited_at 全是空值
|
||||
assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"]
|
||||
|
||||
|
||||
async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
detail = await _notes(client, sid, "visits")
|
||||
gate = _relation(detail["notes"], "gate_id")
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/visits/review",
|
||||
json={"path": f"relations.{gate['id']}", "action": "confirm",
|
||||
"if_version": detail["version"]})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "confirmed")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"])
|
||||
assert finding["status"] == "confirmed" and finding["confirmed"] is True
|
||||
|
||||
|
||||
async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db):
|
||||
"""同一天、数据没变:第二次剖析不改目录、不升版本(冻结进快照的目录不会无故变)。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
first = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
second = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
a, b = _table(first, "visits"), _table(second, "visits")
|
||||
assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0)
|
||||
assert b["version"] == a["version"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 安全约束
|
||||
|
||||
|
||||
async def test_masked_columns_are_never_queried(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options={**_on(), "mask_columns": "member_id, Channel_Type"},
|
||||
draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert watch.all_guarded()
|
||||
assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == []
|
||||
visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels")
|
||||
masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"]
|
||||
assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"}
|
||||
assert all("遮罩" in s["detail"] for s in masked)
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "member_id")["source"] == "name" # 没动
|
||||
assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常
|
||||
assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {})
|
||||
|
||||
|
||||
async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget"
|
||||
visits, channels = _table(body, "visits"), _table(body, "channels")
|
||||
assert "budget" in _reasons(visits)
|
||||
assert channels["queries"] == 0 and _reasons(channels) == {"budget"}
|
||||
assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget")
|
||||
# 没查完的关系不改:原来的命名推断原样留着
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "proposed")
|
||||
|
||||
|
||||
async def test_single_query_timeout_is_recorded_and_profiling_continues(client, make_source, scenic_db, monkeypatch):
|
||||
"""闸口的抽样查询换成一条停不下来的递归查询:1 秒时限到点由数据库掐断,记为超时,其余照常。"""
|
||||
sid = await make_source(scenic_db, options=_on(query_timeout_s=1), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
endless = "WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM r) SELECT COUNT(*) FROM r"
|
||||
|
||||
async def slow_gate(source, sql, *, limits=None):
|
||||
if "gate_id" in sql and "DISTINCT" in sql and "COUNT" not in sql:
|
||||
assert limits is not None and limits.timeout_seconds == 1
|
||||
return await real(source, endless, limits=limits)
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow_gate)
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"]
|
||||
assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"]
|
||||
assert r.json()["stopped"] is None
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改
|
||||
assert _relation(notes, "member_id")["status"] == "verified" # 其余照常
|
||||
|
||||
|
||||
async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
|
||||
async def slow(source, sql, *, limits=None):
|
||||
await asyncio.sleep(0.5)
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
settings = ProfileSettings(enabled=True, max_total_s=1.4)
|
||||
report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None,
|
||||
settings=settings)
|
||||
assert report.stopped == "deadline"
|
||||
assert 1 <= report.queries_used <= 3
|
||||
reasons = {s.reason for t in report.tables for s in t.skipped}
|
||||
assert "deadline" in reasons
|
||||
|
||||
|
||||
async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy):
|
||||
"""行数超过整表统计上限(这里设成 100 行):不做 COUNT(DISTINCT)、MIN/MAX,抽样和取值分布都带上限。"""
|
||||
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101}
|
||||
on_visits = [s for s in watch.executed if "from visits" in s.lower()]
|
||||
assert on_visits and all("LIMIT" in s for s in on_visits), on_visits
|
||||
assert not any("MIN(" in s for s in on_visits)
|
||||
assert "too_large" in _reasons(visits)
|
||||
assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large")
|
||||
# 父表有主键,覆盖率又够:照样有确证;子表一侧没核对唯一,按多对一记
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 接口
|
||||
|
||||
|
||||
async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def held(source, sql, *, limits=None):
|
||||
started.set()
|
||||
await release.wait()
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", held)
|
||||
first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR))
|
||||
await asyncio.wait_for(started.wait(), 10)
|
||||
second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"]
|
||||
release.set()
|
||||
done = await asyncio.wait_for(first, 30)
|
||||
assert done.status_code == 200, done.text
|
||||
# 做完以后可以再来
|
||||
again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert again.status_code == 200, again.text
|
||||
|
||||
|
||||
async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=[])
|
||||
r = await client.post(_url(sid), json={}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
names = [t["table_name"] for t in r.json()["tables"]]
|
||||
assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES
|
||||
async with SessionLocal() as session:
|
||||
entries = await catalog.read_catalog(session, sid)
|
||||
for name in names:
|
||||
rels = entries[name].notes.get("relations") or []
|
||||
assert any(rel["source"] not in ("fk",) for rel in rels), name
|
||||
assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选
|
||||
|
||||
|
||||
async def test_unknown_table_and_missing_source(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR)
|
||||
assert r.status_code == 200
|
||||
assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0
|
||||
r = await client.post(_url("0" * 32), json={}, headers=ACTOR)
|
||||
assert r.status_code == 404
|
||||
@@ -0,0 +1,205 @@
|
||||
"""数据剖析(阶段 2A)里不连库的部分:设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、
|
||||
目录对剖析结果的容纳(待填含义的码值、撤销审阅回到剖析的结论)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data import catalog, catalog_profile, guard
|
||||
from app.data.catalog import make_item
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
|
||||
from app.data.engine import build_url
|
||||
|
||||
AT = "2026-10-03T08:00:00+00:00"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 设置
|
||||
|
||||
|
||||
def test_settings_default_off_and_parse_leniently():
|
||||
s = profile_settings({})
|
||||
assert s == ProfileSettings()
|
||||
assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000
|
||||
assert s.max_scan_rows == 100_000 and s.max_total_s == 120
|
||||
on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}})
|
||||
assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500
|
||||
# 库里存着的值不对(老数据、手改过):那一项用缺省,不因此开启,也不报错
|
||||
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}})
|
||||
assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10
|
||||
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, label", [
|
||||
({"max_queries": 0}, "查询次数上限"),
|
||||
({"max_queries": 501}, "查询次数上限"),
|
||||
({"max_queries": 2.5}, "查询次数上限"),
|
||||
({"query_timeout_s": 61}, "单条查询时限"),
|
||||
({"sample_size": 5}, "抽样键值数"),
|
||||
({"max_scan_rows": -1}, "整表统计行数上限"),
|
||||
({"max_total_s": 5}, "总时长上限"),
|
||||
({"enabled": "true"}, "开启数据剖析"),
|
||||
({"max_queries": True}, "查询次数上限"),
|
||||
])
|
||||
def test_settings_problem_names_the_field_in_chinese(value, label):
|
||||
problem = profile_settings_problem({PROFILE_OPTION: value})
|
||||
assert problem and label in problem and "数据剖析" in problem
|
||||
|
||||
|
||||
def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
|
||||
assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}})
|
||||
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
|
||||
assert profile_settings_problem({}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: None}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5,
|
||||
"sample_size": 10, "max_scan_rows": 0,
|
||||
"max_total_s": 600}}) is None
|
||||
|
||||
|
||||
def test_profile_option_never_reaches_the_connection_url():
|
||||
"""剖析设置是 AgentLab 自己的配置:拼进连接串,驱动会把它当未知参数拒掉。"""
|
||||
source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro",
|
||||
password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}})
|
||||
url = build_url(source)
|
||||
assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 字面量
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind, expected", [
|
||||
(42, "number", "42"),
|
||||
(-7, None, "-7"),
|
||||
(3.0, "number", "3"),
|
||||
("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串
|
||||
("A-01", "text", "'A-01'"),
|
||||
("O'Neil", "text", "'O''Neil'"),
|
||||
("12:30", "text", "'12:30'"),
|
||||
])
|
||||
def test_sql_literal_quotes_safe_values(value, kind, expected):
|
||||
assert catalog_profile.sql_literal(value, kind) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, kind", [
|
||||
(True, "boolean"), # 布尔不当键
|
||||
(2.5, "number"), # 非整数的数不当键:浮点相等比较靠不住
|
||||
("12.50", "number"),
|
||||
("1e3", "number"),
|
||||
("2026-07-01", "date"), # 日期的字面量各家写法不同,不猜
|
||||
("a\\b", "text"), # 反斜杠:MySQL 不同设置下读法不同,守卫会拒
|
||||
("a :b", "text"), # 「:名字」会被查询层当成绑定参数
|
||||
(":b", "text"),
|
||||
("a\nb", "text"), # 控制字符
|
||||
("x" * 201, "text"),
|
||||
(None, "text"),
|
||||
([1], None),
|
||||
])
|
||||
def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind):
|
||||
assert catalog_profile.sql_literal(value, kind) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 各方言的剖析 SQL
|
||||
|
||||
|
||||
_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]},
|
||||
"orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"])
|
||||
def test_profile_sql_passes_readonly_guard_on_every_dialect(kind):
|
||||
"""剖析的每一种查询(含读统计信息的系统视图)都得过只读守卫——守卫不为剖析放宽。"""
|
||||
d = catalog_profile.SqlDialect(kind)
|
||||
sqls = []
|
||||
for table in ("visits", "orders"):
|
||||
stats = d.stats_sql(_META[table], table)
|
||||
if kind == "sqlite":
|
||||
assert stats is None # SQLite 没有可读的统计信息,改为数到上限
|
||||
else:
|
||||
assert stats
|
||||
sqls.append(stats)
|
||||
tbl = d.table(_META[table], table)
|
||||
col = d.quote(_META[table]["columns"][0]["name"])
|
||||
sqls += [
|
||||
d.bounded_count_sql(tbl, 100_001),
|
||||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=None),
|
||||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000),
|
||||
d.match_count_sql(tbl, col, ["1", "'A'"]),
|
||||
d.unique_check_sql(tbl, col),
|
||||
d.value_counts_sql(tbl, col, 21, scan_cap=None),
|
||||
d.value_counts_sql(tbl, col, 21, scan_cap=100_000),
|
||||
d.min_max_sql(tbl, [col]),
|
||||
]
|
||||
for sql in sqls:
|
||||
assert guard.check(sql, readonly=True, dialect=kind) == sql, sql
|
||||
|
||||
|
||||
def test_dialect_quoting_and_row_limits():
|
||||
my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql"))
|
||||
assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"'
|
||||
assert my.table(_META["orders"], "orders") == "sales.orders"
|
||||
assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"'
|
||||
# Oracle 没有 LIMIT(11g 也没有 FETCH FIRST):一律用 ROWNUM
|
||||
sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None)
|
||||
assert "LIMIT" not in sample and "ROWNUM <= 50" in sample
|
||||
assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None)
|
||||
# 统计信息按方言读各自的系统视图
|
||||
assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits")
|
||||
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
|
||||
stats = ora.stats_sql(_META["orders"], "orders")
|
||||
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 目录对剖析结果的容纳
|
||||
|
||||
|
||||
def test_codes_with_pending_meanings_are_valid_and_render_as_pending():
|
||||
"""剖析得到的码值只有取值、含义待人填写:目录要收得下,给模型看时写明含义待填写。"""
|
||||
notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")},
|
||||
"channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}}
|
||||
assert catalog.validate_notes(notes) == []
|
||||
text = catalog.render_table_notes(notes)
|
||||
assert "1、0(含义待填写)" in text
|
||||
assert "WEB=官网,BOX 的含义待填写" in text
|
||||
# 码值仍得是对照表:值不是文字的照样拒收
|
||||
assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}})
|
||||
|
||||
|
||||
def _profile_rel(coverage, cardinality, status):
|
||||
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
|
||||
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
|
||||
"source": "profile", "status": status}
|
||||
|
||||
|
||||
def test_reset_returns_profile_relation_to_what_profiling_concluded():
|
||||
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定:覆盖率够、父键唯一(基数有值)就是有确证。"""
|
||||
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
|
||||
rid = verified["relations"][0]["id"]
|
||||
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
|
||||
assert out["relations"][0]["status"] == "confirmed"
|
||||
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "verified"
|
||||
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
|
||||
f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "verified"
|
||||
# 覆盖率低、父键没核实唯一的:撤销后仍是推断
|
||||
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
|
||||
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
|
||||
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
|
||||
f"relations.{rid}", "reset", at=AT)
|
||||
assert out["relations"][0]["status"] == "proposed"
|
||||
# 剖析提的码值候选、业务日期永远是推断
|
||||
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
|
||||
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
|
||||
"columns.status.codes", "reset", at=AT)
|
||||
assert out["columns"]["status"]["codes"]["status"] == "proposed"
|
||||
|
||||
|
||||
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
|
||||
rel = _profile_rel(1.0, "many_to_one", "confirmed")
|
||||
existing = {"relations": [rel]}
|
||||
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
|
||||
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
|
||||
low = _profile_rel(0.4, "many_to_one", "proposed")
|
||||
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
|
||||
table="visits", at=AT)
|
||||
assert kept["relations"][0]["status"] == "proposed"
|
||||
@@ -24,6 +24,7 @@ from app.db.models import DataSource, Run, RunEvent, Workflow, WorkflowVersion
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
_CACHE: dict[str, dict] = {}
|
||||
SQL = "SELECT COUNT(*) AS n FROM visits v JOIN parks p ON p.id = v.park_id WHERE v.status = 1"
|
||||
@@ -72,9 +73,8 @@ async def source(scenic_db):
|
||||
for wid in made:
|
||||
if (wf := await session.get(Workflow, wid)) is not None:
|
||||
await session.delete(wf)
|
||||
if (row := await session.get(DataSource, sid)) is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
await drop_source(sid)
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
|
||||
@@ -7,6 +7,7 @@ from app.data import catalog
|
||||
from app.data.catalog import make_item
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource, Run
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
MARK = "(推断,未确认)"
|
||||
|
||||
@@ -184,4 +185,5 @@ async def test_table_usage_counts_run_queries_per_table(monkeypatch):
|
||||
await session.commit()
|
||||
source = await session.get(DataSource, sid)
|
||||
usage = await catalog.table_usage(session, source)
|
||||
await drop_source(sid)
|
||||
assert usage == {"visits": 4, "明细": 2, "parks": 1}
|
||||
|
||||
@@ -14,12 +14,12 @@ from types import SimpleNamespace
|
||||
from app.core.artifact_store import canonical_json, content_hash, load
|
||||
from app.data import catalog
|
||||
from app.data.catalog import make_item
|
||||
from app.data.engine import engines
|
||||
from app.data.introspect import describe_table, introspect
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource
|
||||
from app.tools.datasource import _store_schema, build_datasource_tools
|
||||
from app.tools.registry import ToolContext
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
MARK = "(推断,未确认)"
|
||||
|
||||
@@ -182,4 +182,4 @@ async def test_tools_show_and_freeze_catalog_and_record_query_tables(scenic_db):
|
||||
ref = await session.get(Artifact, (result["artifact"], run_id))
|
||||
assert ref.meta == {"source_id": row.id, "source": row.name, "tables": ["visits", "parks"]}
|
||||
finally:
|
||||
await engines.invalidate(row.id)
|
||||
await drop_source(row.id)
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.data.engine import engines
|
||||
from app.data.introspect import introspect
|
||||
from app.data.sqlcheck import SqlChecker
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -177,10 +178,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield name
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
row.enabled = False
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
async def _stream(monkeypatch, model, source):
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.engine.governance import lint_for_publish, publish_issues
|
||||
from app.engine.schema import GraphSpec
|
||||
from app.main import app
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -133,10 +134,7 @@ async def scenic_source(scenic_db):
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
source_id = row.id
|
||||
yield name
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, source_id)
|
||||
row.enabled = False
|
||||
await session.commit()
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -32,6 +32,7 @@ from app.main import app
|
||||
from app.tools.datasource import build_datasource_tools
|
||||
from app.tools.registry import ToolContext
|
||||
from tests.fixtures.catalog import scenic_notes
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
FANOUT_SQL = ("SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id "
|
||||
"WHERE o.status = 1")
|
||||
@@ -54,6 +55,16 @@ async def _schema(path: str) -> dict:
|
||||
return _CACHE[path]
|
||||
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _drop_created_sources():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
name = f"scenic_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
@@ -61,6 +72,7 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
row.schema_cache = await _schema(path)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
_CREATED.append(row.id)
|
||||
if with_catalog:
|
||||
for table, notes in scenic_notes.notes().items():
|
||||
await catalog.write_entry(session, row.id, table, notes, if_version=0, actor="王敏")
|
||||
@@ -69,9 +81,8 @@ async def _source(path: str, *, with_catalog: bool = True) -> tuple[str, str]:
|
||||
|
||||
@pytest.fixture
|
||||
async def scenic_source(scenic_db):
|
||||
source_id, name = await _source(scenic_db)
|
||||
_, name = await _source(scenic_db)
|
||||
yield name
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
async def _query_tool(name: str):
|
||||
|
||||
@@ -254,6 +254,16 @@ async def test_join_unconfirmed_negative(cache, sql):
|
||||
assert "join_unconfirmed" not in codes(cache, sql)
|
||||
|
||||
|
||||
async def test_sibling_join_through_a_shared_parent_is_only_a_note(cache):
|
||||
"""两张表各自按 member_id 指向会员(都是外键),彼此按 member_id 相连:目录里没有这两张表之间的关系,
|
||||
但这是经共同父表推得出来的连法,不是「连错了列」。只提示一句:关联后行数可能成倍增加。"""
|
||||
sql = "SELECT COUNT(*) FROM orders o JOIN member_tag_links t ON t.member_id = o.member_id WHERE o.status = 1"
|
||||
[check] = [c for c in run(cache, sql) if c.code == "join_unconfirmed"]
|
||||
assert check.level == "info"
|
||||
assert "会员" in check.message or "members" in check.message
|
||||
assert "成倍" in check.message
|
||||
|
||||
|
||||
async def test_join_level_follows_status(cache):
|
||||
sql = "SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1"
|
||||
assert "join_unconfirmed" not in codes(cache, sql, "confirmed")
|
||||
|
||||
@@ -34,6 +34,7 @@ from app.engine.expressions import TruncatedUse, eval_expression
|
||||
from app.engine.issuance import decide_tier, incomplete_gaps
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 求值器:整组用到截断结果时记一笔,单格不记
|
||||
@@ -345,7 +346,7 @@ async def park(ticket_db, engine_up):
|
||||
# 连接池按数据源 id 缓存:换了库文件不作废,查的还是上一个测试的库
|
||||
await engines.invalidate(source_id)
|
||||
yield
|
||||
await engines.invalidate(source_id)
|
||||
await drop_source(source_id)
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
|
||||
Reference in New Issue
Block a user