剖析也核实外键关系的基数:外键只保证被指向的键唯一,子表一侧一行还是多行看数据;以前跳过外键,扇出检查对外键来的一对多只能报警告,受管发布挡不住订单关联明细后对订单金额求和。来源、状态、覆盖率不动,只补基数和「用数据核实过」

This commit is contained in:
ThearchyHelios
2026-10-04 03:38:31 +08:00
parent 67136f9798
commit 706f715529
2 changed files with 145 additions and 13 deletions
+67 -7
View File
@@ -536,7 +536,8 @@ class _RelationOutcome:
rid: str
target: str
coverage: float
#: 外键关系不抽样(外键已保证引用完整),没有覆盖率
coverage: float | None
cardinality: str | None
holds: bool
sample: int
@@ -655,10 +656,13 @@ def _relation_target(rel: dict[str, Any]) -> str:
def _relation_candidates(notes: dict[str, Any] | None) -> list[dict[str, Any]]:
"""要核对的关系:没被驳回、不是外键约束来的(外键由数据库保证,剖析也顶不掉它)。人工确认过的也核对,
只补覆盖率和基数。"""
"""要核对的关系:没被驳回的。人工确认过的也核对,只补覆盖率和基数。
外键约束来的也核对,但只核对基数:外键保证了引用完整(不必抽样算覆盖率),却只保证被指向的键唯一,
子表一侧每个键是一行还是多行要看数据。以前跳过外键关系,外键来的一对多永远拿不到「用数据核实过」,
扇出检查只能报 warning——最常见的「订单关联明细后对订单金额求和」在受管发布时也挡不住。"""
return [r for r in (notes or {}).get("relations") or []
if isinstance(r, dict) and r.get("id") and r.get("status") != "rejected" and r.get("source") != "fk"]
if isinstance(r, dict) and r.get("id") and r.get("status") != "rejected"]
async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str,
@@ -680,6 +684,17 @@ async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str,
return total == distinct, "count"
def _fk_note(child: tuple[bool | None, str]) -> str:
"""外键关系的剖析备注:只说子表一侧唯不唯一、依据是什么。不写日期(同 _relation_note)。"""
unique, how = child
if unique is None:
why = "子表太大,没有整表统计" if how == "too_large" else "没能数清子表的取值"
return f"{catalog.PROFILE_NOTE_PREFIX}外键约束已保证引用完整,未抽样核对覆盖率;{why},基数未用数据核实"
basis = "按主键或唯一约束" if how not in ("count", "too_large") else "整表数过取值"
shape = "每个键只有一行(一对一)" if unique else "同一个键有多行(一对多)"
return f"{catalog.PROFILE_NOTE_PREFIX}外键约束已保证引用完整,未抽样核对覆盖率;子表一侧{basis}:{shape}"
def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int | None,
skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None,
holds: bool) -> str:
@@ -741,6 +756,15 @@ async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _T
if cx.is_masked(name):
raise _Skip("masked", f"{owner}.{name} 在数据源设置中被遮罩,不取样", carry=True)
if rel.get("source") == "fk":
await cx.size_with_data(table, result)
child = await _unique(cx, table, meta, col, result)
cardinality = None if child[0] is None else ("one_to_one" if child[0] else "many_to_one")
out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=None,
cardinality=cardinality, holds=cardinality is not None, sample=0,
matched=0, note=_fk_note(child), cardinality_checked=child[0] is not None))
return
d, s = cx.dialect, cx.settings
size = await cx.size_with_data(table, result)
scan_cap = cx.scan_cap(size)
@@ -933,7 +957,11 @@ def _relation_finding(o: _RelationOutcome, rel: dict[str, Any], *, status: str,
(重新剖析、目录没变时不能说升了);cardinality_checked:基数是不是用数据核实的(子表一侧数过是否唯一),
报告据此写「已用数据核实」,不从说明的原话里认。"""
previous = rel.get("status")
if confirmed:
if rel.get("source") == "fk":
summary = ("外键约束," + ("子表每个键只有一行(一对一)" if o.cardinality == "one_to_one"
else "子表同一个键有多行(一对多)") + ",基数已用数据核实") \
if o.cardinality_checked else "外键约束,子表太大或没能数清,基数未用数据核实"
elif confirmed:
summary = "已人工确认,只补充覆盖率和基数"
elif status == "verified":
summary = f"覆盖率 {_percent(o.coverage)}," + ("仍为有确证" if previous == "verified" else "升为有确证")
@@ -990,6 +1018,26 @@ def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool:
return changed
def _patch_fk(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool:
"""外键关系只补基数和「用数据核实过」:外键结构推的多对一可能其实是一对一,数据说了算。来源、状态、
覆盖率都不动;备注同 _patch_confirmed(只是旧备注的日期不同不换)。返回改没改。"""
changed = False
if o.cardinality_checked:
if rel.get("cardinality") != o.cardinality:
rel["cardinality"] = o.cardinality
changed = True
if rel.get("cardinality_checked") is not True:
rel["cardinality_checked"] = True
changed = True
note = rel.get("note")
if (not note or catalog.is_profile_note(note)) and not catalog.same_profile_note(note, o.note):
rel["note"] = o.note
changed = True
if changed:
rel["updated_at"] = at
return changed
def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *,
at: str) -> tuple[dict[str, Any], list[dict[str, Any]], catalog.MergeStats]:
"""把一张表的结论并进写入时的目录:(新目录, 写进去的发现, 计数)。纯函数,写入冲突重来时再调一次。
@@ -997,6 +1045,7 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *,
- 没被确认、驳回的关系:记为 source=profile,覆盖率够、父键唯一的 verified,否则 proposed;按 merge_notes
的规则并入(剖析顶得掉命名推断、模型起草,顶不掉外键和人工)。
- 人工确认过的关系:只补覆盖率和基数(_patch_confirmed)。
- 外键约束来的关系:只补基数和「用数据核实过」(_patch_fk),来源、状态、覆盖率都不动。
- 码值候选:profile / proposed,含义沿用别的来源写过的(_codes_value);整表统计得到的标「已列全」,只看了
前若干行的不标;人工确认、驳回过的码值不碰。
- 业务日期:只有一个日期类列时提议,profile / proposed;已有别的来源定的业务日期不顶掉。
@@ -1007,11 +1056,16 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *,
current = {str(r["id"]): r for r in existing.get("relations") or [] if isinstance(r, dict) and r.get("id")}
relations: list[dict[str, Any]] = []
confirmed: dict[str, _RelationOutcome] = {}
foreign: dict[str, _RelationOutcome] = {}
findings: list[dict[str, Any]] = []
for o in out.relations:
rel = current.get(o.rid)
if rel is None or rel.get("status") == "rejected" or rel.get("source") == "fk":
if rel is None or rel.get("status") == "rejected":
continue # 核对期间被删、被驳回了:不写
if rel.get("source") == "fk":
foreign[o.rid] = o
findings.append(_relation_finding(o, rel, status=str(rel.get("status")), confirmed=False))
continue
if rel.get("status") == "confirmed":
confirmed[o.rid] = o
findings.append(_relation_finding(o, rel, status="confirmed", confirmed=True))
@@ -1060,7 +1114,13 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *,
merged, stats = catalog.merge_notes(existing, draft, covered={"profile"}, at=at)
for rel in merged.get("relations") or []:
o = confirmed.get(str(rel.get("id")))
rid = str(rel.get("id"))
o = foreign.get(rid)
if o is not None and rel.get("source") == "fk":
if _patch_fk(rel, o, at):
stats += catalog.MergeStats(updated=1)
continue
o = confirmed.get(rid)
if o is not None and rel.get("status") == "confirmed" and _patch_confirmed(rel, o, at):
stats += catalog.MergeStats(updated=1)
return merged, findings, stats
+78 -6
View File
@@ -207,10 +207,10 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d
# A7:说明里不写日期,时间看这一项的 updated_at(界面按本地时间显示)
assert rel["note"].startswith("数据剖析:") and not re.search(r"\d{4}-\d{2}-\d{2}", rel["note"])
assert rel["updated_at"]
# 外键约束来的关系不动:外键是外键,剖析是剖析
# 外键约束来的关系:来源、状态、覆盖率都不动(外键已保证引用完整,不抽样),只补用数据核实过的基数
park = _relation(notes, "park_id")
assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None)
assert "cardinality_checked" not in park # 外键的基数是按表结构推的,没用数据核对
assert park.get("cardinality_checked") is True and park["cardinality"] == "many_to_one"
# 状态列:码值候选,值有了、含义留空,推断
codes = notes["columns"]["status"]["codes"]
@@ -227,7 +227,11 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d
visits = _table(body, "visits")
kinds = sorted(f["kind"] for f in visits["findings"])
assert kinds == ["business_date", "codes", "relation", "relation"]
# 两条命名推断的关系(gate_id、member_id),加两条外键关系(park_id、ticket_type_id):外键的只核实基数
assert kinds == ["business_date", "codes", "relation", "relation", "relation", "relation"]
fk = [f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] in (["park_id"], ["ticket_type_id"])]
assert len(fk) == 2 and all(f["cardinality_checked"] and f["coverage"] is None and "外键约束" in f["summary"]
for f in fk)
gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"])
assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}"
assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"]
@@ -237,7 +241,8 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d
blank = sum(1 for v in notes["columns"][codes["column"]]["codes"]["value"].values() if not v.strip())
assert codes["pending"] == blank and (f"{blank} 个含义待填写" in codes["summary"]) == (blank > 0)
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None
# 更新的 4 项:两条命名关系升为有确证,两条外键关系补上用数据核实过的基数
assert visits["added"] == 2 and visits["updated"] == 4 and visits["error"] is None
async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path):
@@ -475,8 +480,7 @@ async def test_default_tables_are_used_tables_with_relation_candidates(client, m
async with SessionLocal() as session:
entries = await catalog.read_catalog(session, sid)
for name in names:
rels = entries[name].notes.get("relations") or []
assert any(rel["source"] not in ("fk",) for rel in rels), name
assert catalog_profile._relation_candidates(entries[name].notes), name
assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选
@@ -550,3 +554,71 @@ async def test_unknown_table_and_missing_source(client, make_source, scenic_db):
assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0
r = await client.post(_url("0" * 32), json={}, headers=ACTOR)
assert r.status_code == 404
def _orders_db(path, *, items_per_order: int) -> str:
"""订单和订单明细,明细按外键指向订单。items_per_order=1 时数据上一对一,>1 时一对多。"""
db = sqlite3.connect(path)
try:
db.execute("CREATE TABLE orders (id INTEGER PRIMARY KEY, total_amount REAL NOT NULL)")
db.execute("CREATE TABLE order_items (id INTEGER PRIMARY KEY, "
"order_id INTEGER NOT NULL REFERENCES orders(id), amount REAL NOT NULL)")
db.executemany("INSERT INTO orders VALUES (?, ?)", [(i, 100.0) for i in range(1, 41)])
db.executemany("INSERT INTO order_items (order_id, amount) VALUES (?, ?)",
[(o, 100.0 / items_per_order) for o in range(1, 41) for _ in range(items_per_order)])
db.commit()
finally:
db.close()
return str(path)
@pytest.mark.parametrize("items, cardinality", [(3, "many_to_one"), (1, "one_to_one")])
async def test_foreign_key_relations_get_their_cardinality_checked_with_data(client, make_source, tmp_path,
items, cardinality):
"""外键只保证被指向的键唯一,子表一侧每个键是一行还是多行要看数据。以前剖析跳过外键关系,于是外键来的
一对多永远拿不到「用数据核实过」,扇出检查只能报 warning——最常见的「订单关联明细后对订单金额求和」
在受管发布时也挡不住。剖析要数子表一侧,来源、状态都不动,也不抽样算覆盖率(外键已经保证了)。"""
sid = await make_source(_orders_db(tmp_path / f"o{items}.db", items_per_order=items), options=_on(),
draft=["order_items"])
before = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id")
assert (before["source"], before["status"]) == ("fk", "verified")
r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR)
assert r.status_code == 200, r.text
rel = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id")
assert (rel["source"], rel["status"]) == ("fk", "verified")
assert rel["cardinality"] == cardinality and rel.get("cardinality_checked") is True
assert rel.get("coverage") is None
finding = next(f for f in _table(r.json(), "order_items")["findings"] if f.get("columns") == ["order_id"])
assert finding["cardinality_checked"] is True and finding["cardinality"] == cardinality
# 重新剖析、数据没变:版本不升
version = (await _notes(client, sid, "order_items"))["version"]
r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR)
assert r.status_code == 200, r.text
assert (await _notes(client, sid, "order_items"))["version"] == version
async def test_fanout_on_a_data_checked_foreign_key_is_an_error_again(client, make_source, tmp_path):
"""剖析核实过的外键一对多:订单关联明细后对订单金额求和,扇出检查报 error;数据上一对一的不报。"""
from app.data.sqlcheck import load_checkers
from app.db.base import SessionLocal
from app.db.models import DataSource
sql = "SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id"
for items, expected in ((3, "error"), (1, None)):
sid = await make_source(_orders_db(tmp_path / f"f{items}.db", items_per_order=items), options=_on(),
draft=["orders", "order_items"])
detail = await _notes(client, sid, "orders")
notes = detail["notes"]
notes.setdefault("columns", {})["total_amount"] = {"measure": {"value": "flow"}}
r = await client.put(f"/api/datasources/{sid}/catalog/orders",
json={"notes": notes, "if_version": detail["version"]}, headers=ACTOR)
assert r.status_code == 200, r.text
r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR)
assert r.status_code == 200, r.text
async with SessionLocal() as session:
source = await session.get(DataSource, sid)
checker = (await load_checkers(session, [source]))[source.name]
levels = [c.level for c in checker.check(sql) if c.code == "fanout_sum"]
assert levels == ([expected] if expected else []), (items, levels)