From 706f7155299414f6b4f4195d06b0556df4b769a2 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 03:38:31 +0800 Subject: [PATCH] =?UTF-8?q?=E5=89=96=E6=9E=90=E4=B9=9F=E6=A0=B8=E5=AE=9E?= =?UTF-8?q?=E5=A4=96=E9=94=AE=E5=85=B3=E7=B3=BB=E7=9A=84=E5=9F=BA=E6=95=B0?= =?UTF-8?q?=EF=BC=9A=E5=A4=96=E9=94=AE=E5=8F=AA=E4=BF=9D=E8=AF=81=E8=A2=AB?= =?UTF-8?q?=E6=8C=87=E5=90=91=E7=9A=84=E9=94=AE=E5=94=AF=E4=B8=80=EF=BC=8C?= =?UTF-8?q?=E5=AD=90=E8=A1=A8=E4=B8=80=E4=BE=A7=E4=B8=80=E8=A1=8C=E8=BF=98?= =?UTF-8?q?=E6=98=AF=E5=A4=9A=E8=A1=8C=E7=9C=8B=E6=95=B0=E6=8D=AE=EF=BC=9B?= =?UTF-8?q?=E4=BB=A5=E5=89=8D=E8=B7=B3=E8=BF=87=E5=A4=96=E9=94=AE=EF=BC=8C?= =?UTF-8?q?=E6=89=87=E5=87=BA=E6=A3=80=E6=9F=A5=E5=AF=B9=E5=A4=96=E9=94=AE?= =?UTF-8?q?=E6=9D=A5=E7=9A=84=E4=B8=80=E5=AF=B9=E5=A4=9A=E5=8F=AA=E8=83=BD?= =?UTF-8?q?=E6=8A=A5=E8=AD=A6=E5=91=8A=EF=BC=8C=E5=8F=97=E7=AE=A1=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E6=8C=A1=E4=B8=8D=E4=BD=8F=E8=AE=A2=E5=8D=95=E5=85=B3?= =?UTF-8?q?=E8=81=94=E6=98=8E=E7=BB=86=E5=90=8E=E5=AF=B9=E8=AE=A2=E5=8D=95?= =?UTF-8?q?=E9=87=91=E9=A2=9D=E6=B1=82=E5=92=8C=E3=80=82=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E3=80=81=E7=8A=B6=E6=80=81=E3=80=81=E8=A6=86=E7=9B=96=E7=8E=87?= =?UTF-8?q?=E4=B8=8D=E5=8A=A8=EF=BC=8C=E5=8F=AA=E8=A1=A5=E5=9F=BA=E6=95=B0?= =?UTF-8?q?=E5=92=8C=E3=80=8C=E7=94=A8=E6=95=B0=E6=8D=AE=E6=A0=B8=E5=AE=9E?= =?UTF-8?q?=E8=BF=87=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog_profile.py | 74 ++++++++++++++++++++--- backend/tests/test_catalog_profile.py | 84 +++++++++++++++++++++++++-- 2 files changed, 145 insertions(+), 13 deletions(-) diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 820bc6d4..6966b26d 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -536,7 +536,8 @@ class _RelationOutcome: rid: str target: str - coverage: float + #: 外键关系不抽样(外键已保证引用完整),没有覆盖率 + coverage: float | None cardinality: str | None holds: bool sample: int @@ -655,10 +656,13 @@ def _relation_target(rel: dict[str, Any]) -> str: def _relation_candidates(notes: dict[str, Any] | None) -> list[dict[str, Any]]: - """要核对的关系:没被驳回、不是外键约束来的(外键由数据库保证,剖析也顶不掉它)。人工确认过的也核对, - 只补覆盖率和基数。""" + """要核对的关系:没被驳回的。人工确认过的也核对,只补覆盖率和基数。 + + 外键约束来的也核对,但只核对基数:外键保证了引用完整(不必抽样算覆盖率),却只保证被指向的键唯一, + 子表一侧每个键是一行还是多行要看数据。以前跳过外键关系,外键来的一对多永远拿不到「用数据核实过」, + 扇出检查只能报 warning——最常见的「订单关联明细后对订单金额求和」在受管发布时也挡不住。""" return [r for r in (notes or {}).get("relations") or [] - if isinstance(r, dict) and r.get("id") and r.get("status") != "rejected" and r.get("source") != "fk"] + if isinstance(r, dict) and r.get("id") and r.get("status") != "rejected"] async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str, @@ -680,6 +684,17 @@ async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str, return total == distinct, "count" +def _fk_note(child: tuple[bool | None, str]) -> str: + """外键关系的剖析备注:只说子表一侧唯不唯一、依据是什么。不写日期(同 _relation_note)。""" + unique, how = child + if unique is None: + why = "子表太大,没有整表统计" if how == "too_large" else "没能数清子表的取值" + return f"{catalog.PROFILE_NOTE_PREFIX}外键约束已保证引用完整,未抽样核对覆盖率;{why},基数未用数据核实" + basis = "按主键或唯一约束" if how not in ("count", "too_large") else "整表数过取值" + shape = "每个键只有一行(一对一)" if unique else "同一个键有多行(一对多)" + return f"{catalog.PROFILE_NOTE_PREFIX}外键约束已保证引用完整,未抽样核对覆盖率;子表一侧{basis}:{shape}" + + def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int | None, skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None, holds: bool) -> str: @@ -741,6 +756,15 @@ async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _T if cx.is_masked(name): raise _Skip("masked", f"{owner}.{name} 在数据源设置中被遮罩,不取样", carry=True) + if rel.get("source") == "fk": + await cx.size_with_data(table, result) + child = await _unique(cx, table, meta, col, result) + cardinality = None if child[0] is None else ("one_to_one" if child[0] else "many_to_one") + out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=None, + cardinality=cardinality, holds=cardinality is not None, sample=0, + matched=0, note=_fk_note(child), cardinality_checked=child[0] is not None)) + return + d, s = cx.dialect, cx.settings size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) @@ -933,7 +957,11 @@ def _relation_finding(o: _RelationOutcome, rel: dict[str, Any], *, status: str, (重新剖析、目录没变时不能说升了);cardinality_checked:基数是不是用数据核实的(子表一侧数过是否唯一), 报告据此写「已用数据核实」,不从说明的原话里认。""" previous = rel.get("status") - if confirmed: + if rel.get("source") == "fk": + summary = ("外键约束," + ("子表每个键只有一行(一对一)" if o.cardinality == "one_to_one" + else "子表同一个键有多行(一对多)") + ",基数已用数据核实") \ + if o.cardinality_checked else "外键约束,子表太大或没能数清,基数未用数据核实" + elif confirmed: summary = "已人工确认,只补充覆盖率和基数" elif status == "verified": summary = f"覆盖率 {_percent(o.coverage)}," + ("仍为有确证" if previous == "verified" else "升为有确证") @@ -990,6 +1018,26 @@ def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: return changed +def _patch_fk(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: + """外键关系只补基数和「用数据核实过」:外键结构推的多对一可能其实是一对一,数据说了算。来源、状态、 + 覆盖率都不动;备注同 _patch_confirmed(只是旧备注的日期不同不换)。返回改没改。""" + changed = False + if o.cardinality_checked: + if rel.get("cardinality") != o.cardinality: + rel["cardinality"] = o.cardinality + changed = True + if rel.get("cardinality_checked") is not True: + rel["cardinality_checked"] = True + changed = True + note = rel.get("note") + if (not note or catalog.is_profile_note(note)) and not catalog.same_profile_note(note, o.note): + rel["note"] = o.note + changed = True + if changed: + rel["updated_at"] = at + return changed + + def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, at: str) -> tuple[dict[str, Any], list[dict[str, Any]], catalog.MergeStats]: """把一张表的结论并进写入时的目录:(新目录, 写进去的发现, 计数)。纯函数,写入冲突重来时再调一次。 @@ -997,6 +1045,7 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, - 没被确认、驳回的关系:记为 source=profile,覆盖率够、父键唯一的 verified,否则 proposed;按 merge_notes 的规则并入(剖析顶得掉命名推断、模型起草,顶不掉外键和人工)。 - 人工确认过的关系:只补覆盖率和基数(_patch_confirmed)。 + - 外键约束来的关系:只补基数和「用数据核实过」(_patch_fk),来源、状态、覆盖率都不动。 - 码值候选:profile / proposed,含义沿用别的来源写过的(_codes_value);整表统计得到的标「已列全」,只看了 前若干行的不标;人工确认、驳回过的码值不碰。 - 业务日期:只有一个日期类列时提议,profile / proposed;已有别的来源定的业务日期不顶掉。 @@ -1007,11 +1056,16 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, current = {str(r["id"]): r for r in existing.get("relations") or [] if isinstance(r, dict) and r.get("id")} relations: list[dict[str, Any]] = [] confirmed: dict[str, _RelationOutcome] = {} + foreign: dict[str, _RelationOutcome] = {} findings: list[dict[str, Any]] = [] for o in out.relations: rel = current.get(o.rid) - if rel is None or rel.get("status") == "rejected" or rel.get("source") == "fk": + if rel is None or rel.get("status") == "rejected": continue # 核对期间被删、被驳回了:不写 + if rel.get("source") == "fk": + foreign[o.rid] = o + findings.append(_relation_finding(o, rel, status=str(rel.get("status")), confirmed=False)) + continue if rel.get("status") == "confirmed": confirmed[o.rid] = o findings.append(_relation_finding(o, rel, status="confirmed", confirmed=True)) @@ -1060,7 +1114,13 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, merged, stats = catalog.merge_notes(existing, draft, covered={"profile"}, at=at) for rel in merged.get("relations") or []: - o = confirmed.get(str(rel.get("id"))) + rid = str(rel.get("id")) + o = foreign.get(rid) + if o is not None and rel.get("source") == "fk": + if _patch_fk(rel, o, at): + stats += catalog.MergeStats(updated=1) + continue + o = confirmed.get(rid) if o is not None and rel.get("status") == "confirmed" and _patch_confirmed(rel, o, at): stats += catalog.MergeStats(updated=1) return merged, findings, stats diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index f9e0c599..9c6a5fe9 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -207,10 +207,10 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d # A7:说明里不写日期,时间看这一项的 updated_at(界面按本地时间显示) assert rel["note"].startswith("数据剖析:") and not re.search(r"\d{4}-\d{2}-\d{2}", rel["note"]) assert rel["updated_at"] - # 外键约束来的关系不动:外键是外键,剖析是剖析 + # 外键约束来的关系:来源、状态、覆盖率都不动(外键已保证引用完整,不抽样),只补用数据核实过的基数 park = _relation(notes, "park_id") assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None) - assert "cardinality_checked" not in park # 外键的基数是按表结构推的,没用数据核对 + assert park.get("cardinality_checked") is True and park["cardinality"] == "many_to_one" # 状态列:码值候选,值有了、含义留空,推断 codes = notes["columns"]["status"]["codes"] @@ -227,7 +227,11 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d visits = _table(body, "visits") kinds = sorted(f["kind"] for f in visits["findings"]) - assert kinds == ["business_date", "codes", "relation", "relation"] + # 两条命名推断的关系(gate_id、member_id),加两条外键关系(park_id、ticket_type_id):外键的只核实基数 + assert kinds == ["business_date", "codes", "relation", "relation", "relation", "relation"] + fk = [f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] in (["park_id"], ["ticket_type_id"])] + assert len(fk) == 2 and all(f["cardinality_checked"] and f["coverage"] is None and "外键约束" in f["summary"] + for f in fk) gate = next(f for f in visits["findings"] if f["kind"] == "relation" and f["columns"] == ["gate_id"]) assert gate["status"] == "verified" and gate["path"] == f"relations.{_relation(notes, 'gate_id')['id']}" assert gate["sample"] == 8 and gate["matched"] == 8 and gate["summary"] @@ -237,7 +241,8 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d blank = sum(1 for v in notes["columns"][codes["column"]]["codes"]["value"].values() if not v.strip()) assert codes["pending"] == blank and (f"{blank} 个含义待填写" in codes["summary"]) == (blank > 0) assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None} - assert visits["added"] == 2 and visits["updated"] == 2 and visits["error"] is None + # 更新的 4 项:两条命名关系升为有确证,两条外键关系补上用数据核实过的基数 + assert visits["added"] == 2 and visits["updated"] == 4 and visits["error"] is None async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path): @@ -475,8 +480,7 @@ async def test_default_tables_are_used_tables_with_relation_candidates(client, m async with SessionLocal() as session: entries = await catalog.read_catalog(session, sid) for name in names: - rels = entries[name].notes.get("relations") or [] - assert any(rel["source"] not in ("fk",) for rel in rels), name + assert catalog_profile._relation_candidates(entries[name].notes), name assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选 @@ -550,3 +554,71 @@ async def test_unknown_table_and_missing_source(client, make_source, scenic_db): assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0 r = await client.post(_url("0" * 32), json={}, headers=ACTOR) assert r.status_code == 404 + + +def _orders_db(path, *, items_per_order: int) -> str: + """订单和订单明细,明细按外键指向订单。items_per_order=1 时数据上一对一,>1 时一对多。""" + db = sqlite3.connect(path) + try: + db.execute("CREATE TABLE orders (id INTEGER PRIMARY KEY, total_amount REAL NOT NULL)") + db.execute("CREATE TABLE order_items (id INTEGER PRIMARY KEY, " + "order_id INTEGER NOT NULL REFERENCES orders(id), amount REAL NOT NULL)") + db.executemany("INSERT INTO orders VALUES (?, ?)", [(i, 100.0) for i in range(1, 41)]) + db.executemany("INSERT INTO order_items (order_id, amount) VALUES (?, ?)", + [(o, 100.0 / items_per_order) for o in range(1, 41) for _ in range(items_per_order)]) + db.commit() + finally: + db.close() + return str(path) + + +@pytest.mark.parametrize("items, cardinality", [(3, "many_to_one"), (1, "one_to_one")]) +async def test_foreign_key_relations_get_their_cardinality_checked_with_data(client, make_source, tmp_path, + items, cardinality): + """外键只保证被指向的键唯一,子表一侧每个键是一行还是多行要看数据。以前剖析跳过外键关系,于是外键来的 + 一对多永远拿不到「用数据核实过」,扇出检查只能报 warning——最常见的「订单关联明细后对订单金额求和」 + 在受管发布时也挡不住。剖析要数子表一侧,来源、状态都不动,也不抽样算覆盖率(外键已经保证了)。""" + sid = await make_source(_orders_db(tmp_path / f"o{items}.db", items_per_order=items), options=_on(), + draft=["order_items"]) + before = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id") + assert (before["source"], before["status"]) == ("fk", "verified") + + r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) + assert r.status_code == 200, r.text + rel = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id") + assert (rel["source"], rel["status"]) == ("fk", "verified") + assert rel["cardinality"] == cardinality and rel.get("cardinality_checked") is True + assert rel.get("coverage") is None + finding = next(f for f in _table(r.json(), "order_items")["findings"] if f.get("columns") == ["order_id"]) + assert finding["cardinality_checked"] is True and finding["cardinality"] == cardinality + + # 重新剖析、数据没变:版本不升 + version = (await _notes(client, sid, "order_items"))["version"] + r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) + assert r.status_code == 200, r.text + assert (await _notes(client, sid, "order_items"))["version"] == version + + +async def test_fanout_on_a_data_checked_foreign_key_is_an_error_again(client, make_source, tmp_path): + """剖析核实过的外键一对多:订单关联明细后对订单金额求和,扇出检查报 error;数据上一对一的不报。""" + from app.data.sqlcheck import load_checkers + from app.db.base import SessionLocal + from app.db.models import DataSource + + sql = "SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id" + for items, expected in ((3, "error"), (1, None)): + sid = await make_source(_orders_db(tmp_path / f"f{items}.db", items_per_order=items), options=_on(), + draft=["orders", "order_items"]) + detail = await _notes(client, sid, "orders") + notes = detail["notes"] + notes.setdefault("columns", {})["total_amount"] = {"measure": {"value": "flow"}} + r = await client.put(f"/api/datasources/{sid}/catalog/orders", + json={"notes": notes, "if_version": detail["version"]}, headers=ACTOR) + assert r.status_code == 200, r.text + r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) + assert r.status_code == 200, r.text + async with SessionLocal() as session: + source = await session.get(DataSource, sid) + checker = (await load_checkers(session, [source]))[source.name] + levels = [c.level for c in checker.check(sql) if c.code == "fanout_sum"] + assert levels == ([expected] if expected else []), (items, levels)