From 46c3fd3093bf26d4b164d15ff902e97e7e907290 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 01:57:01 +0800 Subject: [PATCH] =?UTF-8?q?A6=20=E6=89=87=E5=87=BA=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E5=8F=AA=E5=9C=A8=E4=B8=80=E5=AF=B9=E5=A4=9A=E7=A1=AE=E6=9C=89?= =?UTF-8?q?=E5=85=B6=E4=BA=8B=E6=97=B6=E6=8A=A5=E9=94=99=E8=AF=AF=EF=BC=9A?= =?UTF-8?q?=E5=85=B3=E7=B3=BB=E4=BA=BA=E5=B7=A5=E7=A1=AE=E8=AE=A4=E8=BF=87?= =?UTF-8?q?=EF=BC=8C=E6=88=96=E6=95=B0=E6=8D=AE=E5=89=96=E6=9E=90=E5=9C=A8?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E4=B8=8A=E6=A0=B8=E5=AE=9E=E8=BF=87=E5=AD=90?= =?UTF-8?q?=E8=A1=A8=E4=B8=80=E4=BE=A7=E7=9A=84=E9=94=AE=E4=B8=8D=E5=94=AF?= =?UTF-8?q?=E4=B8=80=EF=BC=88=E5=85=B3=E7=B3=BB=E4=B8=8A=E8=AE=B0=20cardin?= =?UTF-8?q?ality=5Fchecked=EF=BC=8C=E5=AD=90=E8=A1=A8=E5=A4=AA=E5=A4=A7?= =?UTF-8?q?=E6=B2=A1=E6=A0=B8=E5=AF=B9=E7=9A=84=E4=B8=8D=E7=AE=97=EF=BC=89?= =?UTF-8?q?=E6=89=8D=E7=BB=99=20error=EF=BC=9B=E5=8F=AA=E5=87=AD=E5=A4=96?= =?UTF-8?q?=E9=94=AE=E7=BB=93=E6=9E=84=E6=8E=A8=E5=87=BA=E7=9A=84=E4=B8=80?= =?UTF-8?q?=E5=AF=B9=E5=A4=9A=E9=99=8D=E4=B8=BA=20warning=EF=BC=8C?= =?UTF-8?q?=E8=AF=B4=E6=B3=95=E6=94=B9=E6=88=90=E3=80=8C=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E8=AE=A1=E7=AE=97=E3=80=8D=EF=BC=8C=E4=B8=8B?= =?UTF-8?q?=E4=B8=80=E6=AD=A5=E5=86=99=E7=A1=AE=E8=AE=A4=E5=85=B3=E8=81=94?= =?UTF-8?q?=E8=A1=A8=E9=87=8C=E6=AF=8F=E4=B8=AA=E9=94=AE=E5=8F=AA=E6=9C=89?= =?UTF-8?q?=E4=B8=80=E8=A1=8C=E3=80=81=E6=88=96=E5=85=88=E6=B1=87=E6=80=BB?= =?UTF-8?q?=E5=86=8D=E5=85=B3=E8=81=94=EF=BC=9B=E4=BA=BA=E5=B7=A5=E7=BC=96?= =?UTF-8?q?=E8=BE=91=E6=8F=90=E4=BA=A4=E9=87=8C=E5=B8=A6=E7=9A=84=20cardin?= =?UTF-8?q?ality=5Fchecked=20=E4=B8=8D=E4=BD=9C=E6=95=B0=EF=BC=9B=E7=BA=A7?= =?UTF-8?q?=E5=88=AB=E8=AF=B4=E6=98=8E=E3=80=8C=E7=BB=93=E6=9E=9C=E5=BF=85?= =?UTF-8?q?=E7=84=B6=E6=9C=89=E8=AF=AF=E3=80=8D=E6=94=B9=E4=B8=BA=E4=B8=8D?= =?UTF-8?q?=E5=A4=B8=E5=A4=A7=E7=9A=84=E8=AF=B4=E6=B3=95=EF=BC=9B2B?= =?UTF-8?q?=E3=80=81F1=20=E7=9A=84=E6=B5=8B=E8=AF=95=E6=8C=89=E6=96=B0?= =?UTF-8?q?=E8=A7=84=E5=88=99=E6=9B=B4=E6=96=B0=EF=BC=9A=E5=A4=96=E9=94=AE?= =?UTF-8?q?=20+=20verified=20=E2=86=92=20warning=EF=BC=8C=E5=89=96?= =?UTF-8?q?=E6=9E=90=E6=A0=B8=E5=AE=9E=E8=BF=87=20=E2=86=92=20error?= =?UTF-8?q?=EF=BC=8C=E4=BA=BA=E5=B7=A5=E7=A1=AE=E8=AE=A4=20=E2=86=92=20err?= =?UTF-8?q?or?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog.py | 21 +++++++---- backend/app/data/catalog_profile.py | 12 +++++-- backend/app/data/sqlcheck.py | 28 +++++++++++++-- backend/tests/test_catalog_profile.py | 8 ++++- backend/tests/test_catalog_semantics.py | 30 ++++++++++++++++ backend/tests/test_catalog_store.py | 8 ++++- backend/tests/test_sqlcheck.py | 46 +++++++++++++++++++++++-- frontend/src/lib/terms.ts | 7 ++-- frontend/src/types.ts | 7 +++- scripts/check-evidence.mjs | 2 +- 10 files changed, 149 insertions(+), 20 deletions(-) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 29ddb219..a7d559e9 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -18,7 +18,11 @@ notes 结构（每个「项」都是 {value, source, status}，可选 note、upd "codes": 项(dict[码值, 含义]；含义可为空串，表示待填写；项上可带 complete: true， 表示已列出全部取值)}}, "relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage", - "source", "status", "note"?}]} + "source", "status", "note"?, "cardinality_checked"?}]} + +关系上的 cardinality_checked: true 只由数据剖析写：基数是用数据核实的（子表一侧数过 COUNT 和 COUNT(DISTINCT)， +或者有主键、唯一约束）。外键约束、命名推断给的基数是按表结构推的，子表一侧可能每个键恰好一行；SQL 检查只在 +基数用数据核实过、或者人工确认过时，才把一对多关联后的重复计算报成错误（data/sqlcheck.py）。 来源（source）：comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析（data/catalog_profile.py）、 llm 模型起草、human 人工填写。状态（status）：proposed 推断（命名推断、模型起草、数据库注释）、verified 有确证 @@ -88,7 +92,7 @@ _ITEM_KEYS = frozenset({"value", "source", "status", "note", "updated_at"}) #: 没截断地看完了全表取值时才有；没有它的码值表只是「已知的一部分」，SQL 检查不能拿它判断某个值「不在码值表里」 _CODES_ITEM_KEYS = _ITEM_KEYS | {"complete"} _RELATION_KEYS = frozenset({"id", "columns", "to_table", "to_columns", "cardinality", "coverage", "source", - "status", "note", "updated_at"}) + "status", "note", "updated_at", "cardinality_checked"}) _BUSINESS_DATE_KEYS = frozenset({"column", "rule", "timezone"}) #: 文本项的长度上限。目录是给模型和人读的短说明，长篇大论放口径卡或知识库 _TEXT_MAX = 500 @@ -293,6 +297,8 @@ def _relation_problems(rel: Any, seen: set[str]) -> list[str]: problems.append(f"{where}的状态「{rel.get('status')}」不在可选值内") if "note" in rel and not isinstance(rel["note"], str): problems.append(f"{where}的备注应为文字") + if "cardinality_checked" in rel and not isinstance(rel["cardinality_checked"], bool): + problems.append(f"{where}的「基数已用数据核实」应为是或否") if "updated_at" in rel and not isinstance(rel["updated_at"], str): problems.append(f"{where}的修改时间应为文字") return problems @@ -742,7 +748,8 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str, for key, item in new.items(): prev = old.get(key) if prev is None or _edit_value(key, item) != _edit_value(key, prev): - fresh = {k: v for k, v in item.items() if k != "updated_at"} + # 「基数已用数据核实」只由数据剖析写：人工改过的关系记成人工确认，提交里带的这个标记不作数 + fresh = {k: v for k, v in item.items() if k not in ("updated_at", "cardinality_checked")} fresh.update(source="human", status="confirmed", updated_at=at) out[key] = fresh continue @@ -2028,7 +2035,8 @@ _FLIP = {"many_to_one": "one_to_many", "one_to_many": "many_to_one", "one_to_one class JoinEdge: """关系图里的一条有向边：从 from_table 的 from_columns 连到 to_table 的 to_columns。 - 每条关系在图里有正反两条边；cardinality 是从 from_table 看过去的基数。 + 每条关系在图里有正反两条边；cardinality 是从 from_table 看过去的基数。cardinality_checked：基数是数据剖析 + 用数据核实的（关系上的同名字段），不是按表结构推的。 """ from_table: str @@ -2039,10 +2047,11 @@ class JoinEdge: cardinality: str | None source: str status: str + cardinality_checked: bool = False def reversed(self) -> "JoinEdge": return JoinEdge(self.to_table, self.to_columns, self.from_table, self.from_columns, self.relation_id, - _FLIP.get(self.cardinality or ""), self.source, self.status) + _FLIP.get(self.cardinality or ""), self.source, self.status, self.cardinality_checked) def condition(self) -> str: """连接条件：visits.id = channel_visits.visit_id（多列用 AND 连接）。""" @@ -2086,7 +2095,7 @@ def relation_graph(notes_by_table: Mapping[str, Mapping[str, Any]], *, continue edge = JoinEdge(table, tuple(rel.get("columns") or ()), str(rel.get("to_table")), tuple(rel.get("to_columns") or ()), rid, rel.get("cardinality"), str(rel.get("source")), - str(rel.get("status"))) + str(rel.get("status")), rel.get("cardinality_checked") is True) graph.setdefault(edge.from_table, []).append(edge) graph.setdefault(edge.to_table, []).append(edge.reversed()) return graph diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 710b657f..9d7b14fd 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -526,6 +526,9 @@ class _RelationOutcome: sample: int matched: int note: str + #: 基数是用数据核实的：子表一侧是否唯一有了定论（主键、唯一约束，或数过 COUNT 和 COUNT(DISTINCT)）。 + #: 子表太大、没数成的按多对一记，但不算核实——SQL 检查据此不把一对多关联后的重复计算报成错误 + cardinality_checked: bool = False @dataclass @@ -752,9 +755,10 @@ async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _T note = _relation_note(cx.day, sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap, skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique, holds=holds) + checked = child_unique is not None and child_unique[0] is not None out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=coverage, cardinality=cardinality, holds=holds, sample=len(literals), - matched=matched, note=note)) + matched=matched, note=note, cardinality_checked=checked)) # ========================================================================== @@ -953,6 +957,8 @@ def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: changed = True if rel.get("cardinality") is None and o.cardinality is not None: rel["cardinality"] = o.cardinality + if o.cardinality_checked: + rel["cardinality_checked"] = True changed = True note = rel.get("note") if (not note or str(note).startswith(NOTE_PREFIX)) and not catalog.same_profile_note(note, o.note): @@ -992,7 +998,9 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, status = "verified" if o.holds else "proposed" relations.append({"id": o.rid, "columns": list(rel["columns"]), "to_table": rel["to_table"], "to_columns": list(rel["to_columns"]), "cardinality": o.cardinality, - "coverage": o.coverage, "source": "profile", "status": status, "note": o.note}) + "coverage": o.coverage, "source": "profile", "status": status, "note": o.note, + # 只在基数用数据核实过时有这个键：没核实的关系和以前一字不差 + **({"cardinality_checked": True} if o.cardinality_checked else {})}) findings.append(_relation_finding(o, rel, status=status, confirmed=False)) for rid in out.keep_relations: rel = current.get(rid) diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index bbdd492f..c233d430 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -16,7 +16,10 @@ - wrong_date_column：表定义了业务日期，查询却按这张表的另一个时间列分组或筛选。 **级别看依据。** 依据全是 confirmed / verified（人工确认、外键约束、数据剖析）时，fanout_sum、stock_summed 是 error， -其余是 warning；依据里有 proposed（命名推断、模型起草）就一律降为 info。join_unconfirmed 反过来看：对上推断的关系是 +其余是 warning；依据里有 proposed（命名推断、模型起草）就一律降为 info。fanout_sum 另有一道门槛：error 只给「一对多 +确有其事」的——那条一对多的关系人工确认过，或者数据剖析在数据上核实过子表一侧的键不唯一（关系上的 +cardinality_checked）。只凭外键约束推出的一对多降为 warning、说「可能重复计算」：外键只保证父键唯一，子表一侧完全 +可能每个键恰好一行，照「会重复计算」报错就是误报，还会让出具降档。join_unconfirmed 反过来看：对上推断的关系是 info，什么都对不上是 warning，对上有确证的关系不报。rejected 的项一律视而不见（catalog.visible_notes），被驳回的关系 既不算「对上了」，也不拿来判断基数。 @@ -50,7 +53,8 @@ CHECK_CODES = ("fanout_sum", "stock_summed", "join_unconfirmed", "ratio_aggregat "unknown_code", "wrong_date_column") #: 结果的级别，从重到轻 LEVELS = ("error", "warning", "info") -#: 依据有确证时报 error 的规则：它们算出来的数必然是错的。其余规则只是「很可能不对」 +#: 依据有确证时报 error 的规则：照这样写，算出来的数很可能是错的。其余规则只是「可能不对」。 +#: fanout_sum 另要那条一对多的基数人工确认过、或用数据核实过（_ScopeCheck.fanout） _ERROR_CODES = frozenset({"fanout_sum", "stock_summed"}) _STRONG = frozenset({"confirmed", "verified"}) _STATUS_RANK = {"confirmed": 3, "verified": 2, "proposed": 1} @@ -700,6 +704,12 @@ class _ScopeCheck: [status] + [e.status for e in path], counting=isinstance(node, exp.Count))) return out + @staticmethod + def _one_to_many_established(edge: catalog.JoinEdge) -> bool: + """这条一对多是不是确有其事：人工确认过（基数是人定的），或者数据剖析在数据上核实过子表一侧的键不唯一 + （cardinality_checked）。外键约束、命名推断给的基数只是按表结构推的：子表一侧可能每个键恰好一行。""" + return edge.status == "confirmed" or edge.cardinality_checked + def _fanout_basis(self, table: _Table, columns: list[str], *, counting: bool) -> tuple[str, str] | None: """参数里能说明「这是一张表自己的数」的那一列和它的依据：求和看度量类型（流量、存量）， 计数另外认标识列和业务主键（数订单数却数成了明细行数）。""" @@ -717,6 +727,18 @@ class _ScopeCheck: path: list[catalog.JoinEdge], statuses: list[str], *, counting: bool) -> SqlCheck: what = "计数" if counting else "求和" excerpt = self.excerpt(node) + level = _level("fanout_sum", statuses) + if not self._one_to_many_established(path[-1]): + # 只凭表结构推出的一对多：不说「会重复计算」，最多是提醒；下一步先确认是不是真的一对多 + return SqlCheck( + code="fanout_sum", level="warning" if level == "error" else level, table=table.name, column=column, + relation_id=path[-1].relation_id, sql_excerpt=excerpt, + message=(f"按表结构，「{via.label()}」关联「{many.label()}」是一对多，" + f"对「{table.label()}」的「{table.column_label(column)}」{what}可能重复计算。" + f"请确认关联表「{many.label()}」里每个键只有一行，或先汇总再关联"), + for_model=(f"按表结构，{via.name} 关联 {many.name} 是一对多（这个基数没用数据核对过），" + f"{excerpt or what} 可能把 {table.name}.{column} 重复计算。确认 {many.name} 里每个关联键" + f"只有一行；不确定就先在子查询里把 {many.name} 汇总成每条 {via.name} 一行再关联")) if via.alias == table.alias: head = f"「{table.label()}」关联「{many.label()}」是一对多，" model_head = f"{table.name} 关联 {many.name} 是一对多（一行 {table.name} 对多行 {many.name}），" @@ -740,7 +762,7 @@ class _ScopeCheck: model_fix = (f"先在子查询里把 {many.name} 按关联列汇总成每条 {via.name} 一行再关联；用不到 {many.name} 的列" f"就去掉这个关联") return SqlCheck( - code="fanout_sum", level=_level("fanout_sum", statuses), table=table.name, column=column, + code="fanout_sum", level=level, table=table.name, column=column, relation_id=path[-1].relation_id, sql_excerpt=excerpt, message=f"{head}对「{table.label()}」的「{table.column_label(column)}」{what}会重复计算。{fix}", for_model=f"{model_head}{excerpt or what} 会把 {table.name}.{column} 重复计算。{model_fix}") diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index c88d1c82..aae554d7 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -194,9 +194,12 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert (rel["source"], rel["status"]) == ("profile", "verified") assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" assert "数据剖析" in rel["note"] and "抽样" in rel["note"] + # 子表一侧数过 COUNT 和 COUNT(DISTINCT)、确有重复：基数是用数据核实的（A6，一对多关联的检查据此给 error） + assert rel["cardinality_checked"] is True # 外键约束来的关系不动：外键是外键，剖析是剖析 park = _relation(notes, "park_id") assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None) + assert "cardinality_checked" not in park # 外键的基数是按表结构推的，没用数据核对 # 状态列：码值候选，值有了、含义留空，推断 codes = notes["columns"]["status"]["codes"] @@ -265,6 +268,8 @@ async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, mak rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") assert (rel["source"], rel["status"]) == ("name", "confirmed") assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" + # 人定的基数不改；原来就有基数（命名推断记的多对一），剖析也不给它记「用数据核实过」 + assert "cardinality_checked" not in rel finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"]) assert finding["status"] == "confirmed" and finding["confirmed"] is True @@ -376,9 +381,10 @@ async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, sp assert not any("MIN(" in s for s in on_visits) assert "too_large" in _reasons(visits) assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large") - # 父表有主键，覆盖率又够：照样有确证；子表一侧没核对唯一，按多对一记 + # 父表有主键，覆盖率又够：照样有确证；子表一侧没核对唯一，按多对一记，不算用数据核实过基数 rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one" + assert not rel.get("cardinality_checked") # ---------------------------------------------------------------- 接口 diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index a3dd338e..3c5d8416 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -137,6 +137,8 @@ async def test_confirming_a_profiled_relation_by_patch_keeps_the_fanout_check(cl rel = _relation(detail["notes"], "gate_id") assert (rel["source"], rel["status"], rel["cardinality"], rel["coverage"]) == ( "profile", "verified", "many_to_one", 1.0) + # 子表一侧数过、确有重复：基数用数据核实过，一对多确有其事 → error（A6） + assert rel["cardinality_checked"] is True assert ("fanout_sum", "error", "gates", "id") in await _found(sid, sql) changes = [{"path": "relations.new", "value": {"columns": ["gate_id"], "to_table": "gates", "to_columns": ["id"]}, @@ -149,6 +151,21 @@ async def test_confirming_a_profiled_relation_by_patch_keeps_the_fanout_check(cl assert ("fanout_sum", "error", "gates", "id") in await _found(sid, sql) +async def test_profiled_relation_without_child_check_only_warns(client, make_source, scenic_db): + """A6：子表超过整表统计上限、没核对子表一侧唯不唯一的关系，剖析照样记有确证、按多对一记，但基数不算用数据 + 核实过：扇出检查只给 warning（「可能重复计算」），不给 error、不让出具降档。""" + sql = "SELECT COUNT(g.id) AS n FROM gates g JOIN visits v ON v.gate_id = g.id" + sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits", "gates"]) + r = await client.post(_url(sid), json={"tables": ["visits"]}) + assert r.status_code == 200, r.text + rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") + assert (rel["source"], rel["status"], rel["cardinality"]) == ("profile", "verified", "many_to_one") + assert not rel.get("cardinality_checked") + found = await _found(sid, sql) + assert ("fanout_sum", "warning", "gates", "id") in found + assert ("fanout_sum", "error", "gates", "id") not in found + + # ========================================================================== # A2：去掉推断或外键来的关系即驳回 # ========================================================================== @@ -193,6 +210,19 @@ def test_human_edit_rejects_a_removed_human_relation_that_structure_would_bring_ assert catalog.apply_human_edit({"relations": [park]}, {"relations": []}, table="visits", at=AT) == {} +def test_human_edit_cannot_forge_a_data_checked_cardinality(): + """A6：「基数用数据核实过」只能由数据剖析写。人工改了关系（记成人工确认），提交里带的 cardinality_checked 不作数； + 值没动的关系保留剖析原来写的。""" + gate = {**_rel(["gate_id"], "gates", ["id"], "profile", "verified"), "coverage": 1.0, "cardinality_checked": True} + park = _rel(["park_id"], "parks", ["id"], "name") + forged = {**park, "cardinality": "one_to_many", "cardinality_checked": True} + out = catalog.apply_human_edit({"relations": [gate, park]}, {"relations": [gate, forged]}, table="visits", at=AT) + by_id = {r["id"]: r for r in out["relations"]} + assert by_id[gate["id"]]["cardinality_checked"] is True + assert (by_id[park["id"]]["source"], by_id[park["id"]]["status"]) == ("human", "confirmed") + assert "cardinality_checked" not in by_id[park["id"]] + + async def test_removed_or_rejected_relations_stay_out_of_checks_graph_and_redraft(client, make_source, scenic_db): sid = await make_source(scenic_db, draft=["visits"]) detail = await _notes(client, sid, "visits") diff --git a/backend/tests/test_catalog_store.py b/backend/tests/test_catalog_store.py index 000a78ef..0de0e0bf 100644 --- a/backend/tests/test_catalog_store.py +++ b/backend/tests/test_catalog_store.py @@ -116,6 +116,9 @@ async def test_read_catalog_lists_all_tables_of_a_source(): ({"formula": _label("sum(amount)")}, "formula"), # 目录不放公式 ({"relations": [{"id": "r1", "columns": ["a"], "to_table": "t", "to_columns": [], "source": "fk", "status": "verified"}]}, "关联关系 r1"), + # 基数是否用数据核实过（数据剖析写）：只能是 true / false + ({"relations": [{"id": "r1", "columns": ["a"], "to_table": "t", "to_columns": ["id"], "cardinality": "many_to_one", + "source": "profile", "status": "verified", "cardinality_checked": "yes"}]}, "关联关系 r1"), ]) def test_validate_rejects_malformed_notes(notes, fragment): problems = catalog.validate_notes(notes) @@ -134,7 +137,10 @@ def test_validate_accepts_full_example(): "codes": _label({"1": "有效"}, "comment", "proposed")}}, "relations": [{"id": catalog.relation_id("visits", ["store_id"], "stores", ["id"]), "columns": ["store_id"], "to_table": "stores", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": None, - "source": "fk", "status": "verified"}], + "source": "fk", "status": "verified"}, + {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"], + "to_table": "gates", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": 1.0, + "source": "profile", "status": "verified", "cardinality_checked": True}], } assert catalog.validate_notes(notes) == [] diff --git a/backend/tests/test_sqlcheck.py b/backend/tests/test_sqlcheck.py index 329087b8..2f8bf546 100644 --- a/backend/tests/test_sqlcheck.py +++ b/backend/tests/test_sqlcheck.py @@ -113,10 +113,19 @@ async def test_fanout_count_of_the_one_side_key(cache): async def test_fanout_through_two_hops_with_an_inferred_foreign_key(cache): - """支付记录没写目录：订单 → 支付记录的关系按外键现推（verified），明细经订单连支付记录，明细的金额照样重复。""" + """支付记录没写目录：订单 → 支付记录的关系按外键现推（verified），明细经订单连支付记录，明细的金额可能重复。 + 这条一对多只是按外键推的（没用数据核对过子表一侧），所以是 warning（A6）。""" sql = ("SELECT SUM(i.amount) FROM order_items i JOIN orders o ON o.id = i.order_id " "JOIN payments p ON p.order_id = o.id WHERE o.status = 1") [check] = [c for c in run(cache, sql) if c.code == "fanout_sum"] + assert (check.level, check.table, check.column) == ("warning", "order_items", "amount") + assert "可能重复计算" in check.message and "「订单」关联「payments」" in check.message + # 人工确认了支付记录 → 订单这条关系（基数是人定的）：一对多确有其事，error，说清明细怎么随之重复 + notes = scenic_notes.notes() + notes["payments"] = {"relations": [scenic_notes._rel("payments", ["order_id"], "orders", ["id"], "many_to_one", + "confirmed")]} + checker = SqlChecker(kind="sqlite", schema_cache=cache, notes=notes, source_name="scenic") + [check] = [c for c in checker.check(sql) if c.code == "fanout_sum"] assert (check.level, check.table, check.column) == ("error", "order_items", "amount") assert "「订单」关联「payments」是一对多，「订单明细」的每一行会随之重复" in check.message assert "去掉与「payments」的关联" in check.message @@ -156,7 +165,9 @@ async def test_fanout_sum_negative(cache, sql): async def test_fanout_level_follows_the_status_of_its_basis(cache): sql = FANOUT[0] - assert ("fanout_sum", "error", "orders", "total_amount") in found(cache, sql, "verified") + assert ("fanout_sum", "error", "orders", "total_amount") in found(cache, sql, "confirmed") + # 外键 + verified：一对多只是按表结构推的，降为 warning（A6，见下一个测试） + assert ("fanout_sum", "warning", "orders", "total_amount") in found(cache, sql, "verified") assert ("fanout_sum", "info", "orders", "total_amount") in found(cache, sql, "proposed") # 关系确认了、度量类型只是推断：依据里有推断就降为 info assert ("fanout_sum", "info", "orders", "total_amount") in found( @@ -169,6 +180,37 @@ async def test_fanout_level_follows_the_status_of_its_basis(cache): assert "fanout_sum" not in codes(cache, sql, override={("orders", "columns.total_amount.measure"): "rejected"}) +def _checked(table_notes: dict, rid: str, *, source: str = "profile") -> dict: + """把一条关系记成「剖析用数据核实过基数」（cardinality_checked），不改入参。""" + import copy + + out = copy.deepcopy(table_notes) + rel = next(r for r in out["relations"] if r["id"] == rid) + rel.update(source=source, status="verified", cardinality_checked=True) + return out + + +async def test_fanout_error_only_when_one_to_many_is_established(cache): + """A6：error 只给「一对多确有其事」——剖析在数据上核实过子表一侧的键不唯一，或者人工确认过基数。 + 只凭外键结构推出的一对多（子表一侧可能每个键恰好一行）降为 warning，说法是「可能重复计算」。""" + sql = FANOUT[0] + # 外键 + verified：基数是按表结构推的，没用数据核对过 → warning，不说「会重复计算」 + [fk] = [c for c in run(cache, sql, "verified") if c.code == "fanout_sum"] + assert (fk.level, fk.table, fk.column) == ("warning", "orders", "total_amount") + assert "可能重复计算" in fk.message and "会重复计算" not in fk.message + assert "确认关联表" in fk.message and "每个键只有一行" in fk.message and "先汇总再关联" in fk.message + assert "必然" not in fk.message and "必然" not in fk.for_model + # 剖析核实过（关系上记 cardinality_checked）→ error + profiled = scenic_notes.notes("verified") + profiled["order_items"] = _checked(profiled["order_items"], ORDER_ITEMS_TO_ORDERS) + checker = SqlChecker(kind="sqlite", schema_cache=cache, notes=profiled, source_name="scenic") + [hit] = [c for c in checker.check(sql) if c.code == "fanout_sum"] + assert hit.level == "error" and "会重复计算" in hit.message + # 人工确认过基数（关系是 confirmed）→ error + assert ("fanout_sum", "error", "orders", "total_amount") in found( + cache, sql, "verified", override={("order_items", f"relations.{ORDER_ITEMS_TO_ORDERS}"): "confirmed"}) + + # -------------------------------------------------------------------------- # stock_summed：存量跨时间求和 # -------------------------------------------------------------------------- diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index c681ad21..5d3fc031 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2509,7 +2509,8 @@ export const CODES_TEXT = { // // 术语： // - 「SQL 检查」：对照数据目录找出能执行、数却可能不对的写法。七条规则，界面上写中文规则名，不露规则编号。 -// - 级别：错误（依据已确认，结果必然有误）、提醒（依据已确认，结果很可能有误）、提示（依据只是推断，仅供参考）。 +// - 级别：错误（依据已核实，结果很可能有误）、提醒（依据已确认，结果可能有误）、提示（依据只是推断，仅供参考）。 +// 一对多关联后重复计算：只有一对多人工确认过、或数据剖析用数据核实过时才是错误；只凭外键推出的是提醒。 // 颜色走语义令牌：错误 --st-failed，提醒 --st-waiting，提示 --accent。 // =========================================================================== @@ -2540,8 +2541,8 @@ export const SQL_CHECK_LEVEL_LABEL: Record = { } export const SQL_CHECK_LEVEL_HINT: Record = { - error: '依据已确认，结果必然有误', - warning: '依据已确认，结果很可能有误', + error: '依据已核实，结果很可能有误，需要修改', + warning: '依据已确认，结果可能有误，请核对', info: '依据尚未确认（推断），仅供参考', } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 45ef3806..f2e4f915 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2637,6 +2637,11 @@ export interface CatalogRelation { status: CatalogStatus note?: string updated_at?: string + /** + * 基数是数据剖析用数据核实的（子表一侧数过是否唯一），只由剖析写。外键、命名推断的基数没有这个键。 + * 一对多关联后重复计算的检查只在它为真、或者关系人工确认过时报「错误」，否则最多是「提醒」 + */ + cardinality_checked?: boolean } /** 一张表的目录（notes）。字段固定，服务端不认识的一律拒收 */ @@ -2949,7 +2954,7 @@ export interface CatalogProfileOut { export type SqlCheckCode = 'fanout_sum' | 'stock_summed' | 'join_unconfirmed' | 'ratio_aggregated' | 'missing_valid_filter' | 'unknown_code' | 'wrong_date_column' -/** error：依据有确证、结果必然有误；warning：依据有确证、很可能有误；info：依据只是推断 */ +/** error：依据已核实、结果很可能有误；warning：依据有确证、结果可能有误；info：依据只是推断 */ export type SqlCheckLevel = 'error' | 'warning' | 'info' /** 一条检查结果（证据接口查询步骤里的 checks、助手自查和发布前检查的问题）。交回模型改写用的那句不上界面，这里不收 */ diff --git a/scripts/check-evidence.mjs b/scripts/check-evidence.mjs index 1e31f86b..5126e1c0 100644 --- a/scripts/check-evidence.mjs +++ b/scripts/check-evidence.mjs @@ -1073,7 +1073,7 @@ await section('sqlcheck', 'SQL 检查（数据目录阶段 4B）：查询步骤 const want = [await tokenColor(bad.page, 'color', 'var(--st-failed)'), await tokenColor(bad.page, 'color', 'var(--st-waiting)'), await tokenColor(bad.page, 'color', 'var(--accent)')] check('级别的颜色走令牌：错误 --st-failed、提醒 --st-waiting、提示 --accent', tones.join('|') === want.join('|'), tones.join('|')) - check('悬停写明依据的确证程度', (await items.first().locator('[data-sql-level]').getAttribute('title')) === '依据已确认，结果必然有误') + check('悬停写明依据的确证程度', (await items.first().locator('[data-sql-level]').getAttribute('title')) === '依据已核实，结果很可能有误，需要修改') check('不露规则编号', !/fanout_sum|missing_valid_filter|join_unconfirmed/.test(await p.innerText())) await bad.page.keyboard.press('Escape') await openQ(bad.page, '45,678.5元', { wait: '[data-ev-sql-check-failed]' })