mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
A6 扇出检查只在一对多确有其事时报错误:关系人工确认过,或数据剖析在数据上核实过子表一侧的键不唯一(关系上记 cardinality_checked,子表太大没核对的不算)才给 error;只凭外键结构推出的一对多降为 warning,说法改成「可能重复计算」,下一步写确认关联表里每个键只有一行、或先汇总再关联;人工编辑提交里带的 cardinality_checked 不作数;级别说明「结果必然有误」改为不夸大的说法;2B、F1 的测试按新规则更新:外键 + verified → warning,剖析核实过 → error,人工确认 → error
This commit is contained in:
@@ -18,7 +18,11 @@ notes 结构(每个「项」都是 {value, source, status},可选 note、upd
|
||||
"codes": 项(dict[码值, 含义];含义可为空串,表示待填写;项上可带 complete: true,
|
||||
表示已列出全部取值)}},
|
||||
"relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage",
|
||||
"source", "status", "note"?}]}
|
||||
"source", "status", "note"?, "cardinality_checked"?}]}
|
||||
|
||||
关系上的 cardinality_checked: true 只由数据剖析写:基数是用数据核实的(子表一侧数过 COUNT 和 COUNT(DISTINCT),
|
||||
或者有主键、唯一约束)。外键约束、命名推断给的基数是按表结构推的,子表一侧可能每个键恰好一行;SQL 检查只在
|
||||
基数用数据核实过、或者人工确认过时,才把一对多关联后的重复计算报成错误(data/sqlcheck.py)。
|
||||
|
||||
来源(source):comment 数据库注释、fk 外键约束、name 命名推断、profile 数据剖析(data/catalog_profile.py)、
|
||||
llm 模型起草、human 人工填写。状态(status):proposed 推断(命名推断、模型起草、数据库注释)、verified 有确证
|
||||
@@ -88,7 +92,7 @@ _ITEM_KEYS = frozenset({"value", "source", "status", "note", "updated_at"})
|
||||
#: 没截断地看完了全表取值时才有;没有它的码值表只是「已知的一部分」,SQL 检查不能拿它判断某个值「不在码值表里」
|
||||
_CODES_ITEM_KEYS = _ITEM_KEYS | {"complete"}
|
||||
_RELATION_KEYS = frozenset({"id", "columns", "to_table", "to_columns", "cardinality", "coverage", "source",
|
||||
"status", "note", "updated_at"})
|
||||
"status", "note", "updated_at", "cardinality_checked"})
|
||||
_BUSINESS_DATE_KEYS = frozenset({"column", "rule", "timezone"})
|
||||
#: 文本项的长度上限。目录是给模型和人读的短说明,长篇大论放口径卡或知识库
|
||||
_TEXT_MAX = 500
|
||||
@@ -293,6 +297,8 @@ def _relation_problems(rel: Any, seen: set[str]) -> list[str]:
|
||||
problems.append(f"{where}的状态「{rel.get('status')}」不在可选值内")
|
||||
if "note" in rel and not isinstance(rel["note"], str):
|
||||
problems.append(f"{where}的备注应为文字")
|
||||
if "cardinality_checked" in rel and not isinstance(rel["cardinality_checked"], bool):
|
||||
problems.append(f"{where}的「基数已用数据核实」应为是或否")
|
||||
if "updated_at" in rel and not isinstance(rel["updated_at"], str):
|
||||
problems.append(f"{where}的修改时间应为文字")
|
||||
return problems
|
||||
@@ -742,7 +748,8 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
|
||||
for key, item in new.items():
|
||||
prev = old.get(key)
|
||||
if prev is None or _edit_value(key, item) != _edit_value(key, prev):
|
||||
fresh = {k: v for k, v in item.items() if k != "updated_at"}
|
||||
# 「基数已用数据核实」只由数据剖析写:人工改过的关系记成人工确认,提交里带的这个标记不作数
|
||||
fresh = {k: v for k, v in item.items() if k not in ("updated_at", "cardinality_checked")}
|
||||
fresh.update(source="human", status="confirmed", updated_at=at)
|
||||
out[key] = fresh
|
||||
continue
|
||||
@@ -2028,7 +2035,8 @@ _FLIP = {"many_to_one": "one_to_many", "one_to_many": "many_to_one", "one_to_one
|
||||
class JoinEdge:
|
||||
"""关系图里的一条有向边:从 from_table 的 from_columns 连到 to_table 的 to_columns。
|
||||
|
||||
每条关系在图里有正反两条边;cardinality 是从 from_table 看过去的基数。
|
||||
每条关系在图里有正反两条边;cardinality 是从 from_table 看过去的基数。cardinality_checked:基数是数据剖析
|
||||
用数据核实的(关系上的同名字段),不是按表结构推的。
|
||||
"""
|
||||
|
||||
from_table: str
|
||||
@@ -2039,10 +2047,11 @@ class JoinEdge:
|
||||
cardinality: str | None
|
||||
source: str
|
||||
status: str
|
||||
cardinality_checked: bool = False
|
||||
|
||||
def reversed(self) -> "JoinEdge":
|
||||
return JoinEdge(self.to_table, self.to_columns, self.from_table, self.from_columns, self.relation_id,
|
||||
_FLIP.get(self.cardinality or ""), self.source, self.status)
|
||||
_FLIP.get(self.cardinality or ""), self.source, self.status, self.cardinality_checked)
|
||||
|
||||
def condition(self) -> str:
|
||||
"""连接条件:visits.id = channel_visits.visit_id(多列用 AND 连接)。"""
|
||||
@@ -2086,7 +2095,7 @@ def relation_graph(notes_by_table: Mapping[str, Mapping[str, Any]], *,
|
||||
continue
|
||||
edge = JoinEdge(table, tuple(rel.get("columns") or ()), str(rel.get("to_table")),
|
||||
tuple(rel.get("to_columns") or ()), rid, rel.get("cardinality"), str(rel.get("source")),
|
||||
str(rel.get("status")))
|
||||
str(rel.get("status")), rel.get("cardinality_checked") is True)
|
||||
graph.setdefault(edge.from_table, []).append(edge)
|
||||
graph.setdefault(edge.to_table, []).append(edge.reversed())
|
||||
return graph
|
||||
|
||||
@@ -526,6 +526,9 @@ class _RelationOutcome:
|
||||
sample: int
|
||||
matched: int
|
||||
note: str
|
||||
#: 基数是用数据核实的:子表一侧是否唯一有了定论(主键、唯一约束,或数过 COUNT 和 COUNT(DISTINCT))。
|
||||
#: 子表太大、没数成的按多对一记,但不算核实——SQL 检查据此不把一对多关联后的重复计算报成错误
|
||||
cardinality_checked: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -752,9 +755,10 @@ async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _T
|
||||
note = _relation_note(cx.day, sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap,
|
||||
skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique,
|
||||
holds=holds)
|
||||
checked = child_unique is not None and child_unique[0] is not None
|
||||
out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=coverage,
|
||||
cardinality=cardinality, holds=holds, sample=len(literals),
|
||||
matched=matched, note=note))
|
||||
matched=matched, note=note, cardinality_checked=checked))
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
@@ -953,6 +957,8 @@ def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool:
|
||||
changed = True
|
||||
if rel.get("cardinality") is None and o.cardinality is not None:
|
||||
rel["cardinality"] = o.cardinality
|
||||
if o.cardinality_checked:
|
||||
rel["cardinality_checked"] = True
|
||||
changed = True
|
||||
note = rel.get("note")
|
||||
if (not note or str(note).startswith(NOTE_PREFIX)) and not catalog.same_profile_note(note, o.note):
|
||||
@@ -992,7 +998,9 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *,
|
||||
status = "verified" if o.holds else "proposed"
|
||||
relations.append({"id": o.rid, "columns": list(rel["columns"]), "to_table": rel["to_table"],
|
||||
"to_columns": list(rel["to_columns"]), "cardinality": o.cardinality,
|
||||
"coverage": o.coverage, "source": "profile", "status": status, "note": o.note})
|
||||
"coverage": o.coverage, "source": "profile", "status": status, "note": o.note,
|
||||
# 只在基数用数据核实过时有这个键:没核实的关系和以前一字不差
|
||||
**({"cardinality_checked": True} if o.cardinality_checked else {})})
|
||||
findings.append(_relation_finding(o, rel, status=status, confirmed=False))
|
||||
for rid in out.keep_relations:
|
||||
rel = current.get(rid)
|
||||
|
||||
@@ -16,7 +16,10 @@
|
||||
- wrong_date_column:表定义了业务日期,查询却按这张表的另一个时间列分组或筛选。
|
||||
|
||||
**级别看依据。** 依据全是 confirmed / verified(人工确认、外键约束、数据剖析)时,fanout_sum、stock_summed 是 error,
|
||||
其余是 warning;依据里有 proposed(命名推断、模型起草)就一律降为 info。join_unconfirmed 反过来看:对上推断的关系是
|
||||
其余是 warning;依据里有 proposed(命名推断、模型起草)就一律降为 info。fanout_sum 另有一道门槛:error 只给「一对多
|
||||
确有其事」的——那条一对多的关系人工确认过,或者数据剖析在数据上核实过子表一侧的键不唯一(关系上的
|
||||
cardinality_checked)。只凭外键约束推出的一对多降为 warning、说「可能重复计算」:外键只保证父键唯一,子表一侧完全
|
||||
可能每个键恰好一行,照「会重复计算」报错就是误报,还会让出具降档。join_unconfirmed 反过来看:对上推断的关系是
|
||||
info,什么都对不上是 warning,对上有确证的关系不报。rejected 的项一律视而不见(catalog.visible_notes),被驳回的关系
|
||||
既不算「对上了」,也不拿来判断基数。
|
||||
|
||||
@@ -50,7 +53,8 @@ CHECK_CODES = ("fanout_sum", "stock_summed", "join_unconfirmed", "ratio_aggregat
|
||||
"unknown_code", "wrong_date_column")
|
||||
#: 结果的级别,从重到轻
|
||||
LEVELS = ("error", "warning", "info")
|
||||
#: 依据有确证时报 error 的规则:它们算出来的数必然是错的。其余规则只是「很可能不对」
|
||||
#: 依据有确证时报 error 的规则:照这样写,算出来的数很可能是错的。其余规则只是「可能不对」。
|
||||
#: fanout_sum 另要那条一对多的基数人工确认过、或用数据核实过(_ScopeCheck.fanout)
|
||||
_ERROR_CODES = frozenset({"fanout_sum", "stock_summed"})
|
||||
_STRONG = frozenset({"confirmed", "verified"})
|
||||
_STATUS_RANK = {"confirmed": 3, "verified": 2, "proposed": 1}
|
||||
@@ -700,6 +704,12 @@ class _ScopeCheck:
|
||||
[status] + [e.status for e in path], counting=isinstance(node, exp.Count)))
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _one_to_many_established(edge: catalog.JoinEdge) -> bool:
|
||||
"""这条一对多是不是确有其事:人工确认过(基数是人定的),或者数据剖析在数据上核实过子表一侧的键不唯一
|
||||
(cardinality_checked)。外键约束、命名推断给的基数只是按表结构推的:子表一侧可能每个键恰好一行。"""
|
||||
return edge.status == "confirmed" or edge.cardinality_checked
|
||||
|
||||
def _fanout_basis(self, table: _Table, columns: list[str], *, counting: bool) -> tuple[str, str] | None:
|
||||
"""参数里能说明「这是一张表自己的数」的那一列和它的依据:求和看度量类型(流量、存量),
|
||||
计数另外认标识列和业务主键(数订单数却数成了明细行数)。"""
|
||||
@@ -717,6 +727,18 @@ class _ScopeCheck:
|
||||
path: list[catalog.JoinEdge], statuses: list[str], *, counting: bool) -> SqlCheck:
|
||||
what = "计数" if counting else "求和"
|
||||
excerpt = self.excerpt(node)
|
||||
level = _level("fanout_sum", statuses)
|
||||
if not self._one_to_many_established(path[-1]):
|
||||
# 只凭表结构推出的一对多:不说「会重复计算」,最多是提醒;下一步先确认是不是真的一对多
|
||||
return SqlCheck(
|
||||
code="fanout_sum", level="warning" if level == "error" else level, table=table.name, column=column,
|
||||
relation_id=path[-1].relation_id, sql_excerpt=excerpt,
|
||||
message=(f"按表结构,「{via.label()}」关联「{many.label()}」是一对多,"
|
||||
f"对「{table.label()}」的「{table.column_label(column)}」{what}可能重复计算。"
|
||||
f"请确认关联表「{many.label()}」里每个键只有一行,或先汇总再关联"),
|
||||
for_model=(f"按表结构,{via.name} 关联 {many.name} 是一对多(这个基数没用数据核对过),"
|
||||
f"{excerpt or what} 可能把 {table.name}.{column} 重复计算。确认 {many.name} 里每个关联键"
|
||||
f"只有一行;不确定就先在子查询里把 {many.name} 汇总成每条 {via.name} 一行再关联"))
|
||||
if via.alias == table.alias:
|
||||
head = f"「{table.label()}」关联「{many.label()}」是一对多,"
|
||||
model_head = f"{table.name} 关联 {many.name} 是一对多(一行 {table.name} 对多行 {many.name}),"
|
||||
@@ -740,7 +762,7 @@ class _ScopeCheck:
|
||||
model_fix = (f"先在子查询里把 {many.name} 按关联列汇总成每条 {via.name} 一行再关联;用不到 {many.name} 的列"
|
||||
f"就去掉这个关联")
|
||||
return SqlCheck(
|
||||
code="fanout_sum", level=_level("fanout_sum", statuses), table=table.name, column=column,
|
||||
code="fanout_sum", level=level, table=table.name, column=column,
|
||||
relation_id=path[-1].relation_id, sql_excerpt=excerpt,
|
||||
message=f"{head}对「{table.label()}」的「{table.column_label(column)}」{what}会重复计算。{fix}",
|
||||
for_model=f"{model_head}{excerpt or what} 会把 {table.name}.{column} 重复计算。{model_fix}")
|
||||
|
||||
@@ -194,9 +194,12 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d
|
||||
assert (rel["source"], rel["status"]) == ("profile", "verified")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
assert "数据剖析" in rel["note"] and "抽样" in rel["note"]
|
||||
# 子表一侧数过 COUNT 和 COUNT(DISTINCT)、确有重复:基数是用数据核实的(A6,一对多关联的检查据此给 error)
|
||||
assert rel["cardinality_checked"] is True
|
||||
# 外键约束来的关系不动:外键是外键,剖析是剖析
|
||||
park = _relation(notes, "park_id")
|
||||
assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None)
|
||||
assert "cardinality_checked" not in park # 外键的基数是按表结构推的,没用数据核对
|
||||
|
||||
# 状态列:码值候选,值有了、含义留空,推断
|
||||
codes = notes["columns"]["status"]["codes"]
|
||||
@@ -265,6 +268,8 @@ async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, mak
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "confirmed")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
# 人定的基数不改;原来就有基数(命名推断记的多对一),剖析也不给它记「用数据核实过」
|
||||
assert "cardinality_checked" not in rel
|
||||
finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"])
|
||||
assert finding["status"] == "confirmed" and finding["confirmed"] is True
|
||||
|
||||
@@ -376,9 +381,10 @@ async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, sp
|
||||
assert not any("MIN(" in s for s in on_visits)
|
||||
assert "too_large" in _reasons(visits)
|
||||
assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large")
|
||||
# 父表有主键,覆盖率又够:照样有确证;子表一侧没核对唯一,按多对一记
|
||||
# 父表有主键,覆盖率又够:照样有确证;子表一侧没核对唯一,按多对一记,不算用数据核实过基数
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one"
|
||||
assert not rel.get("cardinality_checked")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 接口
|
||||
|
||||
@@ -137,6 +137,8 @@ async def test_confirming_a_profiled_relation_by_patch_keeps_the_fanout_check(cl
|
||||
rel = _relation(detail["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"], rel["cardinality"], rel["coverage"]) == (
|
||||
"profile", "verified", "many_to_one", 1.0)
|
||||
# 子表一侧数过、确有重复:基数用数据核实过,一对多确有其事 → error(A6)
|
||||
assert rel["cardinality_checked"] is True
|
||||
assert ("fanout_sum", "error", "gates", "id") in await _found(sid, sql)
|
||||
|
||||
changes = [{"path": "relations.new", "value": {"columns": ["gate_id"], "to_table": "gates", "to_columns": ["id"]},
|
||||
@@ -149,6 +151,21 @@ async def test_confirming_a_profiled_relation_by_patch_keeps_the_fanout_check(cl
|
||||
assert ("fanout_sum", "error", "gates", "id") in await _found(sid, sql)
|
||||
|
||||
|
||||
async def test_profiled_relation_without_child_check_only_warns(client, make_source, scenic_db):
|
||||
"""A6:子表超过整表统计上限、没核对子表一侧唯不唯一的关系,剖析照样记有确证、按多对一记,但基数不算用数据
|
||||
核实过:扇出检查只给 warning(「可能重复计算」),不给 error、不让出具降档。"""
|
||||
sql = "SELECT COUNT(g.id) AS n FROM gates g JOIN visits v ON v.gate_id = g.id"
|
||||
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits", "gates"])
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]})
|
||||
assert r.status_code == 200, r.text
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"], rel["cardinality"]) == ("profile", "verified", "many_to_one")
|
||||
assert not rel.get("cardinality_checked")
|
||||
found = await _found(sid, sql)
|
||||
assert ("fanout_sum", "warning", "gates", "id") in found
|
||||
assert ("fanout_sum", "error", "gates", "id") not in found
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# A2:去掉推断或外键来的关系即驳回
|
||||
# ==========================================================================
|
||||
@@ -193,6 +210,19 @@ def test_human_edit_rejects_a_removed_human_relation_that_structure_would_bring_
|
||||
assert catalog.apply_human_edit({"relations": [park]}, {"relations": []}, table="visits", at=AT) == {}
|
||||
|
||||
|
||||
def test_human_edit_cannot_forge_a_data_checked_cardinality():
|
||||
"""A6:「基数用数据核实过」只能由数据剖析写。人工改了关系(记成人工确认),提交里带的 cardinality_checked 不作数;
|
||||
值没动的关系保留剖析原来写的。"""
|
||||
gate = {**_rel(["gate_id"], "gates", ["id"], "profile", "verified"), "coverage": 1.0, "cardinality_checked": True}
|
||||
park = _rel(["park_id"], "parks", ["id"], "name")
|
||||
forged = {**park, "cardinality": "one_to_many", "cardinality_checked": True}
|
||||
out = catalog.apply_human_edit({"relations": [gate, park]}, {"relations": [gate, forged]}, table="visits", at=AT)
|
||||
by_id = {r["id"]: r for r in out["relations"]}
|
||||
assert by_id[gate["id"]]["cardinality_checked"] is True
|
||||
assert (by_id[park["id"]]["source"], by_id[park["id"]]["status"]) == ("human", "confirmed")
|
||||
assert "cardinality_checked" not in by_id[park["id"]]
|
||||
|
||||
|
||||
async def test_removed_or_rejected_relations_stay_out_of_checks_graph_and_redraft(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
detail = await _notes(client, sid, "visits")
|
||||
|
||||
@@ -116,6 +116,9 @@ async def test_read_catalog_lists_all_tables_of_a_source():
|
||||
({"formula": _label("sum(amount)")}, "formula"), # 目录不放公式
|
||||
({"relations": [{"id": "r1", "columns": ["a"], "to_table": "t", "to_columns": [],
|
||||
"source": "fk", "status": "verified"}]}, "关联关系 r1"),
|
||||
# 基数是否用数据核实过(数据剖析写):只能是 true / false
|
||||
({"relations": [{"id": "r1", "columns": ["a"], "to_table": "t", "to_columns": ["id"], "cardinality": "many_to_one",
|
||||
"source": "profile", "status": "verified", "cardinality_checked": "yes"}]}, "关联关系 r1"),
|
||||
])
|
||||
def test_validate_rejects_malformed_notes(notes, fragment):
|
||||
problems = catalog.validate_notes(notes)
|
||||
@@ -134,7 +137,10 @@ def test_validate_accepts_full_example():
|
||||
"codes": _label({"1": "有效"}, "comment", "proposed")}},
|
||||
"relations": [{"id": catalog.relation_id("visits", ["store_id"], "stores", ["id"]), "columns": ["store_id"],
|
||||
"to_table": "stores", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": None,
|
||||
"source": "fk", "status": "verified"}],
|
||||
"source": "fk", "status": "verified"},
|
||||
{"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
|
||||
"to_table": "gates", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": 1.0,
|
||||
"source": "profile", "status": "verified", "cardinality_checked": True}],
|
||||
}
|
||||
assert catalog.validate_notes(notes) == []
|
||||
|
||||
|
||||
@@ -113,10 +113,19 @@ async def test_fanout_count_of_the_one_side_key(cache):
|
||||
|
||||
|
||||
async def test_fanout_through_two_hops_with_an_inferred_foreign_key(cache):
|
||||
"""支付记录没写目录:订单 → 支付记录的关系按外键现推(verified),明细经订单连支付记录,明细的金额照样重复。"""
|
||||
"""支付记录没写目录:订单 → 支付记录的关系按外键现推(verified),明细经订单连支付记录,明细的金额可能重复。
|
||||
这条一对多只是按外键推的(没用数据核对过子表一侧),所以是 warning(A6)。"""
|
||||
sql = ("SELECT SUM(i.amount) FROM order_items i JOIN orders o ON o.id = i.order_id "
|
||||
"JOIN payments p ON p.order_id = o.id WHERE o.status = 1")
|
||||
[check] = [c for c in run(cache, sql) if c.code == "fanout_sum"]
|
||||
assert (check.level, check.table, check.column) == ("warning", "order_items", "amount")
|
||||
assert "可能重复计算" in check.message and "「订单」关联「payments」" in check.message
|
||||
# 人工确认了支付记录 → 订单这条关系(基数是人定的):一对多确有其事,error,说清明细怎么随之重复
|
||||
notes = scenic_notes.notes()
|
||||
notes["payments"] = {"relations": [scenic_notes._rel("payments", ["order_id"], "orders", ["id"], "many_to_one",
|
||||
"confirmed")]}
|
||||
checker = SqlChecker(kind="sqlite", schema_cache=cache, notes=notes, source_name="scenic")
|
||||
[check] = [c for c in checker.check(sql) if c.code == "fanout_sum"]
|
||||
assert (check.level, check.table, check.column) == ("error", "order_items", "amount")
|
||||
assert "「订单」关联「payments」是一对多,「订单明细」的每一行会随之重复" in check.message
|
||||
assert "去掉与「payments」的关联" in check.message
|
||||
@@ -156,7 +165,9 @@ async def test_fanout_sum_negative(cache, sql):
|
||||
|
||||
async def test_fanout_level_follows_the_status_of_its_basis(cache):
|
||||
sql = FANOUT[0]
|
||||
assert ("fanout_sum", "error", "orders", "total_amount") in found(cache, sql, "verified")
|
||||
assert ("fanout_sum", "error", "orders", "total_amount") in found(cache, sql, "confirmed")
|
||||
# 外键 + verified:一对多只是按表结构推的,降为 warning(A6,见下一个测试)
|
||||
assert ("fanout_sum", "warning", "orders", "total_amount") in found(cache, sql, "verified")
|
||||
assert ("fanout_sum", "info", "orders", "total_amount") in found(cache, sql, "proposed")
|
||||
# 关系确认了、度量类型只是推断:依据里有推断就降为 info
|
||||
assert ("fanout_sum", "info", "orders", "total_amount") in found(
|
||||
@@ -169,6 +180,37 @@ async def test_fanout_level_follows_the_status_of_its_basis(cache):
|
||||
assert "fanout_sum" not in codes(cache, sql, override={("orders", "columns.total_amount.measure"): "rejected"})
|
||||
|
||||
|
||||
def _checked(table_notes: dict, rid: str, *, source: str = "profile") -> dict:
|
||||
"""把一条关系记成「剖析用数据核实过基数」(cardinality_checked),不改入参。"""
|
||||
import copy
|
||||
|
||||
out = copy.deepcopy(table_notes)
|
||||
rel = next(r for r in out["relations"] if r["id"] == rid)
|
||||
rel.update(source=source, status="verified", cardinality_checked=True)
|
||||
return out
|
||||
|
||||
|
||||
async def test_fanout_error_only_when_one_to_many_is_established(cache):
|
||||
"""A6:error 只给「一对多确有其事」——剖析在数据上核实过子表一侧的键不唯一,或者人工确认过基数。
|
||||
只凭外键结构推出的一对多(子表一侧可能每个键恰好一行)降为 warning,说法是「可能重复计算」。"""
|
||||
sql = FANOUT[0]
|
||||
# 外键 + verified:基数是按表结构推的,没用数据核对过 → warning,不说「会重复计算」
|
||||
[fk] = [c for c in run(cache, sql, "verified") if c.code == "fanout_sum"]
|
||||
assert (fk.level, fk.table, fk.column) == ("warning", "orders", "total_amount")
|
||||
assert "可能重复计算" in fk.message and "会重复计算" not in fk.message
|
||||
assert "确认关联表" in fk.message and "每个键只有一行" in fk.message and "先汇总再关联" in fk.message
|
||||
assert "必然" not in fk.message and "必然" not in fk.for_model
|
||||
# 剖析核实过(关系上记 cardinality_checked)→ error
|
||||
profiled = scenic_notes.notes("verified")
|
||||
profiled["order_items"] = _checked(profiled["order_items"], ORDER_ITEMS_TO_ORDERS)
|
||||
checker = SqlChecker(kind="sqlite", schema_cache=cache, notes=profiled, source_name="scenic")
|
||||
[hit] = [c for c in checker.check(sql) if c.code == "fanout_sum"]
|
||||
assert hit.level == "error" and "会重复计算" in hit.message
|
||||
# 人工确认过基数(关系是 confirmed)→ error
|
||||
assert ("fanout_sum", "error", "orders", "total_amount") in found(
|
||||
cache, sql, "verified", override={("order_items", f"relations.{ORDER_ITEMS_TO_ORDERS}"): "confirmed"})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# stock_summed:存量跨时间求和
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@@ -2509,7 +2509,8 @@ export const CODES_TEXT = {
|
||||
//
|
||||
// 术语:
|
||||
// - 「SQL 检查」:对照数据目录找出能执行、数却可能不对的写法。七条规则,界面上写中文规则名,不露规则编号。
|
||||
// - 级别:错误(依据已确认,结果必然有误)、提醒(依据已确认,结果很可能有误)、提示(依据只是推断,仅供参考)。
|
||||
// - 级别:错误(依据已核实,结果很可能有误)、提醒(依据已确认,结果可能有误)、提示(依据只是推断,仅供参考)。
|
||||
// 一对多关联后重复计算:只有一对多人工确认过、或数据剖析用数据核实过时才是错误;只凭外键推出的是提醒。
|
||||
// 颜色走语义令牌:错误 --st-failed,提醒 --st-waiting,提示 --accent。
|
||||
// ===========================================================================
|
||||
|
||||
@@ -2540,8 +2541,8 @@ export const SQL_CHECK_LEVEL_LABEL: Record<SqlCheckLevel, string> = {
|
||||
}
|
||||
|
||||
export const SQL_CHECK_LEVEL_HINT: Record<SqlCheckLevel, string> = {
|
||||
error: '依据已确认,结果必然有误',
|
||||
warning: '依据已确认,结果很可能有误',
|
||||
error: '依据已核实,结果很可能有误,需要修改',
|
||||
warning: '依据已确认,结果可能有误,请核对',
|
||||
info: '依据尚未确认(推断),仅供参考',
|
||||
}
|
||||
|
||||
|
||||
@@ -2637,6 +2637,11 @@ export interface CatalogRelation {
|
||||
status: CatalogStatus
|
||||
note?: string
|
||||
updated_at?: string
|
||||
/**
|
||||
* 基数是数据剖析用数据核实的(子表一侧数过是否唯一),只由剖析写。外键、命名推断的基数没有这个键。
|
||||
* 一对多关联后重复计算的检查只在它为真、或者关系人工确认过时报「错误」,否则最多是「提醒」
|
||||
*/
|
||||
cardinality_checked?: boolean
|
||||
}
|
||||
|
||||
/** 一张表的目录(notes)。字段固定,服务端不认识的一律拒收 */
|
||||
@@ -2949,7 +2954,7 @@ export interface CatalogProfileOut {
|
||||
export type SqlCheckCode = 'fanout_sum' | 'stock_summed' | 'join_unconfirmed' | 'ratio_aggregated' | 'missing_valid_filter'
|
||||
| 'unknown_code' | 'wrong_date_column'
|
||||
|
||||
/** error:依据有确证、结果必然有误;warning:依据有确证、很可能有误;info:依据只是推断 */
|
||||
/** error:依据已核实、结果很可能有误;warning:依据有确证、结果可能有误;info:依据只是推断 */
|
||||
export type SqlCheckLevel = 'error' | 'warning' | 'info'
|
||||
|
||||
/** 一条检查结果(证据接口查询步骤里的 checks、助手自查和发布前检查的问题)。交回模型改写用的那句不上界面,这里不收 */
|
||||
|
||||
@@ -1073,7 +1073,7 @@ await section('sqlcheck', 'SQL 检查(数据目录阶段 4B):查询步骤
|
||||
const want = [await tokenColor(bad.page, 'color', 'var(--st-failed)'), await tokenColor(bad.page, 'color', 'var(--st-waiting)'),
|
||||
await tokenColor(bad.page, 'color', 'var(--accent)')]
|
||||
check('级别的颜色走令牌:错误 --st-failed、提醒 --st-waiting、提示 --accent', tones.join('|') === want.join('|'), tones.join('|'))
|
||||
check('悬停写明依据的确证程度', (await items.first().locator('[data-sql-level]').getAttribute('title')) === '依据已确认,结果必然有误')
|
||||
check('悬停写明依据的确证程度', (await items.first().locator('[data-sql-level]').getAttribute('title')) === '依据已核实,结果很可能有误,需要修改')
|
||||
check('不露规则编号', !/fanout_sum|missing_valid_filter|join_unconfirmed/.test(await p.innerText()))
|
||||
await bad.page.keyboard.press('Escape')
|
||||
await openQ(bad.page, '45,678.5元', { wait: '[data-ev-sql-check-failed]' })
|
||||
|
||||
Reference in New Issue
Block a user