数据目录阶段 2B:SQL 检查的七条规则——一对多重复计算、存量跨期求和、关联对不上目录、比率求和、漏筛有效记录、码值不在表里、用错时间列,级别随依据的状态变化

This commit is contained in:
ThearchyHelios
2026-10-03 10:16:12 +08:00
parent 8ed0bc23f7
commit 0e689c1b18
2 changed files with 880 additions and 2 deletions
+486 -1
View File
@@ -459,7 +459,457 @@ class _ScopeCheck:
return out
def run(self) -> list[SqlCheck]:
return []
out: list[SqlCheck] = []
if self.tables:
out += self.matched_join_checks()
out += self.fanout()
out += self.stock_summed()
out += self.ratio_aggregated()
out += self.missing_valid_filter()
out += self.unknown_code()
out += self.wrong_date_column()
return out
# ---- 关联:认出每一对表的关联条件,对照目录里的关系
def matched_joins(self) -> list[tuple[str, str, catalog.JoinEdge]]:
"""对上目录关系的关联:[(别名 x, 别名 y, 从 x 看过去的边)]。顺带记下 join_unconfirmed。"""
if self._matched is None:
self._matched = []
self._analyze_joins()
return self._matched
def matched_join_checks(self) -> list[SqlCheck]:
self.matched_joins()
return self._join_checks
def _analyze_joins(self) -> None:
from sqlglot import exp
groups: dict[tuple[str, str], dict[str, Any]] = {}
skip: set[str] = set()
seen: list[str] = []
first = self.select.args.get("from_")
if first is not None:
seen.append(first.this.alias_or_name.lower())
def add(conds: list[Any], *, owner: str | None) -> None:
for cond in conds:
cond = _unparen(cond)
if not isinstance(cond, exp.EQ):
continue
left, right = _unparen(cond.left), _unparen(cond.right)
if not (isinstance(left, exp.Column) and isinstance(right, exp.Column)):
continue
a, b = self.resolve(left), self.resolve(right)
if a is None or b is None:
if owner is not None:
skip.add(owner) # 关联条件里有认不出的列:说不清这一连是按什么连的
continue
if a[0].alias == b[0].alias:
continue
if a[0].order > b[0].order:
a, b = b, a
group = groups.setdefault((a[0].alias, b[0].alias), {"pairs": set(), "conds": []})
group["pairs"].add((a[1].lower(), b[1].lower()))
group["conds"].append(cond)
for join in self.select.args.get("joins") or []:
alias = join.this.alias_or_name.lower()
if join.args.get("method"): # NATURAL JOIN:按同名列连,不去猜
skip.add(alias)
on, using = join.args.get("on"), join.args.get("using")
if on is not None:
conds = _conjuncts(on)
if any(isinstance(_unparen(c), exp.Or) for c in conds):
skip.add(alias)
add(conds, owner=alias)
elif using:
right = self.tables.get(alias)
for ident in using:
name = ident.name.lower()
lefts = [self.tables[s] for s in seen if s in self.tables and name in self.tables[s].columns]
if right is None or len(lefts) != 1 or name not in right.columns:
skip.add(alias)
break
left = lefts[0]
group = groups.setdefault((left.alias, right.alias), {"pairs": set(), "conds": []})
group["pairs"].add((name, name))
group["conds"].append(ident)
seen.append(alias)
# 逗号连接(以及显式 JOIN 外加的等值条件)写在 WHERE 里
add(_conjuncts(self.where()), owner=None)
for (x, y), group in groups.items():
if x in skip or y in skip:
continue
a, b = self.tables[x], self.tables[y]
if a.name.lower() == b.name.lower():
continue # 自己连自己:目录不记这种关系
pairs = group["pairs"]
candidates = self.checker.edges_between(a.name, b.name)
matched = [e for e in candidates
if {(f.lower(), t.lower()) for f, t in zip(e.from_columns, e.to_columns)} <= pairs]
best = max(matched, key=lambda e: _STATUS_RANK.get(e.status, 0), default=None)
if best is not None:
self._matched.append((x, y, best))
if not (a.notes or b.notes):
continue # 两张表都没写目录:没有可对照的,不判
condition = " AND ".join(filter(None, (self.excerpt(c) for c in group["conds"])))
if best is None:
self._join_checks.append(self._join_unknown(a, b, candidates, condition))
elif best.status not in _STRONG:
self._join_checks.append(self._join_proposed(a, b, best, condition))
def _join_unknown(self, a: _Table, b: _Table, candidates: list[catalog.JoinEdge], condition: str) -> SqlCheck:
known = ";".join(dict.fromkeys(e.condition() for e in candidates))
hint = f"目录里这两张表的关系是:{known}。照它改关联条件" if known else \
"目录里没有这两张表之间的关系。确认这样连两边各是什么粒度、会不会一对多"
return SqlCheck(
code="join_unconfirmed", level="warning", table=b.name, sql_excerpt=condition or None,
message=(f"「{a.label()}」与「{b.label()}」的关联条件对不上数据目录中的任何关系,可能连错了列。"
"请对照数据目录修改关联条件,或在数据目录中补充并确认这条关系"),
for_model=(f"{a.name} 和 {b.name} 按 {condition or '这组条件'} 关联,数据目录里没有对应的关系。{hint};"
"确实要这样连的话,避免一对多关联之后对「一」那一侧的列求和"))
def _join_proposed(self, a: _Table, b: _Table, edge: catalog.JoinEdge, condition: str) -> SqlCheck:
return SqlCheck(
code="join_unconfirmed", level="info", table=b.name, relation_id=edge.relation_id,
sql_excerpt=condition or None,
message=(f"「{a.label()}」与「{b.label()}」按推断的关系关联({edge.condition()}),这条关系尚未确认。"
"请在数据目录中确认"),
for_model=(f"{a.name} 和 {b.name} 的关联依据的是推断出来的关系({edge.condition()}),没人确认过。"
"核对两边的列确实对应、基数和你以为的一致"))
# ---- fanout_sum
def _fanned(self, start: str) -> tuple[str, str, list[catalog.JoinEdge]] | None:
"""start 这张表会不会被关联放大:沿多对一、一对一往外走,碰到一条一对多的边就会。返回 (V, W, 路径)。
按 W 的业务主键分组时,每组只有一条 W:这条边不放大,接着往外找。对不上关系的关联不知道基数,不往外走。
"""
adjacency: dict[str, list[tuple[str, catalog.JoinEdge]]] = {}
for x, y, edge in self.matched_joins():
adjacency.setdefault(x, []).append((y, edge))
adjacency.setdefault(y, []).append((x, edge.reversed()))
grouped = {(r[0].alias, r[1].lower()) for r in self.group_refs() if r}
# 反连接(LEFT JOIN 明细 … WHERE 明细.id IS NULL):留下的是没有明细的订单,每条订单至多一行
absent = {r[0].alias for c in _conjuncts(self.where()) if self._is_null(c) for r in self.refs(c) if r}
visited = {start}
queue: list[tuple[str, list[catalog.JoinEdge]]] = [(start, [])]
while queue:
here, path = queue.pop(0)
for there, edge in adjacency.get(here, []):
if there in visited:
continue
keys = self.tables[there].keys()
collapsed = there in absent or (bool(keys) and all((there, k.lower()) in grouped for k in keys[0]))
if edge.cardinality == "one_to_many" and not collapsed:
return here, there, path + [edge]
if edge.cardinality in ("many_to_one", "one_to_one") or collapsed:
visited.add(there)
queue.append((there, path + [edge]))
return None
@staticmethod
def _is_null(cond: Any) -> bool:
"""col IS NULL(不含 IS NOT NULL)。"""
from sqlglot import exp
cond = _unparen(cond)
return isinstance(cond, exp.Is) and isinstance(_unparen(cond.expression), exp.Null) and \
isinstance(_unparen(cond.this), exp.Column)
def fanout(self) -> list[SqlCheck]:
from sqlglot import exp
if not self.matched_joins():
return []
out = []
for node, arg in self.aggregates(exp.Sum, exp.Count):
refs = self.refs(arg)
if not refs or any(r is None for r in refs) or len({r[0].alias for r in refs}) != 1:
continue
table = refs[0][0]
basis = self._fanout_basis(table, [r[1] for r in refs], counting=isinstance(node, exp.Count))
if basis is None:
continue
cause = self._fanned(table.alias)
if cause is None:
continue
column, status = basis
via, many, path = cause
out.append(self._fanout_check(node, table, column, self.tables[via], self.tables[many], path,
[status] + [e.status for e in path], counting=isinstance(node, exp.Count)))
return out
def _fanout_basis(self, table: _Table, columns: list[str], *, counting: bool) -> tuple[str, str] | None:
"""参数里能说明「这是一张表自己的数」的那一列和它的依据:求和看度量类型(流量、存量),
计数另外认标识列和业务主键(数订单数却数成了明细行数)。"""
keys = table.keys()
for column in columns:
measure = table.measure(column)
wanted = ("flow", "stock", "identifier") if counting else ("flow", "stock")
if measure and measure[0] in wanted:
return column, measure[1]
if counting and keys and column.lower() in {k.lower() for k in keys[0]}:
return column, keys[1]
return None
def _fanout_check(self, node: Any, table: _Table, column: str, via: _Table, many: _Table,
path: list[catalog.JoinEdge], statuses: list[str], *, counting: bool) -> SqlCheck:
what = "计数" if counting else "求和"
excerpt = self.excerpt(node)
if via.alias == table.alias:
head = f"「{table.label()}」关联「{many.label()}」是一对多,"
model_head = f"{table.name} 关联 {many.name} 是一对多(一行 {table.name} 对多行 {many.name}),"
else:
head = f"「{via.label()}」关联「{many.label()}」是一对多,「{table.label()}」的每一行会随之重复,"
model_head = (f"{via.name} 关联 {many.name} 是一对多,经 {via.name} 连过去之后 {table.name} 的每一行"
f"都按 {many.name} 的行数重复,")
# 直接连的:常见的本意是「明细的数」,改法之一是换成对明细求和;隔一层连的(明细经订单连支付记录),
# 本意是明细自己的数,要么先把支付记录汇总,要么根本不该连它
if counting:
fix = f"请改为对「{table.label()}」去重计数,或先在子查询中把「{many.label()}」汇总后再关联"
model_fix = (f"改成 COUNT(DISTINCT {table.name} 的主键),或者先在子查询里把 {many.name} 汇总成每条 "
f"{via.name} 一行再关联")
elif via.alias == table.alias:
fix = (f"请先在子查询中把「{many.label()}」按「{table.label()}」汇总后再关联,"
f"或改为对「{many.label()}」上的对应列求和")
model_fix = (f"先在子查询里把 {many.name} 按关联列汇总成每条 {table.name} 一行再关联;或者改成对 {many.name} "
f"上的对应列求和;只要 {table.name} 自己的数就不要关联 {many.name}")
else:
fix = f"请先在子查询中把「{many.label()}」按「{via.label()}」汇总后再关联,或去掉与「{many.label()}」的关联"
model_fix = (f"先在子查询里把 {many.name} 按关联列汇总成每条 {via.name} 一行再关联;用不到 {many.name} 的列"
f"就去掉这个关联")
return SqlCheck(
code="fanout_sum", level=_level("fanout_sum", statuses), table=table.name, column=column,
relation_id=path[-1].relation_id, sql_excerpt=excerpt,
message=f"{head}对「{table.label()}」的「{table.column_label(column)}」{what}会重复计算。{fix}",
for_model=f"{model_head}{excerpt or what} 会把 {table.name}.{column} 重复计算。{model_fix}")
# ---- stock_summed
def _date_equivalents(self, table: _Table, column: str) -> set[tuple[str, str]]:
"""业务日期列,加上关联、筛选条件里和它等值的列(经日期维度表分组也算按日期分组)。"""
out = {(table.alias, column.lower())}
conds = _conjuncts(self.where())
for join in self.select.args.get("joins") or []:
conds += _conjuncts(join.args.get("on"))
from sqlglot import exp
for cond in conds:
cond = _unparen(cond)
if isinstance(cond, exp.EQ):
a, b = self.refs(cond.left), self.refs(cond.right)
if len(a) == 1 and len(b) == 1 and a[0] and b[0]:
ka, kb = (a[0][0].alias, a[0][1].lower()), (b[0][0].alias, b[0][1].lower())
if ka in out:
out.add(kb)
elif kb in out:
out.add(ka)
return out
def _mentions(self, refs: list[_Ref | None], wanted: set[tuple[str, str]], name: str) -> bool:
"""refs 里有没有 wanted 里的列;认不出来、但名字就是 name 的列也算有(宁可漏报)。"""
return any((r[0].alias, r[1].lower()) in wanted if r else False for r in refs) or \
self._unresolved_named(refs, name)
def _unresolved_named(self, refs: list[_Ref | None], name: str) -> bool:
return False if not any(r is None for r in refs) else name.lower() in self.statement_columns
def stock_summed(self) -> list[SqlCheck]:
from sqlglot import exp
out = []
for node, arg in self.aggregates(exp.Sum):
refs = self.refs(arg)
if not refs or any(r is None for r in refs) or len({r[0].alias for r in refs}) != 1:
continue
table = refs[0][0]
stock = next(((c, m[1]) for c in (r[1] for r in refs) if (m := table.measure(c)) and m[0] == "stock"), None)
date = table.business_date()
if stock is None or date is None or any(r[1].lower() == date[0].lower() for r in refs):
continue
equivalents = self._date_equivalents(table, date[0])
if self._mentions(self.group_refs(), equivalents, date[0]):
continue
pinned = False
for cond in (self.where().walk() if self.where() is not None else []):
if isinstance(cond, (exp.EQ, exp.In)) and self._mentions(self.refs(cond), equivalents, date[0]):
pinned = True # 筛到某一天(或某几天):不算跨期加总
break
if pinned:
continue
column, label = stock[0], table.column_label(stock[0])
date_label = table.column_label(date[0])
excerpt = self.excerpt(node)
out.append(SqlCheck(
code="stock_summed", level=_level("stock_summed", [stock[1], date[1]]), table=table.name,
column=column, sql_excerpt=excerpt,
message=(f"「{table.label()}」的「{label}」是存量,跨日期求和会把不同日期的存量加在一起。"
f"请按「{date_label}」分组,或筛选到某一天后再求和"),
for_model=(f"{table.name}.{column} 是存量(某一时点的值),{excerpt or 'SUM'} 把不同日期的存量加在一起了。"
f"按 {date[0]} 分组,或者在 WHERE 里把 {date[0]} 限定到某一天(比如期末那天)再求和;"
"要看一段时间的水平就用 AVG,或取期末那天的值")))
return out
# ---- ratio_aggregated
def ratio_aggregated(self) -> list[SqlCheck]:
from sqlglot import exp
out = []
for node, arg in self.aggregates(exp.Sum, exp.Avg):
column = _bare_column(arg)
ref = self.resolve(column) if column is not None else None
measure = ref[0].measure(ref[1]) if ref else None
if not ref or not measure or measure[0] != "ratio":
continue
table, name = ref
what = "求平均" if isinstance(node, exp.Avg) else "求和"
excerpt = self.excerpt(node)
out.append(SqlCheck(
code="ratio_aggregated", level=_level("ratio_aggregated", [measure[1]]), table=table.name,
column=name, sql_excerpt=excerpt,
message=(f"「{table.label()}」的「{table.column_label(name)}」是比率,直接{what}得不到正确的结果。"
"请分别汇总分子和分母后再相除"),
for_model=(f"{table.name}.{name} 是比率,{excerpt or what} 直接把比率{what}了。分别对分子、分母求和以后"
f"再相除;要按权重平均就写 SUM(权重 * {name}) / SUM(权重)")))
return out
# ---- missing_valid_filter
def missing_valid_filter(self) -> list[SqlCheck]:
out = []
done: set[str] = set()
local = [self.resolve(c) for c in self.columns]
for table in sorted(self.tables.values(), key=lambda t: t.order):
item = table.item("valid_filter")
condition = item.get("value") if item else None
if table.name in done or not isinstance(condition, str) or not condition.strip():
continue
wanted = self._filter_columns(condition, table)
if not wanted:
continue
done.add(table.name)
if any(r and r[0].name == table.name and r[1].lower() in wanted for r in local):
continue
# 没加前缀、认不出是哪张表的同名列:可能就是它,不报
if any(c.name.lower() in wanted for c, r in zip(self.columns, local) if r is None):
continue
# 子查询、CTE 里的表:外层可能按透出的列筛(SELECT … FROM (SELECT * FROM t) x WHERE x.status = 1)
if not self.scope.is_root and wanted & self.statement_columns:
continue
cols = "、".join(table.columns[c] for c in sorted(wanted))
out.append(SqlCheck(
code="missing_valid_filter", level=_level("missing_valid_filter", [item.get("status")]),
table=table.name, sql_excerpt=self.excerpt(table.node),
message=(f"「{table.label()}」定义了有效记录条件「{condition.strip()}」,查询中没有按它筛选,"
"结果会包含无效记录。请在筛选条件中加上该条件,或确认确实需要包含全部记录"),
for_model=(f"{table.name} 的有效记录条件是 {condition.strip()},这条 SQL 没有用到其中的列({cols})。"
"在 WHERE 里加上这个条件(有别名就加上别名前缀);确实要包含无效记录的话,在条件里显式写出这一列")))
return out
def _filter_columns(self, condition: str, table: _Table) -> set[str]:
"""有效记录条件里用到的、这张表上的列(小写)。条件解析不了就返回空集(不查)。"""
import sqlglot
from sqlglot import exp
try:
parsed = sqlglot.parse_one(f"SELECT 1 FROM t WHERE {condition}", read=self.dialect)
except Exception: # noqa: BLE001
return set()
return {c.name.lower() for c in parsed.find_all(exp.Column) if c.name and c.name.lower() in table.columns}
# ---- unknown_code
def unknown_code(self) -> list[SqlCheck]:
from sqlglot import exp
hits: dict[tuple[str, str], tuple[_Table, str, Mapping[str, Any], list[str], Any]] = {}
for node in self.nodes:
if isinstance(node, exp.EQ):
pairs = [(node.left, [node.right]), (node.right, [node.left])]
elif isinstance(node, exp.In) and not node.args.get("query") and node.expressions:
pairs = [(node.this, node.expressions)]
else:
continue
for target, values in pairs:
column = _unparen(target)
if not isinstance(column, exp.Column):
continue
ref = self.resolve(column)
item = ref[0].column_item(ref[1], "codes") if ref else None
codes = item.get("value") if item else None
if not isinstance(codes, Mapping) or not codes:
continue
for value in values:
literal = _literal(value)
if literal is None or _known_code(literal, codes):
continue
key = (ref[0].name, ref[1].lower())
entry = hits.setdefault(key, (ref[0], ref[1], item, [], node))
if literal[0] not in entry[3]:
entry[3].append(literal[0])
out = []
for table, column, item, values, node in hits.values():
codes = item["value"]
shown = "、".join(f"{k}={v}" for k, v in list(codes.items())[:10]) + ("等" if len(codes) > 10 else "")
told = "、".join(f"「{v}」" for v in values[:5])
listed = "、".join(f"{k}({v})" for k, v in list(codes.items())[:10])
hint = f";拿不准就先用 db_schema__{self.checker.source_name} 查这张表的码值" if self.checker.source_name else ""
out.append(SqlCheck(
code="unknown_code", level=_level("unknown_code", [item.get("status")]), table=table.name,
column=column, sql_excerpt=self.excerpt(node),
message=(f"「{table.label()}」的「{table.column_label(column)}」没有码值{told}(已知码值:{shown})。"
"请按数据目录中的码值修改筛选条件"),
for_model=(f"{table.name}.{column} 的码值只有 {listed},SQL 里写的 {'、'.join(values[:5])} 不在其中。"
f"照目录里的码值改{hint}")))
return out
# ---- wrong_date_column
def wrong_date_column(self) -> list[SqlCheck]:
from sqlglot import exp
out = []
where = self.where()
where_refs = [(c, self.resolve(c)) for c in (where.find_all(exp.Column) if where is not None else [])
if not any(isinstance(p, (exp.Subquery, exp.Select)) for p in _parents(c, stop=where))]
group = self.group_refs()
for table in sorted(self.tables.values(), key=lambda t: t.order):
date = table.business_date()
if date is None:
continue
own = {(table.alias, date[0].lower())}
if self._mentions([r for _, r in where_refs] + group, own, date[0]):
continue # 业务日期已经用上了:另一个时间列只是附加条件
used: dict[str, set[str]] = {}
for column, ref in where_refs:
# IS NULL / IS NOT NULL 只是要求有值,不是按这一列划时间范围
if ref and ref[0].alias == table.alias and not isinstance(column.parent, exp.Is):
used.setdefault(ref[1], set()).add("筛选")
for ref in group:
if ref and ref[0].alias == table.alias:
used.setdefault(ref[1], set()).add("分组")
for column, how in used.items():
if column.lower() == date[0].lower() or not table.temporal(column):
continue
verb = "分组和筛选" if len(how) == 2 else next(iter(how))
label, date_label = table.column_label(column), table.column_label(date[0])
item = table.item("business_date")
rule = (item.get("value") or {}).get("rule") if item else None
out.append(SqlCheck(
code="wrong_date_column", level=_level("wrong_date_column", [date[1]]), table=table.name,
column=column, sql_excerpt=self.excerpt(next((c for c, r in where_refs if r and
r[1] == column), None) or exp.column(column)),
message=(f"「{table.label()}」的业务日期按「{date_label}」计,查询却按「{label}」{verb}。"
f"请改用「{date_label}」,或确认确实要按「{label}」统计"),
for_model=(f"{table.name} 的业务日期列是 {date[0]}{f'({rule})' if rule else ''},这条 SQL 却按 "
f"{column} {verb}。按业务日期统计就把 {column} 换成 {date[0]};确实要按 {column} 统计的话"
"保留,并在结果说明里写清口径")))
return out
def _parents(node: Any, *, stop: Any) -> Iterable[Any]:
@@ -470,5 +920,40 @@ def _parents(node: Any, *, stop: Any) -> Iterable[Any]:
parent = parent.parent
def _literal(node: Any) -> tuple[str, bool] | None:
"""字面量 → (文本, 是不是字符串)。负数照常认;模板占位、参数、表达式返回 None(值到运行时才知道)。"""
from sqlglot import exp
node = _unparen(node)
negative = False
if isinstance(node, exp.Neg):
node, negative = _unparen(node.this), True
if not isinstance(node, exp.Literal):
return None
text = str(node.this)
if _SENTINEL in text:
return None
if node.is_string:
return (text, True) if not negative else None
return (f"-{text}" if negative else text), False
def _number(text: str) -> float | None:
try:
return float(text.strip())
except ValueError:
return None
def _known_code(literal: tuple[str, bool], codes: Mapping[str, Any]) -> bool:
"""值在不在码值表里:原样相等、不分大小写相等、或者两边都是数且数值相等(1 和 '1'、1.0 都算 1)。"""
text = literal[0]
keys = [str(k) for k in codes]
if text in keys or text.strip().lower() in {k.strip().lower() for k in keys}:
return True
number = _number(text)
return number is not None and any(_number(k) == number for k in keys)
__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "check_sql",
"dialect_of"]
+394 -1
View File
@@ -12,8 +12,9 @@ import pytest
from app.data.engine import engines
from app.data.introspect import introspect
from app.data.sqlcheck import SqlChecker, check_sql, dialect_of
from app.data.sqlcheck import CHECK_CODES, SqlChecker, check_sql, dialect_of
from tests.fixtures.catalog import scenic_notes
from tests.fixtures.catalog.scenic_notes import ORDER_ITEMS_TO_ORDERS
_CACHE: dict[str, dict] = {}
@@ -47,6 +48,32 @@ def codes(cache, sql, status="confirmed", override=None):
return {c.code for c in run(cache, sql, status, override)}
# --------------------------------------------------------------------------
# 结果的形状
# --------------------------------------------------------------------------
async def test_result_shape_and_wording(cache):
sql = ("SELECT o.channel_id, SUM(o.total_amount) AS amt FROM orders o "
"JOIN order_items i ON i.order_id = o.id WHERE o.status = 1 GROUP BY o.channel_id")
[check] = run(cache, sql)
out = check.as_dict()
assert set(out) == {"code", "level", "message", "for_model", "table", "column", "relation_id", "sql_excerpt"}
assert out["code"] == "fanout_sum" and out["level"] == "error"
assert out["table"] == "orders" and out["column"] == "total_amount"
assert out["relation_id"] == ORDER_ITEMS_TO_ORDERS
assert "SUM(o.total_amount)" in out["sql_excerpt"]
assert out["message"].startswith("「订单」关联「订单明细」是一对多,对「订单」的「订单金额」求和会重复计算。")
# 给模型的写 SQL 里的真名和改法;给人看的不出现工具名和对模型的指令
assert "orders" in out["for_model"] and "order_items" in out["for_model"]
assert "db_schema__" not in out["message"] and "db_" not in out["message"]
# 没有的可选字段不出现
single = run(cache, "SELECT AVG(discount_rate) FROM member_levels")[0].as_dict()
assert "relation_id" not in single and single["column"] == "discount_rate"
assert set(CHECK_CODES) == {"fanout_sum", "stock_summed", "join_unconfirmed", "ratio_aggregated",
"missing_valid_filter", "unknown_code", "wrong_date_column"}
def test_dialects():
assert dialect_of("sqlite") == "sqlite"
assert dialect_of("postgres") == dialect_of("postgresql") == "postgres"
@@ -55,6 +82,326 @@ def test_dialects():
assert dialect_of("mssql") is None and dialect_of(None) is None
# --------------------------------------------------------------------------
# fanout_sum:沿一对多关联之后对「一」那一侧的度量列求和、计数
# --------------------------------------------------------------------------
FANOUT = [
"SELECT SUM(o.total_amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1",
# 先写明细、后连订单:订单照样随明细重复
"SELECT SUM(o.total_amount) FROM order_items i JOIN orders o ON o.id = i.order_id WHERE o.status = 1",
# 包一层 COALESCE、CASE 里只用订单的列:还是订单的数
"SELECT SUM(COALESCE(o.total_amount, 0)) FROM orders o LEFT JOIN order_items i ON o.id = i.order_id "
"WHERE o.status = 1",
"SELECT SUM(CASE WHEN o.status = 1 THEN o.total_amount END) FROM orders o JOIN order_items i "
"ON i.order_id = o.id AND i.qty > 0",
# 逗号连接、条件写在 WHERE 里
"SELECT SUM(o.total_amount) FROM orders o, order_items i WHERE i.order_id = o.id AND o.status = 1",
]
@pytest.mark.parametrize("sql", FANOUT)
async def test_fanout_sum_positive(cache, sql):
assert ("fanout_sum", "error", "orders", "total_amount") in found(cache, sql)
async def test_fanout_count_of_the_one_side_key(cache):
sql = "SELECT COUNT(o.id) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1"
[check] = run(cache, sql)
assert (check.code, check.level, check.table, check.column) == ("fanout_sum", "error", "orders", "id")
assert "计数会重复计算" in check.message
async def test_fanout_through_two_hops_with_an_inferred_foreign_key(cache):
"""支付记录没写目录:订单 → 支付记录的关系按外键现推(verified),明细经订单连支付记录,明细的金额照样重复。"""
sql = ("SELECT SUM(i.amount) FROM order_items i JOIN orders o ON o.id = i.order_id "
"JOIN payments p ON p.order_id = o.id WHERE o.status = 1")
[check] = [c for c in run(cache, sql) if c.code == "fanout_sum"]
assert (check.level, check.table, check.column) == ("error", "order_items", "amount")
assert "「订单」关联「payments」是一对多,「订单明细」的每一行会随之重复" in check.message
assert "去掉与「payments」的关联" in check.message
FANOUT_NEGATIVE = [
# 对「多」那一侧求和:本来就在明细粒度上
"SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1",
"SELECT SUM(i.amount) FROM order_items i JOIN orders o ON o.id = i.order_id WHERE o.status = 1",
# 不关联
"SELECT SUM(o.total_amount) FROM orders o WHERE o.status = 1",
# 去重计数、去重求和:写的人已经在处理重复
"SELECT COUNT(DISTINCT o.id) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1",
# 两边的列一起算:按明细逐行算出来的数
"SELECT SUM(i.qty * o.total_amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1",
# 按明细的主键分组:每组只有一行明细,订单不重复
"SELECT i.id, SUM(o.total_amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1 "
"GROUP BY i.id",
# 明细先在子查询里按订单汇总再连:一对一
"SELECT SUM(o.total_amount), SUM(x.n) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM order_items "
"GROUP BY order_id) x ON x.order_id = o.id WHERE o.status = 1",
# 计数所有行:说不清是要数订单还是数明细
"SELECT COUNT(*) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1",
# 窗口函数:不是分组聚合
"SELECT o.id, SUM(o.total_amount) OVER (PARTITION BY o.channel_id) FROM orders o "
"JOIN order_items i ON i.order_id = o.id WHERE o.status = 1",
# 反连接:只留没有明细的订单,每笔订单至多一行
"SELECT SUM(o.total_amount) FROM orders o LEFT JOIN order_items i ON i.order_id = o.id "
"WHERE i.id IS NULL AND o.status = 1",
]
@pytest.mark.parametrize("sql", FANOUT_NEGATIVE)
async def test_fanout_sum_negative(cache, sql):
assert "fanout_sum" not in codes(cache, sql)
async def test_fanout_level_follows_the_status_of_its_basis(cache):
sql = FANOUT[0]
assert ("fanout_sum", "error", "orders", "total_amount") in found(cache, sql, "verified")
assert ("fanout_sum", "info", "orders", "total_amount") in found(cache, sql, "proposed")
# 关系确认了、度量类型只是推断:依据里有推断就降为 info
assert ("fanout_sum", "info", "orders", "total_amount") in found(
cache, sql, override={("orders", "columns.total_amount.measure"): "proposed"})
# 关系被驳回:不知道是一对多,不报重复计算;关联本身对不上目录,单独提示
rejected = found(cache, sql, override={("order_items", f"relations.{ORDER_ITEMS_TO_ORDERS}"): "rejected"})
assert not [f for f in rejected if f[0] == "fanout_sum"]
assert ("join_unconfirmed", "warning", "order_items", None) in rejected
# 度量类型被驳回:不知道订单金额能不能加,不报
assert "fanout_sum" not in codes(cache, sql, override={("orders", "columns.total_amount.measure"): "rejected"})
# --------------------------------------------------------------------------
# stock_summed:存量跨时间求和
# --------------------------------------------------------------------------
STOCK = [
"SELECT SUM(on_hand) FROM inventory_snapshots",
'SELECT "storeID", SUM(on_hand) FROM inventory_snapshots GROUP BY "storeID"',
"SELECT SUM(s.on_hand) FROM inventory_snapshots s WHERE s.snapshot_date BETWEEN '2026-09-01' AND '2026-09-30'",
]
@pytest.mark.parametrize("sql", STOCK)
async def test_stock_summed_positive(cache, sql):
assert ("stock_summed", "error", "inventory_snapshots", "on_hand") in found(cache, sql)
STOCK_NEGATIVE = [
"SELECT snapshot_date, SUM(on_hand) FROM inventory_snapshots GROUP BY snapshot_date",
"SELECT SUM(on_hand) FROM inventory_snapshots WHERE snapshot_date = '2026-09-30'",
"SELECT substr(snapshot_date, 1, 7) AS m, SUM(on_hand) FROM inventory_snapshots GROUP BY substr(snapshot_date, 1, 7)",
"SELECT AVG(on_hand) FROM inventory_snapshots",
"SELECT MAX(on_hand) FROM inventory_snapshots",
"SELECT snapshot_date, SUM(on_hand) OVER (PARTITION BY snapshot_date) FROM inventory_snapshots",
# 经日期维度表分组:维度表的日期和快照日期相等
"SELECT d.biz_date, SUM(s.on_hand) FROM inventory_snapshots s JOIN dim_dates d ON d.biz_date = s.snapshot_date "
"GROUP BY d.biz_date",
]
@pytest.mark.parametrize("sql", STOCK_NEGATIVE)
async def test_stock_summed_negative(cache, sql):
assert "stock_summed" not in codes(cache, sql)
async def test_stock_level_follows_status(cache):
sql = STOCK[0]
assert ("stock_summed", "error", "inventory_snapshots", "on_hand") in found(cache, sql, "verified")
assert ("stock_summed", "info", "inventory_snapshots", "on_hand") in found(cache, sql, "proposed")
assert "stock_summed" not in codes(cache, sql, "rejected")
# 业务日期被驳回:不知道按哪一列算时间,不报
assert "stock_summed" not in codes(cache, sql, override={("inventory_snapshots", "business_date"): "rejected"})
# --------------------------------------------------------------------------
# join_unconfirmed:关联条件对不上目录里的关系
# --------------------------------------------------------------------------
async def test_join_unconfirmed_positive(cache):
# 列连错了:订单的主键对明细的主键
wrong = found(cache, "SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.id = o.id WHERE o.status = 1")
assert ("join_unconfirmed", "warning", "order_items", None) in wrong
[check] = [c for c in run(cache, "SELECT o.id FROM orders o JOIN members m ON m.id = o.channel_id "
"WHERE o.status = 1") if c.code == "join_unconfirmed"]
assert check.level == "warning" and "o.channel_id" in check.sql_excerpt
assert check.table == "members" and "「订单」与「members」的关联条件" in check.message
# 只对上命名推断出来的关系(入园记录.member_id → 会员):提示一句,级别 info
proposed = found(cache, "SELECT m.name FROM visits v JOIN members m ON m.id = v.member_id WHERE v.status = 1")
assert ("join_unconfirmed", "info", "members", None) in proposed
JOIN_NEGATIVE = [
"SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1",
# 多写的过滤条件不影响对关系
"SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id AND i.qty > 0 WHERE o.status = 1",
# 没写目录的表之间按外键现推(verified)
"SELECT SUM(p.paid_amount) FROM orders o JOIN payments p ON p.order_id = o.id WHERE o.status = 1",
# 两张表都没写目录:没有可对照的目录,不判
"SELECT COUNT(*) FROM tour_groups t JOIN agencies a ON a.channel_id = t.agency_id",
# 子查询、CTE 当一边:不知道它的粒度,不判
"WITH x AS (SELECT order_id, SUM(amount) AS a FROM order_items GROUP BY order_id) "
"SELECT SUM(x.a) FROM orders o JOIN x ON x.order_id = o.id WHERE o.status = 1",
# 自己连自己:目录不记这种关系
"SELECT c.name FROM categories c JOIN categories p ON p.id = c.\"parentId\"",
# 没有等值条件
"SELECT COUNT(*) FROM orders o JOIN order_items i ON i.amount > o.total_amount WHERE o.status = 1",
# OR 连起来的条件:说不清按哪条连
"SELECT COUNT(*) FROM orders o JOIN order_items i ON i.order_id = o.id OR i.id = o.id WHERE o.status = 1",
]
@pytest.mark.parametrize("sql", JOIN_NEGATIVE)
async def test_join_unconfirmed_negative(cache, sql):
assert "join_unconfirmed" not in codes(cache, sql)
async def test_join_level_follows_status(cache):
sql = "SELECT SUM(i.amount) FROM orders o JOIN order_items i ON i.order_id = o.id WHERE o.status = 1"
assert "join_unconfirmed" not in codes(cache, sql, "confirmed")
assert "join_unconfirmed" not in codes(cache, sql, "verified")
# 结果记在 JOIN 引入的那张表上
assert ("join_unconfirmed", "info", "order_items", None) in found(cache, sql, "proposed")
assert ("join_unconfirmed", "warning", "order_items", None) in found(cache, sql, "rejected")
# --------------------------------------------------------------------------
# ratio_aggregated:对比率求和、求平均
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql", [
"SELECT AVG(discount_rate) FROM member_levels",
"SELECT SUM(discount_rate) FROM member_levels",
"SELECT ROUND(AVG(l.discount_rate), 2) FROM member_levels l",
])
async def test_ratio_aggregated_positive(cache, sql):
assert found(cache, sql) == [("ratio_aggregated", "warning", "member_levels", "discount_rate")]
@pytest.mark.parametrize("sql", [
"SELECT level_name, discount_rate FROM member_levels",
"SELECT MAX(discount_rate) FROM member_levels",
# 加权:比率乘上数量再加,是正确的算法
"SELECT SUM(m.id * l.discount_rate) FROM members m JOIN member_levels l ON l.id = m.member_level_id",
])
async def test_ratio_aggregated_negative(cache, sql):
assert "ratio_aggregated" not in codes(cache, sql)
async def test_ratio_level_follows_status(cache):
sql = "SELECT AVG(discount_rate) FROM member_levels"
assert found(cache, sql, "verified") == [("ratio_aggregated", "warning", "member_levels", "discount_rate")]
assert found(cache, sql, "proposed") == [("ratio_aggregated", "info", "member_levels", "discount_rate")]
assert found(cache, sql, "rejected") == []
# --------------------------------------------------------------------------
# missing_valid_filter:没按有效记录条件筛
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql", [
"SELECT COUNT(*) FROM visits",
"SELECT SUM(visitor_count) FROM visits WHERE park_id = 1",
"SELECT v.park_id, SUM(v.visitor_count) FROM visits v GROUP BY v.park_id",
])
async def test_missing_valid_filter_positive(cache, sql):
[check] = run(cache, sql)
assert (check.code, check.level, check.table) == ("missing_valid_filter", "warning", "visits")
assert "有效记录条件「status = 1」" in check.message
@pytest.mark.parametrize("sql", [
"SELECT COUNT(*) FROM visits WHERE status = 1",
"SELECT COUNT(*) FROM visits v WHERE v.status IN (0, 1)",
# 按状态分组:写的人就是要看每种状态
"SELECT status, COUNT(*) FROM visits GROUP BY status",
# 条件写在聚合里
"SELECT SUM(CASE WHEN status = 1 THEN visitor_count ELSE 0 END) FROM visits",
# CTE 里筛过
"WITH ok AS (SELECT * FROM visits WHERE status = 1) SELECT COUNT(*) FROM ok",
# 子查询原样透出、外层再筛
"SELECT COUNT(*) FROM (SELECT * FROM visits) x WHERE x.status = 1",
])
async def test_missing_valid_filter_negative(cache, sql):
assert "missing_valid_filter" not in codes(cache, sql)
async def test_valid_filter_level_follows_status(cache):
sql = "SELECT COUNT(*) FROM visits"
assert found(cache, sql, "verified") == [("missing_valid_filter", "warning", "visits", None)]
assert found(cache, sql, "proposed") == [("missing_valid_filter", "info", "visits", None)]
assert found(cache, sql, "rejected") == []
# --------------------------------------------------------------------------
# unknown_code:写了码值表里没有的值
# --------------------------------------------------------------------------
async def test_unknown_code_positive(cache):
[check] = run(cache, "SELECT COUNT(*) FROM visits WHERE status = 3")
assert (check.code, check.level, check.table, check.column) == ("unknown_code", "warning", "visits", "status")
assert "「3」" in check.message and "1=有效、0=作废" in check.message
[listed] = run(cache, "SELECT COUNT(*) FROM visits WHERE status IN (1, 5, 7)")
assert "「5」、「7」" in listed.message and "「1」" not in listed.message
assert found(cache, "SELECT COUNT(*) FROM channels WHERE channel_type = '线上'") == [
("unknown_code", "warning", "channels", "channel_type")]
@pytest.mark.parametrize("sql", [
"SELECT COUNT(*) FROM visits WHERE status = 1",
"SELECT COUNT(*) FROM visits WHERE status = '1'",
"SELECT COUNT(*) FROM visits WHERE 0 = status OR status = 1",
"SELECT COUNT(*) FROM visits WHERE status IN (0, 1)",
"SELECT COUNT(*) FROM visits WHERE status = 1.0",
"SELECT COUNT(*) FROM channels WHERE channel_type = '分销'",
# 模板和参数:值要到运行时才知道
"SELECT COUNT(*) FROM visits WHERE status = '{{ input.status }}'",
"SELECT COUNT(*) FROM visits WHERE status = {{ input.status }}",
"SELECT COUNT(*) FROM visits WHERE status = ?",
"SELECT COUNT(*) FROM visits WHERE status = :status",
])
async def test_unknown_code_negative(cache, sql):
assert "unknown_code" not in codes(cache, sql)
async def test_unknown_code_level_follows_status(cache):
sql = "SELECT COUNT(*) FROM channels WHERE channel_type = '线上'"
assert found(cache, sql, "verified") == [("unknown_code", "warning", "channels", "channel_type")]
assert found(cache, sql, "proposed") == [("unknown_code", "info", "channels", "channel_type")]
assert found(cache, sql, "rejected") == []
# --------------------------------------------------------------------------
# wrong_date_column:按另一个时间列分组、筛选
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql", [
"SELECT substr(entered_at, 1, 10) AS d, SUM(fee) FROM parking_records GROUP BY substr(entered_at, 1, 10)",
"SELECT SUM(p.fee) FROM parking_records p WHERE p.entered_at >= '2026-09-01'",
])
async def test_wrong_date_column_positive(cache, sql):
[check] = run(cache, sql)
assert (check.code, check.level, check.table, check.column) == (
"wrong_date_column", "warning", "parking_records", "entered_at")
assert "业务日期按「出场时间」计" in check.message and "「入场时间」" in check.message
@pytest.mark.parametrize("sql", [
"SELECT substr(exited_at, 1, 10) AS d, SUM(fee) FROM parking_records GROUP BY substr(exited_at, 1, 10)",
"SELECT SUM(fee) FROM parking_records WHERE exited_at >= '2026-09-01' AND entered_at IS NOT NULL",
# 两个时间都用上了:写的人知道两者的区别
"SELECT SUM(fee) FROM parking_records WHERE exited_at >= '2026-09-01' AND entered_at < '2026-09-30'",
"SELECT entered_at, exited_at, fee FROM parking_records",
])
async def test_wrong_date_column_negative(cache, sql):
assert "wrong_date_column" not in codes(cache, sql)
async def test_wrong_date_level_follows_status(cache):
sql = "SELECT SUM(p.fee) FROM parking_records p WHERE p.entered_at >= '2026-09-01'"
assert found(cache, sql, "verified") == [("wrong_date_column", "warning", "parking_records", "entered_at")]
assert found(cache, sql, "proposed") == [("wrong_date_column", "info", "parking_records", "entered_at")]
assert found(cache, sql, "rejected") == []
# --------------------------------------------------------------------------
# 复杂写法:照样分析得出,或者跳过;不报错、不误报
# --------------------------------------------------------------------------
@@ -84,6 +431,15 @@ async def test_complex_but_clean_sql_reports_nothing(cache, sql):
assert run(cache, sql) == []
async def test_problems_inside_cte_and_union_are_found(cache):
cte = ("WITH t AS (SELECT o.channel_id, SUM(o.total_amount) AS amt FROM orders o "
"JOIN order_items i ON i.order_id = o.id WHERE o.status = 1 GROUP BY o.channel_id) SELECT * FROM t")
assert ("fanout_sum", "error", "orders", "total_amount") in found(cache, cte)
union = ("SELECT SUM(on_hand) FROM inventory_snapshots UNION ALL "
"SELECT SUM(on_hand) FROM inventory_snapshots WHERE snapshot_date = '2026-09-30'")
assert found(cache, union) == [("stock_summed", "error", "inventory_snapshots", "on_hand")]
@pytest.mark.parametrize("sql", [
"SELECT FROM WHERE",
"SELEC total_amount FROM orders",
@@ -104,9 +460,46 @@ async def test_unparseable_or_unsupported_sql_is_skipped(cache, sql):
assert run(cache, sql) == []
async def test_other_dialects(cache):
pg = "SELECT s.on_hand::numeric FROM inventory_snapshots s WHERE s.snapshot_date = $1"
assert run(cache, pg, kind="postgres") == []
assert found(cache, "SELECT SUM(s.on_hand)::numeric FROM inventory_snapshots s", kind="postgres") == [
("stock_summed", "error", "inventory_snapshots", "on_hand")]
mysql = "SELECT `v`.`status`, COUNT(*) FROM `visits` `v` WHERE `v`.`status` = 3 GROUP BY `v`.`status`"
assert found(cache, mysql, kind="mysql") == [("unknown_code", "warning", "visits", "status")]
# 能读就照常分析,读不了就跳过:只要求不抛
assert isinstance(run(cache, "SELECT SUM(on_hand) FROM inventory_snapshots WHERE q'[it''s]' = 'x'",
kind="oracle"), list)
assert run(cache, "SELECT SUM(on_hand) FROM inventory_snapshots", kind="mssql") == []
async def test_no_catalog_means_no_checks(cache):
sql = "SELECT SUM(o.total_amount) FROM orders o JOIN order_items i ON i.id = o.id"
assert SqlChecker(kind="sqlite", schema_cache=cache, notes={}).check(sql) == []
assert check_sql(sql, kind="sqlite", schema_cache=cache, notes=None) == []
# 没探查过结构:对不上表,一样不判
assert check_sql(sql, kind="sqlite", schema_cache={}, notes=scenic_notes.notes()) == []
async def test_duplicates_are_folded_and_errors_come_first(cache):
sql = ("SELECT SUM(o.total_amount), SUM(o.total_amount) * 2, AVG(l.discount_rate) "
"FROM orders o JOIN order_items i ON i.order_id = o.id JOIN members m ON m.id = o.member_id "
"JOIN member_levels l ON l.id = m.member_level_id WHERE o.status = 1")
out = found(cache, sql)
assert [f for f in out if f[0] == "fanout_sum"] == [("fanout_sum", "error", "orders", "total_amount")]
levels = [f[1] for f in out]
assert levels == sorted(levels, key=["error", "warning", "info"].index)
async def test_checker_builds_relation_graph_lazily_and_reuses_it(cache, monkeypatch):
from app.data import catalog
calls = []
real = catalog.relation_graph
monkeypatch.setattr(catalog, "relation_graph", lambda *a, **k: calls.append(1) or real(*a, **k))
checker = SqlChecker(kind="sqlite", schema_cache=cache, notes=scenic_notes.notes())
checker.check("SELECT AVG(discount_rate) FROM member_levels")
assert calls == [] # 单表查询用不到关系图
checker.check(FANOUT[0])
checker.check(FANOUT[1])
assert calls == [1]