Files
AgentLab/backend/app/data/sqlcheck.py
T

475 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""基于数据目录的 SQL 检查:按数据源方言把 SQL 解析成语法树,对照目录里记下的事实找出「跑得通、数却不对」的写法。
**为什么要有它。** 助手自查以前只核对列名在不在(copilot.sql_unknown_columns,正则)。列名全对的 SQL 照样会算错:
订单关联订单明细以后对订单金额求和(每笔订单按明细行数重复相加)、把每天的库存加成「一个月的库存」、对折扣率求
平均、忘了筛掉作废的入园记录……这些错误数据库不会报,结果看起来也像回事。数据目录记下了判断它们要的事实(关系的
基数、列的度量类型、有效记录条件、码值、业务日期),这里把两者对起来。
**七条规则,code 稳定**(界面、修复、出具都按它认):
- fanout_sum:沿一对多方向关联之后,对「一」那一侧的度量列求和或计数(重复计算);
- stock_summed:对存量列求和,分组里却没有这张表的业务日期(跨期加总存量);
- join_unconfirmed:关联条件对不上目录里的任何关系;
- ratio_aggregated:对比率列直接求和或求平均;
- missing_valid_filter:表定义了有效记录条件,查询完全没涉及条件里的列;
- unknown_code:对有码值的列写了 = 或 IN,值不在码值表里;
- wrong_date_column:表定义了业务日期,查询却按这张表的另一个时间列分组或筛选。
**级别看依据。** 依据全是 confirmed / verified(人工确认、外键约束、数据剖析)时,fanout_sum、stock_summed 是 error,
其余是 warning;依据里有 proposed(命名推断、模型起草)就一律降为 info。join_unconfirmed 反过来看:对上推断的关系是
info,什么都对不上是 warning,对上有确证的关系不报。rejected 的项一律视而不见(catalog.visible_notes),被驳回的关系
既不算「对上了」,也不拿来判断基数。
**宁可漏报,不可误报。** error 会把助手搭的图打回去改、挡住受管级别的发布、让出具降档;误报一次,用户就会学会无视它。
所以拿不准一律跳过:解析不了(语法错、方言特性、多条语句、不是查询)、模板里有控制语句、列对不上表(别名不认识、
没加表前缀又有歧义、来自子查询)、关联条件里有 OR 或解析不了的列、分组写成表达式之后看不清粒度……都不报。
写法上能想到的「其实没问题」的情形也放过:按明细主键分组、去重聚合、窗口函数、两边的列一起算、业务日期筛到某一天、
CTE 或子查询里已经筛过有效记录。
这里不改写、不执行 SQL,也和 engine/direct_select.py 无关:那边刻意只用守卫的 SQLite 分词(证据下钻要和执行时的读法
完全一致),这边是多方言的静态检查,读错了最多漏报一条提示。
用法:SqlChecker 按一个数据源建一次(目录读一次、关系图用到时才建、建了就复用),check(sql) 返回 SqlCheck 列表。
助手自查、发布门禁、数据源查询工具三处共用它(load_checkers 从库里读目录,graph_checks 查一张图里写死的 SQL)。
"""
from __future__ import annotations
import logging
import re
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
from app.data import catalog
logger = logging.getLogger(__name__)
#: 规则编号,按严重程度大致排序
CHECK_CODES = ("fanout_sum", "stock_summed", "join_unconfirmed", "ratio_aggregated", "missing_valid_filter",
"unknown_code", "wrong_date_column")
#: 结果的级别,从重到轻
LEVELS = ("error", "warning", "info")
#: 依据有确证时报 error 的规则:它们算出来的数必然是错的。其余规则只是「很可能不对」
_ERROR_CODES = frozenset({"fanout_sum", "stock_summed"})
_STRONG = frozenset({"confirmed", "verified"})
_STATUS_RANK = {"confirmed": 3, "verified": 2, "proposed": 1}
#: 数据源类型 → sqlglot 方言。不认识的类型不查:拿别的方言硬读,读错了就是误报
_DIALECTS = {"sqlite": "sqlite", "postgres": "postgres", "postgresql": "postgres", "mysql": "mysql",
"mariadb": "mysql", "oracle": "oracle"}
#: 太长的 SQL 不查:解析耗时随长度增长,数据源查询工具每次执行都要跑一遍,不能拖慢查询
MAX_SQL_CHARS = 20_000
#: 一条 SQL 最多报这么多条:再多就是 SQL 整体有问题,逐条列出只会淹没重点
MAX_CHECKS = 20
_EXCERPT_MAX = 160
#: 模板占位({{ input.week }})换成这个标识符再解析:在引号里是一个认得出的字符串(码值检查跳过它),
#: 在引号外是一个不属于任何表的列名(各条规则都对不上它)。不换成数字 0:放在引号里的 0 会被当成码值去比
_SENTINEL = "__agentlab_tpl__"
_TEMPLATE = re.compile(r"\{\{.*?\}\}", re.S)
#: 名字看得出是时间的列:xxx_at、xxx_time、xxx_date、xxx_on、createdAt…(SQLite 的日期常存成文本,只看类型认不全)
_TEMPORAL_NAME = re.compile(r"(?:^|_)(?:at|time|date|day|dt|ts|on|timestamp|datetime)$|[a-z0-9](?:At|Time|Date|On)$",
re.IGNORECASE)
_TEMPORAL_TYPE = re.compile(r"DATE|TIME", re.IGNORECASE)
def dialect_of(kind: str | None) -> str | None:
"""数据源类型对应的 sqlglot 方言;不支持的类型返回 None(不查)。"""
return _DIALECTS.get((kind or "").strip().lower())
@dataclass(frozen=True)
class SqlCheck:
"""一条检查结果。
message 给人看(界面、发布前检查、出具降档的原因),for_model 交回模型改写时用(写 SQL 里的真名和改法)。
table 是 schema_cache 里的表名;column、relation_id、sql_excerpt 没有时不出现在 as_dict 里。
"""
code: str
level: str
message: str
for_model: str
table: str
column: str | None = None
relation_id: str | None = None
sql_excerpt: str | None = None
def as_dict(self) -> dict[str, Any]:
out: dict[str, Any] = {"code": self.code, "level": self.level, "message": self.message,
"for_model": self.for_model, "table": self.table}
for key in ("column", "relation_id", "sql_excerpt"):
if (value := getattr(self, key)) is not None:
out[key] = value
return out
def _level(code: str, statuses: Iterable[str | None]) -> str:
"""依据全有确证:error(fanout_sum、stock_summed)或 warning;有一项只是推断:info。"""
if all(s in _STRONG for s in statuses):
return "error" if code in _ERROR_CODES else "warning"
return "info"
# ==========================================================================
# 检查器:一个数据源一个
# ==========================================================================
class SqlChecker:
"""按一个数据源的方言、表结构和目录检查 SQL。
notes 是 {表名: 目录 notes},原样传(含被驳回的项):关系图要知道哪些关系被驳回了,免得按表结构又推回来;
各条规则只看去掉驳回项之后的那份。没有目录(notes 为空)的源什么都不查——没有可对照的东西,
也不让还没起草目录的部署多出一堆提示;这时 check 不解析 SQL,几乎没有开销。
"""
def __init__(self, *, kind: str | None, schema_cache: Mapping[str, Any] | None,
notes: Mapping[str, Mapping[str, Any]] | None, source_name: str | None = None) -> None:
self.dialect = dialect_of(kind)
self.schema_cache: Mapping[str, Any] = schema_cache or {}
self.source_name = source_name
self._raw = {str(k): v for k, v in (notes or {}).items() if isinstance(v, Mapping) and v}
self._visible = {k: catalog.visible_notes(v) for k, v in self._raw.items()}
self._visible_lower = {k.lower(): v for k, v in self._visible.items()}
self._edges: dict[str, list[catalog.JoinEdge]] | None = None
@property
def enabled(self) -> bool:
"""有没有可能报出东西:方言认得、有目录、探查过结构。"""
return bool(self.dialect and self._raw and (self.schema_cache.get("tables") or {}))
# ---- 表、目录、关系
def table_meta(self, table: Any) -> tuple[str, Mapping[str, Any]] | None:
"""SQL 里的一张表 → (schema_cache 的表名, 表结构)。带了别的 schema、库名的不认(宁可当作不认识)。"""
tables = self.schema_cache.get("tables") or {}
name, db = table.name, table.text("db")
if not name or table.text("catalog"):
return None
if db:
qualified = f"{db}.{name}"
if qualified in tables:
return qualified, tables[qualified]
schema = str(self.schema_cache.get("schema") or "")
if not schema or schema.lower() != db.lower():
return None
key = catalog.resolve_table_name(self.schema_cache, name)
meta = tables.get(key) if key else None
return (key, meta) if key and isinstance(meta, Mapping) else None
def notes_of(self, table: str) -> dict[str, Any]:
return self._visible.get(table) or self._visible_lower.get(table.lower()) or {}
def edges_between(self, a: str, b: str) -> list[catalog.JoinEdge]:
"""从 a 连到 b 的边(每条关系正反各一条)。关系图第一次用到时才建,之后复用。
和助手挑表看到的是同一张图(catalog.relation_graph):目录里的关系,加上没起草过的表按外键、命名现推的;
被驳回的关系不进图,也不会被现推回来。
"""
if self._edges is None:
graph = catalog.relation_graph(self._raw, schema_cache=self.schema_cache)
index: dict[str, list[catalog.JoinEdge]] = {}
for table, edges in graph.items():
index.setdefault(table.lower(), []).extend(edges)
self._edges = index
return [e for e in self._edges.get(a.lower(), []) if e.to_table.lower() == b.lower()]
# ---- 检查
def check(self, sql: Any) -> list[SqlCheck]:
"""检查一条 SQL。解析不了、不是单条查询、或者不支持的写法,返回空列表;从不抛异常。"""
if not self.enabled or not isinstance(sql, str):
return []
text = sql.strip()
if not text or len(text) > MAX_SQL_CHARS or "{%" in text:
return []
text = _TEMPLATE.sub(_SENTINEL, text)
try:
import sqlglot
from sqlglot import exp
from sqlglot.optimizer.scope import traverse_scope
statements = [s for s in sqlglot.parse(text, read=self.dialect) if s is not None]
if len(statements) != 1 or not isinstance(statements[0], exp.Query):
return []
root = statements[0]
scopes = traverse_scope(root)
except Exception: # noqa: BLE001 - 语法错、方言特性、sqlglot 不支持的写法:一律不查
return []
names = {c.name.lower() for c in root.find_all(exp.Column) if c.name}
found: list[SqlCheck] = []
for scope in scopes:
if not isinstance(scope.expression, exp.Select):
continue
try:
found += _ScopeCheck(self, scope, statement_columns=names).run()
except Exception: # noqa: BLE001 - 规则本身的缺陷不能连累查询;这一段不报,记下来修
logger.warning("SQL 检查跳过了一段查询", exc_info=True)
return _fold(found)
def check_dicts(self, sql: Any) -> list[dict[str, Any]]:
return [c.as_dict() for c in self.check(sql)]
def check_sql(sql: Any, *, kind: str | None, schema_cache: Mapping[str, Any] | None,
notes: Mapping[str, Mapping[str, Any]] | None, source_name: str | None = None) -> list[SqlCheck]:
"""只查一条时的便捷写法。要查多条(同一个源)就建一个 SqlChecker 复用:关系图只建一次。"""
return SqlChecker(kind=kind, schema_cache=schema_cache, notes=notes, source_name=source_name).check(sql)
def _fold(found: list[SqlCheck]) -> list[SqlCheck]:
"""同一件事只报一次(规则、表、列、关系相同的留最重的那条),重的在前,最多 MAX_CHECKS 条。"""
kept: dict[tuple[Any, ...], SqlCheck] = {}
for check in found:
key = (check.code, check.table.lower(), (check.column or "").lower(), check.relation_id)
if key not in kept or LEVELS.index(check.level) < LEVELS.index(kept[key].level):
kept[key] = check
ordered = sorted(kept.values(), key=lambda c: LEVELS.index(c.level))
return ordered[:MAX_CHECKS]
# ==========================================================================
# 一段 SELECT 里的分析
# ==========================================================================
@dataclass
class _Table:
"""一段 SELECT 里出现的一张实表:别名、schema_cache 表名、表结构、目录(去掉驳回项)。"""
alias: str
name: str
meta: Mapping[str, Any]
notes: Mapping[str, Any]
order: int
#: FROM / JOIN 里写这张表的那个节点(摘录 SQL 用)
node: Any = None
columns: dict[str, str] = field(default_factory=dict)
def __post_init__(self) -> None:
self.columns = {str(c.get("name")).lower(): str(c.get("name"))
for c in self.meta.get("columns") or [] if isinstance(c, Mapping) and c.get("name")}
def item(self, name: str) -> Mapping[str, Any] | None:
value = self.notes.get(name)
return value if isinstance(value, Mapping) and "value" in value else None
def column_item(self, column: str, name: str) -> Mapping[str, Any] | None:
columns = self.notes.get("columns") if isinstance(self.notes.get("columns"), Mapping) else {}
items = columns.get(column)
if not isinstance(items, Mapping):
items = next((v for k, v in columns.items() if str(k).lower() == column.lower()), None)
value = items.get(name) if isinstance(items, Mapping) else None
return value if isinstance(value, Mapping) and "value" in value else None
def label(self) -> str:
item = self.item("label")
return str(item["value"]) if item and isinstance(item.get("value"), str) and item["value"].strip() \
else self.name
def column_label(self, column: str) -> str:
item = self.column_item(column, "label")
return str(item["value"]) if item and isinstance(item.get("value"), str) and item["value"].strip() \
else column
def measure(self, column: str) -> tuple[str, str] | None:
"""(度量类型, 状态)。"""
item = self.column_item(column, "measure")
return (str(item["value"]), str(item.get("status"))) if item else None
def keys(self) -> tuple[list[str], str] | None:
"""业务主键:目录写了用目录的(带状态),没写用表结构的主键(数据库约束,算有确证)。"""
item = self.item("keys")
if item and isinstance(item.get("value"), list) and item["value"]:
return [str(k) for k in item["value"]], str(item.get("status"))
pk = self.meta.get("primary_key")
return ([str(k) for k in pk], "verified") if isinstance(pk, list) and pk else None
def business_date(self) -> tuple[str, str] | None:
"""(业务日期列的真名, 状态)。列不在表结构里(写的是表达式、或者列已改名)就当没有。"""
item = self.item("business_date")
value = item.get("value") if item else None
column = value.get("column") if isinstance(value, Mapping) else None
if not isinstance(column, str) or column.lower() not in self.columns:
return None
return self.columns[column.lower()], str(item.get("status"))
def temporal(self, column: str) -> bool:
meta = next((c for c in self.meta.get("columns") or []
if isinstance(c, Mapping) and str(c.get("name")).lower() == column.lower()), None)
kind = str((meta or {}).get("type") or "")
return bool(_TEMPORAL_TYPE.search(kind) or _TEMPORAL_NAME.search(column))
#: 一列解析到的位置:(表, 列的真名)
_Ref = tuple[_Table, str]
def _unparen(node: Any) -> Any:
from sqlglot import exp
while isinstance(node, exp.Paren):
node = node.this
return node
def _conjuncts(node: Any) -> list[Any]:
"""AND 连起来的各个条件(只拆最外层)。"""
from sqlglot import exp
node = _unparen(node)
if isinstance(node, exp.And):
return _conjuncts(node.left) + _conjuncts(node.right)
return [node] if node is not None else []
def _bare_column(node: Any) -> Any:
"""剥掉不改变「这是哪一列」的外壳(括号、类型转换、COALESCE / IFNULL 给默认值、ROUND),剩下的是列就返回它。"""
from sqlglot import exp
while True:
node = _unparen(node)
if isinstance(node, (exp.Cast, exp.TryCast, exp.Round)):
node = node.this
elif isinstance(node, exp.Coalesce) and all(isinstance(_unparen(e), exp.Literal)
for e in node.expressions):
node = node.this
else:
break
return node if isinstance(node, exp.Column) else None
class _ScopeCheck:
"""一段 SELECT(根查询、CTE、子查询、UNION 的一支各算一段)对照目录检查。"""
def __init__(self, checker: SqlChecker, scope: Any, *, statement_columns: set[str]) -> None:
from sqlglot import exp
self.checker = checker
self.dialect = checker.dialect
self.scope = scope
self.select = scope.expression
self.statement_columns = statement_columns
self.tables: dict[str, _Table] = {}
#: 不是实表的来源(子查询、CTE、认不出的表):{别名: 它透出的列名,说不清就是 None}
self.opaque: dict[str, set[str] | None] = {}
for order, (alias, (node, source)) in enumerate(scope.selected_sources.items()):
key = str(alias).lower()
if isinstance(source, exp.Table):
found = checker.table_meta(source)
if found:
name, meta = found
self.tables[key] = _Table(key, name, meta, checker.notes_of(name), order, node)
continue
self.opaque[key] = None
else:
inner = getattr(source, "expression", None)
exposes = None
if isinstance(inner, exp.Select) and not any(isinstance(s, exp.Star) or
(isinstance(s, exp.Column) and s.is_star)
for s in inner.selects):
exposes = {n.lower() for n in inner.named_selects}
self.opaque[key] = exposes
# FROM / JOIN 里还有没进 selected_sources 的东西(表函数、LATERAL):没加前缀的列可能出自它们
items = ([self.select.args["from_"].this] if self.select.args.get("from_") else []) + \
[j.this for j in self.select.args.get("joins") or []]
if len(items) != len(scope.selected_sources):
self.opaque["\0"] = None
self.nodes = list(scope.walk())
self.columns = [n for n in self.nodes if isinstance(n, exp.Column)]
self._matched: list[tuple[str, str, catalog.JoinEdge]] | None = None
self._join_checks: list[SqlCheck] = []
# ---- 工具
def excerpt(self, node: Any) -> str | None:
try:
text = node.sql(dialect=self.dialect)
except Exception: # noqa: BLE001
return None
text = text.replace(_SENTINEL, "{{…}}")
return text if len(text) <= _EXCERPT_MAX else text[:_EXCERPT_MAX - 1] + "…"
def resolve(self, column: Any) -> _Ref | None:
"""列 → (表, 真名)。别名不认识、列不在那张表里、没加前缀又可能出自别处,都返回 None。"""
name = column.name.lower() if column.name else ""
if not name or column.is_star:
return None
qualifier = column.table.lower() if column.table else ""
if qualifier:
table = self.tables.get(qualifier)
return (table, table.columns[name]) if table and name in table.columns else None
if any(exposes is None or name in exposes for exposes in self.opaque.values()):
return None
hits = [t for t in self.tables.values() if name in t.columns]
return (hits[0], hits[0].columns[name]) if len(hits) == 1 else None
def refs(self, node: Any) -> list[_Ref | None]:
"""node 里每一列的解析结果(不进嵌套的子查询)。"""
from sqlglot import exp
if node is None:
return []
if isinstance(node, exp.Column):
return [self.resolve(node)]
return [self.resolve(c) for c in node.find_all(exp.Column)
if not any(isinstance(p, (exp.Subquery, exp.Select)) for p in _parents(c, stop=node))]
def group_refs(self) -> list[_Ref | None]:
"""分组里用到的列。GROUP BY 别名、GROUP BY 1 换成 SELECT 里对应的表达式(两种读法都算上,宁可多认)。"""
from sqlglot import exp
group = self.select.args.get("group")
if group is None:
return []
out = self.refs(group)
selects = self.select.selects
by_alias = {s.alias.lower(): s.this for s in selects if isinstance(s, exp.Alias) and s.alias}
for item in group.expressions:
item = _unparen(item)
if isinstance(item, exp.Literal) and not item.is_string and str(item.this).isdigit():
index = int(item.this) - 1
if 0 <= index < len(selects):
out += self.refs(selects[index].this if isinstance(selects[index], exp.Alias)
else selects[index])
elif isinstance(item, exp.Column) and not item.table and item.name.lower() in by_alias:
out += self.refs(by_alias[item.name.lower()])
return out
def where(self) -> Any:
where = self.select.args.get("where")
return where.this if where is not None else None
def aggregates(self, *kinds: type) -> list[tuple[Any, Any]]:
"""(聚合节点, 参数):不含窗口函数、去重聚合、COUNT(*)、参数里套子查询的。"""
from sqlglot import exp
out = []
for node in self.nodes:
if not isinstance(node, kinds) or isinstance(node.parent, exp.Window):
continue
arg = node.this
if arg is None or isinstance(arg, (exp.Distinct, exp.Star)) or \
any(isinstance(n, (exp.Subquery, exp.Select)) for n in arg.walk()):
continue
out.append((node, arg))
return out
def run(self) -> list[SqlCheck]:
return []
def _parents(node: Any, *, stop: Any) -> Iterable[Any]:
"""node 到 stop 之间(不含两端)的各层父节点。"""
parent = node.parent
while parent is not None and parent is not stop:
yield parent
parent = parent.parent
__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "check_sql",
"dialect_of"]