From db91c9eaf6746cd31a94d71193a1b5a71d5f8dcc Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 03:36:12 +0800 Subject: [PATCH 1/6] =?UTF-8?q?SQL=20=E8=AF=AD=E4=B9=89=E5=A4=8D=E6=A0=B8?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=88=E7=AC=AC=203=20=E8=BD=AE=EF=BC=89?= =?UTF-8?q?=EF=BC=9A=E4=B8=A4=E8=BE=B9=E7=9A=84=20CTE=E3=80=81=E5=AD=90?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E5=85=88=E6=8C=89=E7=BB=93=E6=9E=84=E9=85=8D?= =?UTF-8?q?=E5=A5=BD=E5=AF=B9=E5=86=8D=E6=AF=94=EF=BC=8C=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E6=8B=BF=E3=80=8C=E4=B8=8D=E5=86=99=E7=BC=96=E5=8F=B7=E3=80=8D?= =?UTF-8?q?=E7=9A=84=E5=86=99=E6=B3=95=E8=A1=A5=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一轮给派生来源(CTE、FROM 里的子查询)只编了位置编号,编号错开时再按不写编号的写法(派生来源一律写「@」)补配一遍。复验发现三处问题,根因相同:派生来源没有结构上的标识。一是漏报:两个 CTE 都算出 total(付款合计、退款合计),编号一致时 SUM(p.total) 对 SUM(r.total)、WHERE p.total > 100 对 r.total > 100、ORDER BY、HAVING、差值写反,都被补配吞掉、返回空列表。二是误报:盲算多一个排在前面的 CTE(vip),原流程的 t(@1)按编号配给了盲算的 vip(@1),报出「关联 t 表的条件是 orders.user_id = users.id,盲算是 users.id = users.id」「盲算关联了 t 表」。三是误报:内连接换了先后(FROM paid JOIN refunded 换成 FROM refunded JOIN paid),编号跟着对调,比较的两边按编号排成两个方向,报出两条自相矛盾的话。 现在 diff_facets 先给两边每个派生来源算一份结构标识(_Derived):里面读到的实表(FROM / JOIN,往里套着的也算)、选出来的各列规范化后的定义式(total 是 SUM(orders.amount))、分组键、里面的条件、整段规范化后的写法和起的名字;按从严到宽几级一对一地配:整段一样,表和各列定义、分组一样,表和列名一样,表和名字一样,最后三级只在两边各剩一个时才配(只是表一样、只是名字一样、只是列名一样);同一级几对都配得上时先配同名的、再配条件重合多的、最后看先后,哪一级都配不上的不配,没用到的 CTE 也参加配对。配好之后第二条 SQL 的派生来源改用第一条里对应的编号(配不上的另起编号),facets 记下原文(Facets._origin,不进比较和 as_dict),按共同的编号重新分析一遍再比:比较两边谁在前、关联条件的先后都按共同编号排,不再受位置编号左右。_Compare 去掉了不写编号的补配(删掉 Part.anon),loose 里派生来源的编号照写,筛选按同一列配对时列也带编号。 连接:内连接、CROSS JOIN 按涉及的来源和条件比(JoinFacet.sides),不按哪一侧是关联进来的,实表的内连接换了先后也不再报;配成一对的两边名字不同时两个名字都说;一边独有的内连接、关联进来的那一侧另一边也有时,报它接上的另一头(FROM stores JOIN orders、另一边只查 orders:多出来的是 stores,不是 orders;Facets.sources 记各段 FROM / JOIN 的来源拿来对照,条件里子查询的来源不算),已经说了「关联了」的表不再另说一遍「用到了」。取行:CTE、子查询里的取行按所属派生来源配好的编号(LimitFacet.scope_key)配对,一边多一个限了行数的 CTE 不再和另一边的 CTE 凑对;LATERAL 子查询照写它的别名。筛选、HAVING:两边各自独有、读起来却一样的条件(p.total > 100 和 r.total > 100 都只读得出 total > 100)配成一对、改用带来源名字的写法说,不再各说一句。 模块说明、Part / JoinFacet / LimitFacet / Facets、unmatched、joins 的文档按现在的做法改写,做不到的照实列出(外层按名字引用的列两边起名不同算两列;读同一张表、各列定义一样的两个派生来源两边都改了条件又改了名时可能配反;全换了的两个派生来源不配对;自连接分不出两个别名;LEFT JOIN 对调成 RIGHT JOIN、内连接的单表条件写在 ON 还是 WHERE、三张以上不带条件的表换先后仍算不同)。测试补上三处问题的反例(CTE、FROM 子查询、改名又改先后的写法)和上面各项,原有 99 项照旧通过。 --- backend/app/data/sql_semantics.py | 548 ++++++++++++++++++++++------ backend/tests/test_sql_semantics.py | 233 +++++++++++- 2 files changed, 652 insertions(+), 129 deletions(-) diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index 613f5e41..f0a78158 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -17,12 +17,29 @@ 写在 WHERE 里的列等式算作关联条件,不算筛选。条件里套着的子查询(EXISTS、IN (SELECT …))既单算一段,也按它自己 那一段的来源整个写进外层那个条件:别名在子查询里定义,拿外层的来源去认,同一个条件只因别名不同就会被报成不同。 -**各自起的名字不进比较。** 表的别名换成真实表名;CTE、FROM 里的子查询没有真实表名可换,比较时按它在语句里的位置 -编号(@1、@2…,从外往里、按第一次用到的先后),给人看时照写名字;条件里的子查询(IN、EXISTS、标量子查询)选出来 -的列名不影响取值,比较时去掉。这些名字都是各自起的,双盲复算的盲算更是:按名字比,同一个口径会被说成换了口径。 -编号只看位置,一边多出一个排在前面的 CTE、子查询,后面的编号就错开了:diff_facets 先按编号比,比不上的再按不写 -编号的写法一对一地配,配上的不算不同。仍按名字比的只有 CTE、子查询选出来、外层按名字引用的列(一边叫 total、 -一边叫 sum_amt,算两列)。 +**各自起的名字不进比较。** 表的别名换成真实表名;CTE、FROM 里的子查询(含 LATERAL)没有真实表名可换,单看一条 +语句时按它在语句里的位置编号(@1、@2…,从外往里、按第一次用到的先后),给人看时照写名字;条件里的子查询(IN、 +EXISTS、标量子查询)选出来的列名不影响取值,比较时去掉。这些名字都是各自起的,双盲复算的盲算更是:按名字比,同一个 +口径会被说成换了口径。 + +**两边的派生来源按结构配对。** 位置编号只看先后:一边多出一个排在前面的 CTE,或者内连接两边换了先后,编号就错开 +了;可也不能干脆不看是哪一个(两个 CTE 都算出 total 时,外层用的是付款合计还是退款合计,正是要报的不同)。所以 +diff_facets 先给两边的每个派生来源算一份结构标识(见 _Derived):里面读到的实表(FROM / JOIN,往里套着的也算)、 +选出来的各列怎么定义(规范化后的定义式,如 total 是 SUM(orders.amount))、按什么分组、整段规范化后的写法,以及 +起的名字;再按从严到宽的几级一对一地配(见 _align):整段一样;读的表、各列定义、分组一样(里面的筛选条件可以不 +同);读的表和选出来的列名一样(某一列的算法可以不同);读的表和名字一样;最后三级只在两边都只剩一个时才配:只是 +读的表一样、只是名字一样、只是选出来的列名一样。同一级有几个都配得上时,先挑同名的,再挑里面的条件重合最多的,最后 +才看位置编号的先后。哪一级都配不上的不配:一边多出来的 CTE 不会被拿去和另一边不相干的 CTE 凑成一对。配好之后, +第二条 SQL 的派生来源改用第一条里对应的那个编号(配不上的另起编号),按这个共同的编号重新分析一遍再逐个方面比: +比较的两边谁在前、关联条件的先后都按共同的编号排,内连接换了先后也排成同一个方向。连接里的内连接(含逗号连接、 +CROSS JOIN)按涉及的来源和条件比,不按哪一侧是关联进来的;一边独有的一次内连接,关联进来的那一侧另一边也有时,报 +的是它接上的另一头(FROM stores JOIN orders、另一边只查 orders:多出来的是 stores)。 + +仍做不到的:两边给 CTE、子查询选出来、外层按名字引用的列起的名字不同(一边叫 total、一边叫 sum_amt),算两列;读同 +一张表、各列定义也一样的两个派生来源,两边都改了里面的条件、又各起了名字时,只能按条件的重合和先后配,可能配反; +读的表、选出来的列名、名字全换了的两个派生来源不配对,外层按它们的列写的条件照报成不同;同一张表、同一个 CTE 自己 +关联自己时两个别名比较用的写法一样,p1.total > p2.total 写反了认不出;连接上做不到的见 _Compare.joins。facets 单独 +返回的 key 仍是位置编号;不是 facets 算出来的 Facets(没记原文)diff_facets 也只能按位置编号比。 **和 sqlcheck 一样宁可漏报。** 解析不了(语法错、方言特性、多条语句、不是查询)就带 parse_error 返回,从不抛 异常。没加表前缀、又有好几张表可能有这一列的,照写列名(给了表结构才去对)。只读语法树,不执行 SQL。 @@ -33,7 +50,7 @@ import logging import re from collections import deque from collections.abc import Iterable, Iterator, Mapping -from dataclasses import dataclass, fields, is_dataclass +from dataclasses import dataclass, field, fields, is_dataclass, replace from itertools import zip_longest from typing import Any @@ -51,9 +68,10 @@ MAX_SQL_CHARS = 100_000 #: 算好的来源(别名、外层):复制品没有父节点,自己找不到外层 _SEGMENT = "sql_semantics_segment" #: 派生来源(引用 CTE 的表和 CTE 的定义、FROM / JOIN 里的子查询)在语法树上记比较用编号的 meta 键(见 -#: _Statement._number_derived)。复制出来的条件里套着子查询时,靠它把 CTE 名、子查询的别名换成编号 +#: _Statement._number_derived、_Statement.relabel)。复制出来的条件里套着子查询时,靠它把 CTE 名、子查询的别名 +#: 换成编号 _DERIVED = "sql_semantics_derived" -#: 比较用的写法里不写编号时,派生来源一律写成这个(见 Part.anon) +#: 不写编号的写法(render 的 anon 模式)里派生来源一律写成这个:算结构标识(_Derived)时用,和编号无关 _ANON = "@" @@ -214,14 +232,14 @@ ColumnRef = tuple[str | None, str] class Part: """一个条件、一个分组键、一个聚合参数…… - key 用来比较:别名换成真实表名(CTE、子查询写编号),表名、列名按 name_key,等式和比较的两边排好顺序;loose 同 - key 但不写表名,两边查的表换了一张(orders 换成 orders_archive)时拿它比其余各方面,免得同一个条件只因表名不同 - 被报成不同;anon 同 key,但 CTE、子查询一律写「@」、不写编号(loose 也不写),编号错开时拿它再配一遍(见 - _Compare.unmatched)。text 给人看:好几张表都有的列写「表.列」(真实表名,不是别名),其余只写列名;full 一律 - 写表名,两边的 text 一样、key 却不同时改用它说明。excludes 只在 WHERE 的「排除」写法上有:<>、NOT、NOT IN、 - IS NOT NULL 去掉否定之后排除的是什么(IS NOT NULL 写成「列 为空」)。column_keys 和 columns 一一对应,是各列 - 比较用的写法(实表.列;CTE、子查询的列写成「@.列」):两边给 CTE、子查询起的名字不同、编号错开,也认得出两个 - 条件说的是同一列,配成一对说。 + key 用来比较:别名换成真实表名,CTE、子查询写编号(facets 单独返回的是位置编号,diff_facets 里是两边按结构配好 + 对的共同编号,见模块说明),表名、列名按 name_key,等式和比较的两边排好顺序;loose 同 key 但不写实表的表名(派生 + 来源的编号照写:它不是表名,换了表也还是配好的那一个),两边查的表换了一张(orders 换成 orders_archive)时拿它 + 比其余各方面,免得同一个条件只因表名不同被报成不同。text 给人看:好几张表都有的列写「表.列」(真实表名,不是 + 别名),其余只写列名;full 一律写表名,两边的 text 一样、key 却不同时改用它说明。excludes 只在 WHERE 的「排除」 + 写法上有:<>、NOT、NOT IN、IS NOT NULL 去掉否定之后排除的是什么(IS NOT NULL 写成「列 为空」)。column_keys 和 + columns 一一对应,是各列比较用的写法(实表.列;CTE、子查询的列写成「编号.列」):两边对同一列的不同条件靠它配成 + 一对说。 """ key: str @@ -231,24 +249,28 @@ class Part: loose: str = "" full: str = "" column_keys: tuple[str, ...] = () - anon: str = "" @staticmethod def plain(key: str, text: str) -> "Part": """没有列的一项(表名、COUNT(*) 的「*」):比较用的写法都是 key,给人看的都是 text。""" - return Part(key=key, text=text, loose=key, full=text, anon=key) + return Part(key=key, text=text, loose=key, full=text) @dataclass(frozen=True) class JoinFacet: """一次连接。kind:inner / left / right / full / cross / semi / anti / natural。table 是关联进来的那一侧,other 是另一侧:关联条件里出现的别的表(给人看的真实表名);没有条件的(CROSS JOIN)是在它之前出现的各个来源。on 是 - 关联条件的合取式排好序后用 AND 连起来的写法,没有条件是 None。""" + 关联条件的合取式排好序后用 AND 连起来的写法,没有条件是 None。 + + sides 是这次连接涉及的来源(比较用的写法,排好序):关联进来的那一侧,加上关联条件里直接引用的这一段的来源(不 + 透过 CTE 找实表);没有条件的加上它之前出现的各个来源。内连接、CROSS JOIN 两边换个先后意思不变(FROM paid JOIN + refunded 和 FROM refunded JOIN paid),diff_facets 比它们时按 sides 和条件比,不按哪一侧是关联进来的。""" kind: str table: Part on: Part | None = None other: tuple[str, ...] = () + sides: tuple[str, ...] = () @dataclass(frozen=True) @@ -277,12 +299,15 @@ class Aggregate: class LimitFacet: """一段查询取哪几行。count、offset 是行数和跳过行数的写法(没写为 None,OFFSET 0 也算没写);order 是这一段的 排序键 (Part, 是否降序):只取前几行时,按什么排决定了取到的是哪几行。scope 是哪一段:None 是最外层,CTE 写它的 - 名字,FROM 里的子查询写它的别名(没有别名写「子查询」)。条件里套着的子查询不单算:它整个写在那个条件里比。""" + 名字,FROM 里的子查询写它的别名(没有别名写「子查询」)。条件里套着的子查询不单算:它整个写在那个条件里比。 + scope_key 是这一段所属派生来源的编号(和 Part.key 里的同一套编号;最外层、UNION 的某一支这类不是派生来源整段的 + 为 None):diff_facets 按它把两边同一个 CTE、子查询里的取行配成一对。""" count: str | None offset: str | None = None order: tuple[tuple[Part, bool], ...] = () scope: str | None = None + scope_key: str | None = None @dataclass(frozen=True) @@ -291,6 +316,14 @@ class Facets: limit 是最外层 LIMIT 的行数(写法);limits 是各段取哪几行(最外层在前,见 LimitFacet),只有写了 LIMIT 或 OFFSET 的段才有。 + + sources 是各段 FROM / JOIN 里的来源(实表、CTE、子查询;key 是比较用的写法,派生来源是编号,text 是给人看的 + 名字),按出现的先后、按 key 去重;条件里套着的子查询(IN (SELECT … FROM users))的来源不算,它们只筛行,不是 + 连接的一头。它本身不单独比较:一边独有的一次内连接要说清多出来的是哪个来源时,拿另一边的 sources 对照(见 + _Compare.added)。 + + _origin 是 facets 记下的 (原文, 表结构),不进比较、不进 as_dict:diff_facets 把两边的派生来源按结构配好对之后, + 拿它按共同的编号重新分析一遍(见模块说明)。 """ dialect: str | None = None @@ -303,7 +336,9 @@ class Facets: having: tuple[Part, ...] = () limit: str | None = None limits: tuple[LimitFacet, ...] = () + sources: tuple[Part, ...] = () parse_error: str | None = None + _origin: tuple[str, Mapping[str, Any] | None] | None = field(default=None, compare=False, repr=False) def as_dict(self) -> dict[str, Any]: return _jsonable(self) @@ -311,7 +346,7 @@ class Facets: def _jsonable(value: Any) -> Any: if is_dataclass(value) and not isinstance(value, type): - return {f.name: _jsonable(getattr(value, f.name)) for f in fields(value)} + return {f.name: _jsonable(getattr(value, f.name)) for f in fields(value) if not f.name.startswith("_")} if isinstance(value, (list, tuple)): return [_jsonable(v) for v in value] return value @@ -322,13 +357,15 @@ def facets(sql: str, dialect: str | None = None, *, schema: Mapping[str, Any] | schema 是数据源的 schema_cache 形状({"tables": {表名: {"columns": [{"name": …}]}}}),可以不给。给了的话, 几张表连在一起、列又没加表前缀时按它认列是哪张表的;不给就照写列名,比较时可能把同一列当成两列。 + + 返回的 Facets 记着原文和表结构(_origin):diff_facets 要把两边的派生来源按结构配好对再重新分析。 """ read = _read(dialect) root, problem = _parse(sql, read) if root is None: return Facets(dialect=read, parse_error=problem) try: - return _Statement(root, read, schema).facets() + return replace(_Statement(root, read, schema).facets(), _origin=(sql, schema)) except Exception: # noqa: BLE001 - 分析本身的缺陷不能连累调用方:当作分析不了,记下来修 logger.warning("SQL 语义分析失败", exc_info=True) return Facets(dialect=read, parse_error="SQL 的结构无法分析") @@ -423,6 +460,17 @@ def _derived_body(item: Any) -> Any: return item.this if isinstance(item, exp.Subquery) else None +def _branches(query: Any) -> list[Any]: + """一段查询的各支 SELECT,按出现的先后:UNION 这类拆开,外面套的括号去掉;认不出的返回空列表。""" + from sqlglot import exp + + while isinstance(query, exp.Subquery): + query = query.this + if isinstance(query, exp.SetOperation): + return _branches(query.this) + _branches(query.expression) + return [query] if isinstance(query, exp.Select) else [] + + def _own_nodes(select: Any) -> Iterator[Any]: """一段 SELECT 自己的节点:不进 CTE、子查询(它们是别的段)。""" from sqlglot import exp @@ -498,6 +546,69 @@ class _Source: query: Any = None # derived:里面那段查询 +@dataclass(frozen=True) +class _Derived: + """一个派生来源(CTE、FROM / JOIN 里的子查询、LATERAL 子查询)的结构标识,和起的名字、位置编号无关的部分都在 + 这里,diff_facets 拿它把两边的派生来源配成对(见 _align)。各项都用不写编号的写法(render 的 anon 模式:实表 + 写表名,往里套着的派生来源一律写「@」),所以两边写法一样就相同,不受先后、名字影响。""" + + number: int # 位置编号(@N 的 N):别的都分不出时才看先后 + name: str # CTE 名、子查询的别名(name_key),没有为空:辅助线索 + tables: tuple[str, ...] # 里面读到的实表:各支 FROM / JOIN,往里套着的也算;排好序 + columns: tuple[tuple[str, str], ...] # 选出来的各列 (列名, 规范化后的定义式),SELECT * 记成 ("*", "*") + group: tuple[str, ...] # 分组键,排好序 + conditions: frozenset[str] # 里面各段 WHERE、HAVING 的合取式:同一级配得上好几个时比重合 + body: str # 整段查询规范化后的写法 + + @property + def names(self) -> tuple[str, ...]: + """选出来的列名(外层按名字引用它们)。""" + return tuple(name for name, _ in self.columns) + + +#: 派生来源配对的各级,从严到宽:(这一级比什么, 是否只在两边都只剩一个时才配)。比什么返回 None 表示这一级没有 +#: 可比的(没有名字、没读实表),不配 +_ALIGN_LEVELS: tuple[tuple[Any, bool], ...] = ( + (lambda d: (d.body, d.columns), False), # 整段一样 + (lambda d: (d.tables, d.columns, d.group), False), # 读的表、各列定义、分组一样 + (lambda d: (d.tables, d.names), False), # 读的表、选出来的列名一样 + (lambda d: (d.tables, d.name) if d.name else None, False), # 读的表、名字一样 + (lambda d: d.tables or None, True), # 只是读的表一样 + (lambda d: d.name or None, True), # 只是名字一样(里面换了表) + (lambda d: d.names if any(n != "*" for n in d.names) else None, True), # 只是选出来的列名一样 +) + + +def _align(xs: list[_Derived], ys: list[_Derived]) -> dict[int, int]: + """两条语句的派生来源配对:{第二条里的位置编号: 第一条里的位置编号},配不上的不在里面。 + + 按 _ALIGN_LEVELS 从严到宽一级一级地配,每一级一对一:严的先配走,宽的只收拾剩下的。同一级几对都配得上时,先配 + 同名的,再配里面的条件重合最多的,最后按位置编号的先后(几个一模一样的派生来源按先后一一对应);这一级所有 + 配得上的对一起排,不是左边挨个挑,免得前一个挑走后一个更合适的。最后三级证据弱,只在两边各只剩一个这样的时才 + 配,免得猜错。哪一级都配不上的就不配:一边多出来的 CTE(盲算先筛一遍 VIP 用户)和另一边不相干的 CTE 凑成一对, + 会把外层不相干的条件说成「关联条件不同」。 + """ + rest_x, rest_y = sorted(xs, key=lambda d: d.number), sorted(ys, key=lambda d: d.number) + pairs: dict[int, int] = {} + for view, unique in _ALIGN_LEVELS: + candidates = [] + for x in rest_x: + seen = view(x) + if seen is None: + continue + found = [y for y in rest_y if view(y) == seen] + if unique and (len(found) != 1 or sum(1 for z in rest_x if view(z) == seen) != 1): + continue + candidates += [((y.name != x.name, -len(y.conditions & x.conditions), x.number, y.number), x, y) + for y in found] + for _, x, y in sorted(candidates, key=lambda c: c[0]): + if x.number not in pairs.values() and y.number not in pairs: + pairs[y.number] = x.number + rest_x = [x for x in rest_x if x.number not in pairs.values()] + rest_y = [y for y in rest_y if y.number not in pairs] + return pairs + + class _Statement: """一条语句的分析:CTE、实表、每段 SELECT 的来源,以及把表达式写成 key / text 的规则。""" @@ -552,14 +663,23 @@ class _Statement: CTE 名、子查询的别名是各自起的(双盲复算的盲算更是),拿它作 key,同一个口径只因起的名字不同就被报成不同; 拿里面那段查询的写法作 key 也不行:里面差一个条件,外层按它的列写的条件就全跟着变,同一处不同报好几遍,还会 - 出现两边各说一句「只保留 total > 100 的行」的自相矛盾。所以只按位置编:从最外层往里一层一层地编,同一层按 - 出现的先后;CTE 在第一次用到的地方编号,在哪儿用到都是这一个编号。写成 CTE 还是 FROM 里的子查询、CTE 定义 - 的先后都不影响编号,里层多一段、少一段也不挪动外层的编号。 + 出现两边各说一句「只保留 total > 100 的行」的自相矛盾。所以单看一条语句时按位置编:从最外层往里一层一层地 + 编,同一层按出现的先后;CTE 在第一次用到的地方编号,在哪儿用到都是这一个编号。写成 CTE 还是 FROM 里的子查询、 + CTE 定义的先后都不影响编号,里层多一段、少一段也不挪动外层的编号。两条语句之间编号错开(一边多一个 CTE、 + 内连接换了先后)由 diff_facets 按结构配对之后改成共同的编号(见 relabel)。 """ from sqlglot import exp - ctes: dict[str, str] = {} + ctes: dict[str, int] = {} count = 0 + #: 编了号的派生来源 [(位置编号, 起的名字, 里面那段查询)],按编号排:diff_facets 按它们的结构配对(见 derived) + self._derived: list[tuple[int, str, Any]] = [] + #: 记了编号的节点 {id: (节点, 位置编号)}:配好对之后按共同的编号改写(见 relabel) + self._marked: dict[int, tuple[Any, int]] = {} + + def mark(node: Any, number: int) -> None: + node.meta[_DERIVED] = f"@{number}" + self._marked[id(node)] = (node, number) def visit(select: Any) -> list[Any]: """给这一段还没编号的 FROM / JOIN 项编号,返回要接着往里看的查询(子查询、第一次用到的 CTE 的定义)。""" @@ -570,17 +690,19 @@ class _Statement: if body is not None: if _DERIVED not in item.meta: count += 1 - item.meta[_DERIVED] = f"@{count}" + mark(item, count) + self._derived.append((count, _alias(item), body)) inner.append(body) elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) and item.name \ and self._is_cte(item): name = name_key(item.name) if name not in ctes: count += 1 - ctes[name] = f"@{count}" - self._cte_defs[name].meta[_DERIVED] = ctes[name] + ctes[name] = count + mark(self._cte_defs[name], count) + self._derived.append((count, self._cte_defs[name].alias, self.ctes[name])) inner.append(self.ctes[name]) - item.meta[_DERIVED] = ctes[name] + mark(item, ctes[name]) return inner queue: deque[Any] = deque([self.root]) @@ -594,6 +716,86 @@ class _Statement: # 没走到的(没用到的 CTE 里的、表函数参数里的)按出现的先后补上:每个派生来源都得有编号 for select in self.root.find_all(exp.Select, bfs=False): visit(select) + # 没用到的 CTE 也编上号(排在最后):外层不引用它,可它照样要参加两边的配对。两边都定义了付款、退款两个 CTE, + # 一边关联付款的、一边关联退款的,没用到的那个不参加的话,两边各剩一个只有列名对得上的,会被凑成一对 + for name, cte in self._cte_defs.items(): + if name not in ctes: + count += 1 + ctes[name] = count + mark(cte, count) + self._derived.append((count, cte.alias, self.ctes[name])) + + def relabel(self, ids: Mapping[int, str]) -> None: + """派生来源改用 ids 给的编号({位置编号: 比较用的编号},没给的照旧写位置编号)。diff_facets 把两边的派生 + 来源配好对之后,第二条语句里配上的写第一条里对应的编号,两边比较用的写法才对得上。算好的来源、共有的列名 + 都带着旧编号,跟着清掉重算。""" + for node, number in self._marked.values(): + node.meta[_DERIVED] = ids.get(number, f"@{number}") + self._memo.clear() + self._shared = None + + def derived(self) -> list[_Derived]: + """各个派生来源的结构标识(见 _Derived),按位置编号排。""" + from sqlglot import exp + + out = [] + for number, name, body in self._derived: + columns: list[tuple[str, str]] = [] + group: list[str] = [] + conditions: set[str] = set() + for i, select in enumerate(_branches(body)): + scope = self.sources(select) + for clause in (select.args.get("where"), select.args.get("having")): + if clause is not None: + conditions.update(self.render(self.canonical(c, scope), scope, "anon") + for c in _conjuncts(clause.this)) + if i: + continue # UNION 选出来的列名、分组按第一支的算 + for item in select.selects: + if isinstance(item, exp.Star) or (isinstance(item, exp.Column) and item.is_star): + columns.append(("*", "*")) + continue + target = item.this if isinstance(item, exp.Alias) else item + columns.append((name_key(item.alias_or_name), + self.render(self.canonical(target, scope), scope, "anon"))) + if (clause := select.args.get("group")) is not None: + selects, aliases = select.selects, self._aliases(select) + group += [self.render(self.canonical(self._target(g, selects, aliases), scope), scope, "anon") + for g in clause.expressions] + out.append(_Derived(number=number, name=name_key(name) if name else "", + tables=tuple(sorted(self._row_tables(body, frozenset()))), + columns=tuple(sorted(columns)), group=tuple(sorted(group)), + conditions=frozenset(conditions), + body=self.render(self.canonical(body, {}), {}, "anon"))) + return out + + def _row_tables(self, query: Any, seen: frozenset[str]) -> set[str]: + """query 的行从哪些实表来:各支 FROM / JOIN 里的实表,往里套着的子查询、CTE 也算。条件里的子查询不算(它只 + 决定留哪些行,算进来的话一边多一个 IN (SELECT … FROM users) 的筛选,结构标识就整个变了)。seen 是这条路上 + 已经进过的 CTE:递归 CTE 引用自己时不再往里走。""" + from sqlglot import exp + + out: set[str] = set() + for select in _branches(query): + for item in self._items(select): + body = _derived_body(item) + if body is not None: + out |= self._row_tables(body, seen) + elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) and item.name: + if not self._is_cte(item): + out.add(self._table(item).key) + elif (name := name_key(item.name)) not in seen: + out |= self._row_tables(self.ctes[name], seen | {name}) + return out + + @classmethod + def reparse(cls, f: Facets) -> "_Statement | None": + """按 facets 记下的原文、表结构重新分析一遍(见 Facets._origin);没记、解析不了返回 None。""" + if f._origin is None: + return None + sql, schema = f._origin + root, _ = _parse(sql, f.dialect) + return cls(root, f.dialect, schema) if root is not None else None @staticmethod def _table(table: Any) -> _Source: @@ -721,9 +923,9 @@ class _Statement: return default if owner is None else self.sources(self.segments[owner.meta[_SEGMENT]]) def render(self, node: Any, sources: dict[str, _Source], mode: str = "text") -> str: - """五种写法(见 Part):key、anon、loose 比较用,名字按 name_key,key、anon 写真实表名、loose 不写(CTE、子查询 - 只有 key 写编号);text、full 给人看,text 只在好几张表都有这一列时写真实表名,full 一律写(关联条件说的就是 - 两张表各出哪一列,用 full)。 + """五种写法:key、loose(见 Part)、anon 比较用,名字按 name_key;key、anon 写实表的表名,loose 不写;CTE、 + 子查询 key 和 loose 写编号,anon 一律写「@」(算结构标识用,和编号无关,见 _Derived)。text、full 给人看 + (见 Part),text 只在好几张表都有这一列时写真实表名,full 一律写(关联条件说的就是两张表各出哪一列,用 full)。 node 里套着的子查询逐段改写:每一列按它所在那一段的来源认,子查询里的表去掉别名(列已经写成了真实表名)。 子查询里拿两张表的列互相比的(相关子查询),给人看时和关联条件一样写表名。比较用的写法里各自起的名字也去掉 @@ -754,7 +956,7 @@ class _Statement: if isinstance(ident, exp.Identifier): table.set(arg, exp.to_identifier(name_key(ident.name))) if mode in ("key", "loose", "anon"): - self._drop_names(copy, numbered=mode == "key") + self._drop_names(copy, numbered=mode != "anon") return copy.sql(dialect=self.read, comments=False) @staticmethod @@ -762,7 +964,8 @@ class _Statement: """比较用的写法里抹掉套着的子查询里各自起的名字(就地改):引用 CTE 的表、FROM 里的子查询(含 LATERAL)、 子查询里自带的 CTE 定义写编号(numbered 为假时写「@」;引用它们的列已经照这样写了);只看取值的子查询(IN、 EXISTS、标量子查询)去掉选出来的列名。FROM 里的子查询、CTE 定义选出来的列名留着:外层按这个名字引用它们, - 去掉了就分不出外层用的是哪一列。""" + 去掉了就分不出外层用的是哪一列。复制出来的那段本身没有父节点,认不出是 FROM 里的子查询,它选出来的列名也会 + 去掉(_Derived.body 另记了各列,见 derived)。""" from sqlglot import exp for node in copy.find_all(exp.Table, exp.Subquery, exp.Lateral, exp.CTE): @@ -788,8 +991,12 @@ class _Statement: quoted = isinstance(col.this, exp.Identifier) and bool(col.this.args.get("quoted")) if mode in ("key", "loose", "anon"): - owner = None if source is None or mode == "loose" else \ - _ANON if mode == "anon" and source.kind == "derived" else source.key + if source is None: + owner = None + elif source.kind == "derived": + owner = _ANON if mode == "anon" else source.key + else: + owner = None if mode == "loose" else source.key col.set("this", exp.to_identifier(name_key(name))) col.set("table", exp.to_identifier(owner) if owner else None) col.set("db", None) @@ -828,23 +1035,23 @@ class _Statement: return node.transform(fix) def _rank(self, node: Any, sources: dict[str, _Source]) -> tuple[str, str]: - """比较的两边谁排前面:先按不写编号的写法,分不出(同名的列)再按编号。只按编号排的话,编号错开的两条语句里 - 同一个比较会排成两个方向,按不写编号的写法也配不上了。""" + """比较的两边谁排前面:先按不写编号的写法(实表的列、派生来源的列分得开),分不出(两个派生来源的同名列) + 再按编号。diff_facets 重新分析时用的是两边按结构配好对的共同编号(见 relabel):p.total > r.total 和换了 + 关联先后、位置编号对调的同一句排出来是同一个方向,配对之前的位置编号决定不了先后。""" return self.render(node, sources, "anon"), self.render(node, sources, "key") def part(self, node: Any, sources: dict[str, _Source], *, excludes: bool = False) -> Part: refs: dict[tuple[str | None, str], str] = {} for col in _own_columns(node): source, name = self.resolve(col, sources) - owner = "" if source is None else _ANON if source.kind == "derived" else source.key - refs.setdefault((source.name if source is not None else None, name), f"{owner}.{name_key(name)}") + refs.setdefault((source.name if source is not None else None, name), + f"{source.key if source is not None else ''}.{name_key(name)}") full = self.render(node, sources, "full") # 拿两张表的列互相比的(相关子查询里的 o.id = r.order_id)和关联条件一样,给人看也写表名 across = len({table for table, _ in refs}) > 1 return Part(key=self.render(node, sources, "key"), text=full if across else self.render(node, sources), columns=tuple(refs), excludes=self._excludes(node, sources) if excludes else None, - loose=self.render(node, sources, "loose"), full=full, column_keys=tuple(refs.values()), - anon=self.render(node, sources, "anon")) + loose=self.render(node, sources, "loose"), full=full, column_keys=tuple(refs.values())) def _excludes(self, node: Any, sources: dict[str, _Source]) -> str | None: """排除写法去掉否定之后排除的是什么:a <> 1 → a = 1;NOT IN、NOT LIKE、NOT BETWEEN 同理;IS NOT NULL → 「a 为空」。 @@ -877,8 +1084,11 @@ class _Statement: group: list[Part] = [] aggregates: list[Aggregate] = [] having: list[Part] = [] + used: list[Part] = [] for select in self.root.find_all(exp.Select, bfs=False): sources = self.sources(select) + if select is self.root or not self._in_condition(select): + used += [Part.plain(s.key, s.name) for s in sources.values()] drafts = self._join_drafts(select, sources) clause = select.args.get("where") for cond in _conjuncts(clause.this) if clause is not None else []: @@ -903,6 +1113,7 @@ class _Statement: having=_dedupe(having, lambda p: p.key), limit=self._limit(), limits=tuple(self._limits()), + sources=_dedupe(used, lambda p: p.key), ) def _join_drafts(self, select: Any, sources: dict[str, _Source]) -> list[dict[str, Any]]: @@ -922,8 +1133,9 @@ class _Statement: conds = [self.canonical(exp.EQ(this=exp.column(ident.name, table=first_alias), expression=exp.column(ident.name, table=alias)), sources) for ident in join.args["using"]] + before = [s for key, s in list(sources.items())[:i + 1] if key != slot] drafts.append({"kind": _join_kind(join), "slot": slot, "source": sources.get(slot), "conds": conds, - "before": [s.name for key, s in list(sources.items())[:i + 1] if key != slot]}) + "before": [s.name for s in before], "before_keys": [s.key for s in before]}) return drafts def _joins_on_where(self, cond: Any, sources: dict[str, _Source], drafts: list[dict[str, Any]]) -> bool: @@ -950,17 +1162,25 @@ class _Statement: def _join(self, draft: dict[str, Any], sources: dict[str, _Source]) -> JoinFacet: source = draft["source"] table = Part.plain("?", "?") if source is None else \ - Part(key=source.key, text=source.name, loose=source.key, full=source.name, - anon=_ANON if source.kind == "derived" else source.key) + Part(key=source.key, text=source.name, loose=source.key, full=source.name) forms = sorted((self.render(c, sources, "key"), self.render(c, sources, "loose"), - self.render(c, sources, "full"), self.render(c, sources, "anon")) for c in draft["conds"]) + self.render(c, sources, "full")) for c in draft["conds"]) on = Part(key=" AND ".join(f[0] for f in forms), text=" AND ".join(f[2] for f in forms), - loose=" AND ".join(sorted(f[1] for f in forms)), full=" AND ".join(f[2] for f in forms), - anon=" AND ".join(sorted(f[3] for f in forms))) if forms else None + loose=" AND ".join(sorted(f[1] for f in forms)), full=" AND ".join(f[2] for f in forms)) \ + if forms else None mentioned = {table for c in draft["conds"] for table, _ in self.part(c, sources).columns} other = [name for name in dict.fromkeys(t for t in mentioned if t) if name != table.text] if draft["conds"] \ else draft["before"] - return JoinFacet(kind=draft["kind"], table=table, on=on, other=tuple(sorted(other))) + # 涉及的来源按条件里直接引用的算(slot 不为 None 的是这一段的来源):透过 CTE 找到的实表两边可能一样, + # 关联的却是两个不同的 CTE + sides = {source.key} if source is not None else set() + if draft["conds"]: + sides |= {found.key for c in draft["conds"] for col in _own_columns(c) + for slot, found in [self.locate(col, sources)] if slot is not None and found is not None} + else: + sides |= set(draft["before_keys"]) + return JoinFacet(kind=draft["kind"], table=table, on=on, other=tuple(sorted(other)), + sides=tuple(sorted(sides))) def _time(self, cond: Any, sources: dict[str, _Source]) -> TimePredicate | None: """时间条件:一边恰好一列(可以套着日期函数、CAST),另一边没有列;列名像日期,或者用了日期函数、像日期的字符串。""" @@ -1119,7 +1339,21 @@ class _Statement: skip = None # OFFSET 0 等于没写 if limit is None and skip is None: return None - return LimitFacet(count=count, offset=skip, order=self._order(query), scope=self._scope_name(query)) + return LimitFacet(count=count, offset=skip, order=self._order(query), scope=self._scope_name(query), + scope_key=self._scope_key(query)) + + @staticmethod + def _scope_key(query: Any) -> str | None: + """query 整段是哪个派生来源(CTE 的定义、FROM 里的子查询、LATERAL 子查询)的编号;最外层、UNION 的某一支、 + SELECT 列表里的子查询这类不是派生来源整段的返回 None。编号记在 CTE 定义、FROM / JOIN 项上,往外找一两层。""" + from sqlglot import exp + + node = query + while isinstance(node.parent, (exp.Subquery, exp.Lateral, exp.CTE)): + node = node.parent + if found := node.meta.get(_DERIVED): + return found + return None def _order(self, query: Any) -> tuple[tuple[Part, bool], ...]: """这一段的排序键 (Part, 是否降序)。ORDER BY 1、ORDER BY 别名换成 SELECT 里的表达式(UNION 按第一支的)。""" @@ -1145,7 +1379,9 @@ class _Statement: if query is self.root: return None parent = query.parent - if isinstance(parent, (exp.CTE, exp.Subquery)) and parent.alias: + if isinstance(parent, exp.Subquery) and not parent.alias and isinstance(parent.parent, exp.Lateral): + parent = parent.parent # LATERAL 的别名挂在 LATERAL 上,不在里面的子查询上 + if isinstance(parent, (exp.CTE, exp.Subquery, exp.Lateral)) and parent.alias: return parent.alias return "子查询" @@ -1232,11 +1468,12 @@ def _labeler(names: Mapping[str, str] | None) -> Any: def _single_column(part: Part, *, loose: bool) -> str | None: - """只涉及一列的条件:那一列(Part.column_keys:实表.列,CTE、子查询的列写「@.列」,两边起的名字不同、编号错开 - 也配得上;loose 时只看列名),用来把两边对同一列的不同写法配成一对。""" + """只涉及一列的条件:那一列(Part.column_keys:实表.列,CTE、子查询的列写「编号.列」),用来把两边对同一列的 + 不同写法配成一对。loose 时实表的列只看列名(表换了一张),派生来源的列照样带编号:两个 CTE 的同名列不是一列。""" if len(part.columns) != 1 or len(part.column_keys) != 1: return None - return name_key(part.columns[0][1]) if loose else part.column_keys[0] + key = part.column_keys[0] + return name_key(part.columns[0][1]) if loose and not key.startswith("@") else key def _verb(g: Aggregate) -> str: @@ -1261,7 +1498,8 @@ class _Compare: 两边各有一张只在自己这边出现、又不是关联进来的表(orders 换成了 orders_archive)时,其余各方面按不写表名的 loose 比:表换了这件事由一条「tables」说清,同一个条件、同一个聚合不再因为表名不同被重复报一遍。 - 每个方面都先按 k 比,比不上的再按 ak(CTE、子查询不写编号)一对一地配一遍(见 unmatched)。 + 派生来源的编号由 diff_facets 先按结构配好(两边配成一对的写同一个编号,见模块说明),这里各方面都只按 k 比: + 编号不同就是不同的来源,不再拿「不写编号」的写法补配(那样会把 p.total 和 r.total 当成同一列)。 """ def __init__(self, a: Facets, b: Facets, labels: tuple[str, str], names: Mapping[str, str] | None) -> None: @@ -1278,31 +1516,49 @@ class _Compare: return None return part.loose if self.swapped else part.key - def ak(self, part: Part | None) -> str | None: - """同 k,但 CTE、子查询不写编号(见 unmatched)。""" - if part is None: - return None - return part.loose if self.swapped else (part.anon or part.key) + def side(self, key: str) -> str: + """比较用的来源 key;换了一张表时,换的那两张记成同一个。""" + return "?" if self.swapped and key in (self.ta[0].key, self.tb[0].key) else key + + def sides(self, j: JoinFacet) -> tuple[str, ...]: + """连接涉及的来源(见 JoinFacet.sides)。""" + return tuple(sorted(self.side(s) for s in j.sides)) + + def added(self, j: JoinFacet, mine: Facets, theirs: Facets) -> tuple[str, set[str]]: + """一边独有的一次连接,多出来的是哪个来源:(给人看的名字, 比较用的 key)。 + + 一般是关联进来的那一侧;可内连接、CROSS JOIN 关联进来的那一侧另一边也有时,多出来的其实是这次连接接上的 + 另一头(FROM stores JOIN orders,另一边只查 orders:多出来的是 stores,不是 orders),说成「关联了 orders + 表」就错了。这时报涉及的来源里另一边没有的那几个。""" + if j.kind in ("inner", "cross"): + have = {self.side(s.key) for s in theirs.sources} + new = [k for k in j.sides if self.side(k) not in have] + if new and self.side(j.table.key) in have: + names = {s.key: s.text for s in mine.sources} + return "、".join(names.get(k, k) for k in new), set(new) + return j.table.text, {j.table.key} @staticmethod - def unmatched(xs: Iterable[Any], ys: Iterable[Any], key: Any, anon: Any) -> tuple[list[Any], list[Any]]: - """两边各自独有的项,各按原来的先后。先按 key 比;剩下的再按 anon(CTE、子查询不写编号)一对一地配,配上的 - 也不算不同。编号只看位置,一边多出一个排在前面的 CTE、子查询,后面的编号就错开了:同一个条件不能因此被说成 - 两边各有一个。编号对得上时 key 就分得清是哪个 CTE 的列,anon 只收拾 key 比不上的;两个 CTE 算出同名的列、 - 编号又错开时可能配错一对,宁可漏报。""" + def unmatched(xs: Iterable[Any], ys: Iterable[Any], key: Any) -> tuple[list[Any], list[Any]]: + """两边各自独有的项(按 key 比),各按原来的先后。派生来源在 key 里已经是两边按结构配好的共同编号(见 + diff_facets),这里不再拿不写编号的写法补配一遍:那样两个 CTE 的同名列(p.total、r.total)会被当成一列, + 换了用哪一个的列也报不出来。""" xs, ys = list(xs), list(ys) kx, ky = {key(x) for x in xs}, {key(y) for y in ys} - only_x = [x for x in xs if key(x) not in ky] - only_y = [y for y in ys if key(y) not in kx] - paired_x: set[int] = set() - paired_y: set[int] = set() - for i, x in enumerate(only_x): - j = next((j for j, y in enumerate(only_y) if j not in paired_y and anon(y) == anon(x)), None) - if j is not None: - paired_x.add(i) - paired_y.add(j) - return [x for i, x in enumerate(only_x) if i not in paired_x], \ - [y for j, y in enumerate(only_y) if j not in paired_y] + return [x for x in xs if key(x) not in ky], [y for y in ys if key(y) not in kx] + + @staticmethod + def lookalikes(only_a: list[Part], only_b: list[Part]) -> list[tuple[Part, Part]]: + """两边各自独有、给人看的写法却一样的条件配成一对,就地从两边拿掉:同一个条件挂在不同的来源上(p.total > 100 + 和 r.total > 100,各自只读得出 total > 100),分开各说一句「只保留 total > 100 的行」读起来自相矛盾。""" + pairs = [] + for p in list(only_a): + q = next((q for q in only_b if q.text == p.text), None) + if q is not None: + pairs.append((p, q)) + only_a.remove(p) + only_b.remove(q) + return pairs @staticmethod def shown(xs: list[str], ys: list[str], full_xs: list[str], full_ys: list[str]) -> tuple[str, str]: @@ -1332,7 +1588,7 @@ class _Compare: a, b, la, lb = self.a, self.b, self.la, self.lb ca = list(_dedupe((t.column for t in a.time), self.k)) cb = list(_dedupe((t.column for t in b.time), self.k)) - if any(self.unmatched(ca, cb, self.k, self.ak)): + if any(self.unmatched(ca, cb, self.k)): wa, wb = self.named(ca), self.named(cb) if wa == wb: wa, wb = self.named(ca, full=True), self.named(cb, full=True) @@ -1346,7 +1602,7 @@ class _Compare: for side, parts in (("a", ca), ("b", cb))} return [FacetDiff("time", "、".join(p.text for p in ca) or None, "、".join(p.text for p in cb) or None, text, detail)] - if any(self.unmatched(a.time, b.time, lambda t: self.k(t.condition), lambda t: self.ak(t.condition))): + if any(self.unmatched(a.time, b.time, lambda t: self.k(t.condition))): ra = " AND ".join(t.condition.text for t in a.time) rb = " AND ".join(t.condition.text for t in b.time) return [FacetDiff("time_range", ra, rb, _say(la, "的时间范围是", ra, ",", lb, "是", rb))] @@ -1356,8 +1612,9 @@ class _Compare: timed = {self.k(t.condition) for t in (*self.a.time, *self.b.time)} fa = [p for p in self.a.where if self.k(p) not in timed] fb = [p for p in self.b.where if self.k(p) not in timed] - only_a, only_b = self.unmatched(fa, fb, self.k, self.ak) + only_a, only_b = self.unmatched(fa, fb, self.k) out: list[FacetDiff] = [] + # 先配对同一列的不同写法(status = 'paid' 和 status IN (…)),再配读起来一样、挂在不同来源上的同一个条件 for p in list(only_a): column = _single_column(p, loose=self.swapped) mine = [q for q in only_a if column and _single_column(q, loose=self.swapped) == column] @@ -1368,6 +1625,9 @@ class _Compare: out.append(FacetDiff("filter", p.text, q.text, _say(self.la, "按", wa, "筛选,", self.lb, "按", wb, "筛选"))) only_a.remove(p) only_b.remove(q) + for p, q in self.lookalikes(only_a, only_b): + wa, wb = self.shown([p.text], [q.text], [p.full], [q.full]) + out.append(FacetDiff("filter", wa, wb, _say(self.la, "按", wa, "筛选,", self.lb, "按", wb, "筛选"))) out += [FacetDiff("filter", p.text, None, self.one_sided(self.la, self.lb, p)) for p in only_a] out += [FacetDiff("filter", None, p.text, self.one_sided(self.lb, self.la, p)) for p in only_b] return out @@ -1378,9 +1638,6 @@ class _Compare: def key(g: Aggregate) -> tuple[str, str | None, bool]: return g.func, self.k(g.arg), g.distinct - def anon(g: Aggregate) -> tuple[str, str | None, bool]: - return g.func, self.ak(g.arg), g.distinct - def pair(xs: list[Aggregate], ys: list[Aggregate], same: Any) -> list[tuple[Aggregate, Aggregate]]: pairs = [] for x in list(xs): @@ -1391,7 +1648,7 @@ class _Compare: ys.remove(y) return pairs - ga, gb = self.unmatched(self.a.aggregates, self.b.aggregates, key, anon) + ga, gb = self.unmatched(self.a.aggregates, self.b.aggregates, key) # 先配参数相同的(同一列:函数或去重不同),再配函数相同的(同一种算法:列不同);各剩一个的也配成一对 pairs = pair(ga, gb, lambda x, y: self.k(x.arg) == self.k(y.arg)) pairs += pair(ga, gb, lambda x, y: x.func == y.func) @@ -1419,15 +1676,26 @@ class _Compare: return out def joins(self) -> list[FacetDiff]: - """连接:先把完全一样的(类型、关联的表、条件)两边配掉,剩下的按关联的是哪张表配成一对、说类型或条件哪里 - 不同,再剩下的是只有一边关联了。直接按表配的话,编号错开时会把两个不相干的 CTE 配成一对,说成「关联条件 - 不同」;先配完全一样的,同一张表关联了两次、先后不同也不算不同。""" + """连接:先把完全一样的两边配掉,剩下的按关联的是哪张表(派生来源按配好的编号)、再按涉及的来源配成一对, + 说类型或条件哪里不同,再剩下的是只有一边关联了(多出来的是哪个来源见 added)。完全一样:内连接、CROSS JOIN + 比类型、涉及的来源和条件(FROM paid JOIN refunded 和 FROM refunded JOIN paid 是同一次连接),别的连接比类型、 + 关联进来的表和条件。先配完全一样的,同一张表关联了两次、先后不同也不算不同。配成一对的两边名字不同(两边给 + CTE 起的名字不同、内连接换了先后)时两个名字都说。派生来源已经按结构配好了编号:一边多出来的 CTE 编号和 + 另一边的都不一样,不会被配给另一边不相干的 CTE、说成「关联条件不同」。 + + 做不到的:LEFT JOIN 和两边对调的 RIGHT JOIN 意思一样,仍报成类型不同;内连接里只涉及一张表的条件写在 ON 里 + 还是 WHERE 里意思一样,仍算两处不同;三张以上不带关联条件的表(逗号连接、CROSS JOIN)换了先后,涉及的来源 + 按「它之前的各个来源」算,仍可能算不同。""" la, lb = self.la, self.lb - rest_a, rest_b = self.unmatched(self.a.joins, self.b.joins, - lambda j: (j.kind, j.table.key, self.k(j.on)), - lambda j: (j.kind, j.table.anon, self.ak(j.on))) + + def same(j: JoinFacet) -> tuple[Any, ...]: + if j.kind in ("inner", "cross"): + return j.kind, self.sides(j), self.k(j.on) + return j.kind, j.table.key, self.k(j.on) + + rest_a, rest_b = self.unmatched(self.a.joins, self.b.joins, same) pairs: list[tuple[JoinFacet, JoinFacet]] = [] - for table in (lambda j: j.table.key, lambda j: j.table.anon): + for table in (lambda j: j.table.key, self.sides): for x in list(rest_a): y = next((y for y in rest_b if table(y) == table(x)), None) if y is not None: @@ -1436,26 +1704,37 @@ class _Compare: rest_b.remove(y) out: list[FacetDiff] = [] for x, y in pairs: + tx, ty = x.table.text, y.table.text if x.kind != y.kind: kx, ky = _JOIN_WORDS.get(x.kind, x.kind), _JOIN_WORDS.get(y.kind, y.kind) - out.append(FacetDiff("join", kx, ky, _say(la, "用", kx, "关联", x.table.text, "表,", lb, "用", ky))) - if self.k(x.on) != self.k(y.on) and self.ak(x.on) != self.ak(y.on): + tail = () if ty == tx else ("关联", ty, "表") + out.append(FacetDiff("join", kx, ky, _say(la, "用", kx, "关联", tx, "表,", lb, "用", ky, *tail))) + if self.k(x.on) != self.k(y.on): xa, ya = (x.on.text if x.on else "(无条件)"), (y.on.text if y.on else "(无条件)") - out.append(FacetDiff("join", xa, ya, _say(la, "关联", x.table.text, "表的条件是", xa, ",", lb, "是", ya))) - out += [FacetDiff("join", x.table.text, None, _say(la, "关联了", x.table.text, "表,", lb, "没有")) for x in rest_a] - out += [FacetDiff("join", None, y.table.text, _say(lb, "关联了", y.table.text, "表,", la, "没有")) for y in rest_b] - # 关联之外的表(FROM 的表、子查询里的表)只在一边出现的 + tail = ("是", ya) if ty == tx else ("关联", ty, "表的条件是", ya) + out.append(FacetDiff("join", xa, ya, _say(la, "关联", tx, "表的条件是", xa, ",", lb, *tail))) + said: dict[str, set[str]] = {"a": set(), "b": set()} + for side, rest, mine, theirs in (("a", rest_a, self.a, self.b), ("b", rest_b, self.b, self.a)): + me, other = (la, lb) if side == "a" else (lb, la) + for j in rest: + name, keys = self.added(j, mine, theirs) + said[side] |= keys + out.append(FacetDiff("join", name if side == "a" else None, name if side == "b" else None, + _say(me, "关联了", name, "表,", other, "没有"))) + # 关联之外的表(FROM 的表、子查询里的表)只在一边出现的;上面已经说过「关联了它」的不再说一遍 if self.swapped: ta, tb = self.ta[0], self.tb[0] return out + [FacetDiff("tables", ta.text, tb.text, _say(la, "查的是", ta.text, "表,", lb, "查的是", tb.text, "表"))] - out += [FacetDiff("tables", t.text, None, _say(la, "用到了", t.text, "表,", lb, "没有")) for t in self.ta] - out += [FacetDiff("tables", None, t.text, _say(lb, "用到了", t.text, "表,", la, "没有")) for t in self.tb] + out += [FacetDiff("tables", t.text, None, _say(la, "用到了", t.text, "表,", lb, "没有")) + for t in self.ta if t.key not in said["a"]] + out += [FacetDiff("tables", None, t.text, _say(lb, "用到了", t.text, "表,", la, "没有")) + for t in self.tb if t.key not in said["b"]] return out def group(self) -> list[FacetDiff]: a, b = self.a.group_by, self.b.group_by - if not any(self.unmatched(a, b, self.k, self.ak)): + if not any(self.unmatched(a, b, self.k)): return [] wa, wb = self.shown([p.text for p in a], [p.text for p in b], [p.full for p in a], [p.full for p in b]) if wa and wb: @@ -1467,30 +1746,45 @@ class _Compare: return [FacetDiff("group_by", wa or None, wb or None, text)] def having(self) -> list[FacetDiff]: - only_a, only_b = self.unmatched(self.a.having, self.b.having, self.k, self.ak) - return [FacetDiff("having", p.text, None, self.one_sided(self.la, self.lb, p, group=True)) for p in only_a] + \ + only_a, only_b = self.unmatched(self.a.having, self.b.having, self.k) + out = [] + for p, q in self.lookalikes(only_a, only_b): + wa, wb = self.shown([p.text], [q.text], [p.full], [q.full]) + out.append(FacetDiff("having", wa, wb, _say(self.la, "分组后只保留", wa, "的组,", self.lb, "只保留", wb, + "的组"))) + return out + \ + [FacetDiff("having", p.text, None, self.one_sided(self.la, self.lb, p, group=True)) for p in only_a] + \ [FacetDiff("having", None, p.text, self.one_sided(self.lb, self.la, p, group=True)) for p in only_b] def limit(self) -> list[FacetDiff]: - """取哪几行:最外层两边比;CTE、子查询里的按出现的先后一一配对(两边给它起的名字可以不同),整组一样就不报。""" + """取哪几行:最外层两边比;CTE、子查询里的按所属派生来源配好的编号(LimitFacet.scope_key)配成一对,两边给 + 它起的名字可以不同。配不上的派生来源各说各的,不按先后凑对:一边多一个限了行数的 CTE,不能拿它和另一边的 + CTE 比。不属于哪个派生来源整段的(UNION 的某一支这类)按出现的先后配。""" def outer(f: Facets) -> LimitFacet | None: return next((x for x in f.limits if x.scope is None), None) out = self.rows(outer(self.a), outer(self.b)) ia = [x for x in self.a.limits if x.scope is not None] ib = [x for x in self.b.limits if x.scope is not None] - if [self.lk(x) for x in ia] != [self.lk(x) for x in ib] and \ - [self.lk(x, anon=True) for x in ia] != [self.lk(x, anon=True) for x in ib]: - for x, y in zip_longest(ia, ib): - out += self.rows(x, y) + pairs: list[tuple[LimitFacet | None, LimitFacet | None]] = [] + for x in list(ia): + y = next((y for y in ib if x.scope_key is not None and y.scope_key == x.scope_key), None) + if y is not None: + pairs.append((x, y)) + ia.remove(x) + ib.remove(y) + pairs += zip_longest([x for x in ia if x.scope_key is None], [y for y in ib if y.scope_key is None]) + pairs += [(x, None) for x in ia if x.scope_key is not None] + pairs += [(None, y) for y in ib if y.scope_key is not None] + for x, y in pairs: + out += self.rows(x, y) return out - def lk(self, x: LimitFacet | None, *, anon: bool = False) -> tuple[Any, ...] | None: - """一段取哪几行的比较用写法:行数、跳过的行数、排序键。anon 时排序键按 ak(见 unmatched)。""" + def lk(self, x: LimitFacet | None) -> tuple[Any, ...] | None: + """一段取哪几行的比较用写法:行数、跳过的行数、排序键。""" if x is None: return None - key = self.ak if anon else self.k - return x.count, x.offset, tuple((key(p), desc) for p, desc in x.order) + return x.count, x.offset, tuple((self.k(p), desc) for p, desc in x.order) def sorting(self, x: LimitFacet, *, full: bool = False) -> str: if not x.order: @@ -1500,7 +1794,7 @@ class _Compare: def rows(self, x: LimitFacet | None, y: LimitFacet | None) -> list[FacetDiff]: """一对段(一边可以没有)取的行有什么不同:行数、跳过的行数,两边都限了行时再比排序。""" - if self.lk(x) == self.lk(y) or self.lk(x, anon=True) == self.lk(y, anon=True): + if self.lk(x) == self.lk(y): return [] sa, sb = (x.scope if x else None), (y.scope if y else None) la = _say(self.la, "在", sa, "里") if sa else self.la @@ -1512,8 +1806,7 @@ class _Compare: oa, ob = (x.offset if x else None), (y.offset if y else None) if oa != ob: out.append(FacetDiff("limit", oa, ob, _say(la, _skip(oa), ",", lb, _skip(ob)))) - if x is not None and y is not None and self.lk(x)[2] != self.lk(y)[2] \ - and self.lk(x, anon=True)[2] != self.lk(y, anon=True)[2]: + if x is not None and y is not None and self.lk(x)[2] != self.lk(y)[2]: wa, wb = self.sorting(x), self.sorting(y) if wa == wb: wa, wb = self.sorting(x, full=True), self.sorting(y, full=True) @@ -1529,6 +1822,8 @@ def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, labels 是两边的叫法(缺省 A / B,双盲复算可以传「原流程」「盲算」);names 是列的中文名({「表.列」或「列」: 中文名},可以用 catalog_labels 从数据目录取),给了的话时间列写成「支付时间 paid_at」。任一边解析不了就只返回 一条 parse,不逐项比较:拿半份结构去比,只会报出一堆不存在的不同。 + + 比之前先把两边的派生来源按结构配好对,按共同的编号重新分析 b(见模块说明、_aligned);给人看的写法不受影响。 """ la, lb = labels if a.parse_error or b.parse_error: @@ -1538,9 +1833,30 @@ def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, side, error = (la, a.parse_error) if a.parse_error else (lb, b.parse_error) text = _say(side, ":", error or "", ",没有逐项比较") return [FacetDiff("parse", a.parse_error, b.parse_error, text)] - cmp = _Compare(a, b, labels, names) + cmp = _Compare(a, _aligned(a, b), labels, names) return cmp.time() + cmp.filters() + cmp.aggregates() + cmp.joins() + cmp.group() + cmp.having() + cmp.limit() +def _aligned(a: Facets, b: Facets) -> Facets: + """b 的派生来源按结构配给 a 的(见 _align),配上的改写成 a 里对应的编号、配不上的另起 a 里没有的编号,按这套 + 编号重新分析 b 的原文。a 的编号就是它的位置编号,不用重算。 + + 任一边不是 facets 算出来的(没记原文)、重新解析不了,或者分析时出了意外,照原样返回 b:两边仍按各自的位置 + 编号比,一边多出排在前面的 CTE 时会多报。 + """ + try: + sa, sb = _Statement.reparse(a), _Statement.reparse(b) + if sa is None or sb is None: + return b + da, db = sa.derived(), sb.derived() + pairs = _align(da, db) + fresh = iter(range(max((d.number for d in da), default=0) + 1, 1 << 30)) + sb.relabel({d.number: f"@{pairs[d.number]}" if d.number in pairs else f"@{next(fresh)}" for d in db}) + return replace(sb.facets(), _origin=b._origin) + except Exception: # noqa: BLE001 - 配对本身的缺陷不能连累比较:退回按各自的编号比,记下来修 + logger.warning("SQL 派生来源配对失败,按各自的位置编号比较", exc_info=True) + return b + + __all__ = ["MAX_SQL_CHARS", "PLACEHOLDER", "Aggregate", "ColumnRef", "FacetDiff", "Facets", "JoinFacet", "LimitFacet", "NormalizedSql", "Part", "TimePredicate", "catalog_labels", "diff_facets", "facets", "normalize_sql"] diff --git a/backend/tests/test_sql_semantics.py b/backend/tests/test_sql_semantics.py index 1e62c3ab..f4afdb2f 100644 --- a/backend/tests/test_sql_semantics.py +++ b/backend/tests/test_sql_semantics.py @@ -4,11 +4,12 @@ 不同,不能算模板变了);关键字大小写、空白、注释也不算不同;解析不了的退回逐词规范化,照样认得出只改了字面量 - facets:表、连接(类型 / 关联进来的表 / ON 条件)、WHERE 合取式、时间条件(列 / 运算符 / 取值)、分组、聚合 (函数 / 参数 / 去重)、HAVING、取哪几行(各段的 LIMIT / OFFSET / 排序)。别名换成真实表名(条件里套着的子查询、 - HAVING 里的别名也是),CTE、子查询按位置编号、不按起的名字比,条件里的子查询选出来的列名不算;比较的两边排好 - 顺序;解析不了带 parse_error,不抛 + HAVING 里的别名也是),CTE、子查询不按起的名字比(单看一条按位置编号),条件里的子查询选出来的列名不算;比较的 + 两边排好顺序;解析不了带 parse_error,不抛 - diff_facets:每处不同一句中文:「A 按支付时间 paid_at 筛时间,B 按下单时间 created_at」「A 排除了 - is_deleted = 1 的行,B 没有」「A 对 order_id 去重计数,B 没去重」……写法不同、口径相同的不报,一边多出一个排在 - 前面的 CTE、编号错开了也不报 + is_deleted = 1 的行,B 没有」「A 对 order_id 去重计数,B 没去重」……写法不同、口径相同的不报。两边的派生来源先按 + 结构(读的表、各列的定义、名字)配好对再比:一边多出一个排在前面的 CTE、内连接换了先后不报,两个 CTE 算出同名 + 的列时外层用的是哪一个照报 夹具全是虚构的表:订单 orders、退款 refunds、门店 stores、用户 users、付款 payments,以及一张中文表「订单」。 """ @@ -273,6 +274,7 @@ def test_facets_are_json_ready(): f = facets("SELECT COUNT(DISTINCT order_id) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY region") data = json.loads(json.dumps(f.as_dict(), ensure_ascii=False)) assert data["aggregates"][0]["distinct"] is True and data["time"][0]["op"] == ">=" + assert not [k for k in data if k.startswith("_")], "拿来重新分析的原文不进 as_dict" # -------------------------------------------------------------------------- @@ -552,8 +554,9 @@ SHIFTED_B = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " def test_a_shifted_derived_number_does_not_split_one_condition_in_two(): - """编号只看位置:一边多出一个排在前面的 CTE(筛 VIP 用户),后面那个 CTE 的编号就错开了。按编号比不上的再按 - 不写编号的写法一对一地配,同一个条件不会被说成两边各有一个;真正的不同(VIP 用 IN 筛还是关联进来)照报。""" + """位置编号只看先后:一边多出一个排在前面的 CTE(筛 VIP 用户),后面那个 CTE 的位置编号就错开了。两边的派生 + 来源先按结构配好对(t 配 t,vip 配不上),按共同的编号比,同一个条件不会被说成两边各有一个;真正的不同(VIP + 用 IN 筛还是关联进来)照报。""" found = diff_facets(facets(SHIFTED_A), facets(SHIFTED_B), labels=("原流程", "盲算")) filters = [d.text for d in found if d.aspect == "filter"] assert filters == ["原流程只保留 user_id IN (SELECT id FROM users WHERE vip = 1) 的行,盲算没有这个条件"] @@ -605,7 +608,8 @@ def test_a_shifted_derived_number_is_no_difference_in_any_aspect(aspect, a, b): def test_a_shifted_number_inside_a_condition_subquery_is_no_difference(): - """条件里的子查询用到的 CTE 编号错开了:子查询整个写进外层条件的那一份,按不写编号的写法也写「@」,照样配得上。""" + """条件里的子查询用到的 CTE 位置编号错开了:两边的 t 按结构配成一对、写同一个编号,子查询整个写进外层条件的 + 那一份照样对得上。""" a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " "SELECT COUNT(*) FROM users u WHERE u.vip = 1 AND u.id IN (SELECT user_id FROM t WHERE total > 100)") b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " @@ -615,8 +619,8 @@ def test_a_shifted_number_inside_a_condition_subquery_is_no_difference(): def test_a_shifted_derived_join_is_paired_with_its_counterpart(): - """关联进来的 CTE 编号错开了(盲算多关联了一个排在前面的 CTE):先把完全一样的关联配掉,再按关联的是哪张表配, - 不会把原流程的 t 和盲算的 vip 当成同一处关联、说成「关联条件不同」。""" + """关联进来的 CTE 位置编号错开了(盲算多关联了一个排在前面的 CTE):两边的派生来源先按结构配好对,原流程的 t + 配盲算的 t,不会和盲算的 vip 当成同一处关联、说成「关联条件不同」。""" a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " "SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id WHERE t.total > 100") b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " @@ -627,8 +631,8 @@ def test_a_shifted_derived_join_is_paired_with_its_counterpart(): def test_swapped_join_order_of_two_derived_tables_is_no_difference(): - """两个 CTE 关联的先后对调,编号也跟着对调:拿它们的列互相比的条件,两边先按不写编号的写法排好两边的先后, - 才配得上(只按编号排,同一个比较会排成两个方向)。""" + """两个 CTE 关联的先后对调,位置编号也跟着对调:两边按结构配好对、写同一个编号之后,拿它们的列互相比的条件 + 排成同一个方向(只按位置编号排,同一个比较会排成两个方向),关联按涉及的两边比,不按哪一侧是关联进来的。""" ctes = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " "u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) ") assert texts(ctes + "SELECT COUNT(*) FROM t JOIN u ON u.user_id = t.user_id WHERE t.total > u.cnt", @@ -636,8 +640,8 @@ def test_swapped_join_order_of_two_derived_tables_is_no_difference(): def test_a_reversed_comparison_between_two_derived_tables_is_still_reported(): - """两个 CTE 算出了同名的列:编号对得上时照样分得清是哪个 CTE 的,比较的方向反了要报出来(不写编号的写法只在 - 按编号比不上时才拿来配,不会把它吞掉)。""" + """两个 CTE 算出了同名的列:按结构配好对之后照样分得清是哪个 CTE 的,比较的方向反了要报出来(只按配好的编号 + 比,不拿不写编号的写法补配,不会把它吞掉)。""" a = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total") @@ -660,6 +664,209 @@ def test_a_real_difference_next_to_renamed_derived_tables_is_reported_once(): assert texts(a, outer, labels=("原流程", "盲算")) == ["原流程按 total > 100 筛选,盲算按 total > 200 筛选"] +# -------------------------------------------------------------------------- +# diff_facets:两边的派生来源按结构配对 +# -------------------------------------------------------------------------- + +#: 两个 CTE 算出了同名的列 total:一个是付款合计,一个是退款合计 +PAID_REFUNDED = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) ") +PR_JOIN = "FROM paid p JOIN refunded r ON r.user_id = p.user_id" +#: 同样两段写成 FROM 里的子查询 +PR_SUBQUERIES = ("FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p " + "JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r " + "ON r.user_id = p.user_id") + + +@pytest.mark.parametrize("form", ["cte", "subquery"]) +@pytest.mark.parametrize("a,b,expected", [ + ("SELECT SUM(p.total) {j}", "SELECT SUM(r.total) {j}", + ["原流程对 paid.total 求和,盲算对 refunded.total 求和"]), + ("SELECT SUM(p.total - r.total) {j}", "SELECT SUM(r.total - p.total) {j}", + ["原流程对 paid.total - refunded.total 求和,盲算对 refunded.total - paid.total 求和"]), + ("SELECT COUNT(*) {j} WHERE p.total > 100", "SELECT COUNT(*) {j} WHERE r.total > 100", + ["原流程按 paid.total > 100 筛选,盲算按 refunded.total > 100 筛选"]), + ("SELECT p.user_id {j} ORDER BY p.total DESC LIMIT 10", "SELECT p.user_id {j} ORDER BY r.total DESC LIMIT 10", + ["原流程取行前按 paid.total 降序排,盲算按 refunded.total 降序排"]), + ("SELECT p.user_id {j} GROUP BY p.user_id HAVING MAX(p.total) > 10", + "SELECT p.user_id {j} GROUP BY p.user_id HAVING MAX(r.total) > 10", + ["原流程对 paid.total 取最大值,盲算对 refunded.total 取最大值", + "原流程分组后只保留 MAX(paid.total) > 10 的组,盲算只保留 MAX(refunded.total) > 10 的组"]), + ("SELECT COUNT(*) {j} WHERE p.total - r.total > 100", "SELECT COUNT(*) {j} WHERE r.total - p.total > 100", + ["原流程只保留 paid.total - refunded.total > 100 的行,盲算没有这个条件", + "盲算只保留 refunded.total - paid.total > 100 的行,原流程没有这个条件"]), +], ids=["sum", "sum-of-difference", "where", "order-by", "having", "where-difference"]) +def test_which_derived_table_a_column_comes_from_is_reported(form, a, b, expected): + """两个派生来源算出了同名的列(付款合计、退款合计都叫 total),编号也对得上:外层用的是哪一个的 total 不同, + 聚合、条件、排序、HAVING 里都得照报,不能因为都是「某个 CTE 的 total」就当成一样。只读得出列名的写法两边 + 一样时,改用带来源名字的写法说。""" + if form == "cte": + head, j = PAID_REFUNDED, PR_JOIN + else: + head, j = "", PR_SUBQUERIES + expected = [t.replace("paid.", "p.").replace("refunded.", "r.") for t in expected] + assert texts(head + a.format(j=j), head + b.format(j=j), labels=("原流程", "盲算")) == expected + + +def test_renamed_and_reordered_derived_tables_still_tell_their_columns_apart(): + """盲算给两个 CTE 另起了名字、定义和关联的先后也反过来:按里面读的表和各列的定义认出谁对谁,外层用的是 + 哪一个的列照样分得清;同一列只是名字、先后不同的不报。""" + b_head = ("WITH ref_sum AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "pay_sum AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) ") + b_join = "FROM ref_sum r JOIN pay_sum p ON p.user_id = r.user_id" + labels = ("原流程", "盲算") + assert texts(PAID_REFUNDED + f"SELECT SUM(p.total) {PR_JOIN}", + b_head + f"SELECT SUM(r.total) {b_join}", labels=labels) == \ + ["原流程对 paid.total 求和,盲算对 ref_sum.total 求和"] + assert texts(PAID_REFUNDED + f"SELECT COUNT(*) {PR_JOIN} WHERE p.total > 100", + b_head + f"SELECT COUNT(*) {b_join} WHERE r.total > 100", labels=labels) == \ + ["原流程按 paid.total > 100 筛选,盲算按 ref_sum.total > 100 筛选"] + assert texts(PAID_REFUNDED + f"SELECT SUM(p.total) {PR_JOIN} WHERE p.total > r.total", + b_head + f"SELECT SUM(p.total) {b_join} WHERE r.total < p.total") == [] + + +def test_an_extra_derived_table_on_one_side_is_not_paired_with_another(): + """盲算多一个排在前面的 CTE(vip),位置编号就错开了:原流程的 t 要和盲算的 t 配成一对(里面读的表、各列的 + 定义都一样),不能按编号配给 vip。""" + a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id") + b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " + "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id JOIN t ON t.user_id = u.referrer_id") + assert texts(a, b, labels=("原流程", "盲算")) == [ + "盲算只保留 vip = 1 的行,原流程没有这个条件", + "原流程关联 t 表的条件是 orders.user_id = users.id,盲算是 orders.user_id = users.referrer_id", + "盲算关联了 vip 表,原流程没有"] + # 两边各一个、结构毫不相干的派生来源也不配成一对:各说各的 + other = ("WITH vip AS (SELECT id FROM users WHERE vip = 1) " + "SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id") + joins = [d.text for d in diff_facets(facets(a), facets(other), labels=("原流程", "盲算")) if d.aspect == "join"] + assert joins == ["原流程关联了 t 表,盲算没有", "盲算关联了 vip 表,原流程没有"] + + +def test_unused_ctes_take_part_in_pairing(): + """两边都定义了付款、退款两个 CTE,原流程关联付款的、盲算关联退款的:没用到的那个也参加配对,付款配付款、退款 + 配退款,不会把两边各自用到的那个只凭列名都叫 user_id、total 凑成一对,说成「关联条件不同」。""" + a = PAID_REFUNDED + "SELECT COUNT(*) FROM users u JOIN paid p ON p.user_id = u.id" + b = PAID_REFUNDED + "SELECT COUNT(*) FROM users u JOIN refunded r ON r.user_id = u.id" + assert texts(a, b, labels=("原流程", "盲算")) == ["原流程关联了 paid 表,盲算没有", "盲算关联了 refunded 表,原流程没有"] + + +@pytest.mark.parametrize("a,b,expected", [ + ("SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE o.status = 'paid'", + ["盲算关联了 stores 表,原流程没有"]), + ("SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM users u JOIN refunds r ON r.user_id = u.id JOIN orders o ON o.id = r.order_id", + ["盲算关联了 users 表,原流程没有"]), + (PAID_REFUNDED + f"SELECT COUNT(*) {PR_JOIN} WHERE p.total > r.total", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), " + "paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " + "SELECT COUNT(*) FROM vip v JOIN refunded r ON r.user_id = v.id JOIN paid p ON p.user_id = r.user_id " + "WHERE p.total > r.total", + ["盲算只保留 vip = 1 的行,原流程没有这个条件", "盲算关联了 vip 表,原流程没有", "盲算用到了 users 表,原流程没有"]), + # 原流程只在条件的子查询里用到 users:那不是连接的一头,盲算多关联的还是 users + ("SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT id FROM users WHERE vip = 1)", + "SELECT SUM(o.amount) FROM users u JOIN orders o ON o.user_id = u.id WHERE u.vip = 1", + ["原流程只保留 user_id IN (SELECT id FROM users WHERE vip = 1) 的行,盲算没有这个条件", + "盲算关联了 users 表,原流程没有"]), +], ids=["real-table", "real-table-chain", "derived-shifted-and-swapped", "in-subquery-is-not-a-join"]) +def test_an_extra_source_written_in_from_is_named_correctly(a, b, expected): + """盲算多出来的来源写在 FROM 里、靠内连接接上原流程也有的表(FROM stores JOIN orders):多出来的是 stores, + 不能说成「盲算关联了 orders 表」(原流程查的就是 orders);已经说了「关联了 stores 表」的,不再另说一遍「用到了 + stores 表」。""" + assert texts(a, b, labels=("原流程", "盲算")) == expected + + +def test_a_lateral_subquery_is_named_by_its_alias(): + """LATERAL 子查询的别名挂在 LATERAL 上:取哪几行的说明里照写它,不写「子查询」。""" + a = ("SELECT o.id FROM orders o, LATERAL (SELECT amount FROM refunds r WHERE r.order_id = o.id " + "ORDER BY amount DESC LIMIT 1) l") + b = ("SELECT o.id FROM orders o CROSS JOIN LATERAL (SELECT amount FROM refunds r WHERE r.order_id = o.id " + "ORDER BY amount DESC LIMIT 2) AS x") + assert texts(a, b, "postgres") == ["A 在 l 里只取前 1 行,B 在 x 里只取前 2 行"] + + +def test_a_paired_derived_join_with_another_name_says_both_names(): + """配成一对的派生来源两边名字不同(原流程叫 t、盲算叫 totals):关联条件不同时两个名字都说出来。""" + a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id") + b = ("WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM users u JOIN totals ON totals.user_id = u.referrer_id") + assert texts(a, b, labels=("原流程", "盲算")) == [ + "原流程关联 t 表的条件是 orders.user_id = users.id,盲算关联 totals 表的条件是 orders.user_id = users.referrer_id"] + left = b.replace("JOIN totals ON totals.user_id = u.referrer_id", "LEFT JOIN totals ON totals.user_id = u.id") + assert texts(a, left, labels=("原流程", "盲算")) == ["原流程用 INNER JOIN 关联 t 表,盲算用 LEFT JOIN 关联 totals 表"] + + +@pytest.mark.parametrize("a,b", [ + (PAID_REFUNDED + f"SELECT COUNT(*) {PR_JOIN} WHERE p.total > r.total", + PAID_REFUNDED + "SELECT COUNT(*) FROM refunded r JOIN paid p ON p.user_id = r.user_id WHERE p.total > r.total"), + (f"SELECT COUNT(*) {PR_SUBQUERIES} WHERE p.total > r.total", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r " + "JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id " + "WHERE p.total > r.total"), + # 两个 CTE 读同一张表、各列的定义也一样,只差筛选条件:按整段的写法认得出谁是谁 + ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " + "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) " + "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > o.total", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " + "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) " + "SELECT COUNT(*) FROM open o JOIN paid p ON p.user_id = o.user_id WHERE o.total < p.total"), + # 实表也一样:内连接换个先后写,意思不变 + ("SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.amount > r.amount", + "SELECT SUM(o.amount) FROM refunds r JOIN orders o ON o.id = r.order_id WHERE o.amount > r.amount"), + ("SELECT SUM(o.amount) FROM orders o, refunds r WHERE r.order_id = o.id", + "SELECT SUM(o.amount) FROM refunds r, orders o WHERE r.order_id = o.id"), +], ids=["cte", "subquery", "same-shape-ctes", "real-tables", "comma-join"]) +def test_swapping_the_order_of_an_inner_join_is_no_difference(a, b): + """内连接两边换个先后(FROM paid JOIN refunded 换成 FROM refunded JOIN paid)意思不变:派生来源按结构配对, + 比较的两边按配好的编号排,不会排成两个方向、报出两条自相矛盾的话;连接按涉及的两边和条件比,不按哪一侧是 + 关联进来的。""" + assert texts(a, b) == [] + + +def test_two_same_shape_derived_tables_are_paired_by_their_conditions(): + """两个 CTE 读同一张表、各列定义也一样(付款的、未付的),盲算两个都另起了名字、都多加了 is_deleted = 0、关联的 + 先后也反过来:结构上只剩里面的条件分得开,按条件的重合配(status = 'paid' 配 status = 'paid'),只报多出来的那个 + 条件,外层拿两者比的条件不跟着报。""" + a = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " + "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) " + "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > o.total") + b = ("WITH unpaid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' AND is_deleted = 0 " + "GROUP BY user_id), " + "settled AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' AND is_deleted = 0 " + "GROUP BY user_id) " + "SELECT COUNT(*) FROM unpaid u JOIN settled s ON s.user_id = u.user_id WHERE u.total < s.total") + assert texts(a, b, labels=("原流程", "盲算")) == ["盲算只保留 is_deleted = 0 的行,原流程没有这个条件"] + + +def test_limits_inside_derived_tables_are_paired_by_structure(): + """CTE 里取哪几行按配好对的派生来源比,不按出现的先后:盲算多一个排在前面、也限了行数的 CTE,不能拿它和 + 原流程的 top 比。""" + a = ("WITH top AS (SELECT user_id, amount FROM orders ORDER BY amount DESC LIMIT 10) " + "SELECT SUM(amount) FROM top") + b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1 LIMIT 5), " + "top AS (SELECT user_id, amount FROM orders ORDER BY amount DESC LIMIT 10) " + "SELECT SUM(top.amount) FROM top JOIN vip ON vip.id = top.user_id") + found = [d.text for d in diff_facets(facets(a), facets(b), labels=("原流程", "盲算")) if d.aspect == "limit"] + assert found == ["原流程不限行数,盲算在 vip 里只取前 5 行"] + + +def test_a_changed_column_inside_a_renamed_derived_table_is_said_once(): + """CTE 里一列的算法变了(对 amount 求和换成对 paid_amount 求和),名字也另起了:按读的表和选出来的列名仍配成 + 一对,只报里面那处不同,外层按这一列写的条件不跟着报成两边各有一个。""" + a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE total > 100") + b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " + "x AS (SELECT user_id, SUM(paid_amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM vip JOIN x ON x.user_id = vip.id WHERE total > 100") + found = diff_facets(facets(a), facets(b), labels=("原流程", "盲算")) + assert "原流程对 amount 求和,盲算对 paid_amount 求和" in [d.text for d in found] + assert not [d for d in found if d.aspect == "filter" and "total" in d.text] + + @pytest.mark.parametrize("a,b,text", [ ("WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM top", "WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 100) SELECT SUM(amount) FROM top", From 3e0ab059b1c9fef35406da20f9f4965ec352ab5b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 04:58:17 +0800 Subject: [PATCH 2/6] =?UTF-8?q?SQL=20=E8=AF=AD=E4=B9=89=E5=A4=8D=E6=A0=B8?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=88=E7=AC=AC=204=20=E8=BD=AE=EF=BC=89?= =?UTF-8?q?=EF=BC=9A=E5=A4=96=E8=BF=9E=E6=8E=A5=E6=8C=89=E4=B8=BB=E8=A1=A8?= =?UTF-8?q?=E6=AF=94=E3=80=81=E7=AE=97=E5=87=BA=E6=9D=A5=E7=9A=84=E5=88=97?= =?UTF-8?q?=E6=8C=89=E5=AE=9A=E4=B9=89=E6=AF=94=E3=80=81=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E5=88=97=E9=80=8F=E8=BF=87=20CTE=20=E8=AE=A4=E3=80=81=E7=AD=9B?= =?UTF-8?q?=E7=9A=84=E5=85=88=E5=90=8E=E7=85=A7=E6=8A=A5=E3=80=81=E6=B2=A1?= =?UTF-8?q?=E7=94=A8=E5=88=B0=E7=9A=84=20CTE=20=E4=B8=8D=E7=AE=97=EF=BC=8C?= =?UTF-8?q?=E5=90=8C=E5=90=8D=E5=8D=B4=E4=B8=8D=E6=98=AF=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E7=9A=84=E6=B4=BE=E7=94=9F=E6=9D=A5=E6=BA=90=E8=AF=B4?= =?UTF-8?q?=E6=B8=85=E5=90=84=E6=98=AF=E4=BB=80=E4=B9=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一轮(db91c9e)给派生来源加了结构标识、按结构配对之后,复验还有一批用例返回空列表或者报得自相矛盾。按根因分几类改: 外连接:joins 第二遍按涉及的来源配对、不分方向,orders LEFT JOIN refunds 和 refunds LEFT JOIN orders 配成一对后类型、条件都一样,一句不说(db91c9e 引入的回归),RIGHT JOIN 对调、两个 CTE 对调、子查询写法同样漏。现在 JoinFacet 记下主表(keep)和可空那一侧(nullable),LEFT / RIGHT 按主表比:对调了报「原流程以 orders 为主表外连接 refunds,盲算以 refunds 为主表外连接 orders」,LEFT JOIN 和两边对调的 RIGHT JOIN 是同一回事、不再报成类型不同(原来列为做不到)。同一段 WHERE 里拿可空那一侧的列比较、IN、LIKE、BETWEEN、IS NOT NULL 的(JoinFacet.rejects),这次外连接按内连接比:盲算把可空一侧的条件写进 WHERE、LEFT JOIN 实际成了内连接的照报;IS NULL、OR、套在 COALESCE / CASE 里的不认。 算出来的列(Computed,新的 expression 一面):聚合只按去重后的 (函数, 参数) 集合比,SUM(p.total) - SUM(r.total) 对反过来、退款率分子分母对调报不出;派生来源里不是聚合的列不进任何方面,net.v 谁减谁、x.total 取的是付款还是退款合计、amount * 0.9 对 0.8、CASE 的口径、窗口函数的分区和排序、RANK 对 ROW_NUMBER 都是空列表。现在最外层选出来的列(先剥掉 ROUND、CAST、COALESCE 给默认值这类外壳)按算法比,用到的聚合一样、位置换了照报;派生来源里被外面用到、又不是原样透出一列的列按定义比。只差在聚合上、聚合那一面已经报了的,按透过去的那一列筛时间、时间那一面已经报了的,不再报第二遍。比较时加法、乘法的两项排好先后,DATE()、CAST(… AS DATE)、::date 统一成一种写法,免得写法不同被报成算法不同。 时间:时间条件的列透过「同一列套着日期函数、CAST、MIN / MAX」算出来的派生列,一直换到实表(TimePredicate.via 记下路过的列):日期表达式写在 CTE 里,按支付时间还是下单时间筛照样报出来,带上表名、列名给调用方对照数据目录(三个方言的 DATE()、CAST … BETWEEN、两个同名 CTE 对调后的 MAX(paid_at) 对 MAX(refunded_at))。 结构标识和配对:_Derived 里往里套着的派生来源原来一律写「@」,net 是 paid.total - refunded.total 还是反过来写出来一样。现在按结构写的 shape 写法里引用派生来源的地方写它自己的结构标识(里面那段按 shape 写出来再取摘要),配对之前比较的两边也按结构标识排先后;配对之后仍按共同编号排(配成一对的两个派生来源里面稍有不同时结构标识就不同,按它排会把同一个比较排成两个方向)。_align 同一级先看条件重合、再看名字:两边都叫 paid、一边筛已付一边筛未付时不再按名字配反。配成一对的派生来源里两边起名不同的列按定义配成同一列(total 对 sum_amt,原来列为做不到;_renamed),两边各剩一列时也配上,定义的不同照报。 筛的先后:派生来源取前几行(LIMIT、OFFSET)或算了窗口函数(这一列又不是分区键,含 DISTINCT ON)时,外面的条件不再透成里面那张表的条件;Part 另记一路透到底的写法(pierce)和挡住它的是什么(stage),两边只差在这里时报「原流程先算 t 的窗口函数、再按 status = 'paid' 筛选,盲算先按 status = 'paid' 筛选、再算 t 的窗口函数」「先取 t 的前 10 行、再筛」,时间条件同样。 没用到的 CTE:原来逐段收集连定义了没用的 CTE 里的条件也收,盲算定义了只筛已付的 CTE 却直接查 orders 报不出来,没用的 CTE 里多一个条件又会被报成多筛了一遍。现在只收从最外层一路引用到的段(_Statement._enliven);例外是两边都定义了同一个 CTE、一边把它关联进来一边没用到,没用到的那边照样收它,免得它里面的聚合、分组被说成一边独有(连接那一面已经说了关联了它)。 说清楚:两边同名的派生来源其实不是同一个时(t1 / t2、a / b 定义对调),「按 t1.total > 100 筛选」两边写法一模一样,读起来自相矛盾。Part 记下用到的派生来源各列的来历(origins,一直换到实表的列),给人看的写法依次退到一律写表名、带上来历(t1.total 即 SUM(orders.amount))、来历也一样(两个 CTE 只差里面的条件)时说「盲算的 paid 对应原流程的 open」。 自己关联自己:同一个 CTE、同一张表在一段里出现两次,两个别名按各自在关联条件里出的列区分(_Source.role),b1.total > b2.total 写反了报得出来,换了别名、换了关联先后不报(原来列为做不到)。CTE、子查询自己写了列名的(WITH t(n) AS …,递归 CTE 常这么写)按写的列名认列。 模块说明、Part / JoinFacet / TimePredicate / Computed / Facets / FacetDiff、unmatched、joins 的文档按现在的做法改写,做不到的照实列出:两边起名不同、各多出不止一列且定义都不同的列仍算两列;全换了的派生来源不配对;条件改得一个都不重合又各起了名字的同形派生来源只能按先后配;自连接两个别名在关联条件里出的列一样时分不出;三项以上的加法乘法换序、括号展开算不同算法;QUALIFY、GROUP BY 之后才筛的先后不认;外连接链里连带变成内连接的、套在函数里或写在外层 WHERE 的不认;不是 facets 算出来的 Facets 只能按位置编号比。 测试:复验清单里的用例逐个补成测试(各方言、CTE 和子查询写法),另加意思一样、不该报的反例(LEFT / RIGHT 对调、WHERE 筛主表、外连接链换序、0.9 * amount、DATE 对 CAST、按分区键在窗口前后筛、两边都先取前 10 行、列起名不同、自连接换别名换先后、没用到的 CTE)。新加 64 项,其中 48 项在 db91c9e 上失败,其余是防误报的,两边都通过;原有 128 项和 test_verify_agent 照旧通过,全量 7113 项通过。复验清单最后一条只到「WITH base AS (SE」就截断了,看不到全文,按开头和已知缺口一并处理了 CTE 自连接。 --- backend/app/data/sql_semantics.py | 1341 +++++++++++++++++++++++---- backend/tests/test_sql_semantics.py | 452 ++++++++- 2 files changed, 1588 insertions(+), 205 deletions(-) diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index f0a78158..e820c4fd 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -6,46 +6,85 @@ - normalize_sql:字面量换成占位符之后的规范文本,加上字面量本身(按规范文本里的顺序)。两条 SQL 只有字面量不同, 规范文本就相同;关键字大小写、空白、注释、不加引号的名字的大小写都不算不同。 -- facets:把一条查询拆成几个方面——用到的表、连接(类型、关联进来的表、ON 条件)、WHERE 的各个合取式、其中的 - 时间条件(列、运算符、取值)、分组、聚合(函数、参数、是否去重)、HAVING、取哪几行(LIMIT、OFFSET、取前几行 - 时的排序)。别名一律换成真实表名(条件里套着的子查询也是),等式、比较的两边排好顺序:o.paid_at >= '2026-09-01' +- facets:把一条查询拆成几个方面——用到的表、连接(类型、关联进来的表、ON 条件、外连接以哪一侧为主表)、WHERE + 的各个合取式、其中的时间条件(列、运算符、取值)、分组、聚合(函数、参数、是否去重)、算出来的列(最外层的算法、 + CTE 和子查询里被外面用到的列的定义,窗口函数也在这里)、HAVING、取哪几行(LIMIT、OFFSET、取前几行时的排序)。 + 别名一律换成真实表名(条件里套着的子查询也是),等式、比较、加法、乘法的两边排好顺序:o.paid_at >= '2026-09-01' 和 '2026-09-01' <= orders.paid_at 是同一个条件。 - diff_facets:逐个方面比两份 facets,每处不同给一句中文说明。 **逐段收集。** CTE、子查询里的条件、聚合和取行也算进来:盲算可能把有效记录条件写在 CTE 里,原流程写在外层,按 -「整条语句里有哪些条件」比才不会把同一个口径报成不同。逗号连接(以及 INNER JOIN 之外又写在 WHERE 里的等值关联) -写在 WHERE 里的列等式算作关联条件,不算筛选。条件里套着的子查询(EXISTS、IN (SELECT …))既单算一段,也按它自己 -那一段的来源整个写进外层那个条件:别名在子查询里定义,拿外层的来源去认,同一个条件只因别名不同就会被报成不同。 +「整条语句里有哪些条件」比才不会把同一个口径报成不同。只收用到了的段:定义了却没被引用的 CTE 不影响结果,不收 +(盲算定义了只筛已付的 CTE 却直接查 orders,少的那个条件要报出来)。逗号连接(以及 INNER JOIN 之外又写在 WHERE +里的等值关联)写在 WHERE 里的列等式算作关联条件,不算筛选。条件里套着的子查询(EXISTS、IN (SELECT …))既单算 +一段,也按它自己那一段的来源整个写进外层那个条件:别名在子查询里定义,拿外层的来源去认,同一个条件只因别名不同 +就会被报成不同。 **各自起的名字不进比较。** 表的别名换成真实表名;CTE、FROM 里的子查询(含 LATERAL)没有真实表名可换,单看一条 语句时按它在语句里的位置编号(@1、@2…,从外往里、按第一次用到的先后),给人看时照写名字;条件里的子查询(IN、 EXISTS、标量子查询)选出来的列名不影响取值,比较时去掉。这些名字都是各自起的,双盲复算的盲算更是:按名字比,同一个 -口径会被说成换了口径。 +口径会被说成换了口径。同一个来源在一段里出现两次(自己关联自己)时,两个别名按各自在关联条件里出的列区分 +(b2.user_id = b1.referrer_id:b1 出 referrer_id、b2 出 user_id),b1.total > b2.total 写反了认得出,换了别名、 +换了先后不算不同。 **两边的派生来源按结构配对。** 位置编号只看先后:一边多出一个排在前面的 CTE,或者内连接两边换了先后,编号就错开 了;可也不能干脆不看是哪一个(两个 CTE 都算出 total 时,外层用的是付款合计还是退款合计,正是要报的不同)。所以 diff_facets 先给两边的每个派生来源算一份结构标识(见 _Derived):里面读到的实表(FROM / JOIN,往里套着的也算)、 -选出来的各列怎么定义(规范化后的定义式,如 total 是 SUM(orders.amount))、按什么分组、整段规范化后的写法,以及 -起的名字;再按从严到宽的几级一对一地配(见 _align):整段一样;读的表、各列定义、分组一样(里面的筛选条件可以不 -同);读的表和选出来的列名一样(某一列的算法可以不同);读的表和名字一样;最后三级只在两边都只剩一个时才配:只是 -读的表一样、只是名字一样、只是选出来的列名一样。同一级有几个都配得上时,先挑同名的,再挑里面的条件重合最多的,最后 -才看位置编号的先后。哪一级都配不上的不配:一边多出来的 CTE 不会被拿去和另一边不相干的 CTE 凑成一对。配好之后, -第二条 SQL 的派生来源改用第一条里对应的那个编号(配不上的另起编号),按这个共同的编号重新分析一遍再逐个方面比: -比较的两边谁在前、关联条件的先后都按共同的编号排,内连接换了先后也排成同一个方向。连接里的内连接(含逗号连接、 -CROSS JOIN)按涉及的来源和条件比,不按哪一侧是关联进来的;一边独有的一次内连接,关联进来的那一侧另一边也有时,报 -的是它接上的另一头(FROM stores JOIN orders、另一边只查 orders:多出来的是 stores)。 +选出来的各列怎么定义(规范化后的定义式,如 total 是 SUM(orders.amount))、按什么分组、里面的条件、整段规范化后的 +写法,以及起的名字。这些写法里往里套着的派生来源写它自己的结构标识(见 _Statement._signature),和名字、编号无关: +net 读的是 paid.total - refunded.total 还是反过来,写出来不一样。再按从严到宽的几级一对一地配(见 _align):整段 +一样;读的表、各列定义、分组一样(里面的筛选条件可以不同);读的表和选出来的列名一样(某一列的算法可以不同);读的 +表和名字一样;最后三级只在两边都只剩一个时才配:只是读的表一样、只是名字一样、只是选出来的列名一样。同一级有几个 +都配得上时,先挑里面的条件重合最多的,再挑同名的(名字是各自起的:两边都叫 paid,一边筛已付、一边筛未付,不是一 +回事),最后才看位置编号的先后。哪一级都配不上的不配:一边多出来的 CTE 不会被拿去和另一边不相干的 CTE 凑成一对。 +配成一对的派生来源里,两边起名不同的列按定义配成同一列(一边叫 total、一边叫 sum_amt,见 _renamed)。配好之后, +第二条 SQL 的派生来源改用第一条里对应的那个编号(配不上的另起编号),按这个共同的编号重新分析两边再逐个方面比: +比较的两边谁在前、关联条件的先后都按共同的编号排,内连接换了先后也排成同一个方向。 -仍做不到的:两边给 CTE、子查询选出来、外层按名字引用的列起的名字不同(一边叫 total、一边叫 sum_amt),算两列;读同 -一张表、各列定义也一样的两个派生来源,两边都改了里面的条件、又各起了名字时,只能按条件的重合和先后配,可能配反; -读的表、选出来的列名、名字全换了的两个派生来源不配对,外层按它们的列写的条件照报成不同;同一张表、同一个 CTE 自己 -关联自己时两个别名比较用的写法一样,p1.total > p2.total 写反了认不出;连接上做不到的见 _Compare.joins。facets 单独 -返回的 key 仍是位置编号;不是 facets 算出来的 Facets(没记原文)diff_facets 也只能按位置编号比。 +**派生来源里的算法。** 外面引用的是「t.net」,两边写的一样,net 在里面怎么算的(amount * 0.9 还是 * 0.8、窗口 +函数按什么分区排序、是 paid.total 还是 refunded.total)只看外面看不出来:派生来源里被外面用到、又不是原样透出 +一列的列,按定义比(见 Computed)。最外层算出来的列(SUM(p.total) - SUM(r.total)、窗口函数、CASE)也按算法比: +聚合两边的集合一样、位置换了,聚合那一面看不出来。 + +**什么时候能透过派生来源。** CTE、子查询原样透出一张表的列(SELECT *,或者直接选出这一列)时,外面按它写的条件、 +聚合换成那张表的列:先在 CTE 里筛一遍再查和直接查是同一列。它要取前几行(LIMIT、OFFSET)、或者算了窗口函数(这一列 +又不是分区键)时不透:先取前 10 行再筛已付,和先筛已付再取前 10 行取到的不是同一批;两边只差在这里的,报筛的先后 +不同。时间条件另外透过「同一列套着日期函数、CAST、MIN / MAX」算出来的列(DATE(paid_at) AS d),日期表达式写在 CTE +里,按支付时间还是下单时间筛照样认得出。 + +**外连接。** 按以哪一侧为主表(保留它的全部行)比,不按写的是 LEFT 还是 RIGHT:orders LEFT JOIN refunds 和 +refunds RIGHT JOIN orders 一样,和 refunds LEFT JOIN orders 不一样。同一段 WHERE 里拿可空那一侧的列比较、IN、 +LIKE 的(空值都不成立),把没关联上的行筛掉了,按内连接比:写的是 LEFT JOIN,实际成了 INNER JOIN,要报出来。连接 +里的内连接(含逗号连接、CROSS JOIN)按涉及的来源和条件比,不按哪一侧是关联进来的;一边独有的一次内连接,关联 +进来的那一侧另一边也有时,报的是它接上的另一头(FROM stores JOIN orders、另一边只查 orders:多出来的是 stores)。 + +**写法一样、意思不同时说清楚。** 两边同名的派生来源其实不是同一个(盲算的 t1 是原流程的 t2)时,光写「t1.total」 +两边一模一样:带上各列的来历(t1.total 即 SUM(orders.amount));来历也一样(两个 CTE 只差里面的条件)时,说 +「盲算的 paid 对应原流程的 open」。 + +仍做不到的: +- 两边起名不同的派生来源的列,两边各自多出不止一列、定义又都不一样时配不上,算不同的列,外层按它们写的条件照报 + 成两边各有一个;读的表、选出来的列名、名字全换了的两个派生来源不配对,同样照报。 +- 读同一张表、各列定义也一样的两个派生来源,两边都改了里面的条件、改得一个都不重合,又各起了名字时,只能按先后配, + 可能配反。 +- 自己关联自己、两个别名在关联条件里出的列一样(b1.user_id = b2.user_id)时分不出谁是谁;这时 WHERE 里对调过来, + 算出来的行数也一样,只有最外层直接选其中一个别名的列(不比,见下)时会漏。 +- 同一个算法写法不同:三项以上的加法、乘法换了先后(a + b + c 对 c + b + a)、括号展开(a * (b + c) 对 a * b + + a * c)、取日期以外的等价函数,算不同的算法。最外层只比算出来的列:光一列、字面量不比;ROUND、CAST、COALESCE + 给默认值这类外壳当作同一个数(最外层取整到分不算口径不同)。派生来源里没被外面用到的列不比;UNION 按各支同一个 + 位置比。 +- 筛的先后只认取前几行(LIMIT、OFFSET)、窗口函数(含 DISTINCT ON)挡住的;QUALIFY、GROUP BY 之后才筛的不认。 + 外连接被 WHERE 变成内连接只认同一段 WHERE 里直接用到可空那一侧列的比较、IN、LIKE、BETWEEN、IS NOT NULL;套在 + COALESCE、CASE 里,或者写在外面一层 WHERE 里的不认。连接上别的做不到的见 _Compare.joins。 +- facets 单独返回的 key 仍是位置编号;不是 facets 算出来的 Facets(没记原文)diff_facets 也只能按位置编号比, + 没用到的 CTE、起名不同的列也不再另配。 **和 sqlcheck 一样宁可漏报。** 解析不了(语法错、方言特性、多条语句、不是查询)就带 parse_error 返回,从不抛 异常。没加表前缀、又有好几张表可能有这一列的,照写列名(给了表结构才去对)。只读语法树,不执行 SQL。 """ from __future__ import annotations +import hashlib import logging import re from collections import deque @@ -71,8 +110,12 @@ _SEGMENT = "sql_semantics_segment" #: _Statement._number_derived、_Statement.relabel)。复制出来的条件里套着子查询时,靠它把 CTE 名、子查询的别名 #: 换成编号 _DERIVED = "sql_semantics_derived" -#: 不写编号的写法(render 的 anon 模式)里派生来源一律写成这个:算结构标识(_Derived)时用,和编号无关 +#: 不写编号的写法(render 的 anon 模式)里派生来源一律写成这个:排比较的两边谁在前时先看实表的列还是派生来源的列 _ANON = "@" +#: render 里比较用的写法(见 render):名字按 name_key,别名、各自起的名字都去掉 +_COMPARING = frozenset({"key", "loose", "anon", "shape", "pierce"}) +#: 外连接写在 WHERE 里会把可空那一侧没关联上的行(那一侧的列都是空)筛掉的条件:拿空值比都不成立 +_NULL_REJECTING = ("EQ", "NEQ", "GT", "GTE", "LT", "LTE", "Like", "ILike", "Between", "In") # ========================================================================== @@ -240,6 +283,12 @@ class Part: 写法上有:<>、NOT、NOT IN、IS NOT NULL 去掉否定之后排除的是什么(IS NOT NULL 写成「列 为空」)。column_keys 和 columns 一一对应,是各列比较用的写法(实表.列;CTE、子查询的列写成「编号.列」):两边对同一列的不同条件靠它配成 一对说。 + + 后面三项不进比较(两个 Part 相等只看前面几项),只用来把不同说清楚: + origins 是这一项用到的派生来源的列各自的来历 [(「t1.total」, 「SUM(orders.amount)」)](见 _Statement._origin): + 两边写法一模一样(都是 t1.total > 100),各自的 t1 却不是同一个时,带上它才分得清。pierce 是一路透过派生来源写到 + 实表的 key,只在中途被挡住时才有(这一项按派生来源的列写,见 _Statement.through);stage 说是被什么挡住的(「取 t + 的前 10 行」「算 t 的窗口函数」):两边 pierce 一样、只有一边被挡住,是同一个条件筛的先后不同。 """ key: str @@ -249,6 +298,9 @@ class Part: loose: str = "" full: str = "" column_keys: tuple[str, ...] = () + origins: tuple[tuple[str, str], ...] = field(default=(), compare=False) + pierce: str = field(default="", compare=False) + stage: str = field(default="", compare=False) @staticmethod def plain(key: str, text: str) -> "Part": @@ -264,30 +316,46 @@ class JoinFacet: sides 是这次连接涉及的来源(比较用的写法,排好序):关联进来的那一侧,加上关联条件里直接引用的这一段的来源(不 透过 CTE 找实表);没有条件的加上它之前出现的各个来源。内连接、CROSS JOIN 两边换个先后意思不变(FROM paid JOIN - refunded 和 FROM refunded JOIN paid),diff_facets 比它们时按 sides 和条件比,不按哪一侧是关联进来的。""" + refunded 和 FROM refunded JOIN paid),diff_facets 比它们时按 sides 和条件比,不按哪一侧是关联进来的。 + + keep、nullable 只在 LEFT / RIGHT JOIN 上有:保留全部行的主表那一侧、关联不上时记空的那一侧(比较用的写法,排好 + 序)。LEFT JOIN 关联进来的那一侧可空,关联条件里的其余来源(没有就是它之前的各个来源)是主表;RIGHT JOIN 反过来。 + orders LEFT JOIN refunds 和 refunds RIGHT JOIN orders 是同一回事,和 refunds LEFT JOIN orders 不是:diff_facets + 按主表比,不按写的是 LEFT 还是 RIGHT。rejects 是同一段 WHERE 里把可空那一侧没关联上的行筛掉的条件(那一侧的列 + 拿来比较、IN、LIKE……,空值都不成立):有它这次外连接实际等于内连接。""" kind: str table: Part on: Part | None = None other: tuple[str, ...] = () sides: tuple[str, ...] = () + keep: tuple[str, ...] = () + nullable: tuple[str, ...] = () + rejects: Part | None = None @dataclass(frozen=True) class TimePredicate: """WHERE 里的一个时间条件。op:= / <> / > / >= / < / <= / between / in。value 是另一侧给人看的写法: - BETWEEN 是「低 AND 高」,IN 是「(…)」。""" + BETWEEN 是「低 AND 高」,IN 是「(…)」。 + + column 是按哪一列筛时间:条件写的是 CTE、子查询算出来的列(WITH t AS (SELECT DATE(paid_at) AS d …) … WHERE + d >= …)时,它在里面是同一列套着日期函数、CAST、MIN / MAX 的,换成里面那一列,一直换到实表(paid_at):按支付 + 时间还是下单时间筛,日期表达式写在 CTE 里也认得出。via 是路过的派生来源的列(「编号.列」,不进比较):时间那一面 + 已经报了按哪一列筛不同,派生来源那一列的定义不同就不再另报一遍。""" column: Part op: str value: str condition: Part + via: tuple[str, ...] = field(default=(), compare=False) @dataclass(frozen=True) class Aggregate: """一个聚合。func 是函数名的小写(sum / count / avg / min / max …);arg 是参数,COUNT(*)、COUNT(1) 的 arg - 是「*」,去重聚合是 DISTINCT 里面的表达式;text 是整个聚合给人看的写法。窗口函数不算。""" + 是「*」,去重聚合是 DISTINCT 里面的表达式;text 是整个聚合给人看的写法。窗口函数不算(它们按所在的那一列比,见 + Computed)。""" func: str arg: Part @@ -295,6 +363,33 @@ class Aggregate: text: str +@dataclass(frozen=True) +class Computed: + """选出来的一列是怎么算的。两种: + + - 最外层(UNION 的各支)选出来的列,不是光一列(分组那一面比)、也不是字面量的,比如 SUM(p.total) - SUM(r.total)、 + SUM(amount) OVER (PARTITION BY user_id)、CASE …;ROUND、CAST、COALESCE 给默认值这类外壳先剥掉(见 + _unwrap_value)。光一个聚合也收:另一边可能拿同一个合计再算了一下(SUM(amount) 对 SUM(amount) / 100)。scope、 + scope_key 为 None,name 为空串。 + - 派生来源(CTE、FROM 里的子查询)里被外面用到、又不是原样透出一列的列:它在里面的定义(amount * 0.9、 + DATE(paid_at)、ROW_NUMBER() OVER (…)、p.total - r.total)。外面按「t.net」引用它,两边写的都是 t.net,算法却可能 + 不同,只比外面的写法看不出来。scope 是派生来源给人看的名字,scope_key 是它的编号(两边配好对的共同编号), + name 是列名的 name_key,shown 是列名的原样写法。 + + expr 是定义(按它那一段的来源写的 Part;UNION 的各支用「|」连起来)。template 是把里面的聚合换成占位符之后的写法、 + aggregates 是里面的聚合(按出现的先后):两边只差在聚合上(SUM(amount) * 100 对 SUM(paid_amount) * 100)、聚合 + 那一面又已经报了,这里不再报一遍;聚合的集合一样、只是位置换了(SUM(a) - SUM(b) 对 SUM(b) - SUM(a))才在这里报。 + """ + + scope: str | None + scope_key: str | None + name: str + shown: str + expr: Part + template: Part + aggregates: tuple[Aggregate, ...] = () + + @dataclass(frozen=True) class LimitFacet: """一段查询取哪几行。count、offset 是行数和跳过行数的写法(没写为 None,OFFSET 0 也算没写);order 是这一段的 @@ -320,7 +415,12 @@ class Facets: sources 是各段 FROM / JOIN 里的来源(实表、CTE、子查询;key 是比较用的写法,派生来源是编号,text 是给人看的 名字),按出现的先后、按 key 去重;条件里套着的子查询(IN (SELECT … FROM users))的来源不算,它们只筛行,不是 连接的一头。它本身不单独比较:一边独有的一次内连接要说清多出来的是哪个来源时,拿另一边的 sources 对照(见 - _Compare.added)。 + _Compare.added),外连接的主表、可空那一侧也按它写名字。 + + expressions 是算出来的列(见 Computed)。derived 是用到了的派生来源(key 是编号,text 是名字,含条件里套着的子 + 查询用到的 CTE):两边写法一样、同名的派生来源却不是同一个时,拿它说清「盲算的 t1 对应原流程的 t2」。 + + 只收用到了的段:没用到的 CTE(定义了、外面没引用)里的条件、聚合、表不影响结果,不收(见 _Statement._enliven)。 _origin 是 facets 记下的 (原文, 表结构),不进比较、不进 as_dict:diff_facets 把两边的派生来源按结构配好对之后, 拿它按共同的编号重新分析一遍(见模块说明)。 @@ -337,6 +437,8 @@ class Facets: limit: str | None = None limits: tuple[LimitFacet, ...] = () sources: tuple[Part, ...] = () + expressions: tuple[Computed, ...] = () + derived: tuple[Part, ...] = () parse_error: str | None = None _origin: tuple[str, Mapping[str, Any] | None] | None = field(default=None, compare=False, repr=False) @@ -413,6 +515,13 @@ def _own_columns(node: Any) -> list[Any]: return [c for c in found if c.name and not c.is_star] +def _is_star(item: Any) -> bool: + """SELECT 列表里的一项是 * 或 t.*。""" + from sqlglot import exp + + return isinstance(item, exp.Star) or (isinstance(item, exp.Column) and item.is_star) + + def _inner_queries(query: Any, skip: Iterable[Any] = ()) -> list[Any]: """query 里直接套着的查询(不再往里找),按出现的先后:条件、SELECT 列表里的子查询,UNION 的各支,子查询里面 那段。WITH 里的 CTE 定义不算(在用到它的地方才算),skip 里的节点整个跳过。""" @@ -471,6 +580,25 @@ def _branches(query: Any) -> list[Any]: return [query] if isinstance(query, exp.Select) else [] +def _output_names(query: Any) -> list[str | None]: + """派生来源选出来的各列叫什么(外面按这个名字引用它们):按第一支 SELECT 的位置,* 为 None。CTE、子查询自己 + 写了列名的(WITH t(n) AS …、(…) AS t(a, b),递归 CTE 常这么写)按写的列名。""" + from sqlglot import exp + + branches = _branches(query) + if not branches: + return [] + names = [None if _is_star(item) else item.alias_or_name for item in branches[0].selects] + holder = query.parent + if isinstance(holder, exp.Subquery) and isinstance(holder.parent, exp.Lateral) and not holder.alias: + holder = holder.parent # LATERAL 的别名(连同列名)挂在 LATERAL 上 + alias = holder.args.get("alias") if isinstance(holder, (exp.CTE, exp.Subquery, exp.Lateral)) else None + columns = [c.name for c in alias.columns] if isinstance(alias, exp.TableAlias) else [] + for i, column in enumerate(columns[:len(names)]): + names[i] = column + return names + + def _own_nodes(select: Any) -> Iterator[Any]: """一段 SELECT 自己的节点:不进 CTE、子查询(它们是别的段)。""" from sqlglot import exp @@ -501,6 +629,68 @@ def _datey(node: Any) -> bool: return False +def _value_column(node: Any) -> Any: + """派生来源一列的定义是不是「还是那一列」:光一列,或者同一列套着日期函数、CAST、MIN / MAX(按支付时间算的日期、 + 每人最后一次付款的时间);是的话返回那一列,不是返回 None。窗口函数(ROW_NUMBER() OVER (ORDER BY paid_at) 是排名, + 不是时间)、别的聚合、好几列算出来的、套着子查询的都不算。""" + from sqlglot import exp + + n = _unparen(node) + if isinstance(n, exp.Column): + return n if n.name and not n.is_star else None + if any(isinstance(x, (exp.Window, exp.Select, exp.Subquery)) or + (isinstance(x, exp.AggFunc) and not isinstance(x, (exp.Min, exp.Max))) for x in n.walk()): + return None + cols = _own_columns(n) + if len(cols) != 1: + return None + if _datey(n) or _bare_column(n) is not None or any(isinstance(x, (exp.Min, exp.Max)) for x in n.walk()): + return cols[0] + return None + + +def _unwrap_value(node: Any) -> Any: + """剥掉只改写法、不改是哪个数的外壳(括号、类型转换、ROUND、COALESCE / IFNULL 给默认值):最外层的 + ROUND(SUM(amount), 2)、COALESCE(SUM(amount), 0) 还是那个合计,只是显示上的取舍。""" + from sqlglot import exp + + while True: + node = _unparen(node) + if isinstance(node, (exp.Cast, exp.TryCast, exp.Round)) and node.this is not None: + node = node.this + elif isinstance(node, exp.Coalesce) and node.expressions and \ + all(isinstance(_unparen(e), exp.Literal) for e in node.expressions): + node = node.this + else: + return node + + +def _as_date(node: Any) -> Any: + """「取日期」的几种写法统一成 CAST(… AS DATE)(比较用,给人看的照原样):DATE(paid_at)、paid_at::date、 + TRY_CAST(paid_at AS DATE),MySQL 的 DATE() 读成 TsOrDsToDate。各方言、各人写法不同,取的都是同一天;带时区偏移、 + 格式的(DATE(paid_at, '+8 hours'))不是同一天,不动。""" + from sqlglot import exp + + def to_date(n: Any) -> bool: + return isinstance(n, exp.DataType) and n.this == exp.DataType.Type.DATE + + if isinstance(node, exp.Date) and node.this is not None and not node.args.get("zone") \ + and not node.args.get("expressions"): + return exp.Cast(this=node.this, to=exp.DataType.build("DATE")) + if isinstance(node, exp.TsOrDsToDate) and node.this is not None and not node.args.get("format"): + return exp.Cast(this=node.this, to=exp.DataType.build("DATE")) + if isinstance(node, exp.TryCast) and to_date(node.args.get("to")): + return exp.Cast(this=node.this, to=exp.DataType.build("DATE")) + return node + + +def _atomic(node: Any) -> bool: + """算式里的一项不是别的加减乘除:两边对调不会改变怎么结合(a * b / c 换成 b / c * a 就不一样了)。""" + from sqlglot import exp + + return not isinstance(_unparen(node), (exp.Add, exp.Sub, exp.Mul, exp.Div, exp.Mod, exp.IntDiv)) + + def _temporal_name(name: str) -> bool: """列名像日期、时间:英文按 catalog.is_date_column 的命名规则(paid_at、order_date),中文看结尾(支付日期)。""" return catalog.is_date_column(name, None) or bool(_CN_TIME.search(name)) @@ -544,13 +734,21 @@ class _Source: key: str # 比较用:实表是 name_key,派生来源是编号(见 _Statement._number_derived) idents: tuple[Any, ...] = () # 表名各段的 Identifier:给人看的写法照原样写回去 query: Any = None # derived:里面那段查询 + role: str = "" # 同一个来源在这一段里出现了不止一次时,这一次是哪一个(见 _Statement._roles) + + @property + def ref(self) -> str: + """比较用的「哪一个来源」:key,自己关联自己时带上 role(orders#parent_id)。""" + return f"{self.key}#{self.role}" if self.role else self.key @dataclass(frozen=True) class _Derived: """一个派生来源(CTE、FROM / JOIN 里的子查询、LATERAL 子查询)的结构标识,和起的名字、位置编号无关的部分都在 - 这里,diff_facets 拿它把两边的派生来源配成对(见 _align)。各项都用不写编号的写法(render 的 anon 模式:实表 - 写表名,往里套着的派生来源一律写「@」),所以两边写法一样就相同,不受先后、名字影响。""" + 这里,diff_facets 拿它把两边的派生来源配成对(见 _align)。各项都用按结构写的写法(render 的 shape 模式:实表 + 写表名,往里套着的派生来源写它自己的结构标识,见 _Statement._signature),所以两边写法一样就相同,不受先后、 + 名字影响;往里套着的两个派生来源(付款合计、退款合计)也分得开:net 是 paid.total - refunded.total 还是反过来, + 写出来不一样。""" number: int # 位置编号(@N 的 N):别的都分不出时才看先后 name: str # CTE 名、子查询的别名(name_key),没有为空:辅助线索 @@ -583,10 +781,11 @@ def _align(xs: list[_Derived], ys: list[_Derived]) -> dict[int, int]: """两条语句的派生来源配对:{第二条里的位置编号: 第一条里的位置编号},配不上的不在里面。 按 _ALIGN_LEVELS 从严到宽一级一级地配,每一级一对一:严的先配走,宽的只收拾剩下的。同一级几对都配得上时,先配 - 同名的,再配里面的条件重合最多的,最后按位置编号的先后(几个一模一样的派生来源按先后一一对应);这一级所有 - 配得上的对一起排,不是左边挨个挑,免得前一个挑走后一个更合适的。最后三级证据弱,只在两边各只剩一个这样的时才 - 配,免得猜错。哪一级都配不上的就不配:一边多出来的 CTE(盲算先筛一遍 VIP 用户)和另一边不相干的 CTE 凑成一对, - 会把外层不相干的条件说成「关联条件不同」。 + 里面的条件重合最多的,再配同名的,最后按位置编号的先后(几个一模一样的派生来源按先后一一对应);这一级所有 + 配得上的对一起排,不是左边挨个挑,免得前一个挑走后一个更合适的。条件排在名字前面:名字是各自起的,两边都叫 + paid 的两个 CTE,一边筛 status = 'paid'、一边筛 status = 'open',算的不是一回事;条件一个都不重合时才看名字。 + 最后三级证据弱,只在两边各只剩一个这样的时才配,免得猜错。哪一级都配不上的就不配:一边多出来的 CTE(盲算先筛 + 一遍 VIP 用户)和另一边不相干的 CTE 凑成一对,会把外层不相干的条件说成「关联条件不同」。 """ rest_x, rest_y = sorted(xs, key=lambda d: d.number), sorted(ys, key=lambda d: d.number) pairs: dict[int, int] = {} @@ -599,7 +798,7 @@ def _align(xs: list[_Derived], ys: list[_Derived]) -> dict[int, int]: found = [y for y in rest_y if view(y) == seen] if unique and (len(found) != 1 or sum(1 for z in rest_x if view(z) == seen) != 1): continue - candidates += [((y.name != x.name, -len(y.conditions & x.conditions), x.number, y.number), x, y) + candidates += [((-len(y.conditions & x.conditions), y.name != x.name, x.number, y.number), x, y) for y in found] for _, x, y in sorted(candidates, key=lambda c: c[0]): if x.number not in pairs.values() and y.number not in pairs: @@ -609,6 +808,34 @@ def _align(xs: list[_Derived], ys: list[_Derived]) -> dict[int, int]: return pairs +def _renamed(xs: list[_Derived], ys: list[_Derived], pairs: Mapping[int, int]) -> dict[tuple[int, str], str]: + """配成一对的派生来源里两边起名不同的列,配成同一列:{(第二条里的位置编号, 列名): 第一条里的列名}。 + + 外面按名字引用这些列,一边叫 total、一边叫 sum_amt,不配起来就是两列,外面按它写的条件、聚合全成了两边各有 + 一个。只在两边各自独有的列名之间配:定义(结构写法)一样的一对一配上;剩下两边各只有一列时也配上(盲算换了名字 + 又改了算法:SUM(amount) AS total 对 SUM(amount - discount) AS net_total,外面用在同一个位置),定义的不同由算出来 + 的列那一面报(见 _Compare.expressions)。""" + by_number = {x.number: x for x in xs} + out: dict[tuple[int, str], str] = {} + for y in ys: + x = by_number.get(pairs.get(y.number, -1)) + if x is None: + continue + cx = {name: shape for name, shape in x.columns if name and name != "*"} + cy = {name: shape for name, shape in y.columns if name and name != "*"} + rest_x = [name for name in cx if name not in cy] + rest_y = [name for name in cy if name not in cx] + for name in list(rest_y): + same = [other for other in rest_x if cx[other] == cy[name]] + if len(same) == 1 and sum(1 for other in rest_y if cy[other] == cy[name]) == 1: + out[(y.number, name)] = same[0] + rest_x.remove(same[0]) + rest_y.remove(name) + if len(rest_x) == 1 and len(rest_y) == 1: + out[(y.number, rest_y[0])] = rest_x[0] + return out + + class _Statement: """一条语句的分析:CTE、实表、每段 SELECT 的来源,以及把表达式写成 key / text 的规则。""" @@ -629,15 +856,6 @@ class _Statement: if cte.alias: self.ctes.setdefault(name_key(cte.alias), cte.this) self._cte_defs.setdefault(name_key(cte.alias), cte) - self._number_derived() - tables: dict[str, _Source] = {} - for table in root.find_all(exp.Table, bfs=False): - if isinstance(table.this, exp.Identifier) and table.name and not self._is_cte(table): - source = self._table(table) - tables.setdefault(source.key, source) - self.tables = list(tables.values()) - #: 整条语句只用到一张实表:给人看的写法一律不写表名 - self.single = len(self.tables) <= 1 self.columns: dict[str, set[str]] = {} for name, meta in ((schema or {}).get("tables") or {}).items(): if isinstance(meta, Mapping): @@ -645,12 +863,61 @@ class _Statement: if isinstance(c, Mapping) and c.get("name")} self._memo: dict[int, _Sources] = {} self._shared: set[str] | None = None + #: 各派生来源的结构标识 {位置编号: 摘要}(见 _signature)、各派生来源一列的来历 {(里面那段的 id, 列名): 来历} + #: (见 _origin):都和编号无关,重新编号不用清 + self._signatures: dict[int, str] = {} + self._origins: dict[tuple[int, str], str | None] = {} + #: 派生来源各列的名字 {里面那段的 id: [列名]}、每段外面筛和里面筛一不一样 {(那段的 id, 列名): 说法}:只看 + #: 语法树,算一次记下来(透视、认列时反复要用) + self._names_memo: dict[int, list[str | None]] = {} + self._blocks_memo: dict[tuple[int, str], str | None] = {} + self._number_derived() + self._enliven(()) # ---- 来源 def _is_cte(self, table: Any) -> bool: return not table.text("db") and not table.text("catalog") and name_key(table.name) in self.ctes + def _reach(self, extra: Iterable[str] = ()) -> set[str]: + """从最外层那段查询出发(再加上 extra 里的 CTE)一路引用到的 CTE 名:FROM / JOIN 里的、条件和 SELECT 列表里的 + 子查询里的都算,用到的 CTE 里面再引用的也算。CTE 的定义本身不算引用:没人用的 CTE 里引用的别的 CTE 也没用到。""" + from sqlglot import exp + + live = {name for name in extra if name in self.ctes} + stack = [self.root, *(self.ctes[name] for name in live)] + while stack: + for node in stack.pop().walk(prune=lambda n: isinstance(n, exp.CTE)): + if isinstance(node, exp.Table) and isinstance(node.this, exp.Identifier) and node.name \ + and self._is_cte(node) and (name := name_key(node.name)) not in live: + live.add(name) + stack.append(self.ctes[name]) + return live + + def _enliven(self, extra: Iterable[str]) -> None: + """定下哪些段算数:最外层那段查询,加上从它一路引用到的 CTE(extra 里的 CTE 也当作用到了,见 _aligned)。没 + 用到的 CTE 里的条件、聚合、表不影响结果:收进来的话,盲算定义了只筛已付的 CTE 却直接查 orders,会被当成也筛了 + 已付;定义了没用的 CTE 里多一个条件,又会被报成多筛了一遍。用到的实表(给人看时要不要写表名)、共有的列名跟着 + 重算。""" + from sqlglot import exp + + live = self._reach(extra) + #: 没用到的 CTE 里的节点(id):facets 只收别的段 + self._dead = {id(n) for name, cte in self._cte_defs.items() if name not in live for n in cte.this.walk()} + tables: dict[str, _Source] = {} + for table in self._alive(exp.Table): + if isinstance(table.this, exp.Identifier) and table.name and not self._is_cte(table): + source = self._table(table) + tables.setdefault(source.key, source) + self.tables = list(tables.values()) + #: 整条语句只用到一张实表:给人看的写法一律不写表名 + self.single = len(self.tables) <= 1 + self._shared = None + + def _alive(self, *types: Any) -> Iterator[Any]: + """用到了的段里这些类型的节点(深度优先、按出现的先后),没用到的 CTE 里的不算(见 _enliven)。""" + return (node for node in self.root.find_all(*types, bfs=False) if id(node) not in self._dead) + @staticmethod def _items(select: Any) -> list[Any]: """一段 SELECT 的 FROM、JOIN 项,按出现的先后。""" @@ -672,8 +939,9 @@ class _Statement: ctes: dict[str, int] = {} count = 0 - #: 编了号的派生来源 [(位置编号, 起的名字, 里面那段查询)],按编号排:diff_facets 按它们的结构配对(见 derived) - self._derived: list[tuple[int, str, Any]] = [] + #: 编了号的派生来源 [(位置编号, 起的名字, 里面那段查询, CTE 的 name_key 或 None)],按编号排:diff_facets 按 + #: 它们的结构配对(见 derived),facets 按它们找各列的定义(见 _expressions) + self._derived: list[tuple[int, str, Any, str | None]] = [] #: 记了编号的节点 {id: (节点, 位置编号)}:配好对之后按共同的编号改写(见 relabel) self._marked: dict[int, tuple[Any, int]] = {} @@ -691,7 +959,7 @@ class _Statement: if _DERIVED not in item.meta: count += 1 mark(item, count) - self._derived.append((count, _alias(item), body)) + self._derived.append((count, _alias(item), body, None)) inner.append(body) elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) and item.name \ and self._is_cte(item): @@ -700,7 +968,7 @@ class _Statement: count += 1 ctes[name] = count mark(self._cte_defs[name], count) - self._derived.append((count, self._cte_defs[name].alias, self.ctes[name])) + self._derived.append((count, self._cte_defs[name].alias, self.ctes[name], name)) inner.append(self.ctes[name]) mark(item, ctes[name]) return inner @@ -723,23 +991,49 @@ class _Statement: count += 1 ctes[name] = count mark(cte, count) - self._derived.append((count, cte.alias, self.ctes[name])) + self._derived.append((count, cte.alias, self.ctes[name], name)) + #: {位置编号: 里面那段查询}、{位置编号: 现在比较用的编号}、反过来 {比较用的编号: 位置编号} + self._bodies = {number: body for number, _, body, _ in self._derived} + self._labels = {number: f"@{number}" for number in self._bodies} + self._numbers = {label: number for number, label in self._labels.items()} + #: 派生来源里改叫另一边那个名字的列 {(比较用的编号, 列名的 name_key): 另一边的列名}(见 relabel) + self._renames: dict[tuple[str, str], str] = {} - def relabel(self, ids: Mapping[int, str]) -> None: + def relabel(self, ids: Mapping[int, str], renames: Mapping[tuple[int, str], str] | None = None) -> None: """派生来源改用 ids 给的编号({位置编号: 比较用的编号},没给的照旧写位置编号)。diff_facets 把两边的派生 来源配好对之后,第二条语句里配上的写第一条里对应的编号,两边比较用的写法才对得上。算好的来源、共有的列名 - 都带着旧编号,跟着清掉重算。""" + 都带着旧编号,跟着清掉重算。 + + renames 是派生来源里改叫第一条里那个名字的列 {(位置编号, 列名的 name_key): 第一条里的列名}(见 _renamed): + 比较用的写法照它写,给人看的照原样。""" for node, number in self._marked.values(): node.meta[_DERIVED] = ids.get(number, f"@{number}") + self._labels = {number: ids.get(number, f"@{number}") for number in self._bodies} + self._numbers = {label: number for number, label in self._labels.items()} + self._renames = {(self._labels[number], column): name for (number, column), name in (renames or {}).items() + if number in self._labels} self._memo.clear() self._shared = None + def column_key(self, source: _Source, column: str) -> str: + """比较用的列名:派生来源里改叫另一边那个名字的列(见 relabel)照改过的写,别的按 name_key。""" + key = name_key(column) + return self._renames.get((source.key, key), key) if source.kind == "derived" else key + + def used_labels(self) -> dict[str, str | None]: + """用到了的派生来源(见 _enliven):{比较用的编号: CTE 的 name_key,FROM 里的子查询为 None}。""" + return {self._labels[number]: cte for number, _, body, cte in self._derived if id(body) not in self._dead} + + def defined_labels(self) -> dict[str, str | None]: + """定义了的派生来源,用没用到都算:{比较用的编号: CTE 的 name_key,FROM 里的子查询为 None}。""" + return {self._labels[number]: cte for number, _, _, cte in self._derived} + def derived(self) -> list[_Derived]: """各个派生来源的结构标识(见 _Derived),按位置编号排。""" from sqlglot import exp out = [] - for number, name, body in self._derived: + for number, name, body, _ in self._derived: columns: list[tuple[str, str]] = [] group: list[str] = [] conditions: set[str] = set() @@ -747,28 +1041,43 @@ class _Statement: scope = self.sources(select) for clause in (select.args.get("where"), select.args.get("having")): if clause is not None: - conditions.update(self.render(self.canonical(c, scope), scope, "anon") - for c in _conjuncts(clause.this)) + conditions.update(self._shape(c, scope) for c in _conjuncts(clause.this)) if i: continue # UNION 选出来的列名、分组按第一支的算 - for item in select.selects: - if isinstance(item, exp.Star) or (isinstance(item, exp.Column) and item.is_star): + for item, called in zip(select.selects, self.names(body)): + if called is None: columns.append(("*", "*")) continue target = item.this if isinstance(item, exp.Alias) else item - columns.append((name_key(item.alias_or_name), - self.render(self.canonical(target, scope), scope, "anon"))) + columns.append((name_key(called), self._shape(target, scope))) if (clause := select.args.get("group")) is not None: selects, aliases = select.selects, self._aliases(select) - group += [self.render(self.canonical(self._target(g, selects, aliases), scope), scope, "anon") - for g in clause.expressions] + group += [self._shape(self._target(g, selects, aliases), scope) for g in clause.expressions] out.append(_Derived(number=number, name=name_key(name) if name else "", tables=tuple(sorted(self._row_tables(body, frozenset()))), columns=tuple(sorted(columns)), group=tuple(sorted(group)), - conditions=frozenset(conditions), - body=self.render(self.canonical(body, {}), {}, "anon"))) + conditions=frozenset(conditions), body=self._shape(body, {}))) return out + def _shape(self, node: Any, sources: dict[str, _Source]) -> str: + """按结构写(render 的 shape 模式,比较的两边也按结构排):配对用,和名字、编号无关。""" + return self.render(self.canonical(node, sources, by="shape"), sources, "shape") + + def _signature(self, number: int) -> str: + """派生来源的结构标识(shape 写法里引用它的地方写这个):里面那段查询按 shape 写法写出来再取摘要。里面再套着 + 的派生来源也写它们各自的结构标识,所以付款合计、退款合计两个 CTE 的同名列 total 在外面一层也分得开,和起的 + 名字、编号无关;取摘要是免得一层套一层越写越长。递归 CTE 引用自己时写「@」。""" + if number not in self._signatures: + self._signatures[number] = _ANON + text = self._shape(self._bodies[number], {}) + self._signatures[number] = "@" + hashlib.sha1(text.encode("utf-8")).hexdigest()[:12] + return self._signatures[number] + + def _signature_of(self, label: str) -> str: + """比较用的编号 → 那个派生来源的结构标识;认不出的编号写「@」。""" + number = self._numbers.get(label) + return self._signature(number) if number is not None else _ANON + def _row_tables(self, query: Any, seen: frozenset[str]) -> set[str]: """query 的行从哪些实表来:各支 FROM / JOIN 里的实表,往里套着的子查询、CTE 也算。条件里的子查询不算(它只 决定留哪些行,算进来的话一边多一个 IN (SELECT … FROM users) 的筛选,结构标识就整个变了)。seen 是这条路上 @@ -813,6 +1122,7 @@ class _Statement: if id(select) in self._memo: return self._memo[id(select)] out = _Sources() + aliases: dict[str, str] = {} for i, node in enumerate(self._items(select)): alias = _alias(node) slot = name_key(alias) if alias else f"#{i}" @@ -827,6 +1137,8 @@ class _Statement: (exp.to_identifier(alias),) if alias else (), body) else: out[slot] = _Source("other", alias, slot) + aliases[slot] = alias + self._roles(select, out, aliases) self._memo[id(select)] = out parent = select.parent while parent is not None and not isinstance(parent, exp.Select): @@ -834,15 +1146,54 @@ class _Statement: out.outer = self.sources(parent) if parent is not None else None return out + @staticmethod + def _roles(select: Any, sources: dict[str, _Source], aliases: Mapping[str, str]) -> None: + """同一个来源在这一段里出现了不止一次(自己关联自己:orders o1 JOIN orders o2、base b1 JOIN base b2)时,各自 + 按在关联条件(ON、WHERE 里两列相等)里出的是哪几列区分,记在 role 上(就地改):比较用的写法写「来源#列」, + 和别名、先后都无关。b2.user_id = b1.referrer_id 里 b1 出 referrer_id、b2 出 user_id,b1.total > b2.total 写反了 + 就认得出来;不然两边都写成 base.total,写反了也一样。出的列一样(b1.user_id = b2.user_id)分不出,也不用分: + 两个对调过来意思一样。给人看的写法这时写别名:都写 orders.amount 读不出是哪一个。""" + from sqlglot import exp + + groups: dict[str, list[str]] = {} + for slot, source in sources.items(): + if source.kind in ("table", "derived"): + groups.setdefault(source.key, []).append(slot) + repeated = [slots for slots in groups.values() if len(slots) > 1] + if not repeated: + return + conds = [j.args["on"] for j in select.args.get("joins") or [] if j.args.get("on") is not None] + if (where := select.args.get("where")) is not None: + conds.append(where.this) + joined: dict[str, set[str]] = {} + for cond in conds: + for eq in _conjuncts(cond): + n = _unparen(eq) + if not isinstance(n, exp.EQ): + continue + cols = [_unparen(n.this), _unparen(n.expression)] + slots = [name_key(c.table) if isinstance(c, exp.Column) and c.table else None for c in cols] + if None in slots or slots[0] == slots[1] or not all(s in sources for s in slots): + continue + for col, slot in zip(cols, slots): + joined.setdefault(slot, set()).add(name_key(col.name)) + for slots in repeated: + roles = {slot: "+".join(sorted(joined.get(slot, ()))) for slot in slots} + for slot in slots: + if roles[slot] and list(roles.values()).count(roles[slot]) == 1: + sources[slot].role = roles[slot] + if aliases.get(slot): + sources[slot].idents = (exp.to_identifier(aliases[slot]),) + def _exposes(self, source: _Source, column: str) -> bool: """这个来源可能有这一列:实表看表结构(没给表结构就当可能有),CTE、子查询看它选出来的列。""" from sqlglot import exp if source.kind == "table": return source.key not in self.columns or name_key(column) in self.columns[source.key] - if source.kind == "derived" and isinstance(source.query, exp.Select): - return any(isinstance(s, exp.Star) or (isinstance(s, exp.Column) and s.is_star) - or name_key(s.alias_or_name) == name_key(column) for s in source.query.selects) + if source.kind == "derived" and isinstance(source.query, exp.Query): + return any(called is None or name_key(called) == name_key(column) + for called in self.names(source.query)) return True def locate(self, column: Any, sources: dict[str, _Source]) -> tuple[str | None, _Source | None]: @@ -862,27 +1213,104 @@ class _Statement: hits = [(slot, s) for slot, s in sources.items() if self._exposes(s, column.name)] return hits[0] if len(hits) == 1 else (None, None) - def through(self, source: _Source | None, column: str, depth: int = 0) -> tuple[_Source | None, str]: - """CTE、子查询只是把一张表的列原样透出来(SELECT *,或者直接选出这一列)时,换成那张表的列: - 原流程直接查 orders、盲算先在 CTE 里筛一遍再查,算的是同一列。""" + def _pass(self, source: _Source | None, column: str) -> tuple[_Source, str] | None: + """派生来源只读一个来源、把它的这一列原样透出来(直接选出这一列,或者 SELECT *)时,里面一层的 (来源, 列名); + 不是原样透出(算出来的列、读好几个来源)返回 None。先认写了名字的列:SELECT *, amount * 0.9 AS net 里的 net + 是算出来的,不是里面那张表的 net。""" from sqlglot import exp - if source is None or source.kind != "derived" or depth > 3 or not isinstance(source.query, exp.Select): - return source, column + if source is None or source.kind != "derived" or not isinstance(source.query, exp.Select): + return None inner = self.sources(source.query) if len(inner) != 1: - return source, column + return None only = next(iter(inner.values())) - for item in source.query.selects: - if isinstance(item, exp.Star) or (isinstance(item, exp.Column) and item.is_star): - return self.through(only, column, depth + 1) - target = _unparen(item.this if isinstance(item, exp.Alias) else item) - if name_key(item.alias_or_name) == name_key(column) and isinstance(target, exp.Column): - return self.through(only, target.name, depth + 1) + names = self.names(source.query) + named = [item for item, called in zip(source.query.selects, names) + if called is not None and name_key(called) == name_key(column)] + if named: + target = _unparen(named[0].this if isinstance(named[0], exp.Alias) else named[0]) + return (only, target.name) if isinstance(target, exp.Column) else None + return (only, column) if None in names else None + + def names(self, query: Any) -> list[str | None]: + """派生来源选出来的各列叫什么(见 _output_names),算一次记下来。""" + if id(query) not in self._names_memo: + self._names_memo[id(query)] = _output_names(query) + return self._names_memo[id(query)] + + def _blocks(self, select: Any, column: str) -> str | None: + """见 _blocking,算一次记下来。""" + memo = (id(select), name_key(column)) + if memo not in self._blocks_memo: + self._blocks_memo[memo] = self._blocking(select, column) + return self._blocks_memo[memo] + + @staticmethod + def _blocking(select: Any, column: str) -> str | None: + """在这一段外面按 column 筛和在它里面筛结果不同时,说它做了什么(「取前几行」「窗口函数」);一样返回 None。 + + 取前几行(LIMIT、OFFSET):先取前 10 行再筛已付,和先筛已付再取前 10 行,取到的不是同一批。窗口函数(含 + DISTINCT ON):先算每人的第一单再筛已付(第一单就是已付的人),和先筛已付再算(有已付订单的人)不是一回事; + 只有 column 是每个窗口的分区键时一样(按分区键筛掉的是整个分区,分区里的排名、合计不受影响)。""" + from sqlglot import exp + + if select.args.get("limit") is not None or select.args.get("offset") is not None: + return "limit" + distinct = select.args.get("distinct") + if isinstance(distinct, exp.Distinct) and distinct.args.get("on") is not None: + keys = {name_key(_unparen(e).name) for e in distinct.args["on"].expressions + if isinstance(_unparen(e), exp.Column)} + if name_key(column) not in keys: + return "window" + for node in _own_nodes(select): + if isinstance(node, exp.Window): + keys = {name_key(_unparen(e).name) for e in node.args.get("partition_by") or [] + if isinstance(_unparen(e), exp.Column)} + if name_key(column) not in keys: + return "window" + return None + + def through(self, source: _Source | None, column: str, *, pierce: bool = False) -> tuple[_Source | None, str]: + """CTE、子查询只是把一张表的列原样透出来(SELECT *,或者直接选出这一列)时,换成那张表的列: + 原流程直接查 orders、盲算先在 CTE 里筛一遍再查,算的是同一列。 + + 这一段取前几行、或者算了窗口函数时不透过去(见 _blocks),照写这个派生来源的列:外面的 t.status = 'paid' + 和写在 t 里面的 status = 'paid' 不是同一个条件,透过去就都成了 orders.status = 'paid',分不出来。pierce 时 + 不管这些,一直透到底:拿来认出两边是不是同一个条件、只是筛的先后不同(见 Part.pierce)。""" + for _ in range(4): + found = self._pass(source, column) + if found is None or (not pierce and self._blocks(source.query, found[1])): + break + source, column = found return source, column - def resolve(self, column: Any, sources: dict[str, _Source]) -> tuple[_Source | None, str]: - return self.through(self.locate(column, sources)[1], column.name) + def resolve(self, column: Any, sources: dict[str, _Source], *, pierce: bool = False) -> tuple[_Source | None, str]: + return self.through(self.locate(column, sources)[1], column.name, pierce=pierce) + + def _definition(self, source: _Source | None, column: str) -> list[tuple[Any, _Sources]] | None: + """派生来源一列在里面的定义:[(定义的表达式, 它那一支的来源)],UNION 每支一项(按列的位置)。只有 SELECT * + 透出来的列,定义就是里面那一支的同名列。认不出(不是派生来源、没有这一列、某一支在那个位置是 *)返回 None。""" + from sqlglot import exp + + if source is None or source.kind != "derived" or source.query is None: + return None + branches = _branches(source.query) + if not branches: + return None + names = [None if called is None else name_key(called) for called in self.names(source.query)] + if name_key(column) not in names: + if None not in names: + return None + return [(exp.column(column), self.sources(branch)) for branch in branches] + position = names.index(name_key(column)) + out = [] + for branch in branches: + if position >= len(branch.selects) or _is_star(branch.selects[position]): + return None + item = branch.selects[position] + out.append((item.this if isinstance(item, exp.Alias) else item, self.sources(branch))) + return out def shared(self) -> set[str]: """好几张表都有的列名(name_key):这条语句里出现时挂在不止一张表上(orders.order_id、refunds.order_id), @@ -891,13 +1319,13 @@ class _Statement: if self._shared is None: owners: dict[str, set[str]] = {} - for select in self.root.find_all(exp.Select, bfs=False): + for select in self._alive(exp.Select): sources = self.sources(select) for node in _own_nodes(select): if isinstance(node, exp.Column) and node.name and not node.is_star: source, name = self.resolve(node, sources) if source is not None: - owners.setdefault(name_key(name), set()).add(source.key) + owners.setdefault(name_key(name), set()).add(source.ref) used = [t.key for t in self.tables] self._shared = {name for name, keys in owners.items() if len(keys) > 1 or sum(1 for t in used if name in self.columns.get(t, ())) > 1} @@ -923,8 +1351,10 @@ class _Statement: return default if owner is None else self.sources(self.segments[owner.meta[_SEGMENT]]) def render(self, node: Any, sources: dict[str, _Source], mode: str = "text") -> str: - """五种写法:key、loose(见 Part)、anon 比较用,名字按 name_key;key、anon 写实表的表名,loose 不写;CTE、 - 子查询 key 和 loose 写编号,anon 一律写「@」(算结构标识用,和编号无关,见 _Derived)。text、full 给人看 + """七种写法。比较用的五种(_COMPARING),名字按 name_key:key、loose 见 Part,key 写实表的表名,loose 不写, + CTE、子查询都写编号;pierce 同 key,只是一路透过取前几行、算窗口函数的派生来源(见 through),拿来认出同一个 + 条件筛的先后不同;anon 写实表的表名、派生来源一律写「@」(排比较的两边谁在前时先分实表的列和派生来源的列); + shape 写实表的表名、派生来源写它的结构标识(见 _signature:配对用,和名字、编号都无关)。text、full 给人看 (见 Part),text 只在好几张表都有这一列时写真实表名,full 一律写(关联条件说的就是两张表各出哪一列,用 full)。 node 里套着的子查询逐段改写:每一列按它所在那一段的来源认,子查询里的表去掉别名(列已经写成了真实表名)。 @@ -934,6 +1364,8 @@ class _Statement: from sqlglot import exp copy = node.copy() + if mode in _COMPARING: + copy = copy.transform(_as_date, copy=False) groups: dict[int, tuple[Any, list[Any]]] = {} for col in copy.find_all(exp.Column): if col.name and not col.is_star: @@ -941,38 +1373,41 @@ class _Statement: groups.setdefault(id(owner), (owner, []))[1].append(col) for owner, cols in groups.values(): scope = sources if owner is None else self.sources(self.segments[owner.meta[_SEGMENT]]) - found = [(col, *self.resolve(col, scope)) for col in cols] + found = [(col, *self.resolve(col, scope, pierce=mode == "pierce")) for col in cols] here = mode - if owner is not None and mode == "text" and len({s.key for _, s, _ in found if s is not None}) > 1: + if owner is not None and mode == "text" and len({s.ref for _, s, _ in found if s is not None}) > 1: here = "full" for col, source, name in found: self._write(col, source, name, here) for table in copy.find_all(exp.Table): if isinstance(table.this, exp.Identifier) and self._owner(table) is not None: table.set("alias", None) - if mode in ("key", "loose", "anon"): + if mode in _COMPARING: for arg in ("catalog", "db", "this"): ident = table.args.get(arg) if isinstance(ident, exp.Identifier): table.set(arg, exp.to_identifier(name_key(ident.name))) - if mode in ("key", "loose", "anon"): - self._drop_names(copy, numbered=mode != "anon") + if mode in _COMPARING: + self._drop_names(copy, mode) return copy.sql(dialect=self.read, comments=False) - @staticmethod - def _drop_names(copy: Any, *, numbered: bool) -> None: + def _drop_names(self, copy: Any, mode: str) -> None: """比较用的写法里抹掉套着的子查询里各自起的名字(就地改):引用 CTE 的表、FROM 里的子查询(含 LATERAL)、 - 子查询里自带的 CTE 定义写编号(numbered 为假时写「@」;引用它们的列已经照这样写了);只看取值的子查询(IN、 - EXISTS、标量子查询)去掉选出来的列名。FROM 里的子查询、CTE 定义选出来的列名留着:外层按这个名字引用它们, - 去掉了就分不出外层用的是哪一列。复制出来的那段本身没有父节点,认不出是 FROM 里的子查询,它选出来的列名也会 - 去掉(_Derived.body 另记了各列,见 derived)。""" + 子查询里自带的 CTE 定义写编号(anon 写「@」,shape 里引用 CTE 的表写它的结构标识、别的写「@」;引用它们的列 + 已经照这样写了);只看取值的子查询(IN、EXISTS、标量子查询)去掉选出来的列名。FROM 里的子查询、CTE 定义选出来 + 的列名留着:外层按这个名字引用它们,去掉了就分不出外层用的是哪一列。复制出来的那段本身没有父节点,认不出是 + FROM 里的子查询,它选出来的列名也会去掉(_Derived.body 另记了各列,见 derived)。""" from sqlglot import exp for node in copy.find_all(exp.Table, exp.Subquery, exp.Lateral, exp.CTE): derived = node.meta.get(_DERIVED) if not derived: continue - mark = exp.to_identifier(derived if numbered else _ANON) + if mode == "shape": + # 引用 CTE 的表写它是哪个(结构标识);FROM 里的子查询、CTE 定义本身整段写在这里,不用再标 + mark = exp.to_identifier(self._signature_of(derived) if isinstance(node, exp.Table) else _ANON) + else: + mark = exp.to_identifier(_ANON if mode == "anon" else derived) if isinstance(node, exp.Table): for arg in ("catalog", "db", "alias"): node.set(arg, None) @@ -990,20 +1425,22 @@ class _Statement: from sqlglot import exp quoted = isinstance(col.this, exp.Identifier) and bool(col.this.args.get("quoted")) - if mode in ("key", "loose", "anon"): + if mode in _COMPARING: if source is None: owner = None elif source.kind == "derived": - owner = _ANON if mode == "anon" else source.key + owner = _ANON if mode == "anon" else source.ref if mode != "shape" else \ + self._signature_of(source.key) + (f"#{source.role}" if source.role else "") else: - owner = None if mode == "loose" else source.key - col.set("this", exp.to_identifier(name_key(name))) + owner = None if mode == "loose" else source.ref + col.set("this", exp.to_identifier(self.column_key(source, name) if source is not None else name_key(name))) col.set("table", exp.to_identifier(owner) if owner else None) col.set("db", None) col.set("catalog", None) return col.set("this", exp.Identifier(this=name, quoted=quoted or not _PLAIN_NAME.match(name))) - if source is None or (source.kind != "other" and mode != "full" + # 自己关联自己的(有 role)一律写别名:只写列名读不出是哪一个 + if source is None or (source.kind != "other" and mode != "full" and not source.role and (self.single or name_key(name) not in self.shared())): col.set("table", None) col.set("db", None) @@ -1014,19 +1451,22 @@ class _Statement: col.set("db", heads[-1].copy() if heads else None) col.set("catalog", heads[-2].copy() if len(heads) > 1 else None) - def canonical(self, node: Any, sources: dict[str, _Source]) -> Any: - """比较的两边排好顺序(列在左、字面量在右;两边都有列的按 _rank 排),IN 的取值排好序。返回新的树。 - 套着的子查询里的比较按它那一段的来源排:拿外层的来源去认子查询的别名,排出来的先后就取决于别名怎么起。""" + def canonical(self, node: Any, sources: dict[str, _Source], *, by: str = "key") -> Any: + """比较的两边排好顺序(列在左、字面量在右;两边都有列的按 _rank 排),IN 的取值排好序;加法、乘法的两项也按 + 这个规矩排(amount * 0.9 和 0.9 * amount 是同一个算法),只排两项都不是别的加减乘除的(见 _atomic)。返回新的 + 树。套着的子查询里的比较按它那一段的来源排:拿外层的来源去认子查询的别名,排出来的先后就取决于别名怎么起。""" from sqlglot import exp def fix(n: Any) -> Any: found = _COMPARE.get(type(n).__name__) - if found is not None and isinstance(n, exp.Binary): + swap = (type(n).__name__, type(n).__name__) if isinstance(n, (exp.Add, exp.Mul)) and \ + _atomic(n.this) and _atomic(n.expression) else found + if swap is not None and isinstance(n, exp.Binary): left, right = n.this, n.expression lc, rc = bool(_own_columns(left)), bool(_own_columns(right)) here = self.scope(n, sources) if lc and rc else sources - if (rc and not lc) or (lc and rc and self._rank(left, here) > self._rank(right, here)): - return getattr(exp, found[1])(this=right, expression=left) + if (rc and not lc) or (lc and rc and self._rank(left, here, by) > self._rank(right, here, by)): + return getattr(exp, swap[1])(this=right, expression=left) elif isinstance(n, exp.In) and n.expressions and not n.args.get("query") \ and not any(_own_columns(e) for e in n.expressions): n.set("expressions", sorted(n.expressions, key=lambda e: e.sql(dialect=self.read))) @@ -1034,24 +1474,105 @@ class _Statement: return node.transform(fix) - def _rank(self, node: Any, sources: dict[str, _Source]) -> tuple[str, str]: + def _rank(self, node: Any, sources: dict[str, _Source], by: str = "key") -> tuple[str, str]: """比较的两边谁排前面:先按不写编号的写法(实表的列、派生来源的列分得开),分不出(两个派生来源的同名列) - 再按编号。diff_facets 重新分析时用的是两边按结构配好对的共同编号(见 relabel):p.total > r.total 和换了 - 关联先后、位置编号对调的同一句排出来是同一个方向,配对之前的位置编号决定不了先后。""" - return self.render(node, sources, "anon"), self.render(node, sources, "key") + 再按 by 的写法分。 + + diff_facets 重新分析时 by 是 key,按两边配好对的共同编号排(见 relabel):p.total > r.total 和换了关联先后、 + 位置编号对调的同一句排出来是同一个方向。不按结构标识排:配成一对的两个派生来源里面稍有不同(一边多一个条件) + 时结构标识就不同,按它排,同一个比较在两边可能排成两个方向,报出两条自相矛盾的话。配对之前(算结构标识时) + 还没有共同的编号,by 是 shape,按结构标识排:位置编号决定不了先后。""" + return self.render(node, sources, "anon"), self.render(node, sources, by) def part(self, node: Any, sources: dict[str, _Source], *, excludes: bool = False) -> Part: - refs: dict[tuple[str | None, str], str] = {} + #: 用到的各列 {(哪一个来源, 列): ((给人看的表名, 列), 比较用的写法)}:按「哪一个来源」分,自己关联自己的两个别名 + #: 是两列(o1.amount、o2.amount) + refs: dict[tuple[str, str], tuple[ColumnRef, str]] = {} + origins: list[tuple[str, str]] = [] + stage = "" for col in _own_columns(node): - source, name = self.resolve(col, sources) - refs.setdefault((source.name if source is not None else None, name), - f"{source.key if source is not None else ''}.{name_key(name)}") + located = self.locate(col, sources)[1] + source, name = self.through(located, col.name) + refs.setdefault((source.ref if source is not None else "", name_key(name)), + ((source.name if source is not None else None, name), + f"{source.ref}.{self.column_key(source, name)}" if source is not None + else f".{name_key(name)}")) + if source is None or source.kind != "derived": + continue + # 透视在这个派生来源上停住了,一路透到底却还能往里走:是它取前几行、算窗口函数挡住的 + if not stage and self.through(located, col.name, pierce=True)[0] is not source: + stage = self._stage(source, name) + if (origin := self._origin(source, name)) is not None: + if (pair := (self.render(col, sources, "full"), origin)) not in origins: + origins.append(pair) full = self.render(node, sources, "full") + key = self.render(node, sources, "key") + pierce = self.render(node, sources, "pierce") if stage else "" + if pierce == key: + pierce = stage = "" + columns = tuple(ref for ref, _ in refs.values()) # 拿两张表的列互相比的(相关子查询里的 o.id = r.order_id)和关联条件一样,给人看也写表名 - across = len({table for table, _ in refs}) > 1 - return Part(key=self.render(node, sources, "key"), text=full if across else self.render(node, sources), - columns=tuple(refs), excludes=self._excludes(node, sources) if excludes else None, - loose=self.render(node, sources, "loose"), full=full, column_keys=tuple(refs.values())) + across = len({table for table, _ in columns}) > 1 + return Part(key=key, text=full if across else self.render(node, sources), + columns=columns, excludes=self._excludes(node, sources) if excludes else None, + loose=self.render(node, sources, "loose"), full=full, + column_keys=tuple(written for _, written in refs.values()), + origins=tuple(origins), pierce=pierce, stage=stage) + + def _stage(self, source: _Source, column: str) -> str: + """挡住了透视的派生来源做了什么(给人看,见 _blocks):「取 t 的前 10 行」「算 t 的窗口函数」。""" + from sqlglot import exp + + query = source.query + if self._blocks(query, column) == "limit": + limit = query.args.get("limit") + count = limit.expression if isinstance(limit, exp.Limit) else limit.args.get("count") \ + if limit is not None else None + written = count.sql(dialect=self.read, comments=False) if count is not None else "" + return _say("取", source.name, "的前", written, "行") if written.isdigit() else \ + _say("按", "LIMIT / OFFSET", "取", source.name, "的行") + return _say("算", source.name, "的窗口函数") + + def _origin(self, source: _Source, column: str, depth: int = 0) -> str | None: + """派生来源一列的来历(给人看):它在里面怎么算的,里面再用到的派生来源的列也换成各自的来历,一直换到实表的 + 列,一律写表名(SUM(orders.amount))。两边同名的派生来源其实不是同一个时(盲算的 t1 是原流程的 t2),光写 + 「t1.total」两边一样,带上来历才说得清各自是付款合计还是退款合计。UNION 只写第一支;认不出定义、套得太深 + 返回 None。""" + if source.kind != "derived" or source.query is None or depth > 4: + return None + memo = (id(source.query), name_key(column)) + if memo not in self._origins: + self._origins[memo] = None # 递归 CTE 绕回自己时到这里为止 + found = self._definition(source, column) + self._origins[memo] = self._expand(*found[0], depth + 1) if found else None + return self._origins[memo] + + def _expand(self, node: Any, sources: dict[str, _Source], depth: int) -> str: + """把 node 写成给人看的 full 写法,里面用到的派生来源的列换成各自的来历(见 _origin)。换进去的是一段 + 写好的文字:先用占位符占住位置,整段写好之后再换回来;放在运算符里的加括号,免得优先级变了。""" + from sqlglot import exp + + bare = _unparen(node) + if isinstance(bare, exp.Column): + source, name = self.resolve(bare, sources) + inner = self._origin(source, name, depth) if source is not None else None + return inner if inner is not None else self.render(bare, sources, "full") + copy = node.copy() + subs: dict[str, str] = {} + for col in _own_columns(copy): + source, name = self.resolve(col, sources) + inner = self._origin(source, name, depth) if source is not None else None + if inner is None: + continue + mark = exp.Placeholder(this=f"sqlsem{len(subs)}") + wrap = " " in inner and isinstance(col.parent, (exp.Binary, exp.Unary)) and \ + not isinstance(col.parent, exp.Paren) + subs[mark.sql(dialect=self.read)] = f"({inner})" if wrap else inner + col.replace(mark) + text = self.render(copy, sources, "full") + for mark, inner in subs.items(): + text = text.replace(mark, inner) + return text def _excludes(self, node: Any, sources: dict[str, _Source]) -> str | None: """排除写法去掉否定之后排除的是什么:a <> 1 → a = 1;NOT IN、NOT LIKE、NOT BETWEEN 同理;IS NOT NULL → 「a 为空」。 @@ -1075,9 +1596,11 @@ class _Statement: # ---- 各个方面 - def facets(self) -> Facets: + def facets(self, extra: Iterable[str] = ()) -> Facets: + """各个方面。extra 是没用到、也照样收进来的 CTE(name_key,见 _aligned),别的没用到的 CTE 不收(见 _enliven)。""" from sqlglot import exp + self._enliven(extra) joins: list[JoinFacet] = [] where: list[Part] = [] time: list[TimePredicate] = [] @@ -1085,23 +1608,27 @@ class _Statement: aggregates: list[Aggregate] = [] having: list[Part] = [] used: list[Part] = [] - for select in self.root.find_all(exp.Select, bfs=False): + for select in self._alive(exp.Select): sources = self.sources(select) if select is self.root or not self._in_condition(select): - used += [Part.plain(s.key, s.name) for s in sources.values()] + used += [Part.plain(s.ref, s.name) for s in sources.values()] drafts = self._join_drafts(select, sources) clause = select.args.get("where") for cond in _conjuncts(clause.this) if clause is not None else []: canon = self.canonical(cond, sources) if self._joins_on_where(canon, sources, drafts): continue - where.append(self.part(canon, sources, excludes=True)) + part = self.part(canon, sources, excludes=True) + where.append(part) + self._rejects(canon, part, sources, drafts) if found := self._time(canon, sources): time.append(found) joins += [self._join(d, sources) for d in drafts] group += self._group(select, sources) aggregates += self._aggregates(select, sources) having += self._having(select, sources) + derived = [Part.plain(self._labels[number], name or "子查询") + for number, name, body, _ in self._derived if id(body) not in self._dead] return Facets( dialect=self.read, tables=tuple(Part.plain(s.key, s.name) for s in self.tables), @@ -1114,8 +1641,39 @@ class _Statement: limit=self._limit(), limits=tuple(self._limits()), sources=_dedupe(used, lambda p: p.key), + expressions=_dedupe(self._expressions(), lambda c: (c.scope_key, c.name, c.expr.key)), + derived=_dedupe(derived, lambda p: p.key), ) + def _rejects(self, cond: Any, part: Part, sources: dict[str, _Source], drafts: list[dict[str, Any]]) -> None: + """同一段 WHERE 里的条件把外连接可空那一侧没关联上的行筛掉了的,记在那次连接上(见 JoinFacet.rejects):LEFT + JOIN 关联进来的那一侧、RIGHT JOIN 之前的各个来源可空,那一侧的列直接拿来比较、IN、LIKE、BETWEEN,或者写 IS NOT + NULL,空值都不成立。IS NULL、OR、套在 COALESCE / IFNULL / CASE 里的不认(宁可漏报)。""" + from sqlglot import exp + + n = _unparen(cond) + if isinstance(n, exp.Not): + n = _unparen(n.this) + if isinstance(n, exp.Is): + if not isinstance(_unparen(n.expression), exp.Null): + return + n = n.this # IS NOT NULL + elif type(n).__name__ not in _NULL_REJECTING: + return + elif type(n).__name__ not in _NULL_REJECTING: + return + hit = set() + for col in _own_columns(n): + if col is not n and any(isinstance(p, (exp.Coalesce, exp.Case, exp.If)) for p in _parents(col, stop=n)): + continue + if (slot := self.locate(col, sources)[0]) is not None: + hit.add(slot) + for draft in drafts: + nullable = {draft["slot"]} if draft["kind"] == "left" else \ + set(draft["before_slots"]) if draft["kind"] == "right" else set() + if draft.get("rejects") is None and hit & nullable: + draft["rejects"] = part + def _join_drafts(self, select: Any, sources: dict[str, _Source]) -> list[dict[str, Any]]: from sqlglot import exp @@ -1133,9 +1691,10 @@ class _Statement: conds = [self.canonical(exp.EQ(this=exp.column(ident.name, table=first_alias), expression=exp.column(ident.name, table=alias)), sources) for ident in join.args["using"]] - before = [s for key, s in list(sources.items())[:i + 1] if key != slot] + before = [(key, s) for key, s in list(sources.items())[:i + 1] if key != slot] drafts.append({"kind": _join_kind(join), "slot": slot, "source": sources.get(slot), "conds": conds, - "before": [s.name for s in before], "before_keys": [s.key for s in before]}) + "before": [s.name for _, s in before], "before_keys": [s.ref for _, s in before], + "before_slots": [key for key, _ in before]}) return drafts def _joins_on_where(self, cond: Any, sources: dict[str, _Source], drafts: list[dict[str, Any]]) -> bool: @@ -1162,7 +1721,7 @@ class _Statement: def _join(self, draft: dict[str, Any], sources: dict[str, _Source]) -> JoinFacet: source = draft["source"] table = Part.plain("?", "?") if source is None else \ - Part(key=source.key, text=source.name, loose=source.key, full=source.name) + Part(key=source.ref, text=source.name, loose=source.ref, full=source.name) forms = sorted((self.render(c, sources, "key"), self.render(c, sources, "loose"), self.render(c, sources, "full")) for c in draft["conds"]) on = Part(key=" AND ".join(f[0] for f in forms), text=" AND ".join(f[2] for f in forms), @@ -1173,14 +1732,22 @@ class _Statement: else draft["before"] # 涉及的来源按条件里直接引用的算(slot 不为 None 的是这一段的来源):透过 CTE 找到的实表两边可能一样, # 关联的却是两个不同的 CTE - sides = {source.key} if source is not None else set() + sides = {source.ref} if source is not None else set() if draft["conds"]: - sides |= {found.key for c in draft["conds"] for col in _own_columns(c) + sides |= {found.ref for c in draft["conds"] for col in _own_columns(c) for slot, found in [self.locate(col, sources)] if slot is not None and found is not None} else: sides |= set(draft["before_keys"]) + # 外连接的主表、可空那一侧:LEFT JOIN 关联进来的那一侧可空,RIGHT JOIN 反过来;另一侧按关联条件里出现的来源 + # 算(没有就是它之前的各个来源),和内连接的 sides 一样不受前面别的连接先后的影响 + keep: set[str] = set() + nullable: set[str] = set() + if draft["kind"] in ("left", "right") and source is not None: + near, far = {source.ref}, (sides - {source.ref}) or set(draft["before_keys"]) + keep, nullable = (far, near) if draft["kind"] == "left" else (near, far) return JoinFacet(kind=draft["kind"], table=table, on=on, other=tuple(sorted(other)), - sides=tuple(sorted(sides))) + sides=tuple(sorted(sides)), keep=tuple(sorted(keep)), nullable=tuple(sorted(nullable)), + rejects=draft.get("rejects")) def _time(self, cond: Any, sources: dict[str, _Source]) -> TimePredicate | None: """时间条件:一边恰好一列(可以套着日期函数、CAST),另一边没有列;列名像日期,或者用了日期函数、像日期的字符串。""" @@ -1203,13 +1770,32 @@ class _Statement: cols = [column] if column is not None else _own_columns(subject) if len(cols) != 1 or any(isinstance(x, (exp.AggFunc, exp.Select)) for x in subject.walk()): return None - if not (_temporal_name(cols[0].name) or _datey(subject) or any(_datey(v) for v in values)): + base, scope, via, datey = self._time_column(cols[0], sources) + if not (_temporal_name(base.name) or _temporal_name(cols[0].name) or datey or _datey(subject) + or any(_datey(v) for v in values)): return None shown = [self.render(v, sources) for v in values] value = f"{shown[0]} AND {shown[1]}" if op == "between" else f"({', '.join(shown)})" if op == "in" \ else shown[0] - return TimePredicate(column=self.part(cols[0], sources), op=op, value=value, - condition=self.part(cond, sources, excludes=True)) + return TimePredicate(column=self.part(base, scope), op=op, value=value, + condition=self.part(cond, sources, excludes=True), via=via) + + def _time_column(self, column: Any, sources: dict[str, _Source]) -> tuple[Any, _Sources, tuple[str, ...], bool]: + """时间条件里的那一列是 CTE、子查询算出来的、在里面是同一列套着日期函数、CAST、MIN / MAX 时(见 _value_column), + 换成里面那一列,一直换到实表:(那一列, 它那一段的来源, 路过的派生来源的列「编号.列」, 路上有没有日期函数)。 + 原样透出的列 resolve 已经透过去了;这里补上算了一下的。""" + via: list[str] = [] + datey = False + for _ in range(4): + source, name = self.resolve(column, sources) + found = self._definition(source, name) if source is not None else None + inner = _value_column(found[0][0]) if found and len(found) == 1 else None + if inner is None: + break + via.append(f"{source.key}.{self.column_key(source, name)}") + datey = datey or _datey(found[0][0]) + column, sources = inner, found[0][1] + return column, sources, tuple(via), datey @staticmethod def _aliases(query: Any) -> dict[str, Any]: @@ -1269,27 +1855,157 @@ class _Statement: def _aggregates(self, select: Any, sources: dict[str, _Source]) -> list[Aggregate]: from sqlglot import exp - out = [] - for node in _own_nodes(select): - if not isinstance(node, exp.AggFunc) or isinstance(node.parent, exp.Window): + return [self._aggregate(node, sources) for node in _own_nodes(select) + if isinstance(node, exp.AggFunc) and not isinstance(node.parent, exp.Window)] + + def _aggregate(self, node: Any, sources: dict[str, _Source]) -> Aggregate: + from sqlglot import exp + + try: + func = str(node.sql_name()).lower() + except Exception: # noqa: BLE001 - 没有固定函数名的聚合:按类型名 + func = node.key + arg = node.this + distinct = isinstance(arg, exp.Distinct) + if distinct: + items = list(arg.expressions) + arg = items[0] if len(items) == 1 else exp.Tuple(expressions=[i.copy() for i in items]) + if arg is None or isinstance(arg, exp.Star) or \ + (func == "count" and not distinct and isinstance(_unparen(arg), exp.Literal)): + part = Part.plain("*", "*") # COUNT(*)、COUNT(1):数的是行数 + else: + part = self.part(self.canonical(arg, sources), sources) + return Aggregate(func=func, arg=part, distinct=distinct, text=self.render(node, sources)) + + # ---- 算出来的列 + + def _expressions(self) -> list[Computed]: + """算出来的列(见 Computed):最外层(UNION 的各支)选出来、不是光一列也不是字面量的;用到了的派生来源里 + 被外面用到、又不是原样透出一列的列(原样透出的,外面的写法 resolve 已经透成了里面那一列,不用再比定义)。""" + from sqlglot import exp + + out: list[Computed] = [] + for select in _branches(self.root): + sources = self.sources(select) + for item in select.selects: + target = _unwrap_value(item.this if isinstance(item, exp.Alias) else item) + if not self._plain_output(target): + out.append(self._computed([(target, sources)], None, None, "", "")) + used = self._used() + for number, name, body, _ in self._derived: + label = self._labels[number] + if id(body) in self._dead or label not in used: continue - try: - func = str(node.sql_name()).lower() - except Exception: # noqa: BLE001 - 没有固定函数名的聚合:按类型名 - func = node.key - arg = node.this - distinct = isinstance(arg, exp.Distinct) - if distinct: - items = list(arg.expressions) - arg = items[0] if len(items) == 1 else exp.Tuple(expressions=[i.copy() for i in items]) - if arg is None or isinstance(arg, exp.Star) or \ - (func == "count" and not distinct and isinstance(_unparen(arg), exp.Literal)): - part = Part.plain("*", "*") # COUNT(*)、COUNT(1):数的是行数 - else: - part = self.part(self.canonical(arg, sources), sources) - out.append(Aggregate(func=func, arg=part, distinct=distinct, text=self.render(node, sources))) + source = _Source("derived", name or "子查询", label, (), body) + columns = [called for called in self.names(body) if called] + columns += sorted(c for c in used[label] if c not in {name_key(x) for x in columns}) + for column in dict.fromkeys(columns): + if name_key(column) not in used[label] or self.through(source, column)[0] is not source: + continue + if found := self._definition(source, column): + out.append(self._computed(found, source.name, label, self.column_key(source, column), column)) return out + @staticmethod + def _plain_output(target: Any) -> bool: + """最外层选出来的一项不用按算法比:光一列(分组那一面比)、*、字面量。光一个聚合照收:另一边可能是拿同一个 + 聚合再算了一下(SUM(amount) 对 SUM(amount) / 100),得配成一对说;两边只差聚合本身的,聚合那一面已经报了。""" + from sqlglot import exp + + return isinstance(_unparen(target), (exp.Column, exp.Star, exp.Literal, exp.Null, exp.Boolean)) + + def _computed(self, branches: list[tuple[Any, _Sources]], scope: str | None, scope_key: str | None, name: str, + shown: str) -> Computed: + """一列的定义(UNION 每支一个)→ Computed:定义本身、聚合换成占位符之后的写法、里面的聚合。""" + from sqlglot import exp + + exprs: list[Part] = [] + templates: list[Part] = [] + aggregates: list[Aggregate] = [] + for node, sources in branches: + canon = self.canonical(node, sources) + exprs.append(self.part(canon, sources)) + # 这一段自己的聚合(窗口函数、套着的子查询里的不算),按出现的先后 + own = [n for n in canon.walk(prune=lambda x: isinstance(x, (exp.Select, exp.Subquery))) + if isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) + and not any(isinstance(p, exp.AggFunc) for p in _parents(n, stop=None))] + aggregates += [self._aggregate(n, sources) for n in own] + if own and own[0] is canon: + templates.append(Part.plain(PLACEHOLDER, PLACEHOLDER)) # 光一个聚合 + continue + for n in own: + n.replace(exp.Placeholder()) + templates.append(Part(key=self.render(canon, sources, "key"), text="", + loose=self.render(canon, sources, "loose"))) + + def joined(parts: list[Part]) -> Part: + if len(parts) == 1: + return parts[0] + return Part(key=" | ".join(p.key for p in parts), text=" | ".join(p.text for p in parts), + loose=" | ".join(p.loose for p in parts), full=" | ".join(p.full for p in parts), + columns=tuple(c for p in parts for c in p.columns), + column_keys=tuple(k for p in parts for k in p.column_keys), + origins=tuple(o for p in parts for o in p.origins)) + + return Computed(scope=scope, scope_key=scope_key, name=name, shown=shown, expr=joined(exprs), + template=joined(templates), aggregates=tuple(aggregates)) + + def _used(self) -> dict[str, set[str]]: + """用到了的段里引用了派生来源的哪些列:{派生来源的编号: {列名的 name_key}}。原样透出的列连同里面那一层也算 + 用到了;最外层的 SELECT *(t.*)用到了那个来源的每一列;USING 的列两边都算。""" + from sqlglot import exp + + used: dict[str, set[str]] = {} + + def mark(source: _Source | None, column: str) -> None: + for _ in range(8): + if source is None or source.kind != "derived": + return + used.setdefault(source.key, set()).add(name_key(column)) + if (found := self._pass(source, column)) is None: + return + source, column = found + + def mark_all(source: _Source | None, depth: int = 0) -> None: + if source is None or source.kind != "derived" or depth > 4: + return + for branch in _branches(source.query)[:1]: + inner = self.sources(branch) + for item, called in zip(branch.selects, self.names(source.query)): + if called is None: + for found in self._starred(item, inner): + mark_all(found, depth + 1) + else: + mark(source, called) + + outermost = {id(s) for s in _branches(self.root)} + for select in self._alive(exp.Select): + sources = self.sources(select) + for node in _own_nodes(select): + if isinstance(node, exp.Column) and node.name and not node.is_star: + mark(self.locate(node, sources)[1], node.name) + if id(select) in outermost: + for item in select.selects: + if _is_star(item): + for found in self._starred(item, sources): + mark_all(found) + for join in select.args.get("joins") or []: + for ident in join.args.get("using") or []: + for found in sources.values(): + if self._exposes(found, ident.name): + mark(found, ident.name) + return used + + @staticmethod + def _starred(item: Any, sources: dict[str, _Source]) -> list[_Source]: + """SELECT 列表里的 *、t.* 选的是哪几个来源。""" + from sqlglot import exp + + if isinstance(item, exp.Column) and item.table: + found = sources.get(name_key(item.table)) + return [found] if found is not None else [] + return list(sources.values()) + def _limit(self) -> str | None: from sqlglot import exp @@ -1305,7 +2021,7 @@ class _Statement: from sqlglot import exp out = [] - for query in self.root.find_all(exp.Select, exp.SetOperation, bfs=False): + for query in self._alive(exp.Select, exp.SetOperation): if query is not self.root and self._in_condition(query): continue if (found := self._limit_of(query)) is not None: @@ -1393,9 +2109,10 @@ class _Statement: @dataclass(frozen=True) class FacetDiff: - """一处不同。aspect:time(时间列)/ time_range(同一时间列、范围不同)/ filter / aggregate / join / tables / - group_by / having / limit(取哪几行:行数、跳过的行数、取前几行时的排序)/ parse。a、b 是两边各自的写法(给人看, - 没有就是 None),text 是一句中文说明。 + """一处不同。aspect:time(时间列)/ time_range(同一时间列、范围不同,或者同一个时间条件筛的先后不同)/ filter / + aggregate / expression(算出来的列:最外层的算法、CTE 和子查询里被用到的列的定义,见 Computed)/ join(类型、 + 外连接的主表、关联条件、只有一边关联了)/ tables / group_by / having / limit(取哪几行:行数、跳过的行数、取前 + 几行时的排序)/ parse。a、b 是两边各自的写法(给人看,没有就是 None),text 是一句中文说明。 detail 只在 time 上有:两边的时间列 [{table, column}],调用方据此对照数据目录(业务日期)。""" aspect: str @@ -1500,6 +2217,9 @@ class _Compare: 派生来源的编号由 diff_facets 先按结构配好(两边配成一对的写同一个编号,见模块说明),这里各方面都只按 k 比: 编号不同就是不同的来源,不再拿「不写编号」的写法补配(那样会把 p.total 和 r.total 当成同一列)。 + + 几个方面之间互相交代,同一处不同只报一遍:聚合那一面报过的聚合(reported)、时间那一面报过的派生来源的列 + (explained),算出来的列那一面不再重复报。 """ def __init__(self, a: Facets, b: Facets, labels: tuple[str, str], names: Mapping[str, str] | None) -> None: @@ -1510,6 +2230,12 @@ class _Compare: self.ta = [t for t in a.tables if t.key not in {u.key for u in b.tables} and t.key not in joined] self.tb = [t for t in b.tables if t.key not in {u.key for u in a.tables} and t.key not in joined] self.swapped = len(self.ta) == 1 and len(self.tb) == 1 + #: 两边的派生来源 {编号: 名字}(同名却不是同一个时拿来说清对应关系)、各段来源 {key: 名字}(外连接的主表) + self.names = ({p.key: p.text for p in a.derived}, {p.key: p.text for p in b.derived}) + self.called = ({p.key: p.text for p in a.sources}, {p.key: p.text for p in b.sources}) + #: 聚合那一面报过的两边独有的聚合、时间那一面已经说了的派生来源的列(「编号.列」) + self.reported: tuple[set[Any], set[Any]] = (set(), set()) + self.explained: set[str] = set() def k(self, part: Part | None) -> str | None: if part is None: @@ -1522,7 +2248,13 @@ class _Compare: def sides(self, j: JoinFacet) -> tuple[str, ...]: """连接涉及的来源(见 JoinFacet.sides)。""" - return tuple(sorted(self.side(s) for s in j.sides)) + return self.keys(j.sides) + + def keys(self, keys: Iterable[str]) -> tuple[str, ...]: + return tuple(sorted(self.side(s) for s in keys)) + + def agg_key(self, g: Aggregate) -> tuple[str, str | None, bool]: + return g.func, self.k(g.arg), g.distinct def added(self, j: JoinFacet, mine: Facets, theirs: Facets) -> tuple[str, set[str]]: """一边独有的一次连接,多出来的是哪个来源:(给人看的名字, 比较用的 key)。 @@ -1542,7 +2274,8 @@ class _Compare: def unmatched(xs: Iterable[Any], ys: Iterable[Any], key: Any) -> tuple[list[Any], list[Any]]: """两边各自独有的项(按 key 比),各按原来的先后。派生来源在 key 里已经是两边按结构配好的共同编号(见 diff_facets),这里不再拿不写编号的写法补配一遍:那样两个 CTE 的同名列(p.total、r.total)会被当成一列, - 换了用哪一个的列也报不出来。""" + 换了用哪一个的列也报不出来。配不上的派生来源(一边多出来的 CTE)编号和另一边的都不一样,按它写的项照样 + 各算各的。""" xs, ys = list(xs), list(ys) kx, ky = {key(x) for x in xs}, {key(y) for y in ys} return [x for x in xs if key(x) not in ky], [y for y in ys if key(y) not in kx] @@ -1561,10 +2294,74 @@ class _Compare: return pairs @staticmethod - def shown(xs: list[str], ys: list[str], full_xs: list[str], full_ys: list[str]) -> tuple[str, str]: - """两边给人看的写法;读起来一样(各自只写列名)却不是同一个东西时,改成一律写表名的写法。""" - wa, wb = "、".join(xs), "、".join(ys) - return ("、".join(full_xs), "、".join(full_ys)) if wa == wb else (wa, wb) + def staged(only_a: list[Part], only_b: list[Part]) -> list[tuple[Part, Part]]: + """两边各自独有、一路透到底却是同一个条件的配成一对,就地从两边拿掉(见 Part.pierce):只有一边被取前几行、 + 算窗口函数的派生来源挡住,是同一个条件筛的先后不同,不是两个条件。""" + pairs = [] + for p in list(only_a): + q = next((q for q in only_b + if bool(p.stage) != bool(q.stage) and (p.pierce or p.key) == (q.pierce or q.key)), None) + if q is not None: + pairs.append((p, q)) + only_a.remove(p) + only_b.remove(q) + return pairs + + def stage_text(self, p: Part, q: Part) -> str: + """同一个条件筛的先后不同:哪一边先取前几行 / 算窗口函数、再筛,哪一边先筛。条件照没被挡住的那一边写:它 + 写的是里面那一列,读起来和原 SQL 一样。""" + cond, stage = (q if p.stage else p).text, p.stage or q.stage + if p.stage: + return _say(self.la, "先", stage, "、再按", cond, "筛选,", self.lb, "先按", cond, "筛选、再", stage) + return _say(self.la, "先按", cond, "筛选、再", stage, ",", self.lb, "先", stage, "、再按", cond, "筛选") + + def shown(self, xs: list[Part], ys: list[Part], *, text: Any = None, full: Any = None) -> tuple[str, str]: + """两边给人看的写法。读起来一样却不是同一个东西时,依次改成:一律写表名的写法(各自只写列名时两张表的 + amount 读起来一样);带上派生来源各列的来历(两边同名的 t1 一个算付款合计、一个算退款合计);两边同名的派生 + 来源各对应另一边的哪一个(两个 CTE 只差里面的条件,来历也一样)。""" + text = text or (lambda ps: "、".join(p.text for p in ps)) + full = full or (lambda ps: "、".join(p.full for p in ps)) + wa, wb = text(xs), text(ys) + if wa != wb: + return wa, wb + wa, wb = full(xs), full(ys) + return (wa, wb) if wa != wb else self.annotated(wa, wb, xs, ys) + + def annotated(self, wa: str, wb: str, xs: list[Part], ys: list[Part]) -> tuple[str, str]: + """两边一律写表名的写法 wa、wb 也一样时(见 shown):带上派生来源各列的来历,来历也一样就说清对应关系。""" + ga, gb = self.gloss(xs), self.gloss(ys) + if ga != gb: + return _say(wa, ga), _say(wb, gb) + return wa, _say(wb, self.correspondence(xs, ys)) + + @staticmethod + def gloss(parts: list[Part]) -> str: + """用到的派生来源的列各自的来历(见 Part.origins):「(即 SUM(orders.amount))」,好几列时「(p.total 即 …、 + r.total 即 …)」。没有返回空串。""" + notes: list[str] = [] + for p in parts: + for column, origin in p.origins: + note = _say("即", origin) if len(p.origins) == 1 and p.full == column else _say(column, "即", origin) + if note not in notes: + notes.append(note) + return f"({'、'.join(notes)})" if notes else "" + + def correspondence(self, xs: list[Part], ys: list[Part]) -> str: + """用到的派生来源两边同名、却各自对应另一边别的派生来源时,说清对应关系:「(盲算的 paid 对应原流程的 open)」; + 一边的在另一边配不上的也说。没有返回空串。""" + keys = dict.fromkeys(ck.split(".", 1)[0].split("#", 1)[0] for p in (*ys, *xs) for ck in p.column_keys + if ck.startswith("@")) + na, nb = self.names + notes = [] + for key in keys: + a, b = na.get(key), nb.get(key) + if a and b and a != b: + notes.append(_say(self.lb, "的", b, "对应", self.la, "的", a)) + elif b and not a: + notes.append(_say(self.lb, "的", b, "在", self.la, "里没有对应的")) + elif a and not b: + notes.append(_say(self.la, "的", a, "在", self.lb, "里没有对应的")) + return f"({'、'.join(notes)})" if notes else "" def named(self, parts: list[Part], *, full: bool = False) -> str: """时间列:有中文名的写「支付时间 paid_at」,几列用顿号隔开。""" @@ -1589,9 +2386,7 @@ class _Compare: ca = list(_dedupe((t.column for t in a.time), self.k)) cb = list(_dedupe((t.column for t in b.time), self.k)) if any(self.unmatched(ca, cb, self.k)): - wa, wb = self.named(ca), self.named(cb) - if wa == wb: - wa, wb = self.named(ca, full=True), self.named(cb, full=True) + wa, wb = self.shown(ca, cb, text=self.named, full=lambda ps: self.named(ps, full=True)) if ca and cb: text = _say(la, "按", wa, "筛时间,", lb, "按", wb) elif ca: @@ -1600,33 +2395,43 @@ class _Compare: text = _say(la, "没有按时间筛选,", lb, "按", wb, "筛时间") detail = {side: [{"table": p.columns[0][0], "column": p.columns[0][1]} for p in parts if p.columns] for side, parts in (("a", ca), ("b", cb))} + # 时间列是从 CTE、子查询算出来的列透过去的(见 TimePredicate.column):那一列的定义不同,这里已经说了 + self.explained |= {v for t in (*a.time, *b.time) for v in t.via} return [FacetDiff("time", "、".join(p.text for p in ca) or None, "、".join(p.text for p in cb) or None, text, detail)] - if any(self.unmatched(a.time, b.time, lambda t: self.k(t.condition))): - ra = " AND ".join(t.condition.text for t in a.time) - rb = " AND ".join(t.condition.text for t in b.time) - return [FacetDiff("time_range", ra, rb, _say(la, "的时间范围是", ra, ",", lb, "是", rb))] - return [] + only_a, only_b = self.unmatched(a.time, b.time, lambda t: self.k(t.condition)) + conds_a, conds_b = [t.condition for t in only_a], [t.condition for t in only_b] + pairs = self.staged(conds_a, conds_b) + out = [FacetDiff("time_range", p.text, q.text, self.stage_text(p, q)) for p, q in pairs] + if conds_a or conds_b: + staged = {id(p) for pair in pairs for p in pair} + ra = [t.condition for t in a.time if id(t.condition) not in staged] + rb = [t.condition for t in b.time if id(t.condition) not in staged] + wa, wb = self.shown(ra, rb, text=lambda ps: " AND ".join(p.text for p in ps), + full=lambda ps: " AND ".join(p.full for p in ps)) + out.append(FacetDiff("time_range", wa, wb, _say(la, "的时间范围是", wa, ",", lb, "是", wb))) + return out def filters(self) -> list[FacetDiff]: timed = {self.k(t.condition) for t in (*self.a.time, *self.b.time)} fa = [p for p in self.a.where if self.k(p) not in timed] fb = [p for p in self.b.where if self.k(p) not in timed] only_a, only_b = self.unmatched(fa, fb, self.k) - out: list[FacetDiff] = [] - # 先配对同一列的不同写法(status = 'paid' 和 status IN (…)),再配读起来一样、挂在不同来源上的同一个条件 + # 先配同一个条件筛的先后不同的,再配同一列的不同写法(status = 'paid' 和 status IN (…)),再配读起来一样、 + # 挂在不同来源上的同一个条件 + out = [FacetDiff("filter", p.text, q.text, self.stage_text(p, q)) for p, q in self.staged(only_a, only_b)] for p in list(only_a): column = _single_column(p, loose=self.swapped) mine = [q for q in only_a if column and _single_column(q, loose=self.swapped) == column] theirs = [q for q in only_b if column and _single_column(q, loose=self.swapped) == column] if len(mine) == 1 and len(theirs) == 1: q = theirs[0] - wa, wb = self.shown([p.text], [q.text], [p.full], [q.full]) + wa, wb = self.shown([p], [q]) out.append(FacetDiff("filter", p.text, q.text, _say(self.la, "按", wa, "筛选,", self.lb, "按", wb, "筛选"))) only_a.remove(p) only_b.remove(q) for p, q in self.lookalikes(only_a, only_b): - wa, wb = self.shown([p.text], [q.text], [p.full], [q.full]) + wa, wb = self.shown([p], [q]) out.append(FacetDiff("filter", wa, wb, _say(self.la, "按", wa, "筛选,", self.lb, "按", wb, "筛选"))) out += [FacetDiff("filter", p.text, None, self.one_sided(self.la, self.lb, p)) for p in only_a] out += [FacetDiff("filter", None, p.text, self.one_sided(self.lb, self.la, p)) for p in only_b] @@ -1635,9 +2440,6 @@ class _Compare: def aggregates(self) -> list[FacetDiff]: la, lb = self.la, self.lb - def key(g: Aggregate) -> tuple[str, str | None, bool]: - return g.func, self.k(g.arg), g.distinct - def pair(xs: list[Aggregate], ys: list[Aggregate], same: Any) -> list[tuple[Aggregate, Aggregate]]: pairs = [] for x in list(xs): @@ -1648,7 +2450,8 @@ class _Compare: ys.remove(y) return pairs - ga, gb = self.unmatched(self.a.aggregates, self.b.aggregates, key) + ga, gb = self.unmatched(self.a.aggregates, self.b.aggregates, self.agg_key) + self.reported = ({self.agg_key(g) for g in ga}, {self.agg_key(g) for g in gb}) # 先配参数相同的(同一列:函数或去重不同),再配函数相同的(同一种算法:列不同);各剩一个的也配成一对 pairs = pair(ga, gb, lambda x, y: self.k(x.arg) == self.k(y.arg)) pairs += pair(ga, gb, lambda x, y: x.func == y.func) @@ -1657,8 +2460,7 @@ class _Compare: out = [] for x, y in pairs: same_arg = self.k(x.arg) == self.k(y.arg) - wa, wb = self.shown([x.arg.text], [y.arg.text], [x.arg.full], [y.arg.full]) if not same_arg \ - else (x.arg.text, y.arg.text) + wa, wb = self.shown([x.arg], [y.arg]) if not same_arg else (x.arg.text, y.arg.text) if same_arg and x.func == y.func: # 只差去重:COUNT(DISTINCT order_id) 和 COUNT(order_id) text = _say(la, "对", wa, _verb(x) + ",", lb, "没去重") if x.distinct \ @@ -1675,23 +2477,101 @@ class _Compare: out += [FacetDiff("aggregate", None, g.text, _say(lb, "多算了", g.text, ",", la, "没有")) for g in gb] return out + def expressions(self) -> list[FacetDiff]: + """算出来的列(见 Computed)。最外层的按算法比,两边各自独有的按先后配成一对说「算的是什么」;派生来源里被 + 用到的列按两边配好对的编号和列名配,定义不同就报。只差在聚合上、聚合那一面已经报了的,按透过去的那一列 + 筛时间、时间那一面已经报了的,不再报一遍。""" + la, lb = self.la, self.lb + out: list[FacetDiff] = [] + + def aggs(c: Computed) -> list[str]: + return sorted(repr(self.agg_key(g)) for g in c.aggregates) + + def pair(xs: list[Computed], ys: list[Computed], same: Any) -> list[tuple[Computed, Computed]]: + pairs = [] + for x in list(xs): + y = next((y for y in ys if same(x, y)), None) + if y is not None: + pairs.append((x, y)) + xs.remove(x) + ys.remove(y) + return pairs + + xa = [c for c in self.a.expressions if c.scope_key is None] + xb = [c for c in self.b.expressions if c.scope_key is None] + only_a, only_b = self.unmatched(xa, xb, lambda c: self.k(c.expr)) + # 先配用到的聚合一样的(同样的合计,算法不同:SUM(amount) 对 SUM(amount) / 100),再配算法一样的(聚合不同), + # 各剩一个的也配成一对 + pairs = pair(only_a, only_b, lambda x, y: aggs(x) == aggs(y)) + pairs += pair(only_a, only_b, lambda x, y: self.k(x.template) == self.k(y.template)) + if len(only_a) == 1 and len(only_b) == 1: + pairs.append((only_a.pop(), only_b.pop())) + for x, y in pairs: + if not self.explained_by_aggregates(x, y): + wa, wb = self.shown([x.expr], [y.expr]) + out.append(FacetDiff("expression", x.expr.text, y.expr.text, + _say(la, "算的是", wa, ",", lb, "算的是", wb))) + # 一边独有的光一个聚合不说:聚合那一面按整条语句比过了(另一边可能在子查询里算了同一个合计、外面只选那一列) + out += [FacetDiff("expression", c.expr.text, None, _say(la, "多算了", c.expr.text, ",", lb, "没有")) + for c in only_a if c.template.key != PLACEHOLDER] + out += [FacetDiff("expression", None, c.expr.text, _say(lb, "多算了", c.expr.text, ",", la, "没有")) + for c in only_b if c.template.key != PLACEHOLDER] + ib = {(c.scope_key, c.name): c for c in self.b.expressions if c.scope_key is not None} + for x in self.a.expressions: + y = ib.get((x.scope_key, x.name)) + if x.scope_key is None or y is None or self.k(x.expr) == self.k(y.expr) or \ + f"{x.scope_key}.{x.name}" in self.explained: + continue + # 两边起名不同、按位置配成同一列的(见 _renamed)照报:聚合那一面只说了合计不同,没说外面用的是这一列 + if name_key(y.shown) == y.name and self.explained_by_aggregates(x, y): + continue + wa, wb = self.shown([x.expr], [y.expr]) + ca, cb = f"{x.scope}.{x.shown}", f"{y.scope}.{y.shown}" + out.append(FacetDiff("expression", wa, wb, + _say(la, "的", ca, "是", wa, ",", lb, *(("是",) if ca == cb else ("的", cb, "是")), wb))) + return out + + def explained_by_aggregates(self, x: Computed, y: Computed) -> bool: + """两边的算法只差在里面的聚合上(聚合换成占位符之后一样),差的那几个聚合聚合那一面又都报了:不再报一遍。 + 聚合的集合一样、只是换了位置(SUM(a) - SUM(b) 对 SUM(b) - SUM(a)),聚合那一面什么也没报,照报。""" + if self.k(x.template) != self.k(y.template) or len(x.aggregates) != len(y.aggregates): + return False + ka, kb = self.reported + differing = [(g, h) for g, h in zip(x.aggregates, y.aggregates) if self.agg_key(g) != self.agg_key(h)] + return bool(differing) and all(self.agg_key(g) in ka or self.agg_key(h) in kb for g, h in differing) + def joins(self) -> list[FacetDiff]: """连接:先把完全一样的两边配掉,剩下的按关联的是哪张表(派生来源按配好的编号)、再按涉及的来源配成一对, - 说类型或条件哪里不同,再剩下的是只有一边关联了(多出来的是哪个来源见 added)。完全一样:内连接、CROSS JOIN - 比类型、涉及的来源和条件(FROM paid JOIN refunded 和 FROM refunded JOIN paid 是同一次连接),别的连接比类型、 - 关联进来的表和条件。先配完全一样的,同一张表关联了两次、先后不同也不算不同。配成一对的两边名字不同(两边给 + 说类型、主表或条件哪里不同,再剩下的是只有一边关联了(多出来的是哪个来源见 added)。 + + 完全一样:内连接、CROSS JOIN 比类型、涉及的来源和条件(FROM paid JOIN refunded 和 FROM refunded JOIN paid 是 + 同一次连接);LEFT / RIGHT JOIN 比主表、涉及的来源和条件,不比写的是 LEFT 还是 RIGHT(orders LEFT JOIN refunds + 和 refunds RIGHT JOIN orders 是同一次连接,和 refunds LEFT JOIN orders 不是);FULL JOIN 比涉及的来源和条件; + 其余(SEMI、ANTI、NATURAL)比类型、关联进来的表和条件。外连接被同一段 WHERE 里的条件筛掉了没关联上的行 + (JoinFacet.rejects)按内连接比:写的是 LEFT JOIN、实际是内连接,和另一边真的 LEFT JOIN 要报出来,和另一边写的 + INNER JOIN 是一回事。先配完全一样的,同一张表关联了两次、先后不同也不算不同。配成一对的两边名字不同(两边给 CTE 起的名字不同、内连接换了先后)时两个名字都说。派生来源已经按结构配好了编号:一边多出来的 CTE 编号和 另一边的都不一样,不会被配给另一边不相干的 CTE、说成「关联条件不同」。 - 做不到的:LEFT JOIN 和两边对调的 RIGHT JOIN 意思一样,仍报成类型不同;内连接里只涉及一张表的条件写在 ON 里 - 还是 WHERE 里意思一样,仍算两处不同;三张以上不带关联条件的表(逗号连接、CROSS JOIN)换了先后,涉及的来源 - 按「它之前的各个来源」算,仍可能算不同。""" + 做不到的:内连接里只涉及一张表的条件写在 ON 里还是 WHERE 里意思一样,仍算两处不同;三张以上不带关联条件的 + 表(逗号连接、CROSS JOIN)换了先后,涉及的来源按「它之前的各个来源」算,仍可能算不同;外连接链(a LEFT JOIN b + LEFT JOIN c)的主表按关联条件里出现的来源算,不按整个左边算;WHERE 筛掉了 c 没关联上的行时,c 是按 b 的列关联的, + b 没关联上的行连带也没了,前面那次 LEFT JOIN 其实也成了内连接,这里只认后面那次;FULL JOIN 被 WHERE 筛掉一边的 + 不认。""" la, lb = self.la, self.lb + def effective(j: JoinFacet) -> str: + return "inner" if j.rejects is not None and j.kind in ("left", "right") else j.kind + def same(j: JoinFacet) -> tuple[Any, ...]: - if j.kind in ("inner", "cross"): - return j.kind, self.sides(j), self.k(j.on) - return j.kind, j.table.key, self.k(j.on) + kind = effective(j) + if kind in ("inner", "cross"): + return kind, self.sides(j), self.k(j.on) + if kind in ("left", "right"): + return "outer", self.keys(j.keep), self.sides(j), self.k(j.on) + if kind == "full": + return kind, self.sides(j), self.k(j.on) + return kind, j.table.key, self.k(j.on) rest_a, rest_b = self.unmatched(self.a.joins, self.b.joins, same) pairs: list[tuple[JoinFacet, JoinFacet]] = [] @@ -1705,7 +2585,19 @@ class _Compare: out: list[FacetDiff] = [] for x, y in pairs: tx, ty = x.table.text, y.table.text - if x.kind != y.kind: + ex, ey = effective(x), effective(y) + outer_x, outer_y = ex in ("left", "right"), ey in ("left", "right") + if outer_x and outer_y: + if self.keys(x.keep) != self.keys(y.keep): + wa, wb = self.outer(x, 0), self.outer(y, 1) + out.append(FacetDiff("join", wa, wb, _say(la, wa, ",", lb, wb))) + elif (outer_x and y.rejects is not None) or (outer_y and x.rejects is not None): + wa = self.outer(x, 0) if outer_x else self.rejected(x, 0) + wb = self.outer(y, 1) if outer_y else self.rejected(y, 1) + out.append(FacetDiff("join", self.outer(x, 0) if outer_x else _JOIN_WORDS[x.kind] + "(实际是 INNER JOIN)", + self.outer(y, 1) if outer_y else _JOIN_WORDS[y.kind] + "(实际是 INNER JOIN)", + _say(la, wa, ",", lb, wb))) + elif ex != ey: kx, ky = _JOIN_WORDS.get(x.kind, x.kind), _JOIN_WORDS.get(y.kind, y.kind) tail = () if ty == tx else ("关联", ty, "表") out.append(FacetDiff("join", kx, ky, _say(la, "用", kx, "关联", tx, "表,", lb, "用", ky, *tail))) @@ -1732,14 +2624,30 @@ class _Compare: for t in self.tb if t.key not in said["b"]] return out + def called_as(self, keys: Iterable[str], side: int) -> str: + """来源 key → 给人看的名字(那一边的叫法),几个用顿号隔开。""" + return "、".join(self.called[side].get(k, k) for k in keys) + + def outer(self, j: JoinFacet, side: int) -> str: + """一次 LEFT / RIGHT JOIN 保留哪一侧:「以 orders 为主表外连接 refunds」。不说 LEFT 还是 RIGHT:orders RIGHT + JOIN refunds 保留的是 refunds,照写关键字反倒容易读反。""" + return _say("以", self.called_as(j.keep, side), "为主表外连接", self.called_as(j.nullable, side)) + + def rejected(self, j: JoinFacet, side: int) -> str: + """写的是外连接、WHERE 里的条件又把没关联上的行筛掉了(见 JoinFacet.rejects)。""" + word = _JOIN_WORDS.get(j.kind, j.kind) + cond = j.rejects.full if j.rejects is not None else "" + return _say("的 WHERE 里有", cond, ",把关联不上", self.called_as(j.nullable, side), "的行筛掉了,", word, + "实际成了 INNER JOIN") + def group(self) -> list[FacetDiff]: a, b = self.a.group_by, self.b.group_by if not any(self.unmatched(a, b, self.k)): return [] - wa, wb = self.shown([p.text for p in a], [p.text for p in b], [p.full for p in a], [p.full for p in b]) - if wa and wb: + wa, wb = self.shown(list(a), list(b)) + if a and b: text = _say(self.la, "按", wa, "分组,", self.lb, "按", wb, "分组") - elif wa: + elif a: text = _say(self.la, "按", wa, "分组,", self.lb, "没有分组") else: text = _say(self.la, "没有分组,", self.lb, "按", wb, "分组") @@ -1749,7 +2657,7 @@ class _Compare: only_a, only_b = self.unmatched(self.a.having, self.b.having, self.k) out = [] for p, q in self.lookalikes(only_a, only_b): - wa, wb = self.shown([p.text], [q.text], [p.full], [q.full]) + wa, wb = self.shown([p], [q]) out.append(FacetDiff("having", wa, wb, _say(self.la, "分组后只保留", wa, "的组,", self.lb, "只保留", wb, "的组"))) return out + \ @@ -1810,20 +2718,22 @@ class _Compare: wa, wb = self.sorting(x), self.sorting(y) if wa == wb: wa, wb = self.sorting(x, full=True), self.sorting(y, full=True) + if wa == wb: + wa, wb = self.annotated(wa, wb, [p for p, _ in x.order], [p for p, _ in y.order]) out.append(FacetDiff("limit", wa, wb, _say(la, "取行前", wa, ",", lb, wb))) return out def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, names: Mapping[str, str] | None = None) -> list[FacetDiff]: - """逐个方面比两份 facets,每处不同一条,按对结果影响大致从大到小排:时间、筛选、聚合、连接和表、分组、HAVING、 - 取哪几行。完全一致返回空列表。 + """逐个方面比两份 facets,每处不同一条,按对结果影响大致从大到小排:时间、筛选、聚合、算出来的列、连接和表、 + 分组、HAVING、取哪几行。完全一致返回空列表。 labels 是两边的叫法(缺省 A / B,双盲复算可以传「原流程」「盲算」);names 是列的中文名({「表.列」或「列」: 中文名},可以用 catalog_labels 从数据目录取),给了的话时间列写成「支付时间 paid_at」。任一边解析不了就只返回 一条 parse,不逐项比较:拿半份结构去比,只会报出一堆不存在的不同。 - 比之前先把两边的派生来源按结构配好对,按共同的编号重新分析 b(见模块说明、_aligned);给人看的写法不受影响。 + 比之前先把两边的派生来源按结构配好对,按共同的编号重新分析两边(见模块说明、_aligned);给人看的写法不受影响。 """ la, lb = labels if a.parse_error or b.parse_error: @@ -1833,30 +2743,53 @@ def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, side, error = (la, a.parse_error) if a.parse_error else (lb, b.parse_error) text = _say(side, ":", error or "", ",没有逐项比较") return [FacetDiff("parse", a.parse_error, b.parse_error, text)] - cmp = _Compare(a, _aligned(a, b), labels, names) - return cmp.time() + cmp.filters() + cmp.aggregates() + cmp.joins() + cmp.group() + cmp.having() + cmp.limit() + cmp = _Compare(*_aligned(a, b), labels, names) + return cmp.time() + cmp.filters() + cmp.aggregates() + cmp.expressions() + cmp.joins() + cmp.group() + \ + cmp.having() + cmp.limit() -def _aligned(a: Facets, b: Facets) -> Facets: +def _aligned(a: Facets, b: Facets) -> tuple[Facets, Facets]: """b 的派生来源按结构配给 a 的(见 _align),配上的改写成 a 里对应的编号、配不上的另起 a 里没有的编号,按这套 - 编号重新分析 b 的原文。a 的编号就是它的位置编号,不用重算。 + 编号重新分析两边的原文(a 的编号就是它的位置编号)。 - 任一边不是 facets 算出来的(没记原文)、重新解析不了,或者分析时出了意外,照原样返回 b:两边仍按各自的位置 - 编号比,一边多出排在前面的 CTE 时会多报。 + 没用到的 CTE 不收进 facets(见 _Statement._enliven),只有一种例外:两边都定义了同一个 CTE(配成了一对),一边 + 用到、一边没用到,而用到的那边是把它关联进来的。连接那一面会说「X 关联了它,Y 没有」;它里面的条件、聚合两边的 + 定义里都有,只收用到的那一边,就会再被说成一边独有的一堆条件、聚合(两边都定义了付款、退款两个 CTE,一边关联 + 付款的、一边关联退款的)。这时没用到的那一边也照样收它。用到的那边是从它取数(FROM 它)而不是关联它时不收: + 盲算定义了只筛已付的 CTE 却直接查 orders,少的那个条件正要报出来。 + + 任一边不是 facets 算出来的(没记原文)、重新解析不了,或者分析时出了意外,照原样返回:两边仍按各自的位置编号 + 比,一边多出排在前面的 CTE 时会多报。 """ try: sa, sb = _Statement.reparse(a), _Statement.reparse(b) if sa is None or sb is None: - return b + return a, b da, db = sa.derived(), sb.derived() pairs = _align(da, db) fresh = iter(range(max((d.number for d in da), default=0) + 1, 1 << 30)) - sb.relabel({d.number: f"@{pairs[d.number]}" if d.number in pairs else f"@{next(fresh)}" for d in db}) - return replace(sb.facets(), _origin=b._origin) + sb.relabel({d.number: f"@{pairs[d.number]}" if d.number in pairs else f"@{next(fresh)}" for d in db}, + _renamed(da, db, pairs)) + fa, fb = sa.facets(), sb.facets() + extra_a = _switched(sb, fb, sa) + extra_b = _switched(sa, fa, sb) + if extra_a or extra_b: + fa, fb = sa.facets(extra_a), sb.facets(extra_b) + return replace(fa, _origin=a._origin), replace(fb, _origin=b._origin) except Exception: # noqa: BLE001 - 配对本身的缺陷不能连累比较:退回按各自的编号比,记下来修 logger.warning("SQL 派生来源配对失败,按各自的位置编号比较", exc_info=True) - return b + return a, b -__all__ = ["MAX_SQL_CHARS", "PLACEHOLDER", "Aggregate", "ColumnRef", "FacetDiff", "Facets", "JoinFacet", "LimitFacet", - "NormalizedSql", "Part", "TimePredicate", "catalog_labels", "diff_facets", "facets", "normalize_sql"] +def _switched(user: "_Statement", facets_of_user: Facets, other: "_Statement") -> set[str]: + """user 用到、而且关联进来了的派生来源里,other 也定义了(配成了一对)却没用到的 CTE:other 照样收它(见 + _aligned)。返回 other 里这些 CTE 的 name_key。""" + joined = {k for j in facets_of_user.joins for k in (*j.sides, j.table.key)} + used, defined = other.used_labels(), other.defined_labels() + return {cte for label in user.used_labels() if label in joined and label not in used + and (cte := defined.get(label)) is not None} + + +__all__ = ["MAX_SQL_CHARS", "PLACEHOLDER", "Aggregate", "ColumnRef", "Computed", "FacetDiff", "Facets", "JoinFacet", + "LimitFacet", "NormalizedSql", "Part", "TimePredicate", "catalog_labels", "diff_facets", "facets", + "normalize_sql"] diff --git a/backend/tests/test_sql_semantics.py b/backend/tests/test_sql_semantics.py index f4afdb2f..6d9d93a1 100644 --- a/backend/tests/test_sql_semantics.py +++ b/backend/tests/test_sql_semantics.py @@ -9,7 +9,10 @@ - diff_facets:每处不同一句中文:「A 按支付时间 paid_at 筛时间,B 按下单时间 created_at」「A 排除了 is_deleted = 1 的行,B 没有」「A 对 order_id 去重计数,B 没去重」……写法不同、口径相同的不报。两边的派生来源先按 结构(读的表、各列的定义、名字)配好对再比:一边多出一个排在前面的 CTE、内连接换了先后不报,两个 CTE 算出同名 - 的列时外层用的是哪一个照报 + 的列时外层用的是哪一个照报;外连接按主表比(LEFT / RIGHT 对调不报,主表换了、被 WHERE 变成内连接照报);算出来 + 的列按算法比(最外层的算式、窗口函数,CTE 里被外面用到的列的定义);时间列透过 CTE 里的日期表达式认;取前几行、 + 窗口函数前后筛的不同照报;没用到的 CTE 不算;两边同名却不是同一个的派生来源说清各是什么;自己关联自己的两个 + 别名分得开 夹具全是虚构的表:订单 orders、退款 refunds、门店 stores、用户 users、付款 payments,以及一张中文表「订单」。 """ @@ -885,6 +888,453 @@ def test_which_rows_are_taken_is_compared(a, b, text): assert texts("SELECT SUM(amount) FROM orders ORDER BY 1 DESC", "SELECT SUM(amount) FROM orders") == [] +# -------------------------------------------------------------------------- +# diff_facets:外连接以哪一侧为主表 +# -------------------------------------------------------------------------- + +LABELS = ("原流程", "盲算") + + +@pytest.mark.parametrize("dialect,a,b,expected", [ + ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id", + ["原流程以 orders 为主表外连接 refunds,盲算以 refunds 为主表外连接 orders"]), + # 「没有退款的订单数」对「order_id 为空的退款数」:IS NULL 不会把外连接变成内连接,差的是主表 + ("sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.order_id IS NULL", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE r.order_id IS NULL", + ["原流程以 orders 为主表外连接 refunds,盲算以 refunds 为主表外连接 orders"]), + ("postgres", "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id", + ["原流程以 refunds 为主表外连接 orders,盲算以 orders 为主表外连接 refunds"]), + ("postgres", PAID_REFUNDED + "SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", + PAID_REFUNDED + "SELECT COUNT(*) FROM refunded r LEFT JOIN paid p ON p.user_id = r.user_id", + ["原流程以 paid 为主表外连接 refunded,盲算以 refunded 为主表外连接 paid"]), + ("mysql", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p " + "LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r " + "LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id", + ["原流程以 p 为主表外连接 r,盲算以 r 为主表外连接 p"]), +], ids=["left", "left-anti", "right", "cte", "subquery"]) +def test_which_side_an_outer_join_keeps_is_compared(dialect, a, b, expected): + """外连接两边对调,保留的就是另一张表的全部行(有没有退款的订单都算,还是有没有订单的退款都算):按主表比, + 关联的表、条件一样也要报出来。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +@pytest.mark.parametrize("dialect,a,b", [ + ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("postgres", PAID_REFUNDED + "SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", + PAID_REFUNDED + "SELECT COUNT(*) FROM refunded r RIGHT JOIN paid p ON p.user_id = r.user_id"), + # 外连接后又在 WHERE 里筛可空那一侧:等于内连接加同一个条件 + ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'"), + # 筛的是主表那一侧,外连接照旧 + ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + "SELECT COUNT(*) FROM orders AS o LEFT OUTER JOIN refunds AS r ON o.id = r.order_id WHERE o.status = 'paid'"), + # 外连接链换了先后,各自的主表不变 + ("sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id LEFT JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM orders o LEFT JOIN users u ON u.id = o.user_id LEFT JOIN refunds r ON r.order_id = o.id"), +], ids=["left-vs-right", "left-vs-right-cte", "rejected-left-vs-inner", "filter-on-kept-side", "chain-reordered"]) +def test_the_same_outer_join_written_differently_is_no_difference(dialect, a, b): + """orders LEFT JOIN refunds 和 refunds RIGHT JOIN orders 是同一次连接:按主表比,不按写的是 LEFT 还是 RIGHT。""" + assert texts(a, b, dialect) == [] + + +@pytest.mark.parametrize("dialect,a,b,expected", [ + ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN (SELECT * FROM refunds WHERE state = 'done') r " + "ON r.order_id = o.id", + "SELECT COUNT(*) FROM orders o LEFT JOIN (SELECT * FROM refunds) r ON r.order_id = o.id WHERE r.state = 'done'", + ["原流程以 orders 为主表外连接 r,盲算的 WHERE 里有 refunds.state = 'done',把关联不上 r 的行筛掉了," + "LEFT JOIN 实际成了 INNER JOIN"]), + ("mysql", "WITH done AS (SELECT * FROM refunds WHERE state = 'done') " + "SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id", + "WITH done AS (SELECT * FROM refunds) " + "SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id WHERE d.state = 'done'", + ["原流程以 orders 为主表外连接 done,盲算的 WHERE 里有 refunds.state = 'done',把关联不上 done 的行筛掉了," + "LEFT JOIN 实际成了 INNER JOIN"]), + # 盲算以 refunds 为主表,WHERE 又筛掉了没对上订单的行:没有退款的已付订单原流程算、盲算不算 + ("postgres", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE o.status = 'paid'", + ["原流程以 orders 为主表外连接 refunds,盲算的 WHERE 里有 orders.status = 'paid',把关联不上 orders 的行筛掉了," + "LEFT JOIN 实际成了 INNER JOIN"]), + ("postgres", "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + ["盲算只保留 status = 'paid' 的行,原流程没有这个条件", + "原流程以 refunds 为主表外连接 orders,盲算的 WHERE 里有 orders.status = 'paid',把关联不上 orders 的行筛掉了," + "RIGHT JOIN 实际成了 INNER JOIN"]), +], ids=["subquery", "cte", "swapped-and-filtered", "right"]) +def test_an_outer_join_turned_inner_by_where_is_reported(dialect, a, b, expected): + """可空那一侧的条件写在 ON / 子查询里,和写在 WHERE 里不是一回事:WHERE 里拿空值比不成立,没关联上的行被筛掉, + LEFT JOIN 实际成了内连接。条件本身两边一样(透过子查询都是 refunds.state = 'done'),只报连接。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +def test_only_null_rejecting_where_conditions_turn_an_outer_join_inner(): + """IS NULL、和 IS NULL 用 OR 连着的、套在 COALESCE 里的,没关联上的行照样留着:外连接不变(宁可漏报)。""" + left = "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id" + inner = "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id" + either = " WHERE r.state = 'done' OR r.id IS NULL" + assert texts(left + either, inner + either) == ["A 用 LEFT JOIN 关联 refunds 表,B 用 INNER JOIN"] + assert aspects(left + " WHERE COALESCE(r.amount, 0) < 100", left) == ["filter"] + + +# -------------------------------------------------------------------------- +# diff_facets:算出来的列 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dialect,a,b,expected", [ + ("sqlite", PAID_REFUNDED + f"SELECT SUM(p.total) - SUM(r.total) {PR_JOIN}", + PAID_REFUNDED + f"SELECT SUM(r.total) - SUM(p.total) {PR_JOIN}", + ["原流程算的是 SUM(paid.total) - SUM(refunded.total),盲算算的是 SUM(refunded.total) - SUM(paid.total)"]), + # 退款率的分子分母对调 + ("postgres", f"SELECT SUM(r.total) / SUM(p.total) {PR_SUBQUERIES}", + f"SELECT SUM(p.total) / SUM(r.total) {PR_SUBQUERIES}", + ["原流程算的是 SUM(r.total) / SUM(p.total),盲算算的是 SUM(p.total) / SUM(r.total)"]), + # 窗口:每个用户的合计还是每家门店的合计 + ("sqlite", "SELECT order_id, SUM(amount) OVER (PARTITION BY user_id) AS s FROM orders", + "SELECT order_id, SUM(amount) OVER (PARTITION BY store_id) AS s FROM orders", + ["原流程算的是 SUM(amount) OVER (PARTITION BY user_id),盲算算的是 SUM(amount) OVER (PARTITION BY store_id)"]), + # 同一个合计再算一下(分换成元):和光一个聚合配成一对说 + ("sqlite", "SELECT SUM(amount) FROM orders", "SELECT SUM(amount) / 100 FROM orders", + ["原流程算的是 SUM(amount),盲算算的是 SUM(amount) / 100"]), + ("sqlite", "SELECT SUM(amount) - SUM(fee) FROM orders", "SELECT SUM(amount), SUM(fee) FROM orders", + ["原流程多算了 SUM(amount) - SUM(fee),盲算没有"]), +], ids=["difference", "ratio", "window-partition", "scaled", "one-sided"]) +def test_how_an_output_column_is_computed_is_compared(dialect, a, b, expected): + """最外层算出来的列按算法比:两边用到的聚合一模一样(SUM(paid.total)、SUM(refunded.total)),只是谁减谁、谁除谁 + 换了,聚合那一面看不出来。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +@pytest.mark.parametrize("dialect,a,b", [ + ("sqlite", PAID_REFUNDED + f"SELECT SUM(p.total) - SUM(r.total) {PR_JOIN}", + PAID_REFUNDED + "SELECT SUM(p.total) - SUM(r.total) FROM refunded r JOIN paid p ON p.user_id = r.user_id"), + ("sqlite", "SELECT region, SUM(amount) * 1.0 / COUNT(*) AS avg_amt FROM orders GROUP BY region", + "SELECT o.region, 1.0 * SUM(o.amount) / COUNT(*) FROM orders o GROUP BY 1"), + ("postgres", "SELECT order_id, RANK() OVER (PARTITION BY user_id ORDER BY amount DESC) AS k FROM orders", + "SELECT o.order_id, RANK() OVER (PARTITION BY o.user_id ORDER BY o.amount DESC) FROM orders o"), + # 最外层取整到分、空值记 0:还是那个合计 + ("sqlite", "SELECT SUM(amount) FROM orders", "SELECT ROUND(SUM(amount), 2) FROM orders"), + ("mysql", "SELECT SUM(amount) FROM orders", "SELECT IFNULL(SUM(amount), 0) FROM orders"), +], ids=["swapped-join", "commutative", "window", "round", "default"]) +def test_the_same_output_computation_written_differently_is_no_difference(dialect, a, b): + """别名、加法乘法的先后、最外层取整、空值记 0 这类写法不同,算的是同一个数。""" + assert texts(a, b, dialect) == [] + + +@pytest.mark.parametrize("dialect,a,b,expected", [ + # 往里挪一层:net 的 v 是谁减谁 + ("sqlite", + PAID_REFUNDED + ", net AS (SELECT p.user_id, p.total - r.total AS v FROM paid p JOIN refunded r " + "ON r.user_id = p.user_id) SELECT SUM(v) FROM net", + PAID_REFUNDED + ", net AS (SELECT p.user_id, r.total - p.total AS v FROM paid p JOIN refunded r " + "ON r.user_id = p.user_id) SELECT SUM(v) FROM net", + ["原流程的 net.v 是 paid.total - refunded.total,盲算是 refunded.total - paid.total"]), + ("postgres", + PAID_REFUNDED + ", net AS (SELECT p.user_id, p.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) " + "SELECT COUNT(*) FROM net WHERE v > 100", + PAID_REFUNDED + ", net AS (SELECT p.user_id, r.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) " + "SELECT COUNT(*) FROM net WHERE v > 100", + ["原流程的 net.v 是 paid.total,盲算是 refunded.total"]), + ("mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total " + PR_SUBQUERIES + ") x", + "SELECT SUM(x.total) FROM (SELECT p.user_id, r.total " + PR_SUBQUERIES + ") x", + ["原流程的 x.total 是 p.total,盲算是 r.total"]), + ("sqlite", "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT SUM(net) FROM t", + "WITH t AS (SELECT user_id, amount * 0.8 AS net FROM orders) SELECT SUM(net) FROM t", + ["原流程的 t.net 是 amount * 0.9,盲算是 amount * 0.8"]), + ("mysql", + "WITH t AS (SELECT user_id, CASE WHEN status = 'paid' THEN amount ELSE 0 END AS paid_amt FROM orders) " + "SELECT SUM(paid_amt) FROM t", + "WITH t AS (SELECT user_id, CASE WHEN status IN ('paid', 'shipped') THEN amount ELSE 0 END AS paid_amt " + "FROM orders) SELECT SUM(paid_amt) FROM t", + ["原流程的 t.paid_amt 是 CASE WHEN status = 'paid' THEN amount ELSE 0 END," + "盲算是 CASE WHEN status IN ('paid', 'shipped') THEN amount ELSE 0 END"]), + ("postgres", "SELECT COUNT(*) FROM (SELECT user_id, amount - fee AS net FROM orders) a WHERE a.net > 100", + "SELECT COUNT(*) FROM (SELECT user_id, amount + fee AS net FROM orders) b WHERE b.net > 100", + ["原流程的 a.net 是 amount - fee,盲算的 b.net 是 amount + fee"]), + # 按支付月份还是下单月份分组 + ("mysql", "WITH t AS (SELECT DATE_FORMAT(paid_at, '%Y-%m') AS m, amount FROM orders) " + "SELECT m, SUM(amount) FROM t GROUP BY m", + "WITH t AS (SELECT DATE_FORMAT(created_at, '%Y-%m') AS m, amount FROM orders) " + "SELECT m, SUM(amount) FROM t GROUP BY m", + ["原流程的 t.m 是 DATE_FORMAT(paid_at, '%Y-%m'),盲算是 DATE_FORMAT(created_at, '%Y-%m')"]), + ("sqlite", "WITH t AS (SELECT strftime('%Y-%m', paid_at) AS m, amount FROM orders) " + "SELECT m, SUM(amount) FROM t GROUP BY m", + "WITH t AS (SELECT strftime('%Y-%m', created_at) AS m, amount FROM orders) " + "SELECT m, SUM(amount) FROM t GROUP BY m", + ["原流程的 t.m 是 STRFTIME('%Y-%m', paid_at),盲算是 STRFTIME('%Y-%m', created_at)"]), + # 窗口:每人第一笔还是最后一笔;有并列时 RANK 和 ROW_NUMBER 取到的行数不同 + ("postgres", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) " + "AS rn FROM orders) t WHERE rn = 1", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at DESC) " + "AS rn FROM orders) t WHERE rn = 1", + ["原流程的 t.rn 是 ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at)," + "盲算是 ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at DESC)"]), + ("mysql", "WITH r AS (SELECT user_id, amount, RANK() OVER (ORDER BY amount DESC) AS k FROM orders) " + "SELECT SUM(amount) FROM r WHERE k <= 10", + "WITH r AS (SELECT user_id, amount, ROW_NUMBER() OVER (ORDER BY amount DESC) AS k FROM orders) " + "SELECT SUM(amount) FROM r WHERE k <= 10", + ["原流程的 r.k 是 RANK() OVER (ORDER BY amount DESC),盲算是 ROW_NUMBER() OVER (ORDER BY amount DESC)"]), + # 外面直接 SELECT *:每一列都用到了 + ("sqlite", "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT * FROM t", + "WITH t AS (SELECT user_id, amount * 0.8 AS net FROM orders) SELECT * FROM t", + ["原流程的 t.net 是 amount * 0.9,盲算是 amount * 0.8"]), + # 递归 CTE 自己写了列名:按列名找各支同一位置的定义 + ("postgres", "WITH RECURSIVE t(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM t WHERE n < 10) SELECT SUM(n) FROM t", + "WITH RECURSIVE t(n) AS (SELECT 1 UNION ALL SELECT n + 2 FROM t WHERE n < 10) SELECT SUM(n) FROM t", + ["原流程的 t.n 是 1 | n + 1,盲算是 1 | n + 2"]), +], ids=["nested-difference", "nested-column", "nested-subquery", "scale", "case", "renamed-subquery", "month", + "month-strftime", "window-order", "window-function", "select-star", "recursive"]) +def test_how_a_derived_column_is_computed_is_reported(dialect, a, b, expected): + """外面写的都是 t.net、net.v、x.total,两边一样;它在 CTE、子查询里怎么算的不一样,要报出来,而且只报这一处: + 外面按它写的条件、聚合两边写法一样,不跟着报。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +@pytest.mark.parametrize("dialect,a,b", [ + ("sqlite", "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT SUM(net) FROM t", + "WITH x AS (SELECT user_id, 0.9 * amount AS net FROM orders) SELECT SUM(x.net) FROM x"), + # 外面没用到的列算法不同,不影响结果 + ("sqlite", "WITH t AS (SELECT user_id, amount, paid_at AS ts FROM orders) SELECT SUM(amount) FROM t", + "WITH t AS (SELECT user_id, amount, created_at AS ts FROM orders) SELECT SUM(amount) FROM t"), + # 取日期的两种写法 + ("postgres", "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT d, SUM(amount) FROM t GROUP BY d", + "WITH t AS (SELECT CAST(paid_at AS DATE) AS d, amount FROM orders) SELECT d, SUM(amount) FROM t GROUP BY d"), + ("mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total " + PR_SUBQUERIES + ") x", + "SELECT SUM(y.total) FROM (SELECT a.user_id, a.total FROM (SELECT user_id, SUM(amount) AS total FROM refunds " + "GROUP BY user_id) b JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) a " + "ON a.user_id = b.user_id) y"), +], ids=["commutative", "unused-column", "date-cast", "nested-renamed"]) +def test_the_same_derived_column_written_differently_is_no_difference(dialect, a, b): + """派生来源里一列的写法不同、算的一样(0.9 * amount 和 amount * 0.9、DATE() 和 CAST(… AS DATE)),或者外面根本 + 没用到那一列:不报。""" + assert texts(a, b, dialect) == [] + + +@pytest.mark.parametrize("dialect,a,b,expected", [ + # 换了名字、算法一样:配成同一列 + ("sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE total > 100", + "WITH x AS (SELECT user_id, SUM(amount) AS sum_amt FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM x WHERE sum_amt > 100", []), + ("sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE total > 100 AND n > 1", + "WITH t AS (SELECT user_id, SUM(amount) AS s, COUNT(*) AS c FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE s > 100 AND c > 1", []), + ("sqlite", "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) " + "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS day FROM orders) " + "SELECT SUM(amount) FROM t WHERE day >= '2026-09-01'", []), + # 换了名字又改了算法:两边各剩一列,配成同一列,外面的条件不报成两边各有一个 + ("sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE total > 100", + "WITH x AS (SELECT user_id, SUM(amount - discount) AS net_total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM x WHERE net_total > 100", + ["原流程对 amount 求和,盲算对 amount - discount 求和", + "原流程的 t.total 是 SUM(amount),盲算的 x.net_total 是 SUM(amount - discount)"]), +], ids=["renamed", "two-renamed", "renamed-date", "renamed-and-changed"]) +def test_derived_columns_renamed_on_one_side_are_paired(dialect, a, b, expected): + """CTE、子查询选出来的列外面按名字引用,盲算常给它另起名字(total、sum_amt):按定义配成同一列,外面按它写的条件 + 不算两边各有一个。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +# -------------------------------------------------------------------------- +# diff_facets:时间列写在 CTE、子查询里 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dialect", DIALECTS) +def test_a_time_column_computed_in_a_cte_is_seen_through(dialect): + """按支付时间还是下单时间筛,日期表达式写在 CTE 里也认得出:时间那一面透过 d = DATE(paid_at) 认出是 paid_at,带上 + 表名、列名给调用方对照数据目录;d 的定义不同不再另报一遍。""" + a = ("WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) " + "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'") + b = a.replace("DATE(paid_at)", "DATE(created_at)") + [diff] = diff_facets(facets(a, dialect), facets(b, dialect), labels=LABELS, + names={"orders.paid_at": "支付时间", "created_at": "下单时间"}) + assert (diff.aspect, diff.text) == ("time", "原流程按支付时间 paid_at 筛时间,盲算按下单时间 created_at") + assert diff.detail == {"a": [{"table": "orders", "column": "paid_at"}], + "b": [{"table": "orders", "column": "created_at"}]} + + +@pytest.mark.parametrize("dialect,a,b,expected", [ + ("postgres", "SELECT SUM(x.amount) FROM (SELECT amount, CAST(paid_at AS DATE) AS day FROM orders) x " + "WHERE x.day BETWEEN '2026-09-01' AND '2026-09-30'", + "SELECT SUM(y.amount) FROM (SELECT amount, CAST(created_at AS DATE) AS day FROM orders) y " + "WHERE y.day BETWEEN '2026-09-01' AND '2026-09-30'", + ["原流程按 paid_at 筛时间,盲算按 created_at"]), + # 两个 CTE 同名、定义对调:原流程筛付款时间,盲算筛退款时间 + ("mysql", "WITH a AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id), " + "b AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id) " + "SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'", + "WITH a AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id), " + "b AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'", + ["原流程按 paid_at 筛时间,盲算按 refunded_at"]), + # 带时区偏移的不是同一天:时间列一样,d 的定义不同照报 + ("sqlite", "WITH t AS (SELECT amount, DATE(paid_at) AS d FROM orders) " + "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT amount, DATE(paid_at, '+8 hours') AS d FROM orders) " + "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + ["原流程的 t.d 是 DATE(paid_at),盲算是 DATE(paid_at, '+8 hours')"]), + ("postgres", "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) " + "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT CAST(paid_at AS DATE) AS d, amount FROM orders) " + "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", []), +], ids=["cast-between", "same-name-swapped", "time-zone", "date-or-cast"]) +def test_time_columns_inside_derived_tables(dialect, a, b, expected): + assert texts(a, b, dialect, labels=LABELS) == expected + + +# -------------------------------------------------------------------------- +# diff_facets:两边写法一样、意思不同时说清楚 +# -------------------------------------------------------------------------- + + +def test_same_named_derived_tables_with_swapped_definitions_are_told_apart(): + """两边的 CTE 同名、定义对调了(盲算的 t1 是原流程的 t2):按结构配对之后 t1.total 两边不是同一列,可只写 + 「t1.total > 100」两边一模一样,自相矛盾。带上各自的来历:原流程筛付款合计,盲算筛退款合计。""" + a = ("WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "t2 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " + "SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100") + b = ("WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "t2 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100") + assert texts(a, b, "postgres", labels=LABELS) == [ + "原流程按 t1.total > 100(t1.total 即 SUM(orders.amount))筛选," + "盲算按 t1.total > 100(t1.total 即 SUM(refunds.amount))筛选"] + a_sum = ("WITH a AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "b AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " + "SELECT SUM(a.total) FROM a JOIN b ON b.user_id = a.user_id") + b_sum = ("WITH a AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "b AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(a.total) FROM a JOIN b ON b.user_id = a.user_id") + assert texts(a_sum, b_sum, labels=LABELS) == [ + "原流程对 a.total(即 SUM(orders.amount))求和,盲算对 a.total(即 SUM(refunds.amount))求和"] + + +def test_same_named_derived_tables_that_differ_only_in_conditions_say_which_is_which(): + """两个 CTE 读同一张表、各列定义一样,只差里面的条件;盲算把两个名字的定义对调了、还都多了 is_deleted = 0。按条件 + 的重合配(不按名字:盲算的 paid 筛的是 status = 'open'),外面的「> 100」筛的就是未付合计;来历也一样(都是 + SUM(orders.amount)),说清两边同名的各对应哪一个。""" + a = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " + "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) " + "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > 100") + b = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' AND is_deleted = 0 " + "GROUP BY user_id), " + "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' AND is_deleted = 0 " + "GROUP BY user_id) " + "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > 100") + assert texts(a, b, labels=LABELS) == [ + "原流程按 paid.total > 100 筛选," + "盲算按 paid.total > 100(盲算的 paid 对应原流程的 open、盲算的 open 对应原流程的 paid)筛选", + "盲算只保留 is_deleted = 0 的行,原流程没有这个条件"] + + +# -------------------------------------------------------------------------- +# diff_facets:同一个条件筛的先后 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dialect,a,b,expected", [ + # 「第一单就是已付的人数」对「有已付订单的人数」 + ("postgres", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn " + "FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn " + "FROM orders WHERE status = 'paid') t WHERE t.rn = 1", + ["原流程先算 t 的窗口函数、再按 status = 'paid' 筛选,盲算先按 status = 'paid' 筛选、再算 t 的窗口函数"]), + ("sqlite", "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t " + "WHERE status = 'paid'", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid' ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t", + ["原流程先取 t 的前 10 行、再按 status = 'paid' 筛选,盲算先按 status = 'paid' 筛选、再取 t 的前 10 行"]), + ("sqlite", "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t " + "WHERE paid_at >= '2026-09-01'", + "WITH t AS (SELECT * FROM orders WHERE paid_at >= '2026-09-01' ORDER BY amount DESC LIMIT 10) " + "SELECT SUM(amount) FROM t", + ["原流程先取 t 的前 10 行、再按 paid_at >= '2026-09-01' 筛选," + "盲算先按 paid_at >= '2026-09-01' 筛选、再取 t 的前 10 行"]), +], ids=["window", "limit", "limit-time"]) +def test_filtering_before_or_after_a_window_or_limit_is_reported(dialect, a, b, expected): + """先取前 10 行再筛已付,和先筛已付再取前 10 行,取到的不是同一批:派生来源取前几行、算窗口函数时,外面的条件 + 不再透成里面那张表的条件,两边只差在筛的先后,照报。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +@pytest.mark.parametrize("dialect,a,b", [ + # 按窗口的分区键筛,窗口里外都一样 + ("postgres", + "SELECT COUNT(*) FROM (SELECT user_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn " + "FROM orders) t WHERE t.rn = 1 AND t.user_id > 100", + "SELECT COUNT(*) FROM (SELECT user_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn " + "FROM orders WHERE user_id > 100) t WHERE t.rn = 1"), + # 两边都是先取前 10 行再筛,只是起的名字不同 + ("sqlite", "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t " + "WHERE status = 'paid'", + "WITH top10 AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(t.amount) FROM top10 t " + "WHERE t.status = 'paid'"), +], ids=["partition-key", "same-order"]) +def test_filtering_at_an_equivalent_stage_is_no_difference(dialect, a, b): + assert texts(a, b, dialect) == [] + + +# -------------------------------------------------------------------------- +# diff_facets:没用到的 CTE +# -------------------------------------------------------------------------- + + +def test_a_cte_defined_but_not_used_does_not_count(): + """盲算定义了只筛已付的 CTE,却直接查 orders:少的那个条件要报出来;定义了没用的 CTE 里多一个条件、多一张表, + 不影响结果,不报。""" + paid = "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') " + assert texts(paid + "SELECT SUM(amount) FROM paid", paid + "SELECT SUM(amount) FROM orders", labels=LABELS) == \ + ["原流程只保留 status = 'paid' 的行,盲算没有这个条件"] + assert texts("SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "WITH junk AS (SELECT * FROM users WHERE vip = 1) " + "SELECT SUM(amount) FROM orders WHERE status = 'paid'") == [] + assert texts(paid + "SELECT SUM(amount) FROM paid", + paid + "SELECT SUM(amount) FROM orders WHERE status = 'paid'") == [] + + +# -------------------------------------------------------------------------- +# diff_facets:自己关联自己 +# -------------------------------------------------------------------------- + +SELF_BASE = ("WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id, referrer_id) ") + + +@pytest.mark.parametrize("dialect,a,b,expected", [ + ("postgres", SELF_BASE + "SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id " + "WHERE b1.total > b2.total", + SELF_BASE + "SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id WHERE b2.total > b1.total", + ["原流程只保留 b1.total > b2.total 的行,盲算没有这个条件", "盲算只保留 b1.total < b2.total 的行,原流程没有这个条件"]), + ("sqlite", "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o1.amount > o2.amount", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o2.amount > o1.amount", + ["原流程只保留 o2.amount < o1.amount 的行,盲算没有这个条件", "盲算只保留 o2.amount > o1.amount 的行,原流程没有这个条件"]), + # 换了别名、换了关联先后:b1 还是出 referrer_id 的那一个 + ("postgres", SELF_BASE + "SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id " + "WHERE b1.total > b2.total", + SELF_BASE + "SELECT COUNT(*) FROM base y JOIN base x ON x.referrer_id = y.user_id WHERE y.total < x.total", []), + ("sqlite", "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o1.amount > o2.amount", + "SELECT COUNT(*) FROM orders c JOIN orders p ON c.parent_id = p.id WHERE c.amount > p.amount", []), +], ids=["cte-reversed", "table-reversed", "cte-renamed", "table-renamed"]) +def test_a_self_join_tells_its_two_aliases_apart(dialect, a, b, expected): + """同一个 CTE、同一张表自己关联自己:两个别名按各自在关联条件里出的列区分(出 referrer_id 的、出 user_id 的), + 比较写反了要报;只是换了别名、换了关联先后不报。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + @pytest.mark.parametrize("dialect", DIALECTS) def test_each_dialect_gives_the_same_three_findings(dialect): q = {"sqlite": '"', "postgres": '"', "mysql": "`"}[dialect] From 007a13d72b8bf2352891795878c0c2d0ed3f3436 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 06:11:40 +0800 Subject: [PATCH 3/6] =?UTF-8?q?SQL=20=E8=AF=AD=E4=B9=89=E5=A4=8D=E6=A0=B8?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=88=E7=AC=AC=205=20=E8=BD=AE=EF=BC=89?= =?UTF-8?q?=EF=BC=9A=E8=AF=BB=E5=90=8C=E4=B8=80=E5=BC=A0=E8=A1=A8=E7=9A=84?= =?UTF-8?q?=E6=B4=BE=E7=94=9F=E6=9D=A5=E6=BA=90=E4=B8=8D=E5=86=8D=E9=80=8F?= =?UTF-8?q?=E6=88=90=E4=B8=80=E5=BC=A0=E8=A1=A8=E3=80=81=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E5=AD=90=E6=9F=A5=E8=AF=A2=E5=88=86=E5=86=85=E5=A4=96=E5=B1=82?= =?UTF-8?q?=E3=80=81=E6=9D=A1=E4=BB=B6=E5=88=86=E7=BB=84=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E8=81=9A=E5=90=88=E6=8C=89=E5=B1=82=E8=AE=A1=E6=95=B0=EF=BC=8C?= =?UTF-8?q?=E5=8E=BB=E9=87=8D=E3=80=81UNION=E3=80=81=E6=9C=80=E5=90=8E?= =?UTF-8?q?=E5=87=BA=E5=87=A0=E8=A1=8C=E8=BF=9B=E6=AF=94=E8=BE=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一轮(3e0ab05)之后复验清单里还有一批用例返回空列表(A24 只报了一半)。最初的问题 1–3(paid / refunded 两个 CTE 换用哪一个的漏报、多一个 vip CTE 时配错、内连接换先后报出自相矛盾的两句)在第 3、4 轮已经修好,这次仍失败的是另外几类,根因各不相同: 读同一张表的几个派生来源:两个原样透出的 CTE(已付、未付,SELECT * FROM orders WHERE …)透过去,p.amount、o.amount 都成了 orders.amount,SUM、WHERE、时间列、ORDER BY、GROUP BY、关联条件、派生列的定义里用的是哪一份都分不出来(CTE 链、子查询套子查询同样);CTE 和它自己读的 orders 关联(父单、子单)也一样;同一个 CTE 关联两次时 _roles 设的 role 透过去就丢了。现在算完一段的来源后看各来源一路透过「只读一个来源」的派生来源最后读的是谁(_base),同一段里有几个读到同一个来源的,其中的派生来源标 opaque(_seal):through、时间列都不透过去,照写两边按结构配好的共同编号(已付配已付、未付配未付),给人看写名字(paid.amount),单看一条语句「只用到一张实表」时也照样写出是哪一份。时间条件给调用方对照数据目录的 detail 另一直换到实表(TimePredicate.table)。只有一个来源读那张表时照旧透过去。 相关子查询读外层那张表:EXISTS、标量子查询、LATERAL 里 c.parent_id = o.id 和 c.id = o.parent_id 都写成 orders.id = orders.parent_id(有子单还是有父单、之前还是之后有过订单分不出)。现在比较用的写法里外层的列在列名后面带「^」(_depth,往外几层带几个;带在列名上,排比较两边的先后只在两边表名一样时才受影响),给人看时内外读同一个来源的照写别名、留着表别名;两边各剩一个 EXISTS(NOT EXISTS)时配成一对说。 按整条语句去重:各段的条件、分组键、连接、聚合原来去重成一个集合,外层的 status = 'paid' 和 IN / EXISTS 子查询、关联进来的汇总 CTE、UNION 另一支里的同一个条件合成了一个,外层少了它报不出来(分组键、连接、聚合同理;聚合去重后只剩「多算了 SUM」一句,算出来的列又被压掉)。现在每一项记下在哪一层(Part / JoinFacet / Aggregate 的 scope,见 _layers:最外层、UNION 第几支、派生来源的共同编号、条件子查询「?」;只筛行的 CTE 和用到它的那一段同层,读同一张表的几个同段 CTE 各算各的层),按(层, key)去重;比的时候同一项在一边出现在几层算几个(_Compare.leftovers):同一层的先配,剩下的不管层也配(条件挪一层常是同一个口径),配不完的才报,另一边也有、只是层少了的说清是哪一层(「原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件」)。条件子查询那一层多出来的不单说:外层那个条件已经报了,原来还会多出一句读起来像外层也这么筛的「只保留 state = 'done' 的行」。时间条件在列、范围都一样时也按层数比。 去重、合并、最后出几行:DISTINCT、UNION / UNION ALL 原来不进任何一面,分组键挪一层也看不出来。SELECT DISTINCT(这一段没有分组、聚合时)当这一段的分组键收:和按同几列 GROUP BY 一样的不报,一边在子查询里先去重报「去重」;各段的 UNION / INTERSECT / EXCEPT 记下合并方式和支数(SetOpFacet,同层的先配,挪了层的同样写法也配上);最后出来的行按什么分(Facets.grain:分组键 / 只出一行 / 逐行)在分组键都配得上、两边都汇总了时比(先在子查询里按用户分组、外层数用户数,对外层按用户分组)。 说清楚:自己关联自己、关联的方向反了,两边都写 a.amount 时说清各自的 a 是关联条件里出哪一列的那个(Part.roles,实表自连接同样)。 模块说明、Part / JoinFacet / TimePredicate / Aggregate / Facets / FacetDiff、unmatched / leftovers / joins / render 的文档按现在的做法改写,做不到的照实列出:读同一张表的几个派生来源同段时只认配好的编号(一边两个 CTE、另一边把其中一个换成关联实表再在 WHERE 里筛,算不同;同一个 CTE 关联两次对定义成两个一样的 CTE 也算不同);挪了层又多一层时挑哪一个说成多出来的有顺序约定;条件子查询那一层多出来的不单说;UNION 各支按位置配;不加表前缀引用外层的列认不出;LATERAL 里的相关条件两边各说一句;最后出几行只在两边都汇总时比;COUNT(DISTINCT x) 对数 DISTINCT 子查询行数照报不同(有空值时确实不一样)。 测试:复验清单里的用例逐个补成测试(三个方言,CTE 和子查询写法),另加意思一样、不该报的反例(关联换先后、改名、单个透出的 CTE、自连接换别名、相关子查询换别名、两边都两层、条件挪进汇总 CTE、分组写在 CTE 里、GROUP BY 对 DISTINCT、一样的 UNION、最外层 UNION 对包一层)。新加 132 项,其中 119 项在 3e0ab05 上失败,其余 13 项防误报的两边都通过;原有 192 项和 test_verify_agent 照旧通过,全量 7245 项通过。另拿测试里出现过的每条 SQL 在三个方言下和自己比(1051 次),都没有不同。复验清单最后一条只到「b_s」就截断了,看不到全文,按开头补了 DISTINCT 子查询对不去重、对 COUNT(DISTINCT) 的对照。 --- backend/app/data/sql_semantics.py | 781 ++++++++++++++++++++++++---- backend/tests/test_sql_semantics.py | 340 +++++++++++- 2 files changed, 1014 insertions(+), 107 deletions(-) diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index e820c4fd..edad1a8e 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -7,18 +7,26 @@ - normalize_sql:字面量换成占位符之后的规范文本,加上字面量本身(按规范文本里的顺序)。两条 SQL 只有字面量不同, 规范文本就相同;关键字大小写、空白、注释、不加引号的名字的大小写都不算不同。 - facets:把一条查询拆成几个方面——用到的表、连接(类型、关联进来的表、ON 条件、外连接以哪一侧为主表)、WHERE - 的各个合取式、其中的时间条件(列、运算符、取值)、分组、聚合(函数、参数、是否去重)、算出来的列(最外层的算法、 - CTE 和子查询里被外面用到的列的定义,窗口函数也在这里)、HAVING、取哪几行(LIMIT、OFFSET、取前几行时的排序)。 - 别名一律换成真实表名(条件里套着的子查询也是),等式、比较、加法、乘法的两边排好顺序:o.paid_at >= '2026-09-01' - 和 '2026-09-01' <= orders.paid_at 是同一个条件。 + 的各个合取式、其中的时间条件(列、运算符、取值)、分组(当分组用的 SELECT DISTINCT 也算)、聚合(函数、参数、是否 + 去重)、算出来的列(最外层的算法、CTE 和子查询里被外面用到的列的定义,窗口函数也在这里)、HAVING、取哪几行 + (LIMIT、OFFSET、取前几行时的排序)、几支查询怎么合并(UNION、INTERSECT、EXCEPT)、最后出来的行按什么分。条件、 + 分组键、连接、聚合、HAVING 都记下在哪一层(见下)。别名一律换成真实表名(条件里套着的子查询也是),等式、比较、 + 加法、乘法的两边排好顺序:o.paid_at >= '2026-09-01' 和 '2026-09-01' <= orders.paid_at 是同一个条件。 - diff_facets:逐个方面比两份 facets,每处不同给一句中文说明。 -**逐段收集。** CTE、子查询里的条件、聚合和取行也算进来:盲算可能把有效记录条件写在 CTE 里,原流程写在外层,按 -「整条语句里有哪些条件」比才不会把同一个口径报成不同。只收用到了的段:定义了却没被引用的 CTE 不影响结果,不收 -(盲算定义了只筛已付的 CTE 却直接查 orders,少的那个条件要报出来)。逗号连接(以及 INNER JOIN 之外又写在 WHERE -里的等值关联)写在 WHERE 里的列等式算作关联条件,不算筛选。条件里套着的子查询(EXISTS、IN (SELECT …))既单算 -一段,也按它自己那一段的来源整个写进外层那个条件:别名在子查询里定义,拿外层的来源去认,同一个条件只因别名不同 -就会被报成不同。 +**逐段收集,按层计数。** CTE、子查询里的条件、聚合和取行也算进来:盲算可能把有效记录条件写在 CTE 里,原流程写在 +外层,同一个口径不能因为写的位置不同就报成不同。只收用到了的段:定义了却没被引用的 CTE 不影响结果,不收(盲算 +定义了只筛已付的 CTE 却直接查 orders,少的那个条件要报出来)。可也不能按「整条语句里有哪些条件」去重成一个集合: +外层的 status = 'paid' 和 IN 子查询(关联进来的汇总 CTE、UNION 的另一支)里的同一个条件会合成一个,外层少了它就 +报不出来。所以每一项记下它在哪一层(见 _Statement._layers):最外层、UNION 的第几支、某个派生来源里(两边配好的 +共同编号),条件里的子查询是一层「?」;只从一个来源里筛出一部分行的 CTE、子查询(不分组、不聚合、不去重)和用到它 +的那一段算同一层。比的时候同一项在一边出现在几层就算几个(见 _Compare.leftovers):同一层的先配,剩下的不管在哪 +一层也配(条件挪了一层常常是同一个口径),配不完的才报;另一边也有、只是在的层少了的,说清多出来的是哪一层 +(「原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件」)。逗号连接(以及 INNER JOIN 之外又写在 +WHERE 里的等值关联)写在 WHERE 里的列等式算作关联条件,不算筛选。条件里套着的子查询(EXISTS、IN (SELECT …)、 +标量子查询)按它自己那一段的来源整个写进外层那个条件、那一列:别名在子查询里定义,拿外层的来源去认,同一个条件 +只因别名不同就会被报成不同。它里面的几段也收,可那一层多出来的条件、分组、连接、聚合不单说:外层那个条件已经报了, +再说一句「只保留 state = 'done' 的行」,读的人会以为外层也这么筛。 **各自起的名字不进比较。** 表的别名换成真实表名;CTE、FROM 里的子查询(含 LATERAL)没有真实表名可换,单看一条 语句时按它在语句里的位置编号(@1、@2…,从外往里、按第一次用到的先后),给人看时照写名字;条件里的子查询(IN、 @@ -27,6 +35,11 @@ EXISTS、标量子查询)选出来的列名不影响取值,比较时去掉 (b2.user_id = b1.referrer_id:b1 出 referrer_id、b2 出 user_id),b1.total > b2.total 写反了认得出,换了别名、 换了先后不算不同。 +**相关子查询。** 子查询里引用外层的列(o.id)在比较用的写法里另带标记(见 _Statement._depth):相关子查询读的表 +和外层是同一张时(orders 里找子单、父单、同一用户更早的单),c.parent_id = o.id 和 c.id = o.parent_id 不带标记都 +成了 orders.id = orders.parent_id。给人看的写法这时照写别名(EXISTS(SELECT 1 FROM orders AS c WHERE o.id = +c.parent_id));两边各只剩一个 EXISTS(NOT EXISTS)的条件时配成一对说。 + **两边的派生来源按结构配对。** 位置编号只看先后:一边多出一个排在前面的 CTE,或者内连接两边换了先后,编号就错开 了;可也不能干脆不看是哪一个(两个 CTE 都算出 total 时,外层用的是付款合计还是退款合计,正是要报的不同)。所以 diff_facets 先给两边的每个派生来源算一份结构标识(见 _Derived):里面读到的实表(FROM / JOIN,往里套着的也算)、 @@ -49,8 +62,17 @@ net 读的是 paid.total - refunded.total 还是反过来,写出来不一样 **什么时候能透过派生来源。** CTE、子查询原样透出一张表的列(SELECT *,或者直接选出这一列)时,外面按它写的条件、 聚合换成那张表的列:先在 CTE 里筛一遍再查和直接查是同一列。它要取前几行(LIMIT、OFFSET)、或者算了窗口函数(这一列 又不是分区键)时不透:先取前 10 行再筛已付,和先筛已付再取前 10 行取到的不是同一批;两边只差在这里的,报筛的先后 -不同。时间条件另外透过「同一列套着日期函数、CAST、MIN / MAX」算出来的列(DATE(paid_at) AS d),日期表达式写在 CTE -里,按支付时间还是下单时间筛照样认得出。 +不同。同一段里有几个来源透过去读的是同一个来源时也不透(见 _Statement._seal):两个原样透出的 CTE 都读 orders +(已付的一份、未付的一份),透过去 p.amount、o.amount 都成了 orders.amount,用的是哪一份就分不出来了;CTE 和它 +自己读的那张表关联、同一个 CTE 关联两次也一样。这时照写派生来源配好对的编号,给人看写它的名字(paid.amount)。 +时间条件另外透过「同一列套着日期函数、CAST、MIN / MAX」算出来的列(DATE(paid_at) AS d),日期表达式写在 CTE 里, +按支付时间还是下单时间筛照样认得出;给调用方对照数据目录的表、列一直换到实表。 + +**去重、合并、最后出几行。** SELECT DISTINCT(这一段没有 GROUP BY、没有聚合时)当这一段的分组键收:和按同几列 +GROUP BY 出的行一样,一边在子查询里先去重、另一边没有照报(去重后的用户数对订单行数)。UNION、INTERSECT、EXCEPT +记下合并的方式和支数(见 SetOpFacet):同一批行 UNION ALL 了两遍、UNION 换成 UNION ALL 要报;挪了一层(最外层直接 +UNION,对包进子查询再原样取)不报。分组键挪了一层按层宽松地配得上,另比最后出来的行按什么分(见 Facets.grain): +先在子查询里按用户分组、外层数用户数,对外层按用户分组,一个出一行、一个每个用户一行。 **外连接。** 按以哪一侧为主表(保留它的全部行)比,不按写的是 LEFT 还是 RIGHT:orders LEFT JOIN refunds 和 refunds RIGHT JOIN orders 一样,和 refunds LEFT JOIN orders 不一样。同一段 WHERE 里拿可空那一侧的列比较、IN、 @@ -59,8 +81,9 @@ LIKE 的(空值都不成立),把没关联上的行筛掉了,按内连接 进来的那一侧另一边也有时,报的是它接上的另一头(FROM stores JOIN orders、另一边只查 orders:多出来的是 stores)。 **写法一样、意思不同时说清楚。** 两边同名的派生来源其实不是同一个(盲算的 t1 是原流程的 t2)时,光写「t1.total」 -两边一模一样:带上各列的来历(t1.total 即 SUM(orders.amount));来历也一样(两个 CTE 只差里面的条件)时,说 -「盲算的 paid 对应原流程的 open」。 +两边一模一样:带上各列的来历(t1.total 即 SUM(orders.amount));自己关联自己、关联的方向反了,两边都写 a.amount +时,说清各自的 a 是关联条件里出哪一列的那一个;来历也一样(两个 CTE 只差里面的条件)时,说「盲算的 paid 对应 +原流程的 open」。 仍做不到的: - 两边起名不同的派生来源的列,两边各自多出不止一列、定义又都不一样时配不上,算不同的列,外层按它们写的条件照报 @@ -76,6 +99,16 @@ LIKE 的(空值都不成立),把没关联上的行筛掉了,按内连接 - 筛的先后只认取前几行(LIMIT、OFFSET)、窗口函数(含 DISTINCT ON)挡住的;QUALIFY、GROUP BY 之后才筛的不认。 外连接被 WHERE 变成内连接只认同一段 WHERE 里直接用到可空那一侧列的比较、IN、LIKE、BETWEEN、IS NOT NULL;套在 COALESCE、CASE 里,或者写在外面一层 WHERE 里的不认。连接上别的做不到的见 _Compare.joins。 +- 读同一个来源的几个派生来源同在一段时不透过去,只认配好对的编号:一边写成两个原样透出的 CTE(已付、未付),另一边 + 把其中一个换成直接关联那张表、再在 WHERE 里筛,列一边是 CTE 的、一边是实表的,算不同;同一个 CTE 在一边关联了 + 两次、另一边定义成两个一模一样的 CTE 各关联一次,也算不同。 +- 按层计数时,同一项挪了一层又多了一层,挑哪一个说成多出来的按「同一层的先配、不在条件子查询里的先配」定,说的那 + 一层不一定是读的人心里的那一层。条件里的子查询那一层多出来的不单说,要从外层那个条件的不同里看;UNION 的各支按 + 位置配(第 2 支对第 2 支),两边把各支换了先后算不同。时间条件只在两边用到的时间列、范围都一样时才按层数比。 +- 子查询里不加表前缀引用外层的列(WHERE parent_id = id)认不出是外层的,不带标记,内外读同一张表时分不出来。 + LATERAL 子查询里的相关条件按那一层的条件比,两边不同时各说一句,不配成一对。 +- 最后出来的行按什么分只在两边都汇总了(分组或者算了聚合)、分组键又都配得上时比;最外层是 UNION 的不比。 + COUNT(DISTINCT x) 和数「SELECT DISTINCT x」子查询行数的 COUNT(*) 照报不同(x 有空值时确实不一样)。 - facets 单独返回的 key 仍是位置编号;不是 facets 算出来的 Facets(没记原文)diff_facets 也只能按位置编号比, 没用到的 CTE、起名不同的列也不再另配。 @@ -282,13 +315,20 @@ class Part: 别名),其余只写列名;full 一律写表名,两边的 text 一样、key 却不同时改用它说明。excludes 只在 WHERE 的「排除」 写法上有:<>、NOT、NOT IN、IS NOT NULL 去掉否定之后排除的是什么(IS NOT NULL 写成「列 为空」)。column_keys 和 columns 一一对应,是各列比较用的写法(实表.列;CTE、子查询的列写成「编号.列」):两边对同一列的不同条件靠它配成 - 一对说。 + 一对说。相关子查询里引用外层的列,比较用的写法(key、loose、column_keys)在列名后面带「^」(往外几层带几个,见 + _Statement._depth):内外读同一张表时,c.parent_id = o.id 和 c.id = o.parent_id 才分得开。 - 后面三项不进比较(两个 Part 相等只看前面几项),只用来把不同说清楚: + scope 是这一项在哪一层(条件、分组键、HAVING 上有,见 _Statement._layers):空串是最外层,「|2」是最外层 UNION + 的第 2 支,「@3」是派生来源 @3 里(「@3|2」是它的 UNION 第 2 支),「?」是条件、SELECT 列表里的子查询。facets 按 + (scope, key)去重:同一个条件在外层和子查询里各写了一次是两项(见 _Compare.leftovers)。 + + 后面四项不进比较(两个 Part 相等只看前面几项),只用来把不同说清楚: origins 是这一项用到的派生来源的列各自的来历 [(「t1.total」, 「SUM(orders.amount)」)](见 _Statement._origin): 两边写法一模一样(都是 t1.total > 100),各自的 t1 却不是同一个时,带上它才分得清。pierce 是一路透过派生来源写到 实表的 key,只在中途被挡住时才有(这一项按派生来源的列写,见 _Statement.through);stage 说是被什么挡住的(「取 t - 的前 10 行」「算 t 的窗口函数」):两边 pierce 一样、只有一边被挡住,是同一个条件筛的先后不同。 + 的前 10 行」「算 t 的窗口函数」):两边 pierce 一样、只有一边被挡住,是同一个条件筛的先后不同。roles 是用到的自己 + 关联自己的别名各是哪一个 [(别名, 在关联条件里出的列, 来源的名字)](见 _Statement._roles):关联的方向反了,两边都 + 写 a.amount,各自的 a 却不是同一个时,带上它才分得清。 """ key: str @@ -298,9 +338,11 @@ class Part: loose: str = "" full: str = "" column_keys: tuple[str, ...] = () + scope: str = "" origins: tuple[tuple[str, str], ...] = field(default=(), compare=False) pierce: str = field(default="", compare=False) stage: str = field(default="", compare=False) + roles: tuple[tuple[str, str, str], ...] = field(default=(), compare=False) @staticmethod def plain(key: str, text: str) -> "Part": @@ -322,7 +364,8 @@ class JoinFacet: 序)。LEFT JOIN 关联进来的那一侧可空,关联条件里的其余来源(没有就是它之前的各个来源)是主表;RIGHT JOIN 反过来。 orders LEFT JOIN refunds 和 refunds RIGHT JOIN orders 是同一回事,和 refunds LEFT JOIN orders 不是:diff_facets 按主表比,不按写的是 LEFT 还是 RIGHT。rejects 是同一段 WHERE 里把可空那一侧没关联上的行筛掉的条件(那一侧的列 - 拿来比较、IN、LIKE……,空值都不成立):有它这次外连接实际等于内连接。""" + 拿来比较、IN、LIKE……,空值都不成立):有它这次外连接实际等于内连接。scope 是这次连接在哪一层(见 Part.scope): + 外层关联了 refunds,和 EXISTS 子查询里同样关联了 refunds,是两次连接。""" kind: str table: Part @@ -332,6 +375,7 @@ class JoinFacet: keep: tuple[str, ...] = () nullable: tuple[str, ...] = () rejects: Part | None = None + scope: str = "" @dataclass(frozen=True) @@ -342,13 +386,16 @@ class TimePredicate: column 是按哪一列筛时间:条件写的是 CTE、子查询算出来的列(WITH t AS (SELECT DATE(paid_at) AS d …) … WHERE d >= …)时,它在里面是同一列套着日期函数、CAST、MIN / MAX 的,换成里面那一列,一直换到实表(paid_at):按支付 时间还是下单时间筛,日期表达式写在 CTE 里也认得出。via 是路过的派生来源的列(「编号.列」,不进比较):时间那一面 - 已经报了按哪一列筛不同,派生来源那一列的定义不同就不再另报一遍。""" + 已经报了按哪一列筛不同,派生来源那一列的定义不同就不再另报一遍。table 是这一列一直换到实表的 (表, 列)(不进 + 比较):读同一张表的几个派生来源同在一段时,column 照写派生来源的列(已付那份的 created_at、未付那份的 created_at + 是两列,见 _Statement._seal),给调用方对照数据目录的还是 orders.created_at。""" column: Part op: str value: str condition: Part via: tuple[str, ...] = field(default=(), compare=False) + table: ColumnRef | None = field(default=None, compare=False) @dataclass(frozen=True) @@ -361,6 +408,7 @@ class Aggregate: arg: Part distinct: bool text: str + scope: str = "" # 在哪一层(见 Part.scope):外层算的和标量子查询里算的同一个聚合是两项 @dataclass(frozen=True) @@ -405,6 +453,19 @@ class LimitFacet: scope_key: str | None = None +@dataclass(frozen=True) +class SetOpFacet: + """一段查询把几支 SELECT 合起来(UNION、UNION ALL、INTERSECT、EXCEPT)。ops 是各处合并的写法,按从左到右的先后 + (「UNION」「UNION ALL」…);branches 是一共几支。scope 是哪一段:None 是最外层,CTE、FROM 里的子查询写它的名字; + scope_key 是那一段所在的层(见 Part.scope:最外层是空串,派生来源是两边配好的共同编号)。条件里的子查询不单算: + 它整个写在那个条件里比。""" + + ops: tuple[str, ...] + branches: int + scope: str | None = None + scope_key: str = "" + + @dataclass(frozen=True) class Facets: """一条查询的各个方面。每一项都按在 SQL 里出现的先后排、按 key 去重;parse_error 不为空时其余都是空的。 @@ -420,6 +481,13 @@ class Facets: expressions 是算出来的列(见 Computed)。derived 是用到了的派生来源(key 是编号,text 是名字,含条件里套着的子 查询用到的 CTE):两边写法一样、同名的派生来源却不是同一个时,拿它说清「盲算的 t1 对应原流程的 t2」。 + where、group_by、having、joins、aggregates、time 的每一项都记着在哪一层(Part.scope、JoinFacet.scope、 + Aggregate.scope),按(层, key)去重;scopes 是这条语句里有的各层(key 是层的标记,text 是给人看的说法:「外层」 + 「t」「t 的 UNION 第 2 支」「条件的子查询」),diff_facets 说多出来的是哪一层时用。group_by 里也有当分组用的 + SELECT DISTINCT 选出来的各项(这一段没有 GROUP BY、没有聚合时,和按它们分组出的行一样),distinct 是这样收进来的 + 那几层:说「去重」不说「分组」。set_ops 是各段的 UNION、INTERSECT、EXCEPT(见 SetOpFacet)。grain 是最后出来 + 的行按什么分(key 为「group:分组键…」「one」「rows」,见 _Statement._grain;最外层是 UNION 的为 None)。 + 只收用到了的段:没用到的 CTE(定义了、外面没引用)里的条件、聚合、表不影响结果,不收(见 _Statement._enliven)。 _origin 是 facets 记下的 (原文, 表结构),不进比较、不进 as_dict:diff_facets 把两边的派生来源按结构配好对之后, @@ -439,6 +507,10 @@ class Facets: sources: tuple[Part, ...] = () expressions: tuple[Computed, ...] = () derived: tuple[Part, ...] = () + set_ops: tuple[SetOpFacet, ...] = () + distinct: tuple[str, ...] = () + grain: Part | None = None + scopes: tuple[Part, ...] = () parse_error: str | None = None _origin: tuple[str, Mapping[str, Any] | None] | None = field(default=None, compare=False, repr=False) @@ -735,6 +807,8 @@ class _Source: idents: tuple[Any, ...] = () # 表名各段的 Identifier:给人看的写法照原样写回去 query: Any = None # derived:里面那段查询 role: str = "" # 同一个来源在这一段里出现了不止一次时,这一次是哪一个(见 _Statement._roles) + alias: str = "" # 这一段里引用它的名字(别名,没有别名是表名、CTE 名):给人看时分内外层用 + opaque: bool = False # derived:同一段里别的来源透过去读的也是同一个来源,不透过去(见 _Statement._seal) @property def ref(self) -> str: @@ -1137,6 +1211,7 @@ class _Statement: (exp.to_identifier(alias),) if alias else (), body) else: out[slot] = _Source("other", alias, slot) + out[slot].alias = alias aliases[slot] = alias self._roles(select, out, aliases) self._memo[id(select)] = out @@ -1144,8 +1219,40 @@ class _Statement: while parent is not None and not isinstance(parent, exp.Select): parent = parent.parent out.outer = self.sources(parent) if parent is not None else None + self._seal(out) return out + def _base(self, source: _Source) -> str: + """一路透过「只读一个来源」的派生来源(through 能透过去的那一路),最后读的是哪个来源:「种类:key」。""" + from sqlglot import exp + + for _ in range(8): + if source.kind != "derived" or not isinstance(source.query, exp.Select): + break + inner = self.sources(source.query) + if len(inner) != 1: + break + source = next(iter(inner.values())) + return f"{source.kind}:{source.key}" + + def _seal(self, sources: dict[str, _Source]) -> None: + """同一段里有几个来源透过去读的是同一个来源时,其中的派生来源都不透过去(就地标 opaque,见 through)。 + + 两个原样透出的 CTE 都读 orders(已付的一份、未付的一份):透过去 p.amount、o.amount 都成了 orders.amount,外层 + 用的是哪一份就分不出来了(SUM(p.amount) 对 SUM(o.amount)、p.amount > 100 对 o.amount > 100 都成了一样);CTE 和 + 它自己读的那张表关联(父单、子单)、同一个原样透出的 CTE 关联两次也一样,自己关联自己时 _roles 设的 role 透过去 + 也就丢了。这时这几个派生来源的列照写它们自己的编号(两边按结构配好的共同编号):已付配已付、未付配未付,换了 + 用哪一份照报,关联换了先后、起的名字不同不报。只有一个来源读那张表时照旧透过去:先在 CTE 里筛一遍再查和直接查 + 是同一列。""" + groups: dict[str, list[_Source]] = {} + for source in sources.values(): + groups.setdefault(self._base(source), []).append(source) + for found in groups.values(): + if len(found) > 1: + for source in found: + if source.kind == "derived": + source.opaque = True + @staticmethod def _roles(select: Any, sources: dict[str, _Source], aliases: Mapping[str, str]) -> None: """同一个来源在这一段里出现了不止一次(自己关联自己:orders o1 JOIN orders o2、base b1 JOIN base b2)时,各自 @@ -1213,6 +1320,26 @@ class _Statement: hits = [(slot, s) for slot, s in sources.items() if self._exposes(s, column.name)] return hits[0] if len(hits) == 1 else (None, None) + @staticmethod + def _depth(column: Any, sources: dict[str, _Source]) -> int: + """列是往外第几层那段的来源的(相关子查询引用外层的别名 o.id 是 1);这一段的、认不出的是 0。 + + 比较用的写法里外层的列另带「^」(orders."id^"):相关子查询读的表和外层是同一张时(orders 里找子单、父单), + c.parent_id = o.id 和 c.id = o.parent_id 不带标记都成了 orders.id = orders.parent_id,分不出来。标记按列所在 + 那一段算,从哪一段开始写都一样。""" + qualifier = column.table + if not qualifier: + return 0 + slot = name_key(qualifier) + if slot in sources: + return 0 + outer, depth = getattr(sources, "outer", None), 1 + while outer is not None: + if slot in outer: + return depth + outer, depth = outer.outer, depth + 1 + return 0 + def _pass(self, source: _Source | None, column: str) -> tuple[_Source, str] | None: """派生来源只读一个来源、把它的这一列原样透出来(直接选出这一列,或者 SELECT *)时,里面一层的 (来源, 列名); 不是原样透出(算出来的列、读好几个来源)返回 None。先认写了名字的列:SELECT *, amount * 0.9 AS net 里的 net @@ -1271,22 +1398,29 @@ class _Statement: return "window" return None - def through(self, source: _Source | None, column: str, *, pierce: bool = False) -> tuple[_Source | None, str]: + def through(self, source: _Source | None, column: str, *, pierce: bool = False, + unseal: bool = False) -> tuple[_Source | None, str]: """CTE、子查询只是把一张表的列原样透出来(SELECT *,或者直接选出这一列)时,换成那张表的列: 原流程直接查 orders、盲算先在 CTE 里筛一遍再查,算的是同一列。 这一段取前几行、或者算了窗口函数时不透过去(见 _blocks),照写这个派生来源的列:外面的 t.status = 'paid' 和写在 t 里面的 status = 'paid' 不是同一个条件,透过去就都成了 orders.status = 'paid',分不出来。pierce 时 - 不管这些,一直透到底:拿来认出两边是不是同一个条件、只是筛的先后不同(见 Part.pierce)。""" + 不管这些,一直透到底:拿来认出两边是不是同一个条件、只是筛的先后不同(见 Part.pierce)。 + + 同一段里别的来源透过去读的也是同一个来源时(opaque,见 _seal)不透过去,pierce 也不透:透过去就分不出是 + 哪一份了。unseal 时连这个也不管,只拿来给人指出最后读的是哪张表的哪一列(时间条件的 detail)。""" for _ in range(4): + if source is not None and source.opaque and not unseal: + break found = self._pass(source, column) if found is None or (not pierce and self._blocks(source.query, found[1])): break source, column = found return source, column - def resolve(self, column: Any, sources: dict[str, _Source], *, pierce: bool = False) -> tuple[_Source | None, str]: - return self.through(self.locate(column, sources)[1], column.name, pierce=pierce) + def resolve(self, column: Any, sources: dict[str, _Source], *, pierce: bool = False, + unseal: bool = False) -> tuple[_Source | None, str]: + return self.through(self.locate(column, sources)[1], column.name, pierce=pierce, unseal=unseal) def _definition(self, source: _Source | None, column: str) -> list[tuple[Any, _Sources]] | None: """派生来源一列在里面的定义:[(定义的表达式, 它那一支的来源)],UNION 每支一项(按列的位置)。只有 SELECT * @@ -1358,8 +1492,9 @@ class _Statement: (见 Part),text 只在好几张表都有这一列时写真实表名,full 一律写(关联条件说的就是两张表各出哪一列,用 full)。 node 里套着的子查询逐段改写:每一列按它所在那一段的来源认,子查询里的表去掉别名(列已经写成了真实表名)。 - 子查询里拿两张表的列互相比的(相关子查询),给人看时和关联条件一样写表名。比较用的写法里各自起的名字也去掉 - (见 _drop_names)。 + 子查询里拿两张表的列互相比的(相关子查询),给人看时和关联条件一样写表名;内外读的是同一个来源(orders 里找 + 子单)时写表名分不出谁是谁,照写别名、留着表别名。比较用的写法里外层的列在列名后面带「^」(见 _depth),各自 + 起的名字去掉(见 _drop_names)。 """ from sqlglot import exp @@ -1371,17 +1506,30 @@ class _Statement: if col.name and not col.is_star: owner = self._owner(col) groups.setdefault(id(owner), (owner, []))[1].append(col) + #: 给人看时照写别名的段(id):里面的列和外层的列读的是同一个来源(见下) + keep: set[int] = set() for owner, cols in groups.values(): scope = sources if owner is None else self.sources(self.segments[owner.meta[_SEGMENT]]) - found = [(col, *self.resolve(col, scope, pierce=mode == "pierce")) for col in cols] + found = [(col, *self.resolve(col, scope, pierce=mode == "pierce"), self._depth(col, scope)) + for col in cols] here = mode - if owner is not None and mode == "text" and len({s.ref for _, s, _ in found if s is not None}) > 1: + seen = {(s.ref, depth) for _, s, _, depth in found if s is not None} + # 同一个来源既是这一段的、又是外层的(相关子查询读的表和外层是同一张):写表名分不出谁是谁,写别名 + clash = {ref for ref, _ in seen if sum(1 for other, _ in seen if other == ref) > 1} + if owner is not None and mode == "text" and len(seen) > 1: here = "full" - for col, source, name in found: - self._write(col, source, name, here) + for col, source, name, depth in found: + alias = None + if mode not in _COMPARING and source is not None and source.ref in clash: + located = self.locate(col, scope)[1] + alias = col.args["table"].copy() if col.table else \ + exp.to_identifier(located.alias) if located is not None and located.alias else None + keep.add(id(owner)) + self._write(col, source, name, here, depth=depth, alias=alias) for table in copy.find_all(exp.Table): - if isinstance(table.this, exp.Identifier) and self._owner(table) is not None: - table.set("alias", None) + if isinstance(table.this, exp.Identifier) and (owner := self._owner(table)) is not None: + if id(owner) not in keep: + table.set("alias", None) if mode in _COMPARING: for arg in ("catalog", "db", "this"): ident = table.args.get(arg) @@ -1420,8 +1568,10 @@ class _Statement: if not _named_output(select): select.set("expressions", [e.this if isinstance(e, exp.Alias) else e for e in select.expressions]) - def _write(self, col: Any, source: _Source | None, name: str, mode: str) -> None: - """按 render 的写法改写一列(就地改)。""" + def _write(self, col: Any, source: _Source | None, name: str, mode: str, *, depth: int = 0, + alias: Any = None) -> None: + """按 render 的写法改写一列(就地改)。depth 是它属于往外第几层的来源(见 _depth):比较用的写法里另带 + 「^」。alias 是给人看时照写的别名(内外层读同一个来源时,见 render):这时列名也照原样写,不写透过去的。""" from sqlglot import exp quoted = isinstance(col.this, exp.Identifier) and bool(col.this.args.get("quoted")) @@ -1433,15 +1583,24 @@ class _Statement: self._signature_of(source.key) + (f"#{source.role}" if source.role else "") else: owner = None if mode == "loose" else source.ref - col.set("this", exp.to_identifier(self.column_key(source, name) if source is not None else name_key(name))) + # 外层的列在列名后面带标记(不带在表名上:表名带了标记要加引号,排比较的两边时外层的列就都排到前面去了, + # 只该在两边表名一样时才影响先后) + column = self.column_key(source, name) if source is not None else name_key(name) + col.set("this", exp.to_identifier(column + "^" * depth)) col.set("table", exp.to_identifier(owner) if owner else None) col.set("db", None) col.set("catalog", None) return + if alias is not None: + col.set("table", alias) + col.set("db", None) + col.set("catalog", None) + return col.set("this", exp.Identifier(this=name, quoted=quoted or not _PLAIN_NAME.match(name))) - # 自己关联自己的(有 role)一律写别名:只写列名读不出是哪一个 + # 自己关联自己的(有 role)一律写别名:只写列名读不出是哪一个。不透过去的派生来源(opaque)不算「整条语句只用到 + # 一张实表」:两个 CTE 都读 orders,好几个都有的列(shared)得写出是哪一个的 if source is None or (source.kind != "other" and mode != "full" and not source.role - and (self.single or name_key(name) not in self.shared())): + and ((self.single and not source.opaque) or name_key(name) not in self.shared())): col.set("table", None) col.set("db", None) col.set("catalog", None) @@ -1489,14 +1648,20 @@ class _Statement: #: 是两列(o1.amount、o2.amount) refs: dict[tuple[str, str], tuple[ColumnRef, str]] = {} origins: list[tuple[str, str]] = [] + roles: list[tuple[str, str, str]] = [] stage = "" for col in _own_columns(node): located = self.locate(col, sources)[1] + # 外层的列另带标记(见 _depth):相关子查询里 c.amount 和外层的 o.amount 是两列 + mark = "^" * self._depth(col, sources) source, name = self.through(located, col.name) - refs.setdefault((source.ref if source is not None else "", name_key(name)), + refs.setdefault((source.ref if source is not None else "", name_key(name) + mark), ((source.name if source is not None else None, name), - f"{source.ref}.{self.column_key(source, name)}" if source is not None - else f".{name_key(name)}")) + f"{source.ref}.{self.column_key(source, name)}{mark}" if source is not None + else f".{name_key(name)}{mark}")) + if source is not None and source.role and source.idents: + if (note := (source.idents[-1].name, source.role, source.name)) not in roles: + roles.append(note) if source is None or source.kind != "derived": continue # 透视在这个派生来源上停住了,一路透到底却还能往里走:是它取前几行、算窗口函数挡住的 @@ -1517,7 +1682,7 @@ class _Statement: columns=columns, excludes=self._excludes(node, sources) if excludes else None, loose=self.render(node, sources, "loose"), full=full, column_keys=tuple(written for _, written in refs.values()), - origins=tuple(origins), pierce=pierce, stage=stage) + origins=tuple(origins), pierce=pierce, stage=stage, roles=tuple(roles)) def _stage(self, source: _Source, column: str) -> str: """挡住了透视的派生来源做了什么(给人看,见 _blocks):「取 t 的前 10 行」「算 t 的窗口函数」。""" @@ -1597,19 +1762,24 @@ class _Statement: # ---- 各个方面 def facets(self, extra: Iterable[str] = ()) -> Facets: - """各个方面。extra 是没用到、也照样收进来的 CTE(name_key,见 _aligned),别的没用到的 CTE 不收(见 _enliven)。""" + """各个方面。extra 是没用到、也照样收进来的 CTE(name_key,见 _aligned),别的没用到的 CTE 不收(见 _enliven)。 + 条件、分组键、连接、聚合、HAVING 都记下在哪一层(见 _layers),按(层, key)去重。""" from sqlglot import exp self._enliven(extra) + tags = self._layers(extra) joins: list[JoinFacet] = [] where: list[Part] = [] time: list[TimePredicate] = [] group: list[Part] = [] + distinct: list[str] = [] aggregates: list[Aggregate] = [] having: list[Part] = [] used: list[Part] = [] for select in self._alive(exp.Select): sources = self.sources(select) + # 没走到的段(条件、SELECT 列表里的子查询,以及只在它们里面用到的派生来源)在「?」那一层 + layer = tags.get(id(select), "?") if select is self.root or not self._in_condition(select): used += [Part.plain(s.ref, s.name) for s in sources.values()] drafts = self._join_drafts(select, sources) @@ -1618,33 +1788,208 @@ class _Statement: canon = self.canonical(cond, sources) if self._joins_on_where(canon, sources, drafts): continue - part = self.part(canon, sources, excludes=True) + part = replace(self.part(canon, sources, excludes=True), scope=layer) where.append(part) self._rejects(canon, part, sources, drafts) if found := self._time(canon, sources): - time.append(found) - joins += [self._join(d, sources) for d in drafts] - group += self._group(select, sources) - aggregates += self._aggregates(select, sources) - having += self._having(select, sources) + time.append(replace(found, condition=replace(found.condition, scope=layer))) + joins += [replace(self._join(d, sources), scope=layer) for d in drafts] + keys = self._group(select, sources) + if not keys and (keys := self._distinct_keys(select, sources)): + distinct.append(layer) + group += [replace(p, scope=layer) for p in keys] + aggregates += [replace(g, scope=layer) for g in self._aggregates(select, sources)] + having += [replace(p, scope=layer) for p in self._having(select, sources)] derived = [Part.plain(self._labels[number], name or "子查询") for number, name, body, _ in self._derived if id(body) not in self._dead] return Facets( dialect=self.read, tables=tuple(Part.plain(s.key, s.name) for s in self.tables), - joins=_dedupe(joins, lambda j: (j.kind, j.table.key, j.on.key if j.on else None)), - where=_dedupe(where, lambda p: p.key), - time=_dedupe(time, lambda t: t.condition.key), - group_by=_dedupe(group, lambda p: p.key), - aggregates=_dedupe(aggregates, lambda a: (a.func, a.arg.key, a.distinct)), - having=_dedupe(having, lambda p: p.key), + joins=_dedupe(joins, lambda j: (j.scope, j.kind, j.table.key, j.on.key if j.on else None)), + where=_dedupe(where, lambda p: (p.scope, p.key)), + time=_dedupe(time, lambda t: (t.condition.scope, t.condition.key)), + group_by=_dedupe(group, lambda p: (p.scope, p.key)), + aggregates=_dedupe(aggregates, lambda a: (a.scope, a.func, a.arg.key, a.distinct)), + having=_dedupe(having, lambda p: (p.scope, p.key)), limit=self._limit(), limits=tuple(self._limits()), sources=_dedupe(used, lambda p: p.key), expressions=_dedupe(self._expressions(), lambda c: (c.scope_key, c.name, c.expr.key)), derived=_dedupe(derived, lambda p: p.key), + set_ops=tuple(self._set_ops(tags)), + distinct=tuple(dict.fromkeys(distinct)), + grain=self._grain(self.root), + scopes=self._scope_parts(tags), ) + # ---- 层 + + def _layers(self, extra: Iterable[str] = ()) -> dict[int, str]: + """每段 SELECT 在哪一层:{段的 id: 层的标记}(见 Part.scope)。 + + 最外层是空串,UNION 的第 k 支(k ≥ 2)在后面加「|k」。FROM / JOIN 里的派生来源(CTE、子查询、LATERAL 子查询) + 是它自己的编号(两边按结构配好的共同编号,所以两边对得上);只从一个来源里筛出一部分行的(见 _transparent) + 和用到它的那一段算同一层:写在这种 CTE 里的条件和写在外层是一回事。读同一个来源的几个派生来源同在一段时 + (见 _seal)各算各的层:两个 CTE 里各有一个 is_deleted = 0 是两个条件。派生来源在哪一层按在 FROM / JOIN 里 + 第一次用到它的地方定,从外往里一层一层地走。没走到的段(条件、SELECT 列表里的子查询,以及只在它们里面用到 + 的派生来源)由 facets 记成「?」。extra 是没用到、也照样收进来的 CTE(见 _aligned),在它自己的编号那一层。""" + from sqlglot import exp + + tags: dict[int, str] = {} + queue: deque[tuple[Any, str]] = deque() + + def push(query: Any, tag: str) -> None: + for i, branch in enumerate(_branches(query)): + queue.append((branch, tag if i == 0 else f"{tag}|{i + 1}")) + + def drain() -> None: + while queue: + select, tag = queue.popleft() + if id(select) in tags: + continue + tags[id(select)] = tag + sources = self.sources(select) + for i, item in enumerate(self._items(select)): + body, label = _derived_body(item), item.meta.get(_DERIVED) + if body is None and isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) \ + and item.name and self._is_cte(item): + body = self.ctes[name_key(item.name)] + if body is None or not label: + continue + alias = _alias(item) + source = sources.get(name_key(alias) if alias else f"#{i}") + lifted = source is not None and not source.opaque and self._transparent(item, body) + push(body, tag if lifted else label) + + push(self.root, "") + drain() + for name in extra: + if name in self._cte_defs and (label := self._cte_defs[name].meta.get(_DERIVED)): + push(self.ctes[name], label) + drain() + return tags + + def _transparent(self, item: Any, body: Any) -> bool: + """派生来源只是从一个来源里筛出一部分行(不分组、不聚合、不去重、不是 UNION,也不是 LATERAL):写在它里面的 + 条件和写在用到它的那一段是一回事,算同一层(见 _layers)。各列怎么算不影响有哪些行;取前几行、算窗口函数的 + 也算同一层,筛的先后另由 Part.pierce / stage 说。""" + from sqlglot import exp + + if isinstance(item, exp.Lateral) or not isinstance(body, exp.Select) or len(self._items(body)) != 1: + return False + if any(body.args.get(k) is not None for k in ("group", "having", "distinct")): + return False + return not self._aggregating(body) + + @staticmethod + def _aggregating(select: Any) -> bool: + """这一段自己算了聚合(窗口函数、套着的子查询里的不算)。""" + from sqlglot import exp + + return any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) for n in _own_nodes(select)) + + def _scope_parts(self, tags: Mapping[int, str]) -> tuple[Part, ...]: + """各层给人看的说法(Part.plain(层的标记, 说法)):最外层「外层」,派生来源写它的名字,UNION 的各支写「t 的 + UNION 第 2 支」,条件里的子查询「条件的子查询」。""" + names = {self._labels[number]: name or "子查询" for number, name, _, _ in self._derived} + unions = {self._labels[number] for number, _, body, _ in self._derived if len(_branches(body)) > 1} + root_union = len(_branches(self.root)) > 1 + out: dict[str, Part] = {} + for tag in dict.fromkeys([*tags.values(), "?"]): + head, _, branch = tag.partition("|") + if head == "?": + text = "条件的子查询" + elif head == "": + text = f"UNION 第 {branch or 1} 支" if root_union else "外层" + else: + name = names.get(head, "子查询") + text = _say(name, "的 UNION 第", branch or "1", "支") if head in unions else name + out[tag] = Part.plain(tag, text) + return tuple(out.values()) + + def _set_ops(self, tags: Mapping[int, str]) -> list[SetOpFacet]: + """各段的 UNION、INTERSECT、EXCEPT(见 SetOpFacet),条件里的子查询里的不算。""" + from sqlglot import exp + + out = [] + for node in self._alive(exp.SetOperation): + parent = node.parent + while isinstance(parent, exp.Subquery): + parent = parent.parent + branches = _branches(node) + if isinstance(parent, exp.SetOperation) or not branches: + continue # 只看一串合并最上面的那一个 + tag = tags.get(id(branches[0]), "?") + if tag != "?": + out.append(SetOpFacet(ops=tuple(self._ops(node)), branches=len(branches), + scope=self._scope_name(node), scope_key=tag)) + return out + + @classmethod + def _ops(cls, node: Any) -> list[str]: + """一串合并从左到右各处的写法:UNION、UNION ALL、INTERSECT、EXCEPT ALL……""" + from sqlglot import exp + + while isinstance(node, exp.Subquery): + node = node.this + if not isinstance(node, exp.SetOperation): + return [] + word = {"Union": "UNION", "Intersect": "INTERSECT", "Except": "EXCEPT"}.get(type(node).__name__, + type(node).__name__.upper()) + if not node.args.get("distinct"): + word += " ALL" + return cls._ops(node.this) + [word] + cls._ops(node.expression) + + def _distinct_keys(self, select: Any, sources: dict[str, _Source]) -> list[Part]: + """SELECT DISTINCT(不带 ON)选出来的各项。没有 GROUP BY、没有聚合时,对这几项去重和按它们分组出的行一模一样, + 当这一段的分组键收(见 Facets.distinct):SELECT DISTINCT user_id 和 GROUP BY user_id 不算不同,一边在子查询里 + 先去重、另一边没有照报。DISTINCT ON 是每组取一行,按窗口函数算(见 _blocking)。""" + from sqlglot import exp + + distinct = select.args.get("distinct") + if not isinstance(distinct, exp.Distinct) or distinct.args.get("on") is not None \ + or select.args.get("group") is not None or self._aggregating(select): + return [] + out = [] + for item in select.selects: + if _is_star(item): + out.append(Part.plain("*", "*")) + continue + target = item.this if isinstance(item, exp.Alias) else item + out.append(self.part(self.canonical(target, sources), sources)) + return out + + def _grain(self, query: Any, depth: int = 0) -> Part | None: + """最后出来的行按什么分:分组(GROUP BY、当分组用的 DISTINCT)是「group:各分组键」,没分组却算了聚合是 + 「one」(只出一行),都没有、只从一个派生来源取数的看那个派生来源,别的是「rows」(逐行)。最外层是 UNION + 的不算(返回 None)。分组键挪了一层(先在子查询里按 user_id 分组、外层数用户数,对外层按 user_id 分组)按 + 整条语句比分组键看不出来:外层出一行还是每个用户一行,看它。""" + from sqlglot import exp + + while isinstance(query, exp.Subquery): + query = query.this + if not isinstance(query, exp.Select) or depth > 6: + return None + sources = self.sources(query) + keys = self._group(query, sources) + if not keys and self._aggregating(query): + return Part.plain("one", "") + keys = keys or self._distinct_keys(query, sources) + if keys: + return Part(key="group:" + ", ".join(sorted({p.key for p in keys})), + text="、".join(dict.fromkeys(p.text for p in keys)), + loose="group:" + ", ".join(sorted({p.loose for p in keys})), + full="、".join(dict.fromkeys(p.full for p in keys))) + items = self._items(query) + if len(items) == 1 and not isinstance(items[0], exp.Lateral): + item, body = items[0], _derived_body(items[0]) + if body is None and isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) \ + and item.name and self._is_cte(item): + body = self.ctes[name_key(item.name)] + if body is not None: + return self._grain(body, depth + 1) + return Part.plain("rows", "") + def _rejects(self, cond: Any, part: Part, sources: dict[str, _Source], drafts: list[dict[str, Any]]) -> None: """同一段 WHERE 里的条件把外连接可空那一侧没关联上的行筛掉了的,记在那次连接上(见 JoinFacet.rejects):LEFT JOIN 关联进来的那一侧、RIGHT JOIN 之前的各个来源可空,那一侧的列直接拿来比较、IN、LIKE、BETWEEN,或者写 IS NOT @@ -1777,17 +2122,27 @@ class _Statement: shown = [self.render(v, sources) for v in values] value = f"{shown[0]} AND {shown[1]}" if op == "between" else f"({', '.join(shown)})" if op == "in" \ else shown[0] + # 给调用方对照数据目录的那一列一直换到实表:不透过去的派生来源(见 _seal)比较时照写它自己,可它的这一列 + # 读的还是那张表的那一列 + real, real_scope, _, _ = self._time_column(cols[0], sources, unseal=True) + found = self.resolve(real, real_scope, pierce=True, unseal=True) + table = found[0].name if found[0] is not None and found[0].kind == "table" else None return TimePredicate(column=self.part(base, scope), op=op, value=value, - condition=self.part(cond, sources, excludes=True), via=via) + condition=self.part(cond, sources, excludes=True), via=via, + table=(table, found[1]) if table else None) - def _time_column(self, column: Any, sources: dict[str, _Source]) -> tuple[Any, _Sources, tuple[str, ...], bool]: + def _time_column(self, column: Any, sources: dict[str, _Source], *, + unseal: bool = False) -> tuple[Any, _Sources, tuple[str, ...], bool]: """时间条件里的那一列是 CTE、子查询算出来的、在里面是同一列套着日期函数、CAST、MIN / MAX 时(见 _value_column), 换成里面那一列,一直换到实表:(那一列, 它那一段的来源, 路过的派生来源的列「编号.列」, 路上有没有日期函数)。 - 原样透出的列 resolve 已经透过去了;这里补上算了一下的。""" + 原样透出的列 resolve 已经透过去了;这里补上算了一下的。不透过去的派生来源(见 _seal)到它为止:两个 CTE 都读 + orders 时,按已付那份的 created_at 筛还是按未付那份的筛,正是要分清的;unseal 时不管这个(见 through)。""" via: list[str] = [] datey = False for _ in range(4): - source, name = self.resolve(column, sources) + source, name = self.resolve(column, sources, unseal=unseal) + if source is not None and source.opaque and not unseal: + break found = self._definition(source, name) if source is not None else None inner = _value_column(found[0][0]) if found and len(found) == 1 else None if inner is None: @@ -2111,9 +2466,10 @@ class _Statement: class FacetDiff: """一处不同。aspect:time(时间列)/ time_range(同一时间列、范围不同,或者同一个时间条件筛的先后不同)/ filter / aggregate / expression(算出来的列:最外层的算法、CTE 和子查询里被用到的列的定义,见 Computed)/ join(类型、 - 外连接的主表、关联条件、只有一边关联了)/ tables / group_by / having / limit(取哪几行:行数、跳过的行数、取前 - 几行时的排序)/ parse。a、b 是两边各自的写法(给人看,没有就是 None),text 是一句中文说明。 - detail 只在 time 上有:两边的时间列 [{table, column}],调用方据此对照数据目录(业务日期)。""" + 外连接的主表、关联条件、只有一边关联了)/ tables / set_op(几支查询怎么合并:UNION 还是 UNION ALL、几支)/ + group_by(分组键、去重,以及最后出来的行按什么分)/ having / limit(取哪几行:行数、跳过的行数、取前几行时的 + 排序)/ parse。a、b 是两边各自的写法(给人看,没有就是 None),text 是一句中文说明。 + detail 只在 time 上有:两边的时间列 [{table, column}](一直换到实表),调用方据此对照数据目录(业务日期)。""" aspect: str a: Any @@ -2190,7 +2546,10 @@ def _single_column(part: Part, *, loose: bool) -> str | None: if len(part.columns) != 1 or len(part.column_keys) != 1: return None key = part.column_keys[0] - return name_key(part.columns[0][1]) if loose and not key.startswith("@") else key + if not loose or key.startswith("@"): + return key + # 外层的列(相关子查询里引用外层,见 _Statement._depth)不写表名也带着标记 + return name_key(part.columns[0][1]) + "^" * (len(key) - len(key.rstrip("^"))) def _verb(g: Aggregate) -> str: @@ -2218,8 +2577,12 @@ class _Compare: 派生来源的编号由 diff_facets 先按结构配好(两边配成一对的写同一个编号,见模块说明),这里各方面都只按 k 比: 编号不同就是不同的来源,不再拿「不写编号」的写法补配(那样会把 p.total 和 r.total 当成同一列)。 + 条件、分组键、连接、聚合、HAVING 按层计数(见 leftovers):同一项在一边出现在几层就算几个,另一边也有、只是在的 + 层少了的,说清多出来的是哪一层(layered)。 + 几个方面之间互相交代,同一处不同只报一遍:聚合那一面报过的聚合(reported)、时间那一面报过的派生来源的列 - (explained),算出来的列那一面不再重复报。 + (explained),算出来的列那一面不再重复报;合并了几支、怎么合并的那一面(set_ops)报了的,另一边没有的那几支里 + 多出来的条件不再单说;分组键那一面报了的,最后出来的行按什么分(grain)不再说。 """ def __init__(self, a: Facets, b: Facets, labels: tuple[str, str], names: Mapping[str, str] | None) -> None: @@ -2236,6 +2599,10 @@ class _Compare: #: 聚合那一面报过的两边独有的聚合、时间那一面已经说了的派生来源的列(「编号.列」) self.reported: tuple[set[Any], set[Any]] = (set(), set()) self.explained: set[str] = set() + #: 两边各有哪些层 {层的标记: 说法}(见 Part.scope、Facets.scopes);另一边也有、只是在的层少了的项多出来的 + #: 那几个 {id(项): 另一边有它的那几层}(见 leftovers) + self.layers = ({p.key: p.text for p in a.scopes}, {p.key: p.text for p in b.scopes}) + self.elsewhere: dict[int, tuple[str, ...]] = {} def k(self, part: Part | None) -> str | None: if part is None: @@ -2272,14 +2639,85 @@ class _Compare: @staticmethod def unmatched(xs: Iterable[Any], ys: Iterable[Any], key: Any) -> tuple[list[Any], list[Any]]: - """两边各自独有的项(按 key 比),各按原来的先后。派生来源在 key 里已经是两边按结构配好的共同编号(见 - diff_facets),这里不再拿不写编号的写法补配一遍:那样两个 CTE 的同名列(p.total、r.total)会被当成一列, - 换了用哪一个的列也报不出来。配不上的派生来源(一边多出来的 CTE)编号和另一边的都不一样,按它写的项照样 - 各算各的。""" + """两边各自独有的项(按 key 比,不管在哪一层),各按原来的先后。派生来源在 key 里已经是两边按结构配好的共同 + 编号(见 diff_facets),这里不再拿不写编号的写法补配一遍:那样两个 CTE 的同名列(p.total、r.total)会被当成 + 一列,换了用哪一个的列也报不出来。配不上的派生来源(一边多出来的 CTE)编号和另一边的都不一样,按它写的项照样 + 各算各的。时间列、算出来的列按它比;条件、分组键、连接、聚合、HAVING 按 leftovers 比(还看在哪一层)。""" xs, ys = list(xs), list(ys) kx, ky = {key(x) for x in xs}, {key(y) for y in ys} return [x for x in xs if key(x) not in ky], [y for y in ys if key(y) not in kx] + def leftovers(self, xs: Iterable[Any], ys: Iterable[Any], key: Any) -> tuple[list[Any], list[Any]]: + """两边各自多出来的项,各按原来的先后。项都记着在哪一层(scope,见 Part.scope),同一项(key 一样)在一边 + 出现在几层就算几个,一对一地配: + + - 同一层的先配掉;剩下的不管在哪一层也配(条件从 CTE 挪到外层、从 IN 子查询换成关联进来的 CTE 筛,常常是同一个 + 口径),要单说的层先配(不单说的见下,配不完剩下的尽量是它们)。配不完的才是多出来的:外层的 status = 'paid' + 和 IN 子查询里的同一个条件原来按整条语句去重成了一个,外层少了它报不出来;现在一边两层、一边一层,多出来外层 + 那一个。这种另一边也有、只是在的层少了的,记进 elsewhere(另一边有它的那几层),说的时候带上是哪一层。 + - 另一边根本没有这一项的,只报一个(在好几层也只报一次,挑要单说的那一层的),照旧不说哪一层。 + + 条件里的子查询那一层(「?」)多出来的不报:它整个写在外层那个条件、那一列里,那边已经报了,再说一句「只保留 + state = 'done' 的行」,读的人会以为外层也这么筛。另一边没有的 UNION 分支里多出来的也不报:合并了几支、怎么合并 + 的那一面(set_ops)已经报了。""" + xs, ys = list(xs), list(ys) + by_x: dict[Any, list[Any]] = {} + by_y: dict[Any, list[Any]] = {} + for x in xs: + by_x.setdefault(key(x), []).append(x) + for y in ys: + by_y.setdefault(key(y), []).append(y) + left_x: list[Any] = [] + left_y: list[Any] = [] + for k in dict.fromkeys([*by_x, *by_y]): + ax, ay = by_x.get(k, []), by_y.get(k, []) + if not ax or not ay: + # 另一边根本没有:挑一个要单说的报(全在不单说的层里就不报) + mine, other = (ax, 1) if ax else (ay, 0) + if (first := next((m for m in mine if self.visible(m.scope, other)), None)) is not None: + (left_x if ax else left_y).append(first) + continue + sx, sy = {x.scope for x in ax}, {y.scope for y in ay} + # 不单说的层(条件子查询里、另一边没有的 UNION 分支里)排在后面:配不完剩下的尽量是它们 + rx = sorted((x for x in ax if x.scope not in sy), key=lambda x: not self.visible(x.scope, 1)) + ry = sorted((y for y in ay if y.scope not in sx), key=lambda y: not self.visible(y.scope, 0)) + n = min(len(rx), len(ry)) + for x in rx[n:]: + left_x.append(x) + self.elsewhere[id(x)] = tuple(sorted(sy)) + for y in ry[n:]: + left_y.append(y) + self.elsewhere[id(y)] = tuple(sorted(sx)) + order_x = {id(x): i for i, x in enumerate(xs)} + order_y = {id(y): i for i, y in enumerate(ys)} + return ([x for x in sorted(left_x, key=lambda x: order_x[id(x)]) if self.visible(x.scope, 1)], + [y for y in sorted(left_y, key=lambda y: order_y[id(y)]) if self.visible(y.scope, 0)]) + + def visible(self, scope: str, other: int) -> bool: + """这一层多出来的项要不要单说(见 leftovers):条件里的子查询那一层不说,另一边(other)没有的 UNION 分支里的 + 不说。""" + head, bar, _ = scope.partition("|") + return head != "?" and (not bar or not self.layers[other] or scope in self.layers[other]) + + def place(self, scope: str, side: int) -> str: + """一层在那一边的说法(见 Facets.scopes),接在「在」后面:「外层」「t 里」「t 的 UNION 第 2 支里」「条件的 + 子查询里」。""" + text = self.layers[side].get(scope) or ("条件的子查询" if scope.startswith("?") else + "外层" if not scope.partition("|")[0] else "子查询") + return text if text == "外层" else _say(text, "里") + + def layered(self, side: int, item: Any, did: tuple[str, ...], lack: str, have: str) -> str: + """另一边也有、只是在的层少了的一项(item,见 leftovers)多出来的那一个怎么说:「原流程在外层也只保留 X 的行, + 盲算在外层没有这个条件」。did 是这一边做了什么(「也只保留」X「的行」);lack 是另一边在这一层没有时怎么说, + have 是另一边根本没有这一层时、说它在哪几层有的说法(「有这个条件」)。""" + me, other = (self.la, self.lb) if side == 0 else (self.lb, self.la) + if item.scope in self.layers[1 - side]: + tail: tuple[str, ...] = ("在", self.place(item.scope, 1 - side), lack) + else: + seen = self.elsewhere.get(id(item), ()) + tail = ("只在", "、".join(self.place(s, 1 - side) for s in seen), have) if seen else (lack,) + return _say(me, "在", self.place(item.scope, side), *did, ",", other, *tail) + @staticmethod def lookalikes(only_a: list[Part], only_b: list[Part]) -> list[tuple[Part, Part]]: """两边各自独有、给人看的写法却一样的条件配成一对,就地从两边拿掉:同一个条件挂在不同的来源上(p.total > 100 @@ -2328,12 +2766,28 @@ class _Compare: return (wa, wb) if wa != wb else self.annotated(wa, wb, xs, ys) def annotated(self, wa: str, wb: str, xs: list[Part], ys: list[Part]) -> tuple[str, str]: - """两边一律写表名的写法 wa、wb 也一样时(见 shown):带上派生来源各列的来历,来历也一样就说清对应关系。""" + """两边一律写表名的写法 wa、wb 也一样时(见 shown):带上派生来源各列的来历;来历也一样、用到了自己关联自己的 + 别名时说清各自的别名是哪一个(关联的方向反了,两边都写 a.amount);再不然说清对应关系。""" ga, gb = self.gloss(xs), self.gloss(ys) if ga != gb: return _say(wa, ga), _say(wb, gb) + ra, rb = self.role_note(xs), self.role_note(ys) + if ra != rb: + return _say(wa, ra), _say(wb, rb) return wa, _say(wb, self.correspondence(xs, ys)) + @staticmethod + def role_note(parts: list[Part]) -> str: + """用到的自己关联自己的别名各是哪一个(见 Part.roles、_Statement._roles):「(a 是关联条件里出 referrer_id 的 + 那个 t)」。没有返回空串。""" + notes: list[str] = [] + for p in parts: + for alias, role, name in p.roles: + note = _say(alias, "是关联条件里出", "、".join(role.split("+")), "的那个", name) + if note not in notes: + notes.append(note) + return f"({'、'.join(notes)})" if notes else "" + @staticmethod def gloss(parts: list[Part]) -> str: """用到的派生来源的列各自的来历(见 Part.origins):「(即 SUM(orders.amount))」,好几列时「(p.total 即 …、 @@ -2372,13 +2826,28 @@ class _Compare: out.append(_say(name, text) if name else text) return "、".join(out) - def one_sided(self, side: str, other: str, part: Part, *, group: bool = False) -> str: + def one_sided(self, index: int, part: Part, *, group: bool = False) -> str: + """一边独有的一个条件(index 是哪一边:0 是第一条)。另一边也有、只是在的层少了的(见 leftovers),带上是哪一层。""" + side, other = (self.la, self.lb) if index == 0 else (self.lb, self.la) + if id(part) in self.elsewhere: + did = ("分组后也只保留", part.text, "的组") if group else ("也排除了", part.excludes, "的行") \ + if part.excludes else ("也只保留", part.text, "的行") + return self.layered(index, part, did, "没有这个条件", "有这个条件") if group: return _say(side, "分组后只保留", part.text, "的组,", other, "没有这个条件") if part.excludes: return _say(side, "排除了", part.excludes, "的行,", other, "没有") return _say(side, "只保留", part.text, "的行,", other, "没有这个条件") + def at(self, side: int, item: Any, other: Any) -> tuple[str, ...]: + """配成一对的两项不在同一层时,这一边说「在哪一层」(见 place);在同一层不说。""" + return ("在", self.place(item.scope, side)) if item.scope != other.scope else () + + def paired(self, p: Part, q: Part, verb: str = "筛选") -> str: + """两边各一个、配成一对的条件:「原流程按 X 筛选,盲算按 Y 筛选」。""" + wa, wb = self.shown([p], [q]) + return _say(self.la, *self.at(0, p, q), "按", wa, verb + ",", self.lb, *self.at(1, q, p), "按", wb, verb) + # ---- 各个方面 def time(self) -> list[FacetDiff]: @@ -2393,7 +2862,10 @@ class _Compare: text = _say(la, "按", wa, "筛时间,", lb, "没有按时间筛选") else: text = _say(la, "没有按时间筛选,", lb, "按", wb, "筛时间") - detail = {side: [{"table": p.columns[0][0], "column": p.columns[0][1]} for p in parts if p.columns] + # 对照数据目录用的表、列一直换到实表(TimePredicate.table):不透过去的派生来源比较时照写它自己 + tables = {self.k(t.column): t.table for t in (*a.time, *b.time) if t.table} + detail = {side: [{"table": real[0], "column": real[1]} if (real := tables.get(self.k(p))) + else {"table": p.columns[0][0], "column": p.columns[0][1]} for p in parts if p.columns] for side, parts in (("a", ca), ("b", cb))} # 时间列是从 CTE、子查询算出来的列透过去的(见 TimePredicate.column):那一列的定义不同,这里已经说了 self.explained |= {v for t in (*a.time, *b.time) for v in t.via} @@ -2405,18 +2877,28 @@ class _Compare: out = [FacetDiff("time_range", p.text, q.text, self.stage_text(p, q)) for p, q in pairs] if conds_a or conds_b: staged = {id(p) for pair in pairs for p in pair} - ra = [t.condition for t in a.time if id(t.condition) not in staged] - rb = [t.condition for t in b.time if id(t.condition) not in staged] + # 同一个时间条件在好几层都有的,范围里只写一次 + ra = list(_dedupe((t.condition for t in a.time if id(t.condition) not in staged), self.k)) + rb = list(_dedupe((t.condition for t in b.time if id(t.condition) not in staged), self.k)) wa, wb = self.shown(ra, rb, text=lambda ps: " AND ".join(p.text for p in ps), full=lambda ps: " AND ".join(p.full for p in ps)) out.append(FacetDiff("time_range", wa, wb, _say(la, "的时间范围是", wa, ",", lb, "是", wb))) + return out + # 两边的时间条件一样,只是一边在更多的层里筛了(外层筛了一遍、IN 子查询里又筛一遍,另一边只在子查询里筛) + more_a, more_b = self.leftovers([t.condition for t in a.time], [t.condition for t in b.time], self.k) + out += [FacetDiff("time_range", p.text, None, + self.layered(0, p, ("也按", p.text, "筛了时间"), "没有这个条件", "有这个条件")) + for p in more_a if id(p) in self.elsewhere] + out += [FacetDiff("time_range", None, p.text, + self.layered(1, p, ("也按", p.text, "筛了时间"), "没有这个条件", "有这个条件")) + for p in more_b if id(p) in self.elsewhere] return out def filters(self) -> list[FacetDiff]: timed = {self.k(t.condition) for t in (*self.a.time, *self.b.time)} fa = [p for p in self.a.where if self.k(p) not in timed] fb = [p for p in self.b.where if self.k(p) not in timed] - only_a, only_b = self.unmatched(fa, fb, self.k) + only_a, only_b = self.leftovers(fa, fb, self.k) # 先配同一个条件筛的先后不同的,再配同一列的不同写法(status = 'paid' 和 status IN (…)),再配读起来一样、 # 挂在不同来源上的同一个条件 out = [FacetDiff("filter", p.text, q.text, self.stage_text(p, q)) for p, q in self.staged(only_a, only_b)] @@ -2426,15 +2908,24 @@ class _Compare: theirs = [q for q in only_b if column and _single_column(q, loose=self.swapped) == column] if len(mine) == 1 and len(theirs) == 1: q = theirs[0] - wa, wb = self.shown([p], [q]) - out.append(FacetDiff("filter", p.text, q.text, _say(self.la, "按", wa, "筛选,", self.lb, "按", wb, "筛选"))) + out.append(FacetDiff("filter", p.text, q.text, self.paired(p, q))) only_a.remove(p) only_b.remove(q) for p, q in self.lookalikes(only_a, only_b): wa, wb = self.shown([p], [q]) - out.append(FacetDiff("filter", wa, wb, _say(self.la, "按", wa, "筛选,", self.lb, "按", wb, "筛选"))) - out += [FacetDiff("filter", p.text, None, self.one_sided(self.la, self.lb, p)) for p in only_a] - out += [FacetDiff("filter", None, p.text, self.one_sided(self.lb, self.la, p)) for p in only_b] + out.append(FacetDiff("filter", wa, wb, self.paired(p, q))) + # 两边各剩一个 EXISTS(或 NOT EXISTS):差在子查询里(有子单还是有父单),外层没有列可配,配成一对说 + for kind in ("EXISTS(", "NOT EXISTS("): + mine = [p for p in only_a if (self.k(p) or "").startswith(kind)] + theirs = [q for q in only_b if (self.k(q) or "").startswith(kind)] + if len(mine) == 1 and len(theirs) == 1: + p, q = mine[0], theirs[0] + wa, wb = self.shown([p], [q]) + out.append(FacetDiff("filter", wa, wb, self.paired(p, q))) + only_a.remove(p) + only_b.remove(q) + out += [FacetDiff("filter", p.text, None, self.one_sided(0, p)) for p in only_a] + out += [FacetDiff("filter", None, p.text, self.one_sided(1, p)) for p in only_b] return out def aggregates(self) -> list[FacetDiff]: @@ -2450,7 +2941,7 @@ class _Compare: ys.remove(y) return pairs - ga, gb = self.unmatched(self.a.aggregates, self.b.aggregates, self.agg_key) + ga, gb = self.leftovers(self.a.aggregates, self.b.aggregates, self.agg_key) self.reported = ({self.agg_key(g) for g in ga}, {self.agg_key(g) for g in gb}) # 先配参数相同的(同一列:函数或去重不同),再配函数相同的(同一种算法:列不同);各剩一个的也配成一对 pairs = pair(ga, gb, lambda x, y: self.k(x.arg) == self.k(y.arg)) @@ -2461,20 +2952,25 @@ class _Compare: for x, y in pairs: same_arg = self.k(x.arg) == self.k(y.arg) wa, wb = self.shown([x.arg], [y.arg]) if not same_arg else (x.arg.text, y.arg.text) + ha, hb = self.at(0, x, y), self.at(1, y, x) if same_arg and x.func == y.func: # 只差去重:COUNT(DISTINCT order_id) 和 COUNT(order_id) - text = _say(la, "对", wa, _verb(x) + ",", lb, "没去重") if x.distinct \ - else _say(la, "对", wa, _verb(x) + "时没去重,", lb, _verb(y)) + text = _say(la, *ha, "对", wa, _verb(x) + ",", lb, *hb, "没去重") if x.distinct \ + else _say(la, *ha, "对", wa, _verb(x) + "时没去重,", lb, *hb, _verb(y)) elif same_arg and x.arg.key != "*": # 同一列、算法不同:后半句不再重复列名 - text = _say(la, "对", wa, _verb(x) + ",", lb, _verb(y)) + text = _say(la, *ha, "对", wa, _verb(x) + ",", lb, *hb, _verb(y)) else: what_a = ["数的是行数"] if x.func == "count" and x.arg.key == "*" else ["对", wa, _verb(x)] what_b = ["数的是行数"] if y.func == "count" and y.arg.key == "*" else ["对", wb, _verb(y)] - text = _say(la, *what_a, ",", lb, *what_b) + text = _say(la, *ha, *what_a, ",", lb, *hb, *what_b) out.append(FacetDiff("aggregate", x.text, y.text, text)) - out += [FacetDiff("aggregate", g.text, None, _say(la, "多算了", g.text, ",", lb, "没有")) for g in ga] - out += [FacetDiff("aggregate", None, g.text, _say(lb, "多算了", g.text, ",", la, "没有")) for g in gb] + for side, rest in ((0, ga), (1, gb)): + me, other = (la, lb) if side == 0 else (lb, la) + for g in rest: + text = self.layered(side, g, ("也算了", g.text), "没有", "算了它") if id(g) in self.elsewhere \ + else _say(me, "多算了", g.text, ",", other, "没有") + out.append(FacetDiff("aggregate", g.text if side == 0 else None, g.text if side == 1 else None, text)) return out def expressions(self) -> list[FacetDiff]: @@ -2541,8 +3037,9 @@ class _Compare: return bool(differing) and all(self.agg_key(g) in ka or self.agg_key(h) in kb for g, h in differing) def joins(self) -> list[FacetDiff]: - """连接:先把完全一样的两边配掉,剩下的按关联的是哪张表(派生来源按配好的编号)、再按涉及的来源配成一对, - 说类型、主表或条件哪里不同,再剩下的是只有一边关联了(多出来的是哪个来源见 added)。 + """连接:先把完全一样的两边配掉(按层计数,见 leftovers:外层关联了 refunds、EXISTS 子查询里又关联了一次, + 另一边只在子查询里关联,多出来外层那一次),剩下的按关联的是哪张表(派生来源按配好的编号)、再按涉及的来源配成 + 一对,说类型、主表或条件哪里不同,再剩下的是只有一边关联了(多出来的是哪个来源见 added)。 完全一样:内连接、CROSS JOIN 比类型、涉及的来源和条件(FROM paid JOIN refunded 和 FROM refunded JOIN paid 是 同一次连接);LEFT / RIGHT JOIN 比主表、涉及的来源和条件,不比写的是 LEFT 还是 RIGHT(orders LEFT JOIN refunds @@ -2573,7 +3070,7 @@ class _Compare: return kind, self.sides(j), self.k(j.on) return kind, j.table.key, self.k(j.on) - rest_a, rest_b = self.unmatched(self.a.joins, self.b.joins, same) + rest_a, rest_b = self.leftovers(self.a.joins, self.b.joins, same) pairs: list[tuple[JoinFacet, JoinFacet]] = [] for table in (lambda j: j.table.key, self.sides): for x in list(rest_a): @@ -2611,8 +3108,10 @@ class _Compare: for j in rest: name, keys = self.added(j, mine, theirs) said[side] |= keys - out.append(FacetDiff("join", name if side == "a" else None, name if side == "b" else None, - _say(me, "关联了", name, "表,", other, "没有"))) + # 另一边也有这次连接、只是在的层少了(外层关联了 refunds,另一边只在 EXISTS 子查询里关联):说是哪一层 + text = self.layered(0 if side == "a" else 1, j, ("也关联了", name, "表"), "没有", "关联了它") \ + if id(j) in self.elsewhere else _say(me, "关联了", name, "表,", other, "没有") + out.append(FacetDiff("join", name if side == "a" else None, name if side == "b" else None, text)) # 关联之外的表(FROM 的表、子查询里的表)只在一边出现的;上面已经说过「关联了它」的不再说一遍 if self.swapped: ta, tb = self.ta[0], self.tb[0] @@ -2641,28 +3140,98 @@ class _Compare: "实际成了 INNER JOIN") def group(self) -> list[FacetDiff]: - a, b = self.a.group_by, self.b.group_by - if not any(self.unmatched(a, b, self.k)): + """分组键按层比(见 leftovers):有不同的那一层,说两边在这一层各按什么分组(当分组用的 DISTINCT 说「去重」, + 见 Facets.distinct)。最外层之外的层、或者另一边在别的层也有这个分组键的,带上是哪一层。分组键都配得上(挪了 + 一层也算配上)时再看最后出来的行按什么分(见 grain)。""" + left_a, left_b = self.leftovers(self.a.group_by, self.b.group_by, self.k) + if not left_a and not left_b: + return self.grain() + distinct = (set(self.a.distinct), set(self.b.distinct)) + out = [] + for scope in dict.fromkeys(p.scope for p in (*left_a, *left_b)): + keys = ([p for p in self.a.group_by if p.scope == scope], [p for p in self.b.group_by if p.scope == scope]) + noted = scope != "" or any(id(p) in self.elsewhere for p in (*left_a, *left_b) if p.scope == scope) + shown = self.shown(*keys) + + def say(side: int) -> tuple[str, ...]: + here = ("在", self.place(scope, side)) if noted and scope in self.layers[side] else () + if keys[side]: + return (*here, "对", shown[side], "去重") if scope in distinct[side] else \ + (*here, "按", shown[side], "分组") + return (*here, "没有去重" if scope in distinct[1 - side] else "没有分组") + + out.append(FacetDiff("group_by", shown[0] or None, shown[1] or None, + _say(self.la, *say(0), ",", self.lb, *say(1)))) + return out + + def grain(self) -> list[FacetDiff]: + """最后出来的行按什么分(见 Facets.grain):分组键挪了一层,按层配得上,可一边外层只出一行(先在子查询里按用户 + 分组、外层数用户数),另一边每个用户一行。只在两边都汇总了(分组或者算了聚合)时比:一边逐行出、一边汇总的, + 聚合那一面已经报了。""" + ga, gb = self.a.grain, self.b.grain + if ga is None or gb is None or "rows" in (ga.key, gb.key) or self.k(ga) == self.k(gb): return [] - wa, wb = self.shown(list(a), list(b)) - if a and b: - text = _say(self.la, "按", wa, "分组,", self.lb, "按", wb, "分组") - elif a: - text = _say(self.la, "按", wa, "分组,", self.lb, "没有分组") - else: - text = _say(self.la, "没有分组,", self.lb, "按", wb, "分组") - return [FacetDiff("group_by", wa or None, wb or None, text)] + wa, wb = self.shown([ga], [gb]) + + def say(g: Part, written: str) -> tuple[str, ...]: + return ("最后只出一行",) if g.key == "one" else ("按", written, "分组、每组一行") + + return [FacetDiff("group_by", ga.text or None, gb.text or None, + _say(self.la, *say(ga, wa), ",", self.lb, *say(gb, wb)))] + + def set_ops(self) -> list[FacetDiff]: + """各段的 UNION、INTERSECT、EXCEPT(见 SetOpFacet):同一层的配成一对,剩下的合并方式、支数都一样的不管在哪 + 一层也配掉(最外层直接 UNION,和把同一个 UNION 包进子查询再 SELECT * 是一回事)。配成一对的报合并方式(UNION + 去掉重复的行、UNION ALL 都留着)或者支数不同,配不上的报一边合并了、另一边没有。""" + la, lb = self.la, self.lb + rest_a, rest_b = list(self.a.set_ops), list(self.b.set_ops) + pairs: list[tuple[SetOpFacet, SetOpFacet]] = [] + for same in (lambda x, y: x.scope_key == y.scope_key, lambda x, y: (x.ops, x.branches) == (y.ops, y.branches)): + for x in list(rest_a): + y = next((y for y in rest_b if same(x, y)), None) + if y is not None: + pairs.append((x, y)) + rest_a.remove(x) + rest_b.remove(y) + out = [] + for x, y in [*pairs, *((x, None) for x in rest_a), *((None, y) for y in rest_b)]: + if x is not None and y is not None and (x.ops, x.branches) == (y.ops, y.branches): + continue + hx = ("在", x.scope, "里") if x is not None and x.scope else () + hy = ("在", y.scope, "里") if y is not None and y.scope and (x is None or y.scope != x.scope) else () + if x is not None and y is not None and x.branches == y.branches: + text = _say(la, *hx, "用", self.merging(x), ",", lb, *hy, "用", self.merging(y)) + elif x is not None and y is not None: + text = _say(la, *hx, "合并了", str(x.branches), "支查询,", lb, *hy, "合并了", str(y.branches), "支") + elif x is not None: + text = _say(la, *hx, "把", str(x.branches), "支查询用", "、".join(dict.fromkeys(x.ops)), "合起来,", + lb, "没有") + else: + text = _say(lb, *hy, "把", str(y.branches), "支查询用", "、".join(dict.fromkeys(y.ops)), "合起来,", + la, "没有") + out.append(FacetDiff("set_op", " / ".join(x.ops) if x else None, " / ".join(y.ops) if y else None, text)) + return out + + @staticmethod + def merging(x: SetOpFacet) -> str: + """合并的方式:只有 UNION 的说去掉重复的行,只有 UNION ALL 的说重复的行都留着,别的照写。""" + ops = set(x.ops) + if ops == {"UNION"}: + return "UNION 合并(去掉重复的行)" + if ops == {"UNION ALL"}: + return "UNION ALL(重复的行都留着)" + return "、".join(x.ops) def having(self) -> list[FacetDiff]: - only_a, only_b = self.unmatched(self.a.having, self.b.having, self.k) + only_a, only_b = self.leftovers(self.a.having, self.b.having, self.k) out = [] for p, q in self.lookalikes(only_a, only_b): wa, wb = self.shown([p], [q]) - out.append(FacetDiff("having", wa, wb, _say(self.la, "分组后只保留", wa, "的组,", self.lb, "只保留", wb, - "的组"))) + out.append(FacetDiff("having", wa, wb, _say(self.la, *self.at(0, p, q), "分组后只保留", wa, "的组,", self.lb, + *self.at(1, q, p), "只保留", wb, "的组"))) return out + \ - [FacetDiff("having", p.text, None, self.one_sided(self.la, self.lb, p, group=True)) for p in only_a] + \ - [FacetDiff("having", None, p.text, self.one_sided(self.lb, self.la, p, group=True)) for p in only_b] + [FacetDiff("having", p.text, None, self.one_sided(0, p, group=True)) for p in only_a] + \ + [FacetDiff("having", None, p.text, self.one_sided(1, p, group=True)) for p in only_b] def limit(self) -> list[FacetDiff]: """取哪几行:最外层两边比;CTE、子查询里的按所属派生来源配好的编号(LimitFacet.scope_key)配成一对,两边给 @@ -2744,8 +3313,8 @@ def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, text = _say(side, ":", error or "", ",没有逐项比较") return [FacetDiff("parse", a.parse_error, b.parse_error, text)] cmp = _Compare(*_aligned(a, b), labels, names) - return cmp.time() + cmp.filters() + cmp.aggregates() + cmp.expressions() + cmp.joins() + cmp.group() + \ - cmp.having() + cmp.limit() + return cmp.time() + cmp.filters() + cmp.aggregates() + cmp.expressions() + cmp.joins() + cmp.set_ops() + \ + cmp.group() + cmp.having() + cmp.limit() def _aligned(a: Facets, b: Facets) -> tuple[Facets, Facets]: diff --git a/backend/tests/test_sql_semantics.py b/backend/tests/test_sql_semantics.py index 6d9d93a1..eb57882b 100644 --- a/backend/tests/test_sql_semantics.py +++ b/backend/tests/test_sql_semantics.py @@ -12,7 +12,9 @@ 的列时外层用的是哪一个照报;外连接按主表比(LEFT / RIGHT 对调不报,主表换了、被 WHERE 变成内连接照报);算出来 的列按算法比(最外层的算式、窗口函数,CTE 里被外面用到的列的定义);时间列透过 CTE 里的日期表达式认;取前几行、 窗口函数前后筛的不同照报;没用到的 CTE 不算;两边同名却不是同一个的派生来源说清各是什么;自己关联自己的两个 - 别名分得开 + 别名分得开;读同一张表的几个派生来源同在一段时不透成那张表的列(用的是已付那份还是未付那份照报);相关子查询里 + 外层的列另带标记(子单、父单分得开);条件、分组、连接、聚合按层计数(外层少了一个和子查询里一样的条件照报); + 去重、UNION 还是 UNION ALL、最后出一行还是每组一行照报 夹具全是虚构的表:订单 orders、退款 refunds、门店 stores、用户 users、付款 payments,以及一张中文表「订单」。 """ @@ -1369,3 +1371,339 @@ def test_catalog_labels_come_from_visible_catalog_items(): def test_a_facets_object_compares_with_itself_cleanly(): f = facets(BASE) assert isinstance(f, Facets) and diff_facets(f, f) == [] + + +# -------------------------------------------------------------------------- +# diff_facets:读同一张表的几个派生来源(原样透出的 CTE、子查询) +# -------------------------------------------------------------------------- + +#: 两个原样透出的 CTE 读同一张表:一个是已付的订单,一个是未付的 +PASS_PO = ("WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), " + "open AS (SELECT * FROM orders WHERE status = 'open') ") +PO_JOIN = "FROM paid p JOIN open o ON o.user_id = p.user_id" +#: 同样两段写成 FROM 里的子查询 +PO_SUBQUERIES = ("FROM (SELECT * FROM orders WHERE status = 'paid') p " + "JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id") + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("form", ["cte", "subquery"]) +@pytest.mark.parametrize("a,b,expected", [ + ("SELECT SUM(p.amount) {j}", "SELECT SUM(o.amount) {j}", ["原流程对 paid.amount 求和,盲算对 open.amount 求和"]), + ("SELECT COUNT(*) {j} WHERE p.amount > 100", "SELECT COUNT(*) {j} WHERE o.amount > 100", + ["原流程按 paid.amount > 100 筛选,盲算按 open.amount > 100 筛选"]), + ("SELECT COUNT(*) {j} WHERE p.created_at >= '2026-09-01'", + "SELECT COUNT(*) {j} WHERE o.created_at >= '2026-09-01'", + ["原流程按 paid.created_at 筛时间,盲算按 open.created_at"]), + ("SELECT p.id {j} ORDER BY p.created_at DESC LIMIT 10", "SELECT p.id {j} ORDER BY o.created_at DESC LIMIT 10", + ["原流程取行前按 paid.created_at 降序排,盲算按 open.created_at 降序排"]), + ("SELECT COUNT(*) {j} GROUP BY p.store_id", "SELECT COUNT(*) {j} GROUP BY o.store_id", + ["原流程按 paid.store_id 分组,盲算按 open.store_id 分组"]), +], ids=["sum", "where", "time", "order-by", "group-by"]) +def test_which_pass_through_derived_table_a_column_comes_from_is_reported(dialect, form, a, b, expected): + """两个原样透出的 CTE(SELECT * FROM orders WHERE …)读同一张表:透过去都成了 orders 的列,外层用的是已付的那份 + 还是未付的那份就分不出来了(SQLite 实测求和 230 对 400)。同一段里有几个来源最后读的是同一张表时,这几个派生来源 + 不透过去,按配好对的编号认列。""" + if form == "cte": + head, j = PASS_PO, PO_JOIN + else: + head, j = "", PO_SUBQUERIES + expected = [t.replace("paid.", "p.").replace("open.", "o.") for t in expected] + assert texts(head + a.format(j=j), head + b.format(j=j), dialect, labels=LABELS) == expected + + +def test_a_time_column_of_a_pass_through_derived_table_names_the_real_table(): + """按哪个派生来源的同名时间列筛,说明里写派生来源的名字;给调用方对照数据目录的 detail 写透到底的实表和列。""" + [diff] = diff_facets(facets(PASS_PO + f"SELECT COUNT(*) {PO_JOIN} WHERE p.created_at::date >= '2026-09-01'", + "postgres"), + facets(PASS_PO + f"SELECT COUNT(*) {PO_JOIN} WHERE o.created_at::date >= '2026-09-01'", + "postgres"), labels=LABELS) + assert (diff.aspect, diff.text) == ("time", "原流程按 paid.created_at 筛时间,盲算按 open.created_at") + assert diff.detail == {"a": [{"table": "orders", "column": "created_at"}], + "b": [{"table": "orders", "column": "created_at"}]} + + +PO_COLUMNS = ("WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), " + "open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') ") +PO_CHAIN = ("WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), " + "paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') ") +#: 原样透出的 CTE 自己关联自己 +SELF_T = "WITH t AS (SELECT * FROM orders WHERE status = 'paid') " + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b,expected", [ + # 直接选出各列的 CTE:关联条件透过去两边都是 orders.referrer_id = orders.user_id + (PO_COLUMNS + "SELECT SUM(p.amount) FROM paid p JOIN open o ON p.user_id = o.referrer_id", + PO_COLUMNS + "SELECT SUM(p.amount) FROM paid p JOIN open o ON p.referrer_id = o.user_id", + ["原流程关联 open 表的条件是 open.referrer_id = paid.user_id,盲算是 paid.referrer_id = open.user_id"]), + # CTE 套 CTE:paid、open 都从 base 取数 + (PO_CHAIN + "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + PO_CHAIN + "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100", + ["原流程按 paid.amount > 100 筛选,盲算按 open.amount > 100 筛选"]), + # 子查询里再套子查询 + ("SELECT COUNT(*) FROM (SELECT p.user_id " + PO_SUBQUERIES + " WHERE p.amount > 100) x", + "SELECT COUNT(*) FROM (SELECT p.user_id " + PO_SUBQUERIES + " WHERE o.amount > 100) x", + ["原流程按 p.amount > 100 筛选,盲算按 o.amount > 100 筛选"]), + # 派生列的定义里谁减谁 + (PASS_PO + "SELECT SUM(d) FROM (SELECT p.amount - o.amount AS d " + PO_JOIN + ") x", + PASS_PO + "SELECT SUM(d) FROM (SELECT o.amount - p.amount AS d " + PO_JOIN + ") x", + ["原流程的 x.d 是 paid.amount - open.amount,盲算是 open.amount - paid.amount"]), + # CTE 和它自己读的那张表关联:父单的金额还是子单的金额 + (SELF_T + "SELECT SUM(p.amount) FROM t p JOIN orders c ON c.parent_id = p.id", + SELF_T + "SELECT SUM(c.amount) FROM t p JOIN orders c ON c.parent_id = p.id", + ["原流程对 t.amount 求和,盲算对 orders.amount 求和"]), + # 原样透出的 CTE 自己关联自己:两个别名按各自在关联条件里出的列区分,透过去也不丢 + (SELF_T + "SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + SELF_T + "SELECT SUM(b.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + ["原流程对 a.amount 求和,盲算对 b.amount 求和"]), + # 自己关联自己、关联的方向反了:两边都写 a.amount,说清各自的 a 是哪一个 + (SELF_T + "SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + SELF_T + "SELECT SUM(a.amount) FROM t a JOIN t b ON b.referrer_id = a.user_id", + ["原流程对 a.amount(a 是关联条件里出 referrer_id 的那个 t)求和," + "盲算对 a.amount(a 是关联条件里出 user_id 的那个 t)求和"]), + ("SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.referrer_id = a.user_id", + ["原流程对 a.amount(a 是关联条件里出 referrer_id 的那个 orders)求和," + "盲算对 a.amount(a 是关联条件里出 user_id 的那个 orders)求和"]), +], ids=["columns-join", "cte-chain", "nested-subquery", "derived-column", "with-own-table", "self-join", + "self-join-reversed", "table-self-join-reversed"]) +def test_derived_tables_reading_the_same_table_keep_apart(dialect, a, b, expected): + """读同一张表的几个派生来源(还有派生来源和它自己读的那张表)在同一段里,各自的列不透成那张表的列:关联条件、 + CTE 链、往里套的子查询、派生列的定义、自己关联自己都分得清是哪一份。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +@pytest.mark.parametrize("dialect,a,b", [ + # 关联先后对调、比较两边对调:意思一样 + ("sqlite", PASS_PO + f"SELECT COUNT(*) {PO_JOIN} WHERE p.amount > o.amount", + PASS_PO + "SELECT COUNT(*) FROM open o JOIN paid p ON p.user_id = o.user_id WHERE o.amount < p.amount"), + ("postgres", f"SELECT SUM(p.amount) {PO_SUBQUERIES}", + "SELECT SUM(x.amount) FROM (SELECT * FROM orders WHERE status = 'open') y " + "JOIN (SELECT * FROM orders WHERE status = 'paid') x ON x.user_id = y.user_id"), + # 两边给 CTE 起的名字不同 + ("mysql", PASS_PO + f"SELECT SUM(p.amount) {PO_JOIN}", + "WITH settled AS (SELECT * FROM orders WHERE status = 'paid'), " + "pending AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT SUM(s.amount) FROM settled s JOIN pending q ON q.user_id = s.user_id"), + # 只有一个原样透出的 CTE:照旧透成那张表的列,写在 CTE 里还是外层的条件是同一个 + ("sqlite", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT SUM(p.amount) FROM paid p JOIN refunds r ON r.order_id = p.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'"), + # 自己关联自己换了别名、换了关联先后 + ("sqlite", SELF_T + "SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + SELF_T + "SELECT SUM(y.amount) FROM t x JOIN t y ON y.referrer_id = x.user_id"), +], ids=["swapped", "subquery-renamed-swapped", "cte-renamed", "single-pass-through", "self-join-renamed"]) +def test_derived_tables_reading_the_same_table_written_differently_are_no_difference(dialect, a, b): + assert texts(a, b, dialect) == [] + + +# -------------------------------------------------------------------------- +# diff_facets:相关子查询读的是外层那张表 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b,expected", [ + # 有子单的订单数对有父单的订单数:内外两个 orders 写出来一样,得分清哪一列是外层的 + ("SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)", + ["原流程按 EXISTS(SELECT 1 FROM orders AS c WHERE o.id = c.parent_id) 筛选," + "盲算按 EXISTS(SELECT 1 FROM orders AS c WHERE o.parent_id = c.id) 筛选"]), + # 之前有过订单对之后还有订单 + ("SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id " + "AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id " + "AND o.created_at < p.created_at)", + ["原流程按 EXISTS(SELECT 1 FROM orders AS p WHERE o.user_id = p.user_id AND o.created_at > p.created_at) 筛选," + "盲算按 EXISTS(SELECT 1 FROM orders AS p WHERE o.user_id = p.user_id AND o.created_at < p.created_at) 筛选"]), + # 外层是透过 CTE 的 orders + ("WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = p.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = p.parent_id)", + ["原流程按 EXISTS(SELECT 1 FROM orders AS c WHERE p.id = c.parent_id) 筛选," + "盲算按 EXISTS(SELECT 1 FROM orders AS c WHERE p.parent_id = c.id) 筛选"]), + # SELECT 列表里的标量子查询:子单数对父单数 + ("SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o", + ["原流程算的是 (SELECT COUNT(*) FROM orders AS c WHERE o.id = c.parent_id)," + "盲算算的是 (SELECT COUNT(*) FROM orders AS c WHERE o.parent_id = c.id)"]), +], ids=["exists-parent-child", "exists-before-after", "exists-through-cte", "scalar"]) +def test_a_correlated_subquery_on_the_outer_table_tells_inner_from_outer(dialect, a, b, expected): + """相关子查询读的表和外层是同一张(orders 里找子单、父单、同一用户更早的单):比较用的写法里外层的列另带标记, + c.parent_id = o.id 和 c.id = o.parent_id 才不会都成了 orders.id = orders.parent_id;给人看的写法照写别名(两边 + 表名一样时外层的列排在前面)。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +def test_a_correlated_lateral_subquery_on_the_outer_table_is_reported(): + a = ("SELECT o.id, x.s FROM orders o CROSS JOIN LATERAL " + "(SELECT SUM(c.amount) AS s FROM orders c WHERE c.parent_id = o.id) x") + b = a.replace("c.parent_id = o.id", "c.id = o.parent_id") + assert texts(a, b, "postgres", labels=LABELS) == ["原流程只保留 o.id = c.parent_id 的行,盲算没有这个条件", + "盲算只保留 o.parent_id = c.id 的行,原流程没有这个条件"] + + +@pytest.mark.parametrize("a,b", [ + ("SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders x WHERE EXISTS (SELECT 1 FROM orders y WHERE x.id = y.parent_id)"), + ("SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), +], ids=["renamed", "other-table"]) +def test_the_same_correlated_subquery_written_differently_is_no_difference(a, b): + assert texts(a, b) == [] + + +def test_a_difference_inside_a_condition_subquery_is_said_once(): + """条件里的子查询整个写进外层那个条件:里面多一个条件,外层那个条件已经报了,不再另说一句「只保留 state = + 'done' 的行」(它不筛外层的行,是 NOT EXISTS 里面的)。""" + a = "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)" + b = ("SELECT COUNT(*) FROM orders o WHERE NOT EXISTS " + "(SELECT 1 FROM refunds r WHERE r.order_id = o.id AND r.state = 'done')") + assert texts(a, b, labels=LABELS) == [ + "原流程按 NOT EXISTS(SELECT 1 FROM refunds WHERE orders.id = refunds.order_id) 筛选," + "盲算按 NOT EXISTS(SELECT 1 FROM refunds WHERE orders.id = refunds.order_id AND refunds.state = 'done') 筛选"] + + +# -------------------------------------------------------------------------- +# diff_facets:同一个条件、分组、连接出现在不同的层 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b,expected", [ + # 外层的条件和 IN 子查询里的同一个条件 + ("SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN " + "(SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + ["原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件"]), + ("SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders x " + "WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders x " + "WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", + ["原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件"]), + # 关联进来的汇总 CTE 里有,外层没有 + ("WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) " + "SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id WHERE o.status = 'paid'", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) " + "SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id", + ["原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件"]), + # UNION 的第二支少了条件 + ("SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' " + "UNION SELECT referrer_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t", + ["原流程在 t 的 UNION 第 2 支里也只保留 status = 'paid' 的行,盲算在 t 的 UNION 第 2 支里没有这个条件"]), + # 外层的分组和 IN 子查询里的分组 + ("SELECT user_id, COUNT(*) FROM orders WHERE user_id IN " + "(SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1) GROUP BY user_id", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1)", + ["原流程在外层按 user_id 分组,盲算在外层没有分组"]), + # 外层的关联和 EXISTS 子查询里的关联 + ("SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE EXISTS " + "(SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS " + "(SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", + ["原流程在外层也关联了 refunds 表,盲算在外层没有"]), + # 外层算的聚合和标量子查询里的聚合 + ("SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", + "SELECT AVG(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", + ["原流程对 amount 求和,盲算求平均"]), + # 外层的时间条件和 IN 子查询里的同一个时间条件 + ("SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND user_id IN " + "(SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", + ["原流程在外层也按 paid_at >= '2026-09-01' 筛了时间,盲算在外层没有这个条件"]), + # 读同一张表的两个 CTE 各有一个 is_deleted = 0,盲算的 b 里少了:两个条件,不是一个 + ("WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), " + "b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'open') " + "SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), " + "b AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + ["原流程在 b 里也只保留 is_deleted = 0 的行,盲算在 b 里没有这个条件"]), + # 最外层就是 UNION:第二支少了条件 + ("SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", + ["原流程在 UNION 第 2 支里也只保留 status = 'paid' 的行,盲算在 UNION 第 2 支里没有这个条件"]), +], ids=["in-subquery", "exists", "joined-cte", "union-branch", "group-by", "join", "aggregate", "time", + "two-ctes", "root-union"]) +def test_the_same_item_on_another_layer_is_counted_once_per_layer(dialect, a, b, expected): + """各段收集到的条件、分组、连接、聚合原来按整条语句去重成一个集合:外层的 status = 'paid' 和 IN 子查询(关联进来的 + 汇总 CTE、UNION 的另一支)里的同一个条件合成了一个,外层少了它报不出来(实测 350 对 700)。现在每一项记下在哪 + 一层,同一项在一边出现在几层就算几个:另一边层数少了,报多出来的那一层。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +@pytest.mark.parametrize("a,b", [ + # 有效记录条件写在原样透出的 CTE 里、外层又写了一遍:还是同一层的同一个条件 + ("WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT SUM(amount) FROM v WHERE is_deleted = 0", + "SELECT SUM(amount) FROM orders WHERE is_deleted = 0"), + # 两边外层都有、子查询里也都有 + ("SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN " + "(SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT x.user_id FROM orders x WHERE x.status = 'paid') " + "AND o.status = 'paid'"), + # 条件从汇总的 CTE 里挪到外层(按分组键筛,先筛后筛一样) + ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 100 GROUP BY user_id) " + "SELECT SUM(total) FROM t", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(total) FROM t WHERE user_id > 100"), +], ids=["repeated-in-pass-through", "both-layers", "moved-group-key-filter"]) +def test_the_same_item_on_equivalent_layers_is_no_difference(a, b): + assert texts(a, b) == [] + + +# -------------------------------------------------------------------------- +# diff_facets:去重、UNION、最后出几行 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b,expected", [ + # 先在子查询里按用户分组、外层数用户数,对外层按用户分组、每人一行(实测 [(4,)] 对 [(3,),(2,),(1,),(1,)]) + ("SELECT COUNT(t.user_id) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT COUNT(user_id) FROM orders GROUP BY user_id", + ["原流程最后只出一行,盲算按 user_id 分组、每组一行"]), + # 同一批行 UNION ALL 了两遍(实测 8 对 4) + ("SELECT COUNT(*) FROM (SELECT id FROM orders WHERE status = 'paid' " + "UNION ALL SELECT id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'", + ["原流程在 t 里把 2 支查询用 UNION ALL 合起来,盲算没有"]), + # 去重后的用户数对订单行数(实测 3 对 4) + ("SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'", + ["原流程在 t 里对 user_id 去重,盲算没有去重"]), + ("SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid') t", + ["原流程在 t 里对 user_id 去重,盲算没有去重"]), + ("SELECT DISTINCT region FROM orders", "SELECT region FROM orders", ["原流程对 region 去重,盲算没有去重"]), + ("SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT referrer_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT referrer_id FROM orders) t", + ["原流程在 t 里用 UNION 合并(去掉重复的行),盲算用 UNION ALL(重复的行都留着)"]), +], ids=["grain", "union-all-twice", "distinct-vs-rows", "distinct-vs-plain", "distinct-outer", "union-vs-all"]) +def test_dedup_union_and_how_many_rows_come_out_are_compared(dialect, a, b, expected): + """DISTINCT、UNION / UNION ALL 不进原来的任何一面,分组键挪了一层也只按整条语句比:同一批行合了两遍、去重后的 + 用户数对订单行数、外层出一行还是每个用户一行,都报不出来。""" + assert texts(a, b, dialect, labels=LABELS) == expected + + +@pytest.mark.parametrize("a,b", [ + # 分组写在 CTE 里、外面原样取出来,和直接分组出的行一样 + ("SELECT region, SUM(amount) FROM orders GROUP BY region", + "WITH t AS (SELECT region, SUM(amount) AS s FROM orders GROUP BY region) SELECT region, s FROM t"), + # 按某列分组不算别的,和对这一列去重一样 + ("SELECT user_id FROM orders GROUP BY user_id", "SELECT DISTINCT user_id FROM orders"), + # 两边一样的 UNION,派生来源起的名字不同 + ("SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION ALL " + "SELECT referrer_id FROM orders WHERE status = 'paid') t", + "WITH ids AS (SELECT user_id FROM orders WHERE status = 'paid' UNION ALL " + "SELECT referrer_id FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM ids"), + # 最外层直接 UNION,和包进子查询再原样取出来 + ("SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", + "SELECT * FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + # 分组后的结果包一层原样取出来 + ("SELECT user_id, COUNT(*) FROM orders GROUP BY user_id", + "SELECT user_id, n FROM (SELECT user_id, COUNT(*) AS n FROM orders GROUP BY user_id) t"), +], ids=["group-in-cte", "group-or-distinct", "same-union", "union-wrapped", "grouped-wrapped"]) +def test_the_same_rows_written_differently_are_no_difference(a, b): + assert texts(a, b) == [] From c55b1983f629e133610586163cf3de4276c5e732 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 09:28:08 +0800 Subject: [PATCH 4/6] =?UTF-8?q?SQL=20=E5=B7=AE=E5=BC=82=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E9=87=8D=E5=81=9A=EF=BC=88=E7=AC=AC=201=20=E8=BD=AE=EF=BC=89?= =?UTF-8?q?=EF=BC=9A=E5=85=88=E5=88=A4=E7=AD=89=E4=BB=B7=E3=80=81=E5=86=8D?= =?UTF-8?q?=E8=AF=B4=E5=B7=AE=E5=9C=A8=E5=93=AA=E2=80=94=E2=80=94=E8=A7=84?= =?UTF-8?q?=E8=8C=83=E5=BD=A2=E5=BC=8F=E4=B8=80=E5=AD=97=E4=B8=8D=E5=B7=AE?= =?UTF-8?q?=E6=89=8D=E7=AE=97=E7=AD=89=E4=BB=B7=EF=BC=8C=E5=85=B7=E4=BD=93?= =?UTF-8?q?=E8=AF=B4=E6=B3=95=E5=8F=AA=E8=AF=B4=E6=9C=89=E6=8A=8A=E6=8F=A1?= =?UTF-8?q?=E7=9A=84=EF=BC=8C=E5=85=B6=E4=BD=99=E7=BB=99=E5=85=9C=E5=BA=95?= =?UTF-8?q?=E5=B9=B6=E9=99=84=E8=A7=84=E8=8C=83=E5=8C=96=E5=90=8E=E4=B8=8D?= =?UTF-8?q?=E5=90=8C=E7=9A=84=E8=A1=8C=EF=BC=9B=E7=AD=89=E4=BB=B7=E7=BB=93?= =?UTF-8?q?=E8=AE=BA=E6=8B=BF=20SQLite=20=E6=89=A7=E8=A1=8C=E6=AF=94?= =?UTF-8?q?=E5=AF=B9=E5=AE=88=E9=97=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么重做:过去五轮按方面归纳任意 SQL,每轮修好一批又冒出一批(第 5 轮还回退过「只在 IN / EXISTS / 标量子查询里用到的 CTE 里的不同」),收不拢。改成先判两条 SQL 是不是一回事,不是的话只说有把握的不同。 做了什么(backend/app/data/sql_semantics.py): - 新增 explain_sql_diff(a, b, dialect=None, *, schema=None, names=None, labels=("A", "B")) -> SqlDiff (equivalent、items、fallback、text_diff、level)。两边各自规范化,每一步只做保持语义的变换;同一级的两个 规范形式语法树逐节点一样才算等价(不比生成的 SQL:sqlglot 生成 SQLite 时把 CAST(x AS DATE) 写成 DATE(x)), 否则一律不等价、条目一定不为空。三级:optimizer(sqlglot 的 qualify、eliminate_subqueries、merge_subqueries、 eliminate_ctes、pushdown_projections、simplify,给了数据源的 schema_cache 就按它认列、展开 SELECT *)、 light(不认列)、parsed(只统一大小写、空白、引号)。 - 不用的 sqlglot 规则:unnest_subqueries(OR 里的标量子查询改成 CROSS JOIN,子查询没有行时整条都没了)、 canonicalize(日期比较加 CAST,SQLite 里 CAST(… AS DATE) 取的是数字)、normalize(合取范式人读不懂); simplify 去掉同一列上比较的合并、字符串常量比较、等式移项、DATE_TRUNC 改区间(取决于列类型、排序规则、方言); merge_subqueries 不并外连接可空一侧算出来的列。 - 本模块自己的几步:去括号、ASC、OFFSET 0,COUNT(1)→COUNT(*),IN 列表去重排序,EXISTS 写成 SELECT 1, DISTINCT↔GROUP BY,原样透列的子查询换掉,RIGHT JOIN→LEFT JOIN,WHERE 筛掉了没关联上的行的外连接写成内连接, 内连接写成逗号连接,起规范的名字(CTE 叫 cte1…、CTE 的列叫 c1…、别名按表名和层数,同名来源排列着试), 交换律成立的地方排序。 - 具体说法的门槛:两段读的来源对得上(一样,或只差一处说得清的)才逐项比时间列、筛选条件、聚合与去重计数、 选出来的列、分组键、HAVING、取行和排序、UNION 还是 UNION ALL;条件里的子查询两边各一个、外面一样的往里比 (第 5 轮回退的 IN-CTE 反例在这里说出来);原文是 LEFT / RIGHT JOIN、规范化成内连接或调了方向的照原文说; 牵涉 CTE 算出来的列、自己关联自己、说不准的子查询的不说具体的;两边给人看一样的不说。 - 兜底:一条具体说法都没有时给 aspect="other"「两条 SQL 的写法不同,未能归纳成具体差异,请对照原文」,只覆盖 一部分时追加「另有未能归纳的差异,请对照原文」,fallback 为真;text_diff 是 difflib 比两边 sqlglot pretty 的输出,最多 40 行。 - diff_facets 签名不变,交给 explain_sql_diff 比两份 facets 记下的原文,返回它的 items。normalize_sql 判 「只有参数不同」不变,补了测试。 守门(tests/test_sql_equivalence.py、tests/fixtures/sql_pairs.py):虚构库(users、stores、orders、 orders_archive、order_items、refunds、payments、shipments)上 5 组固定种子的数据加 1 组极端数据(空值、重复、 边界值、悬空外键、空表)。305 对 SQL(113 对已知等价、192 对不许判等价,含复验者以前造的、在这几组数据上结果 不同的 91 对)在不给和给表结构两种情况下各判一次:结果不同的绝不判等价,标 diff 的不判等价,换个先后问结论 一样。旧测试五轮复核里报过不同的 234 对、意思一样的 56 对(SQLite / PostgreSQL / MySQL)原样收进来只查结论。 压测(变异 + 等价改写,每个种子约 1.87 万条)时找到、已经堵上的不保持语义的改写: - merge_subqueries 把外连接可空一侧的常量列并进外层;merge 时同一个 CTE 并进外层和相关子查询两处互相遮住; - 没给表结构时 qualify 把 WHERE / GROUP BY / HAVING 里和列别名同名的名字当成别名(SQLite 先认列),把 SELECT 列表里和前面别名同名的名字展开(库里不认 SELECT 列表里的别名):这时不出 optimizer 这一级; - UNION(去重)、INTERSECT 调换各支:去重留先出现的值(SQLite 里 2 和 2.0、MySQL 不分大小写时 'SH' 和 'sh'); UNION ALL 带 LIMIT、按名字 ORDER BY 的也要看哪支在前,SQLite 合并结果的排序规则取最左一支:现在只在调换 不改变结果时排;SQLite 从子查询读 UNION 的类型亲和; - 一元加号在 SQLite 里去掉类型亲和、解析时却丢了;ORDER BY 别名换成表达式后撞上另一个别名; - 没有 ORDER BY 的 LIMIT、聚合里既不分组也不聚合的列:调换来源、DISTINCT 写成 GROUP BY、RIGHT 写成 LEFT 都会改变实际取到的行,这样的一段不做;两边都写了 COLLATE 的比较不对调。 验证:tests/test_sql_semantics.py + tests/test_verify_agent.py 519 passed,tests/test_sql_equivalence.py 307 passed,全量 -n 8:7706 passed、11 skipped。已知等价的 113 对认出 111 对(98%),不给表结构时 109 对 (96%)。压测 4 个种子各约 1.5 万条可执行:规范形式相同的组执行结果全一致,能执行的规范形式结果和原文一致 (带 CAST 的不比:生成 SQLite 时写法变了),生成的等价改写认出 2113/2115;复验者留下的 1201 对里可执行的 440 对没有误判等价,结果相同的 302 对认出 276 对。 仍做不到的(只给兜底或认不出等价)见模块说明:IN 子查询和 EXISTS 互换、窗口函数前后按分区键筛、三项以上的 加乘换结合、和方言有关的同义写法、SQLite 里套一层子查询的 UNION、去重 UNION 调换各支、LATERAL、递归 CTE、 没给表结构时别名和列同名的。假设列没有声明非默认的排序规则(SQLite 比较两列取左边那列的,表结构里看不到)。 --- backend/app/data/sql_semantics.py | 3797 +++++++++++++++++-------- backend/tests/fixtures/sql_pairs.py | 1865 ++++++++++++ backend/tests/test_sql_equivalence.py | 82 + backend/tests/test_sql_semantics.py | 1930 ++++--------- 4 files changed, 5164 insertions(+), 2510 deletions(-) create mode 100644 backend/tests/fixtures/sql_pairs.py create mode 100644 backend/tests/test_sql_equivalence.py diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index edad1a8e..99f668b8 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -1,129 +1,93 @@ """SQL 语义:两条 SQL 算的是不是一回事,不是的话差在哪。 -双盲复算要回答「盲算和原流程的 SQL 差在哪」(A 按支付时间筛、B 按下单时间筛;A 排除了作废的行、B 没有),运行 -对比要回答「这一期和上一期的 SQL 是不是同一个模板」(定时运行每次的日期不同,不能算模板变了),反方审稿要看自己 -的查询和原查询的口径是否一致。三处共用这里: +双盲复算要回答「盲算和原流程的 SQL 差在哪」(A 按支付时间筛、B 按下单时间筛;A 排除了作废的行、B 没有),人据此选 +A / B,只差一个方面、又对得上数据目录里的事实(时间归属、有效行过滤、去重)时选择会写进数据目录;运行对比要回答 +「这一期和上一期的 SQL 是不是同一个模板」(定时运行每次的日期不同,不能算模板变了);反方审稿要看自己的查询和原 +查询的口径是否一致。几处共用这里: - normalize_sql:字面量换成占位符之后的规范文本,加上字面量本身(按规范文本里的顺序)。两条 SQL 只有字面量不同, - 规范文本就相同;关键字大小写、空白、注释、不加引号的名字的大小写都不算不同。 -- facets:把一条查询拆成几个方面——用到的表、连接(类型、关联进来的表、ON 条件、外连接以哪一侧为主表)、WHERE - 的各个合取式、其中的时间条件(列、运算符、取值)、分组(当分组用的 SELECT DISTINCT 也算)、聚合(函数、参数、是否 - 去重)、算出来的列(最外层的算法、CTE 和子查询里被外面用到的列的定义,窗口函数也在这里)、HAVING、取哪几行 - (LIMIT、OFFSET、取前几行时的排序)、几支查询怎么合并(UNION、INTERSECT、EXCEPT)、最后出来的行按什么分。条件、 - 分组键、连接、聚合、HAVING 都记下在哪一层(见下)。别名一律换成真实表名(条件里套着的子查询也是),等式、比较、 - 加法、乘法的两边排好顺序:o.paid_at >= '2026-09-01' 和 '2026-09-01' <= orders.paid_at 是同一个条件。 -- diff_facets:逐个方面比两份 facets,每处不同给一句中文说明。 + 规范文本就相同;关键字大小写、空白、注释、不加引号的名字的大小写都不算不同;结构变了的一定不同。 +- explain_sql_diff:两条 SQL 是不是等价(SqlDiff.equivalent),不是的话差在哪(items、text_diff)。 +- diff_facets:照旧收两份 facets,交给 explain_sql_diff 比两份记下的原文,返回它的 items。 +- facets:把一条查询拆成表、连接、WHERE 合取式、时间条件、分组、聚合、算出来的列、HAVING、取哪几行(给调用方取 + 结构用;CTE、子查询按位置编号,见 _Statement)。 -**逐段收集,按层计数。** CTE、子查询里的条件、聚合和取行也算进来:盲算可能把有效记录条件写在 CTE 里,原流程写在 -外层,同一个口径不能因为写的位置不同就报成不同。只收用到了的段:定义了却没被引用的 CTE 不影响结果,不收(盲算 -定义了只筛已付的 CTE 却直接查 orders,少的那个条件要报出来)。可也不能按「整条语句里有哪些条件」去重成一个集合: -外层的 status = 'paid' 和 IN 子查询(关联进来的汇总 CTE、UNION 的另一支)里的同一个条件会合成一个,外层少了它就 -报不出来。所以每一项记下它在哪一层(见 _Statement._layers):最外层、UNION 的第几支、某个派生来源里(两边配好的 -共同编号),条件里的子查询是一层「?」;只从一个来源里筛出一部分行的 CTE、子查询(不分组、不聚合、不去重)和用到它 -的那一段算同一层。比的时候同一项在一边出现在几层就算几个(见 _Compare.leftovers):同一层的先配,剩下的不管在哪 -一层也配(条件挪了一层常常是同一个口径),配不完的才报;另一边也有、只是在的层少了的,说清多出来的是哪一层 -(「原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件」)。逗号连接(以及 INNER JOIN 之外又写在 -WHERE 里的等值关联)写在 WHERE 里的列等式算作关联条件,不算筛选。条件里套着的子查询(EXISTS、IN (SELECT …)、 -标量子查询)按它自己那一段的来源整个写进外层那个条件、那一列:别名在子查询里定义,拿外层的来源去认,同一个条件 -只因别名不同就会被报成不同。它里面的几段也收,可那一层多出来的条件、分组、连接、聚合不单说:外层那个条件已经报了, -再说一句「只保留 state = 'done' 的行」,读的人会以为外层也这么筛。 +**等价怎么判。** 过去几轮按方面归纳任意 SQL,每轮修好一批又冒出一批(还回退过),收不拢;现在先判等价、再说差在哪。 +两边各自规范化,规范化的每一步都只做保持语义的变换,同一级的两个规范形式一字不差(比语法树逐个节点的类型和参数, +见 _fingerprint,不比生成的 SQL:sqlglot 按方言生成时会把不同的写法写成一样,SQLite 的 CAST(x AS DATE) 生成 +DATE(x))才算等价,否则一律不等价、条目一定不为空。规范形式分三级(见 _canonical_forms),哪一级对上都算: -**各自起的名字不进比较。** 表的别名换成真实表名;CTE、FROM 里的子查询(含 LATERAL)没有真实表名可换,单看一条 -语句时按它在语句里的位置编号(@1、@2…,从外往里、按第一次用到的先后),给人看时照写名字;条件里的子查询(IN、 -EXISTS、标量子查询)选出来的列名不影响取值,比较时去掉。这些名字都是各自起的,双盲复算的盲算更是:按名字比,同一个 -口径会被说成换了口径。同一个来源在一段里出现两次(自己关联自己)时,两个别名按各自在关联条件里出的列区分 -(b2.user_id = b1.referrer_id:b1 出 referrer_id、b2 出 user_id),b1.total > b2.total 写反了认得出,换了别名、 -换了先后不算不同。 +- optimizer:sqlglot 优化器里保持语义的几步——qualify(认列、按表结构展开 SELECT *、展开 GROUP BY 1 和别名), + eliminate_subqueries(FROM 里的子查询写成 CTE),merge_subqueries(只筛行、透列的 CTE、子查询并进外层;外连接 + 可空一侧选出算出来的列的不并),eliminate_ctes(去掉没用的 CTE),pushdown_projections(去掉没用到的列),simplify + (去掉和类型、排序规则、方言有关的三种化简,见 _simplify);之前把整条语句的别名改成互不相同的(merge_subqueries + 只在同一段里避重名,同一个 CTE 并进外层和相关子查询两处会互相遮住),没给表结构时把从一张表 SELECT * 的 CTE 换成 + 外面用到的那几列。不用的:unnest_subqueries(把 OR 里的标量子查询改成 CROSS JOIN,子查询没有行时整条都没了; + SQLite 的标量子查询取第一行,改成连接就成了每行都比),canonicalize(给日期比较加 CAST,SQLite 里 CAST(… AS DATE) + 取的是数字),normalize(合取范式改出来的条件人读不懂,也用不上)。认不全列(没给表结构、几张表又都没写表名)、 + WHERE / GROUP BY / HAVING 里有和列别名同名、又可能是表的列的名字(库里先认列,qualify 却当成别名)、SELECT 列表里 + 有和前面列别名同名、又不确定是表的列的名字(库里不认 SELECT 列表里的别名,qualify 没有表结构时却展开)、有递归 + CTE、LATERAL、表函数、不确定的函数(RANDOM)的不出这一级(见 _alias_or_column)。 +- light:轻量规范化,不认列:统一标识符大小写和引号、去注释、只引用一次的非递归 CTE 写回用到它的地方,再走下面不 + 需要认列的几步。 +- parsed:解析后重新生成(只统一大小写、空白、注释、引号)。 -**相关子查询。** 子查询里引用外层的列(o.id)在比较用的写法里另带标记(见 _Statement._depth):相关子查询读的表 -和外层是同一张时(orders 里找子单、父单、同一用户更早的单),c.parent_id = o.id 和 c.id = o.parent_id 不带标记都 -成了 orders.id = orders.parent_id。给人看的写法这时照写别名(EXISTS(SELECT 1 FROM orders AS c WHERE o.id = -c.parent_id));两边各只剩一个 EXISTS(NOT EXISTS)的条件时配成一对说。 +两级共用的本模块自己的几步(见 _finish):不影响结合的括号、ORDER BY 的 ASC、OFFSET 0 去掉;COUNT(1) 写成 COUNT(*); +IN 列表去重排序、只有一项写成 =;EXISTS 子查询写成 SELECT 1;IN 子查询里的 DISTINCT 去掉;SELECT DISTINCT 写成按 +选出来的列 GROUP BY,已经按选出来的列分组的 DISTINCT 去掉;原样选出一个子查询全部列的一段换成那个子查询(SQLite 里 +UNION 不换:从子查询读合并的结果会按第一支的类型亲和转换别的支的值);外层 WHERE 里只涉及一个分组 CTE 的条件挪进它的 +HAVING、HAVING 里只涉及分组键的挪进 WHERE;只有两个来源的 RIGHT JOIN 写成 LEFT JOIN;WHERE 把外连接可空一侧没关联上的 +行筛掉了的写成内连接;内连接的条件统一挪进 WHERE、来源写成逗号连接;最外层和条件里子查询的列名去掉;最后起规范的名字 +(CTE 按定义排序叫 cte1、cte2…,CTE 的列按定义排序叫 c1、c2…,来源的别名叫表名、内层带 @层数、同名第二个起带 #序号, +同名来源谁先谁后逐个排列着试、取写出来最小的;取到哪几行、哪一行的值看扫描先后的一段——没有 ORDER BY 的 LIMIT、 +聚合里既不分组也不聚合的列——来源不调换先后,见 _unstable),交换律成立的地方(AND、OR、比较两边、两项的加乘、IN +列表、GROUP BY、PARTITION BY)排好序。合并的各支只有 UNION ALL、而且调换先后不改变结果时才排(见 _Namer._reorderable):去重的 UNION +和 INTERSECT 留先出现的那个值(SQLite 里 2 和 2.0、MySQL 不分大小写时 'SH' 和 'sh' 算重复),带 LIMIT、按名字 ORDER BY +的要看哪支在前,SQLite 合并结果的排序规则取最左一支的(只排整条语句最外层、不排序的)。 -**两边的派生来源按结构配对。** 位置编号只看先后:一边多出一个排在前面的 CTE,或者内连接两边换了先后,编号就错开 -了;可也不能干脆不看是哪一个(两个 CTE 都算出 total 时,外层用的是付款合计还是退款合计,正是要报的不同)。所以 -diff_facets 先给两边的每个派生来源算一份结构标识(见 _Derived):里面读到的实表(FROM / JOIN,往里套着的也算)、 -选出来的各列怎么定义(规范化后的定义式,如 total 是 SUM(orders.amount))、按什么分组、里面的条件、整段规范化后的 -写法,以及起的名字。这些写法里往里套着的派生来源写它自己的结构标识(见 _Statement._signature),和名字、编号无关: -net 读的是 paid.total - refunded.total 还是反过来,写出来不一样。再按从严到宽的几级一对一地配(见 _align):整段 -一样;读的表、各列定义、分组一样(里面的筛选条件可以不同);读的表和选出来的列名一样(某一列的算法可以不同);读的 -表和名字一样;最后三级只在两边都只剩一个时才配:只是读的表一样、只是名字一样、只是选出来的列名一样。同一级有几个 -都配得上时,先挑里面的条件重合最多的,再挑同名的(名字是各自起的:两边都叫 paid,一边筛已付、一边筛未付,不是一 -回事),最后才看位置编号的先后。哪一级都配不上的不配:一边多出来的 CTE 不会被拿去和另一边不相干的 CTE 凑成一对。 -配成一对的派生来源里,两边起名不同的列按定义配成同一列(一边叫 total、一边叫 sum_amt,见 _renamed)。配好之后, -第二条 SQL 的派生来源改用第一条里对应的那个编号(配不上的另起编号),按这个共同的编号重新分析两边再逐个方面比: -比较的两边谁在前、关联条件的先后都按共同的编号排,内连接换了先后也排成同一个方向。 +等价的结论另有执行比对守门(tests/test_sql_equivalence.py):虚构库上五组随机数据加一组极端数据,判等价的两条每组 +结果都得一样,结果不一样的绝不能判等价。给了表结构时 SELECT * 按它展开,表结构得和库对得上。原文里有 sqlglot 解析时 +会丢掉、语义却不同的写法(一元加号)的,不在语法树上判等价。假设列没有声明非默认的排序规则:SQLite 比较两列时取左边 +那列声明的排序规则,表结构里看不到,比较两边对调当成保持语义(两边都写了 COLLATE 的不对调)。 -**派生来源里的算法。** 外面引用的是「t.net」,两边写的一样,net 在里面怎么算的(amount * 0.9 还是 * 0.8、窗口 -函数按什么分区排序、是 paid.total 还是 refunded.total)只看外面看不出来:派生来源里被外面用到、又不是原样透出 -一列的列,按定义比(见 Computed)。最外层算出来的列(SUM(p.total) - SUM(r.total)、窗口函数、CASE)也按算法比: -聚合两边的集合一样、位置换了,聚合那一面看不出来。 +**具体说法的门槛(见 _Explainer)。** 在两边的规范形式上逐段比,只把有把握的不同写成具体的中文句子:两段读的来源 +对得上(一模一样,或者只差一处说得清的:换了一张表、一边多关联了一张表、同一张表的关联类型或条件不同、外连接的 +主表对调)之后,才逐项比时间条件(实表上的时间列不同、同一列的范围不同、只有一边按时间筛)、筛选条件(只涉及 +这一段实表列的条件一边有一边没有、同一列上两边的条件不同)、选出来的列(聚合函数、去重计数、算法、列数和先后)、 +分组键、HAVING、取行和排序、UNION 还是 UNION ALL。条件里套着的子查询两边各一个、外面写法一样的,往里比(第 5 轮 +漏报的「只在 IN / EXISTS / 标量子查询里用到的 CTE」并进子查询之后就在这里说出来);CTE 两边定义一样的对齐,各剩 +一个对不上的配成一对往里比。说的都是「这一段的这一项两边各写的是什么」,不推断对结果的影响;原文写的是 LEFT / +RIGHT JOIN、规范化成了内连接或者调了方向的,照原文说(「写的是 LEFT JOIN,实际是 INNER JOIN」「以 X 为主表外连接」)。 +牵涉 CTE 算出来的列、外层的列、自己关联自己的表、说不准的子查询,以及 CTE 里选出来的列(规范化去掉、重排过,和原文 +对不上)的,宁可不说具体的。两边给人看的写法一样的话不说(读起来自相矛盾)。 -**什么时候能透过派生来源。** CTE、子查询原样透出一张表的列(SELECT *,或者直接选出这一列)时,外面按它写的条件、 -聚合换成那张表的列:先在 CTE 里筛一遍再查和直接查是同一列。它要取前几行(LIMIT、OFFSET)、或者算了窗口函数(这一列 -又不是分区键)时不透:先取前 10 行再筛已付,和先筛已付再取前 10 行取到的不是同一批;两边只差在这里的,报筛的先后 -不同。同一段里有几个来源透过去读的是同一个来源时也不透(见 _Statement._seal):两个原样透出的 CTE 都读 orders -(已付的一份、未付的一份),透过去 p.amount、o.amount 都成了 orders.amount,用的是哪一份就分不出来了;CTE 和它 -自己读的那张表关联、同一个 CTE 关联两次也一样。这时照写派生来源配好对的编号,给人看写它的名字(paid.amount)。 -时间条件另外透过「同一列套着日期函数、CAST、MIN / MAX」算出来的列(DATE(paid_at) AS d),日期表达式写在 CTE 里, -按支付时间还是下单时间筛照样认得出;给调用方对照数据目录的表、列一直换到实表。 +**兜底。** 规范形式不一致、一条具体说法都没有时,给一条 aspect="other" 的「两条 SQL 的写法不同,未能归纳成具体差异, +请对照原文」;具体说法只覆盖了一部分时追加「另有未能归纳的差异,请对照原文」;fallback 为真。text_diff 给出规范化后 +不同的行(difflib 比两边 sqlglot pretty 的输出,最多 40 行)。 -**去重、合并、最后出几行。** SELECT DISTINCT(这一段没有 GROUP BY、没有聚合时)当这一段的分组键收:和按同几列 -GROUP BY 出的行一样,一边在子查询里先去重、另一边没有照报(去重后的用户数对订单行数)。UNION、INTERSECT、EXCEPT -记下合并的方式和支数(见 SetOpFacet):同一批行 UNION ALL 了两遍、UNION 换成 UNION ALL 要报;挪了一层(最外层直接 -UNION,对包进子查询再原样取)不报。分组键挪了一层按层宽松地配得上,另比最后出来的行按什么分(见 Facets.grain): -先在子查询里按用户分组、外层数用户数,对外层按用户分组,一个出一行、一个每个用户一行。 +仍做不到的(只给兜底说明,或者认不出等价): +- 认不出的等价改写:IN 子查询和 EXISTS、NOT IN 和 NOT EXISTS 互换(空值时本来就不等价)、IN 子查询和关联同一张表 + (关联的键不唯一时会多出行,不知道主键就不能当成一回事)、窗口函数前后按分区键筛、三项以上的加乘换了结合方式、 + 括号展开、和方言有关的同义写法(PostgreSQL 的 DATE(x) 和 CAST(x AS DATE))、SQLite 里套一层子查询的 UNION、 + 去重的 UNION 和 INTERSECT 调换各支、SQLite 里套在子查询里或带 ORDER BY 的 UNION ALL 调换各支、LATERAL、递归 CTE; + 没给表结构时 WHERE、GROUP BY、HAVING 里用列别名的(GROUP BY r 的 r 可能是表的列)、SELECT 列表里和前面的列别名 + 同名的;没有 ORDER BY 的 LIMIT、聚合里既不分组也不聚合的列的一段调换关联先后。 +- 说不出具体差异、只给兜底的:自己关联自己的方向或角色不同、两个 CTE 算出同名的列用了哪一个、相关子查询内外层分不清 + 的、读的来源对不上(一边多一个分组 CTE 再关联、子查询里换了表又改了条件)、CTE 里选出来的列不同、NOT IN 对 NOT + EXISTS、几处不同交错在一起的。 -**外连接。** 按以哪一侧为主表(保留它的全部行)比,不按写的是 LEFT 还是 RIGHT:orders LEFT JOIN refunds 和 -refunds RIGHT JOIN orders 一样,和 refunds LEFT JOIN orders 不一样。同一段 WHERE 里拿可空那一侧的列比较、IN、 -LIKE 的(空值都不成立),把没关联上的行筛掉了,按内连接比:写的是 LEFT JOIN,实际成了 INNER JOIN,要报出来。连接 -里的内连接(含逗号连接、CROSS JOIN)按涉及的来源和条件比,不按哪一侧是关联进来的;一边独有的一次内连接,关联 -进来的那一侧另一边也有时,报的是它接上的另一头(FROM stores JOIN orders、另一边只查 orders:多出来的是 stores)。 - -**写法一样、意思不同时说清楚。** 两边同名的派生来源其实不是同一个(盲算的 t1 是原流程的 t2)时,光写「t1.total」 -两边一模一样:带上各列的来历(t1.total 即 SUM(orders.amount));自己关联自己、关联的方向反了,两边都写 a.amount -时,说清各自的 a 是关联条件里出哪一列的那一个;来历也一样(两个 CTE 只差里面的条件)时,说「盲算的 paid 对应 -原流程的 open」。 - -仍做不到的: -- 两边起名不同的派生来源的列,两边各自多出不止一列、定义又都不一样时配不上,算不同的列,外层按它们写的条件照报 - 成两边各有一个;读的表、选出来的列名、名字全换了的两个派生来源不配对,同样照报。 -- 读同一张表、各列定义也一样的两个派生来源,两边都改了里面的条件、改得一个都不重合,又各起了名字时,只能按先后配, - 可能配反。 -- 自己关联自己、两个别名在关联条件里出的列一样(b1.user_id = b2.user_id)时分不出谁是谁;这时 WHERE 里对调过来, - 算出来的行数也一样,只有最外层直接选其中一个别名的列(不比,见下)时会漏。 -- 同一个算法写法不同:三项以上的加法、乘法换了先后(a + b + c 对 c + b + a)、括号展开(a * (b + c) 对 a * b + - a * c)、取日期以外的等价函数,算不同的算法。最外层只比算出来的列:光一列、字面量不比;ROUND、CAST、COALESCE - 给默认值这类外壳当作同一个数(最外层取整到分不算口径不同)。派生来源里没被外面用到的列不比;UNION 按各支同一个 - 位置比。 -- 筛的先后只认取前几行(LIMIT、OFFSET)、窗口函数(含 DISTINCT ON)挡住的;QUALIFY、GROUP BY 之后才筛的不认。 - 外连接被 WHERE 变成内连接只认同一段 WHERE 里直接用到可空那一侧列的比较、IN、LIKE、BETWEEN、IS NOT NULL;套在 - COALESCE、CASE 里,或者写在外面一层 WHERE 里的不认。连接上别的做不到的见 _Compare.joins。 -- 读同一个来源的几个派生来源同在一段时不透过去,只认配好对的编号:一边写成两个原样透出的 CTE(已付、未付),另一边 - 把其中一个换成直接关联那张表、再在 WHERE 里筛,列一边是 CTE 的、一边是实表的,算不同;同一个 CTE 在一边关联了 - 两次、另一边定义成两个一模一样的 CTE 各关联一次,也算不同。 -- 按层计数时,同一项挪了一层又多了一层,挑哪一个说成多出来的按「同一层的先配、不在条件子查询里的先配」定,说的那 - 一层不一定是读的人心里的那一层。条件里的子查询那一层多出来的不单说,要从外层那个条件的不同里看;UNION 的各支按 - 位置配(第 2 支对第 2 支),两边把各支换了先后算不同。时间条件只在两边用到的时间列、范围都一样时才按层数比。 -- 子查询里不加表前缀引用外层的列(WHERE parent_id = id)认不出是外层的,不带标记,内外读同一张表时分不出来。 - LATERAL 子查询里的相关条件按那一层的条件比,两边不同时各说一句,不配成一对。 -- 最后出来的行按什么分只在两边都汇总了(分组或者算了聚合)、分组键又都配得上时比;最外层是 UNION 的不比。 - COUNT(DISTINCT x) 和数「SELECT DISTINCT x」子查询行数的 COUNT(*) 照报不同(x 有空值时确实不一样)。 -- facets 单独返回的 key 仍是位置编号;不是 facets 算出来的 Facets(没记原文)diff_facets 也只能按位置编号比, - 没用到的 CTE、起名不同的列也不再另配。 - -**和 sqlcheck 一样宁可漏报。** 解析不了(语法错、方言特性、多条语句、不是查询)就带 parse_error 返回,从不抛 -异常。没加表前缀、又有好几张表可能有这一列的,照写列名(给了表结构才去对)。只读语法树,不执行 SQL。 +**和 sqlcheck 一样宁可漏报。** 解析不了(语法错、方言特性、多条语句、不是查询)就只给一条 aspect="parse";从不抛异常, +规范化或说明本身出了意外就当作这一级没有、说不清,记日志。只读语法树,不执行 SQL。 """ from __future__ import annotations import hashlib +import itertools import logging import re from collections import deque from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field, fields, is_dataclass, replace -from itertools import zip_longest from typing import Any from app.data import catalog @@ -140,8 +104,7 @@ MAX_SQL_CHARS = 100_000 #: 算好的来源(别名、外层):复制品没有父节点,自己找不到外层 _SEGMENT = "sql_semantics_segment" #: 派生来源(引用 CTE 的表和 CTE 的定义、FROM / JOIN 里的子查询)在语法树上记比较用编号的 meta 键(见 -#: _Statement._number_derived、_Statement.relabel)。复制出来的条件里套着子查询时,靠它把 CTE 名、子查询的别名 -#: 换成编号 +#: _Statement._number_derived)。复制出来的条件里套着子查询时,靠它把 CTE 名、子查询的别名换成编号 _DERIVED = "sql_semantics_derived" #: 不写编号的写法(render 的 anon 模式)里派生来源一律写成这个:排比较的两边谁在前时先看实表的列还是派生来源的列 _ANON = "@" @@ -308,9 +271,9 @@ ColumnRef = tuple[str | None, str] class Part: """一个条件、一个分组键、一个聚合参数…… - key 用来比较:别名换成真实表名,CTE、子查询写编号(facets 单独返回的是位置编号,diff_facets 里是两边按结构配好 - 对的共同编号,见模块说明),表名、列名按 name_key,等式和比较的两边排好顺序;loose 同 key 但不写实表的表名(派生 - 来源的编号照写:它不是表名,换了表也还是配好的那一个),两边查的表换了一张(orders 换成 orders_archive)时拿它 + key 用来比较:别名换成真实表名,CTE、子查询写位置编号(见 _Statement._number_derived),表名、列名按 + name_key,等式和比较的两边排好顺序;loose 同 key 但不写实表的表名(派生来源的编号照写:它不是表名,换了表也还是 + 那一个),两边查的表换了一张(orders 换成 orders_archive)时拿它 比其余各方面,免得同一个条件只因表名不同被报成不同。text 给人看:好几张表都有的列写「表.列」(真实表名,不是 别名),其余只写列名;full 一律写表名,两边的 text 一样、key 却不同时改用它说明。excludes 只在 WHERE 的「排除」 写法上有:<>、NOT、NOT IN、IS NOT NULL 去掉否定之后排除的是什么(IS NOT NULL 写成「列 为空」)。column_keys 和 @@ -320,7 +283,7 @@ class Part: scope 是这一项在哪一层(条件、分组键、HAVING 上有,见 _Statement._layers):空串是最外层,「|2」是最外层 UNION 的第 2 支,「@3」是派生来源 @3 里(「@3|2」是它的 UNION 第 2 支),「?」是条件、SELECT 列表里的子查询。facets 按 - (scope, key)去重:同一个条件在外层和子查询里各写了一次是两项(见 _Compare.leftovers)。 + (scope, key)去重:同一个条件在外层和子查询里各写了一次是两项。 后面四项不进比较(两个 Part 相等只看前面几项),只用来把不同说清楚: origins 是这一项用到的派生来源的列各自的来历 [(「t1.total」, 「SUM(orders.amount)」)](见 _Statement._origin): @@ -358,12 +321,12 @@ class JoinFacet: sides 是这次连接涉及的来源(比较用的写法,排好序):关联进来的那一侧,加上关联条件里直接引用的这一段的来源(不 透过 CTE 找实表);没有条件的加上它之前出现的各个来源。内连接、CROSS JOIN 两边换个先后意思不变(FROM paid JOIN - refunded 和 FROM refunded JOIN paid),diff_facets 比它们时按 sides 和条件比,不按哪一侧是关联进来的。 + refunded 和 FROM refunded JOIN paid),比它们时按 sides 和条件比,不按哪一侧是关联进来的。 keep、nullable 只在 LEFT / RIGHT JOIN 上有:保留全部行的主表那一侧、关联不上时记空的那一侧(比较用的写法,排好 序)。LEFT JOIN 关联进来的那一侧可空,关联条件里的其余来源(没有就是它之前的各个来源)是主表;RIGHT JOIN 反过来。 - orders LEFT JOIN refunds 和 refunds RIGHT JOIN orders 是同一回事,和 refunds LEFT JOIN orders 不是:diff_facets - 按主表比,不按写的是 LEFT 还是 RIGHT。rejects 是同一段 WHERE 里把可空那一侧没关联上的行筛掉的条件(那一侧的列 + orders LEFT JOIN refunds 和 refunds RIGHT JOIN orders 是同一回事,和 refunds LEFT JOIN orders 不是:按主表比, + 不按写的是 LEFT 还是 RIGHT。rejects 是同一段 WHERE 里把可空那一侧没关联上的行筛掉的条件(那一侧的列 拿来比较、IN、LIKE……,空值都不成立):有它这次外连接实际等于内连接。scope 是这次连接在哪一层(见 Part.scope): 外层关联了 refunds,和 EXISTS 子查询里同样关联了 refunds,是两次连接。""" @@ -421,7 +384,7 @@ class Computed: scope_key 为 None,name 为空串。 - 派生来源(CTE、FROM 里的子查询)里被外面用到、又不是原样透出一列的列:它在里面的定义(amount * 0.9、 DATE(paid_at)、ROW_NUMBER() OVER (…)、p.total - r.total)。外面按「t.net」引用它,两边写的都是 t.net,算法却可能 - 不同,只比外面的写法看不出来。scope 是派生来源给人看的名字,scope_key 是它的编号(两边配好对的共同编号), + 不同,只比外面的写法看不出来。scope 是派生来源给人看的名字,scope_key 是它的位置编号, name 是列名的 name_key,shown 是列名的原样写法。 expr 是定义(按它那一段的来源写的 Part;UNION 的各支用「|」连起来)。template 是把里面的聚合换成占位符之后的写法、 @@ -444,7 +407,7 @@ class LimitFacet: 排序键 (Part, 是否降序):只取前几行时,按什么排决定了取到的是哪几行。scope 是哪一段:None 是最外层,CTE 写它的 名字,FROM 里的子查询写它的别名(没有别名写「子查询」)。条件里套着的子查询不单算:它整个写在那个条件里比。 scope_key 是这一段所属派生来源的编号(和 Part.key 里的同一套编号;最外层、UNION 的某一支这类不是派生来源整段的 - 为 None):diff_facets 按它把两边同一个 CTE、子查询里的取行配成一对。""" + 为 None)。""" count: str | None offset: str | None = None @@ -457,7 +420,7 @@ class LimitFacet: class SetOpFacet: """一段查询把几支 SELECT 合起来(UNION、UNION ALL、INTERSECT、EXCEPT)。ops 是各处合并的写法,按从左到右的先后 (「UNION」「UNION ALL」…);branches 是一共几支。scope 是哪一段:None 是最外层,CTE、FROM 里的子查询写它的名字; - scope_key 是那一段所在的层(见 Part.scope:最外层是空串,派生来源是两边配好的共同编号)。条件里的子查询不单算: + scope_key 是那一段所在的层(见 Part.scope:最外层是空串,派生来源是它的位置编号)。条件里的子查询不单算: 它整个写在那个条件里比。""" ops: tuple[str, ...] @@ -475,23 +438,21 @@ class Facets: sources 是各段 FROM / JOIN 里的来源(实表、CTE、子查询;key 是比较用的写法,派生来源是编号,text 是给人看的 名字),按出现的先后、按 key 去重;条件里套着的子查询(IN (SELECT … FROM users))的来源不算,它们只筛行,不是 - 连接的一头。它本身不单独比较:一边独有的一次内连接要说清多出来的是哪个来源时,拿另一边的 sources 对照(见 - _Compare.added),外连接的主表、可空那一侧也按它写名字。 + 连接的一头。外连接的主表、可空那一侧按它写名字。 expressions 是算出来的列(见 Computed)。derived 是用到了的派生来源(key 是编号,text 是名字,含条件里套着的子 查询用到的 CTE):两边写法一样、同名的派生来源却不是同一个时,拿它说清「盲算的 t1 对应原流程的 t2」。 where、group_by、having、joins、aggregates、time 的每一项都记着在哪一层(Part.scope、JoinFacet.scope、 Aggregate.scope),按(层, key)去重;scopes 是这条语句里有的各层(key 是层的标记,text 是给人看的说法:「外层」 - 「t」「t 的 UNION 第 2 支」「条件的子查询」),diff_facets 说多出来的是哪一层时用。group_by 里也有当分组用的 + 「t」「t 的 UNION 第 2 支」「条件的子查询」),给人说是哪一层时用。group_by 里也有当分组用的 SELECT DISTINCT 选出来的各项(这一段没有 GROUP BY、没有聚合时,和按它们分组出的行一样),distinct 是这样收进来的 那几层:说「去重」不说「分组」。set_ops 是各段的 UNION、INTERSECT、EXCEPT(见 SetOpFacet)。grain 是最后出来 的行按什么分(key 为「group:分组键…」「one」「rows」,见 _Statement._grain;最外层是 UNION 的为 None)。 只收用到了的段:没用到的 CTE(定义了、外面没引用)里的条件、聚合、表不影响结果,不收(见 _Statement._enliven)。 - _origin 是 facets 记下的 (原文, 表结构),不进比较、不进 as_dict:diff_facets 把两边的派生来源按结构配好对之后, - 拿它按共同的编号重新分析一遍(见模块说明)。 + _origin 是 facets 记下的 (原文, 表结构),不进比较、不进 as_dict:diff_facets 拿它交给 explain_sql_diff 比原文。 """ dialect: str | None = None @@ -532,7 +493,7 @@ def facets(sql: str, dialect: str | None = None, *, schema: Mapping[str, Any] | schema 是数据源的 schema_cache 形状({"tables": {表名: {"columns": [{"name": …}]}}}),可以不给。给了的话, 几张表连在一起、列又没加表前缀时按它认列是哪张表的;不给就照写列名,比较时可能把同一列当成两列。 - 返回的 Facets 记着原文和表结构(_origin):diff_facets 要把两边的派生来源按结构配好对再重新分析。 + 返回的 Facets 记着原文和表结构(_origin):diff_facets 拿它交给 explain_sql_diff 比原文。 """ read = _read(dialect) root, problem = _parse(sql, read) @@ -550,9 +511,6 @@ def facets(sql: str, dialect: str | None = None, *, schema: Mapping[str, Any] | #: 比较:(运算符的写法, 两边对调之后的类型) _COMPARE: dict[str, tuple[str, str]] = {"EQ": ("=", "EQ"), "NEQ": ("<>", "NEQ"), "GT": (">", "LT"), "GTE": (">=", "LTE"), "LT": ("<", "GT"), "LTE": ("<=", "GTE")} -#: 连接类型的写法(diff_facets 里用) -_JOIN_WORDS = {"inner": "INNER JOIN", "left": "LEFT JOIN", "right": "RIGHT JOIN", "full": "FULL JOIN", - "cross": "CROSS JOIN", "semi": "SEMI JOIN", "anti": "ANTI JOIN", "natural": "NATURAL JOIN"} #: 日期、时间函数(sqlglot 把各方言的写法归成这些类型:strftime、DATE_FORMAT 都是 TimeToStr) _DATE_FUNCS = ("CurrentDate", "CurrentDatetime", "CurrentTimestamp", "CurrentTime", "Date", "Datetime", "DateAdd", "DateSub", "DateDiff", "DateTrunc", "TimestampTrunc", "DatetimeTrunc", "TimeToStr", "StrToDate", @@ -816,100 +774,6 @@ class _Source: return f"{self.key}#{self.role}" if self.role else self.key -@dataclass(frozen=True) -class _Derived: - """一个派生来源(CTE、FROM / JOIN 里的子查询、LATERAL 子查询)的结构标识,和起的名字、位置编号无关的部分都在 - 这里,diff_facets 拿它把两边的派生来源配成对(见 _align)。各项都用按结构写的写法(render 的 shape 模式:实表 - 写表名,往里套着的派生来源写它自己的结构标识,见 _Statement._signature),所以两边写法一样就相同,不受先后、 - 名字影响;往里套着的两个派生来源(付款合计、退款合计)也分得开:net 是 paid.total - refunded.total 还是反过来, - 写出来不一样。""" - - number: int # 位置编号(@N 的 N):别的都分不出时才看先后 - name: str # CTE 名、子查询的别名(name_key),没有为空:辅助线索 - tables: tuple[str, ...] # 里面读到的实表:各支 FROM / JOIN,往里套着的也算;排好序 - columns: tuple[tuple[str, str], ...] # 选出来的各列 (列名, 规范化后的定义式),SELECT * 记成 ("*", "*") - group: tuple[str, ...] # 分组键,排好序 - conditions: frozenset[str] # 里面各段 WHERE、HAVING 的合取式:同一级配得上好几个时比重合 - body: str # 整段查询规范化后的写法 - - @property - def names(self) -> tuple[str, ...]: - """选出来的列名(外层按名字引用它们)。""" - return tuple(name for name, _ in self.columns) - - -#: 派生来源配对的各级,从严到宽:(这一级比什么, 是否只在两边都只剩一个时才配)。比什么返回 None 表示这一级没有 -#: 可比的(没有名字、没读实表),不配 -_ALIGN_LEVELS: tuple[tuple[Any, bool], ...] = ( - (lambda d: (d.body, d.columns), False), # 整段一样 - (lambda d: (d.tables, d.columns, d.group), False), # 读的表、各列定义、分组一样 - (lambda d: (d.tables, d.names), False), # 读的表、选出来的列名一样 - (lambda d: (d.tables, d.name) if d.name else None, False), # 读的表、名字一样 - (lambda d: d.tables or None, True), # 只是读的表一样 - (lambda d: d.name or None, True), # 只是名字一样(里面换了表) - (lambda d: d.names if any(n != "*" for n in d.names) else None, True), # 只是选出来的列名一样 -) - - -def _align(xs: list[_Derived], ys: list[_Derived]) -> dict[int, int]: - """两条语句的派生来源配对:{第二条里的位置编号: 第一条里的位置编号},配不上的不在里面。 - - 按 _ALIGN_LEVELS 从严到宽一级一级地配,每一级一对一:严的先配走,宽的只收拾剩下的。同一级几对都配得上时,先配 - 里面的条件重合最多的,再配同名的,最后按位置编号的先后(几个一模一样的派生来源按先后一一对应);这一级所有 - 配得上的对一起排,不是左边挨个挑,免得前一个挑走后一个更合适的。条件排在名字前面:名字是各自起的,两边都叫 - paid 的两个 CTE,一边筛 status = 'paid'、一边筛 status = 'open',算的不是一回事;条件一个都不重合时才看名字。 - 最后三级证据弱,只在两边各只剩一个这样的时才配,免得猜错。哪一级都配不上的就不配:一边多出来的 CTE(盲算先筛 - 一遍 VIP 用户)和另一边不相干的 CTE 凑成一对,会把外层不相干的条件说成「关联条件不同」。 - """ - rest_x, rest_y = sorted(xs, key=lambda d: d.number), sorted(ys, key=lambda d: d.number) - pairs: dict[int, int] = {} - for view, unique in _ALIGN_LEVELS: - candidates = [] - for x in rest_x: - seen = view(x) - if seen is None: - continue - found = [y for y in rest_y if view(y) == seen] - if unique and (len(found) != 1 or sum(1 for z in rest_x if view(z) == seen) != 1): - continue - candidates += [((-len(y.conditions & x.conditions), y.name != x.name, x.number, y.number), x, y) - for y in found] - for _, x, y in sorted(candidates, key=lambda c: c[0]): - if x.number not in pairs.values() and y.number not in pairs: - pairs[y.number] = x.number - rest_x = [x for x in rest_x if x.number not in pairs.values()] - rest_y = [y for y in rest_y if y.number not in pairs] - return pairs - - -def _renamed(xs: list[_Derived], ys: list[_Derived], pairs: Mapping[int, int]) -> dict[tuple[int, str], str]: - """配成一对的派生来源里两边起名不同的列,配成同一列:{(第二条里的位置编号, 列名): 第一条里的列名}。 - - 外面按名字引用这些列,一边叫 total、一边叫 sum_amt,不配起来就是两列,外面按它写的条件、聚合全成了两边各有 - 一个。只在两边各自独有的列名之间配:定义(结构写法)一样的一对一配上;剩下两边各只有一列时也配上(盲算换了名字 - 又改了算法:SUM(amount) AS total 对 SUM(amount - discount) AS net_total,外面用在同一个位置),定义的不同由算出来 - 的列那一面报(见 _Compare.expressions)。""" - by_number = {x.number: x for x in xs} - out: dict[tuple[int, str], str] = {} - for y in ys: - x = by_number.get(pairs.get(y.number, -1)) - if x is None: - continue - cx = {name: shape for name, shape in x.columns if name and name != "*"} - cy = {name: shape for name, shape in y.columns if name and name != "*"} - rest_x = [name for name in cx if name not in cy] - rest_y = [name for name in cy if name not in cx] - for name in list(rest_y): - same = [other for other in rest_x if cx[other] == cy[name]] - if len(same) == 1 and sum(1 for other in rest_y if cy[other] == cy[name]) == 1: - out[(y.number, name)] = same[0] - rest_x.remove(same[0]) - rest_y.remove(name) - if len(rest_x) == 1 and len(rest_y) == 1: - out[(y.number, rest_y[0])] = rest_x[0] - return out - - class _Statement: """一条语句的分析:CTE、实表、每段 SELECT 的来源,以及把表达式写成 key / text 的规则。""" @@ -969,7 +833,7 @@ class _Statement: return live def _enliven(self, extra: Iterable[str]) -> None: - """定下哪些段算数:最外层那段查询,加上从它一路引用到的 CTE(extra 里的 CTE 也当作用到了,见 _aligned)。没 + """定下哪些段算数:最外层那段查询,加上从它一路引用到的 CTE(extra 里的 CTE 也当作用到了)。没 用到的 CTE 里的条件、聚合、表不影响结果:收进来的话,盲算定义了只筛已付的 CTE 却直接查 orders,会被当成也筛了 已付;定义了没用的 CTE 里多一个条件,又会被报成多筛了一遍。用到的实表(给人看时要不要写表名)、共有的列名跟着 重算。""" @@ -1006,22 +870,19 @@ class _Statement: 拿里面那段查询的写法作 key 也不行:里面差一个条件,外层按它的列写的条件就全跟着变,同一处不同报好几遍,还会 出现两边各说一句「只保留 total > 100 的行」的自相矛盾。所以单看一条语句时按位置编:从最外层往里一层一层地 编,同一层按出现的先后;CTE 在第一次用到的地方编号,在哪儿用到都是这一个编号。写成 CTE 还是 FROM 里的子查询、 - CTE 定义的先后都不影响编号,里层多一段、少一段也不挪动外层的编号。两条语句之间编号错开(一边多一个 CTE、 - 内连接换了先后)由 diff_facets 按结构配对之后改成共同的编号(见 relabel)。 + CTE 定义的先后都不影响编号,里层多一段、少一段也不挪动外层的编号。这套编号只给 facets 用;比两条语句不靠 + 它(见 explain_sql_diff:两边各自规范化,CTE 按定义起名)。 """ from sqlglot import exp ctes: dict[str, int] = {} count = 0 - #: 编了号的派生来源 [(位置编号, 起的名字, 里面那段查询, CTE 的 name_key 或 None)],按编号排:diff_facets 按 - #: 它们的结构配对(见 derived),facets 按它们找各列的定义(见 _expressions) + #: 编了号的派生来源 [(位置编号, 起的名字, 里面那段查询, CTE 的 name_key 或 None)],按编号排:facets 按它们 + #: 找各列的定义(见 _expressions) self._derived: list[tuple[int, str, Any, str | None]] = [] - #: 记了编号的节点 {id: (节点, 位置编号)}:配好对之后按共同的编号改写(见 relabel) - self._marked: dict[int, tuple[Any, int]] = {} def mark(node: Any, number: int) -> None: node.meta[_DERIVED] = f"@{number}" - self._marked[id(node)] = (node, number) def visit(select: Any) -> list[Any]: """给这一段还没编号的 FROM / JOIN 项编号,返回要接着往里看的查询(子查询、第一次用到的 CTE 的定义)。""" @@ -1058,8 +919,7 @@ class _Statement: # 没走到的(没用到的 CTE 里的、表函数参数里的)按出现的先后补上:每个派生来源都得有编号 for select in self.root.find_all(exp.Select, bfs=False): visit(select) - # 没用到的 CTE 也编上号(排在最后):外层不引用它,可它照样要参加两边的配对。两边都定义了付款、退款两个 CTE, - # 一边关联付款的、一边关联退款的,没用到的那个不参加的话,两边各剩一个只有列名对得上的,会被凑成一对 + # 没用到的 CTE 也编上号(排在最后):每个派生来源都得有编号 for name, cte in self._cte_defs.items(): if name not in ctes: count += 1 @@ -1070,68 +930,10 @@ class _Statement: self._bodies = {number: body for number, _, body, _ in self._derived} self._labels = {number: f"@{number}" for number in self._bodies} self._numbers = {label: number for number, label in self._labels.items()} - #: 派生来源里改叫另一边那个名字的列 {(比较用的编号, 列名的 name_key): 另一边的列名}(见 relabel) - self._renames: dict[tuple[str, str], str] = {} - - def relabel(self, ids: Mapping[int, str], renames: Mapping[tuple[int, str], str] | None = None) -> None: - """派生来源改用 ids 给的编号({位置编号: 比较用的编号},没给的照旧写位置编号)。diff_facets 把两边的派生 - 来源配好对之后,第二条语句里配上的写第一条里对应的编号,两边比较用的写法才对得上。算好的来源、共有的列名 - 都带着旧编号,跟着清掉重算。 - - renames 是派生来源里改叫第一条里那个名字的列 {(位置编号, 列名的 name_key): 第一条里的列名}(见 _renamed): - 比较用的写法照它写,给人看的照原样。""" - for node, number in self._marked.values(): - node.meta[_DERIVED] = ids.get(number, f"@{number}") - self._labels = {number: ids.get(number, f"@{number}") for number in self._bodies} - self._numbers = {label: number for number, label in self._labels.items()} - self._renames = {(self._labels[number], column): name for (number, column), name in (renames or {}).items() - if number in self._labels} - self._memo.clear() - self._shared = None def column_key(self, source: _Source, column: str) -> str: - """比较用的列名:派生来源里改叫另一边那个名字的列(见 relabel)照改过的写,别的按 name_key。""" - key = name_key(column) - return self._renames.get((source.key, key), key) if source.kind == "derived" else key - - def used_labels(self) -> dict[str, str | None]: - """用到了的派生来源(见 _enliven):{比较用的编号: CTE 的 name_key,FROM 里的子查询为 None}。""" - return {self._labels[number]: cte for number, _, body, cte in self._derived if id(body) not in self._dead} - - def defined_labels(self) -> dict[str, str | None]: - """定义了的派生来源,用没用到都算:{比较用的编号: CTE 的 name_key,FROM 里的子查询为 None}。""" - return {self._labels[number]: cte for number, _, _, cte in self._derived} - - def derived(self) -> list[_Derived]: - """各个派生来源的结构标识(见 _Derived),按位置编号排。""" - from sqlglot import exp - - out = [] - for number, name, body, _ in self._derived: - columns: list[tuple[str, str]] = [] - group: list[str] = [] - conditions: set[str] = set() - for i, select in enumerate(_branches(body)): - scope = self.sources(select) - for clause in (select.args.get("where"), select.args.get("having")): - if clause is not None: - conditions.update(self._shape(c, scope) for c in _conjuncts(clause.this)) - if i: - continue # UNION 选出来的列名、分组按第一支的算 - for item, called in zip(select.selects, self.names(body)): - if called is None: - columns.append(("*", "*")) - continue - target = item.this if isinstance(item, exp.Alias) else item - columns.append((name_key(called), self._shape(target, scope))) - if (clause := select.args.get("group")) is not None: - selects, aliases = select.selects, self._aliases(select) - group += [self._shape(self._target(g, selects, aliases), scope) for g in clause.expressions] - out.append(_Derived(number=number, name=name_key(name) if name else "", - tables=tuple(sorted(self._row_tables(body, frozenset()))), - columns=tuple(sorted(columns)), group=tuple(sorted(group)), - conditions=frozenset(conditions), body=self._shape(body, {}))) - return out + """比较用的列名:按 name_key。""" + return name_key(column) def _shape(self, node: Any, sources: dict[str, _Source]) -> str: """按结构写(render 的 shape 模式,比较的两边也按结构排):配对用,和名字、编号无关。""" @@ -1152,34 +954,6 @@ class _Statement: number = self._numbers.get(label) return self._signature(number) if number is not None else _ANON - def _row_tables(self, query: Any, seen: frozenset[str]) -> set[str]: - """query 的行从哪些实表来:各支 FROM / JOIN 里的实表,往里套着的子查询、CTE 也算。条件里的子查询不算(它只 - 决定留哪些行,算进来的话一边多一个 IN (SELECT … FROM users) 的筛选,结构标识就整个变了)。seen 是这条路上 - 已经进过的 CTE:递归 CTE 引用自己时不再往里走。""" - from sqlglot import exp - - out: set[str] = set() - for select in _branches(query): - for item in self._items(select): - body = _derived_body(item) - if body is not None: - out |= self._row_tables(body, seen) - elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) and item.name: - if not self._is_cte(item): - out.add(self._table(item).key) - elif (name := name_key(item.name)) not in seen: - out |= self._row_tables(self.ctes[name], seen | {name}) - return out - - @classmethod - def reparse(cls, f: Facets) -> "_Statement | None": - """按 facets 记下的原文、表结构重新分析一遍(见 Facets._origin);没记、解析不了返回 None。""" - if f._origin is None: - return None - sql, schema = f._origin - root, _ = _parse(sql, f.dialect) - return cls(root, f.dialect, schema) if root is not None else None - @staticmethod def _table(table: Any) -> _Source: from sqlglot import exp @@ -1241,8 +1015,8 @@ class _Statement: 两个原样透出的 CTE 都读 orders(已付的一份、未付的一份):透过去 p.amount、o.amount 都成了 orders.amount,外层 用的是哪一份就分不出来了(SUM(p.amount) 对 SUM(o.amount)、p.amount > 100 对 o.amount > 100 都成了一样);CTE 和 它自己读的那张表关联(父单、子单)、同一个原样透出的 CTE 关联两次也一样,自己关联自己时 _roles 设的 role 透过去 - 也就丢了。这时这几个派生来源的列照写它们自己的编号(两边按结构配好的共同编号):已付配已付、未付配未付,换了 - 用哪一份照报,关联换了先后、起的名字不同不报。只有一个来源读那张表时照旧透过去:先在 CTE 里筛一遍再查和直接查 + 也就丢了。这时这几个派生来源的列照写它们自己的编号:用的是已付那份 + 还是未付那份分得开。只有一个来源读那张表时照旧透过去:先在 CTE 里筛一遍再查和直接查 是同一列。""" groups: dict[str, list[_Source]] = {} for source in sources.values(): @@ -1544,7 +1318,7 @@ class _Statement: 子查询里自带的 CTE 定义写编号(anon 写「@」,shape 里引用 CTE 的表写它的结构标识、别的写「@」;引用它们的列 已经照这样写了);只看取值的子查询(IN、EXISTS、标量子查询)去掉选出来的列名。FROM 里的子查询、CTE 定义选出来 的列名留着:外层按这个名字引用它们,去掉了就分不出外层用的是哪一列。复制出来的那段本身没有父节点,认不出是 - FROM 里的子查询,它选出来的列名也会去掉(_Derived.body 另记了各列,见 derived)。""" + FROM 里的子查询,它选出来的列名也会去掉。""" from sqlglot import exp for node in copy.find_all(exp.Table, exp.Subquery, exp.Lateral, exp.CTE): @@ -1637,10 +1411,7 @@ class _Statement: """比较的两边谁排前面:先按不写编号的写法(实表的列、派生来源的列分得开),分不出(两个派生来源的同名列) 再按 by 的写法分。 - diff_facets 重新分析时 by 是 key,按两边配好对的共同编号排(见 relabel):p.total > r.total 和换了关联先后、 - 位置编号对调的同一句排出来是同一个方向。不按结构标识排:配成一对的两个派生来源里面稍有不同(一边多一个条件) - 时结构标识就不同,按它排,同一个比较在两边可能排成两个方向,报出两条自相矛盾的话。配对之前(算结构标识时) - 还没有共同的编号,by 是 shape,按结构标识排:位置编号决定不了先后。""" + by 是 key 时按位置编号排;算结构标识时 by 是 shape,按结构标识排:位置编号决定不了先后。""" return self.render(node, sources, "anon"), self.render(node, sources, by) def part(self, node: Any, sources: dict[str, _Source], *, excludes: bool = False) -> Part: @@ -1762,7 +1533,7 @@ class _Statement: # ---- 各个方面 def facets(self, extra: Iterable[str] = ()) -> Facets: - """各个方面。extra 是没用到、也照样收进来的 CTE(name_key,见 _aligned),别的没用到的 CTE 不收(见 _enliven)。 + """各个方面。extra 是没用到、也照样收进来的 CTE(name_key),别的没用到的 CTE 不收(见 _enliven)。 条件、分组键、连接、聚合、HAVING 都记下在哪一层(见 _layers),按(层, key)去重。""" from sqlglot import exp @@ -1828,11 +1599,11 @@ class _Statement: """每段 SELECT 在哪一层:{段的 id: 层的标记}(见 Part.scope)。 最外层是空串,UNION 的第 k 支(k ≥ 2)在后面加「|k」。FROM / JOIN 里的派生来源(CTE、子查询、LATERAL 子查询) - 是它自己的编号(两边按结构配好的共同编号,所以两边对得上);只从一个来源里筛出一部分行的(见 _transparent) + 是它自己的位置编号;只从一个来源里筛出一部分行的(见 _transparent) 和用到它的那一段算同一层:写在这种 CTE 里的条件和写在外层是一回事。读同一个来源的几个派生来源同在一段时 (见 _seal)各算各的层:两个 CTE 里各有一个 is_deleted = 0 是两个条件。派生来源在哪一层按在 FROM / JOIN 里 第一次用到它的地方定,从外往里一层一层地走。没走到的段(条件、SELECT 列表里的子查询,以及只在它们里面用到 - 的派生来源)由 facets 记成「?」。extra 是没用到、也照样收进来的 CTE(见 _aligned),在它自己的编号那一层。""" + 的派生来源)由 facets 记成「?」。extra 是没用到、也照样收进来的 CTE,在它自己的编号那一层。""" from sqlglot import exp tags: dict[int, str] = {} @@ -2458,18 +2229,1606 @@ class _Statement: # ========================================================================== -# 比较 +# 等价判定:只做保持语义的变换,比规范形式 +# ========================================================================== + +#: 规范形式的几级(见 _canonical_forms)。等价的结论只在同一级的两个规范形式一字不差时才下 +_OPTIMIZER, _LIGHT, _PARSED = "optimizer", "light", "parsed" +#: 同一段里同名的几个来源(自己关联自己)谁先谁后逐个排列着试,取写出来最小的那个;排列数超过这个就只按原来的 +#: 先后(认得出的等价改写少一些,不会错判) +_MAX_ORDERINGS = 64 +#: 一条语句里一共试多少种排列(里层的子查询在外层每试一种时都要重排一遍):用完了就都只按原来的先后 +_ORDERING_BUDGET = 2000 +#: 原来的名字(CTE 名、FROM 里子查询的别名)记在里面那段查询的 meta 里:起了规范的名字之后,差异说明还要照原样说 +_ORIGINAL = "sql_semantics_name" +#: GROUP BY 是从 SELECT DISTINCT 改写来的:差异说明里说「去重」 +_FROM_DISTINCT = "sql_semantics_distinct" +#: 写的是 LEFT / RIGHT JOIN、WHERE 把没关联上的行筛掉了、按内连接比的来源(记在那张表上,值是原来写的连接):差异 +#: 说明里照实说「写的是 LEFT JOIN,实际是 INNER JOIN」 +_REJECTED = "sql_semantics_rejected" +#: 从 RIGHT JOIN 改写成的 LEFT JOIN(记在 JOIN 上):差异说明里不说「用 LEFT JOIN」,说以哪一侧为主表 +_FLIPPED = "sql_semantics_flipped" +#: 不确定的函数:每次求值结果都可能不同,合并子查询会把一次求值复制成几次,不走优化器那一级 +_VOLATILE = frozenset({"random", "rand", "randn", "newid", "uuid", "gen_random_uuid", "uuid_generate_v4", "random_uuid", + "randomblob", "setseed", "nextval", "sys_guid"}) +#: 拿空值比较、算术都得空值(外连接没关联上的那一侧整行是空值)的写法,WHERE 里用到它就把没关联上的行筛掉了 +_STRICT_UNARY = ("Paren", "Neg", "Cast", "Lower", "Upper", "Length", "Abs") +_STRICT_BINARY = ("Add", "Sub", "Mul", "Div", "Mod", "DPipe") + + +class _Unsupported(Exception): + """这一级规范化处理不了这条语句的结构(递归 CTE、LATERAL、表函数……),这一级不出规范形式。""" + + +#: 一元加号前面可能是这些词(不是一个值的结尾) +_OPERAND_KEYWORDS = frozenset({"SELECT", "WHERE", "AND", "OR", "NOT", "ON", "WHEN", "THEN", "ELSE", "HAVING", "BY", "IN", + "IS", "LIKE", "GLOB", "BETWEEN", "CASE", "AS", "DISTINCT", "ALL", "SET", "VALUES", + "LIMIT", "OFFSET", "RETURN", "ESCAPE", "FILTER", "OVER", "PARTITION", "USING", "IF", + "EXISTS", "ANY", "SOME", "FROM", "JOIN", "WITH", "ORDER", "GROUP"}) + + +def _lost_in_parsing(text: str, read: str | None) -> bool: + """原文里有 sqlglot 解析时丢掉、语义却不一样的写法:一元加号(SQLite 里 +a 去掉了列的类型亲和,+a = '10' 和 + a = '10' 结果不同,解析出来却是同一棵树)。有就不在语法树上判等价。""" + try: + tokens = _tokens(text, read) + except Exception: # noqa: BLE001 - 切词都不行的,解析也过不了,那边会说 + return False + prev = None + for token in tokens: + if token.token_type.name == "PLUS": + if prev is None: + return True + kind, word = prev.token_type.name, prev.text + operand = kind in ("VAR", "IDENTIFIER", "NUMBER", "R_PAREN", "R_BRACKET", "NULL", "TRUE", "FALSE", "END", + "STAR") or kind.endswith("STRING") or ( + _PLAIN_NAME.match(word) is not None and word.upper() not in _OPERAND_KEYWORDS) + if not operand: + return True + prev = token + return False + + +@dataclass +class _Form: + """一条语句在某一级的规范形式。key 一样才算等价,text 是生成的 SQL(给人看);pre 是起规范名字之前的树(差异 + 说明按另一边的 CTE 名重新起名时用,见 _Explainer.statement)。""" + + level: str + text: str + tree: Any + pre: Any = None + qualified: bool = False + + @property + def key(self) -> str: + """比等价用的写法:语法树逐个节点的类型和参数(见 _fingerprint)。不拿生成的 SQL 比:sqlglot 按方言生成时 + 会把不同的写法写成同一个样子(SQLite 的 CAST(x AS DATE) 生成 DATE(x),两者在 SQLite 里结果不同)。""" + return _fingerprint(self.tree) + + +def _fingerprint(node: Any) -> str: + """语法树的确定写法:每个节点写类名和全部参数(按参数名排序,字符串原样),不同的树一定写得不一样。类型标注、 + 注释、meta 不算。""" + from sqlglot import exp + + if isinstance(node, exp.Expr): + parts = [f"{k}={_fingerprint(v)}" for k, v in sorted(node.args.items()) + if v is not None and v is not False and v != []] + return f"{type(node).__name__}({','.join(parts)})" + if isinstance(node, list): + return "[" + ",".join(_fingerprint(v) for v in node) + "]" + return repr(node) + + +def _key(node: Any, read: str | None) -> tuple[str, str]: + """排序、去重用的键:先按生成的 SQL 排(好读),同样写法的再按 _fingerprint 分开。""" + return node.sql(dialect=read, comments=False), _fingerprint(node) + + +def _schema_mapping(schema: Mapping[str, Any] | None) -> dict[str, dict[str, str]] | None: + """schema_cache → sqlglot 的表结构 {表: {列: 类型}}。类型一律记 UNKNOWN:只要列名(认列属于哪张表、展开 + SELECT *);按类型做的改写和方言有关(canonicalize 给日期比较加 CAST,SQLite 里 CAST(… AS DATE) 取的是数字), + 不做。""" + tables = schema.get("tables") if isinstance(schema, Mapping) else None + if not isinstance(tables, Mapping): + return None + out: dict[str, dict[str, str]] = {} + for name, meta in tables.items(): + columns = meta.get("columns") if isinstance(meta, Mapping) else None + names = [c["name"] for c in columns or [] if isinstance(c, Mapping) and isinstance(c.get("name"), str) + and c["name"]] + if names: + out[str(name)] = {n: "UNKNOWN" for n in names} + return out or None + + +def _canonical_forms(root: Any, read: str | None, schema: dict[str, dict[str, str]] | None) -> dict[str, _Form]: + """一条语句的几级规范形式: + + - optimizer:sqlglot 优化器的几步(qualify 认列、展开 SELECT *、别名展开;eliminate_subqueries 把 FROM 里的子查询 + 写成 CTE;merge_subqueries 把只用一次、只是筛行透列的 CTE、子查询并进外层;eliminate_ctes 去掉没用的 CTE; + pushdown_projections 去掉没用到的列;simplify 化简条件),加上本模块自己的几步(见 _finish)。认不全列(没给表 + 结构、几张表又都没写表名)、有递归 CTE、LATERAL 之类的不出这一级。 + - light:轻量规范化,不认列:统一标识符大小写、去注释、只引用一次的非递归 CTE 写回用到它的地方,再走 _finish 里 + 不需要认列的几步。 + - parsed:解析后重新生成的写法(只统一了大小写、空白、注释),总是有。 + + 每一级的每一步都只做保持语义的变换,某一级出不来就没有这一级,从不抛异常。""" + out: dict[str, _Form] = {} + for level, build in ((_OPTIMIZER, _optimized), (_LIGHT, _light)): + try: + out[level] = build(root, read, schema) + except _Unsupported as e: + logger.debug("SQL 规范化(%s)跳过:%s", level, e) + except Exception as e: # noqa: BLE001 - sqlglot 认不了列、写法不支持:这一级不出,别的级照样比 + from sqlglot.errors import SqlglotError + + if isinstance(e, SqlglotError): + logger.debug("SQL 规范化(%s)跳过:%s", level, e) + else: + logger.warning("SQL 规范化(%s)失败", level, exc_info=True) + try: + from sqlglot.optimizer.normalize_identifiers import normalize_identifiers + + tree = normalize_identifiers(root.copy(), dialect=read) + _unquote(tree, read) + out[_PARSED] = _Form(_PARSED, tree.sql(dialect=read, comments=False), tree) + except Exception: # noqa: BLE001 + logger.warning("SQL 重新生成失败", exc_info=True) + return out + + +def _optimized(root: Any, read: str | None, schema: dict[str, dict[str, str]] | None) -> _Form: + from sqlglot.errors import OptimizeError + from sqlglot.optimizer.eliminate_ctes import eliminate_ctes + from sqlglot.optimizer.eliminate_subqueries import eliminate_subqueries + from sqlglot.optimizer.merge_subqueries import merge_subqueries + from sqlglot.optimizer.pushdown_projections import pushdown_projections + from sqlglot.optimizer.qualify import qualify + + tree = root.copy() + why = _unsupported(tree) or ("不确定的函数" if _volatile(tree) else None) + if why: + raise _Unsupported(why) + _remember_names(tree) + # 给了表结构先按表结构认列;表结构里没有的列(表结构旧了)再按不给表结构认一遍:两遍都认不全就不出这一级 + qualified = None + for mapping in ([schema, None] if schema else [None]): + if _alias_or_column(tree, read, mapping): + continue + try: + qualified = qualify(tree.copy(), dialect=read, schema=mapping, validate_qualify_columns=True, + quote_identifiers=False, identify=False, expand_stars=True, expand_alias_refs=True) + break + except OptimizeError as e: + logger.debug("SQL 认列失败:%s", e) + if qualified is None: + raise _Unsupported("认不全列") + tree = qualified + if _has_star(tree): + _narrow_stars(tree, read) + if not _has_star(tree) and not _correlated_derived(tree): + tree = eliminate_subqueries(tree) + _split_simple_ctes(tree) + # merge_subqueries 只在同一段里避免重名:同一个 CTE 并进外层和相关子查询两处,两处都叫 orders,子查询里本来 + # 指外层的 orders.id 就被里层遮住了。先把整条语句的别名都改成互不相同的 + _unique_aliases(tree, read) + # merge_subqueries 会把外连接可空一侧的子查询里算出来的列(常量、COALESCE)原样并进外层:没关联上的行本该 + # 是空值,并进来就成了常量。这种写法不并 + if _merge_safe(tree): + tree = merge_subqueries(tree) + tree = eliminate_ctes(tree) + _push_into_groups(tree) + tree = pushdown_projections(tree) + else: + tree = eliminate_ctes(tree) + return _finish(tree, read, level=_OPTIMIZER, qualified=True) + + +def _light(root: Any, read: str | None, schema: Any = None) -> _Form: + from sqlglot.optimizer.normalize_identifiers import normalize_identifiers + from sqlglot.optimizer.qualify_tables import qualify_tables + + tree = normalize_identifiers(root.copy(), dialect=read) + why = _unsupported(tree) + if why: + raise _Unsupported(why) + _remember_names(tree) + _inline_ctes(tree) + tree = qualify_tables(tree, dialect=read) + return _finish(tree, read, level=_LIGHT, qualified=False) + + +def _finish(tree: Any, read: str | None, *, level: str, qualified: bool) -> _Form: + """两级共用的本模块自己的几步,都只做保持语义的变换(为什么保持语义见各函数的说明): + + 1. 和名字无关的改写:不需要的引号、不影响结合的括号、ORDER BY 的 ASC、OFFSET 0 去掉;COUNT(1) 写成 COUNT(*); + IN 列表去重排序、只有一项写成 =;EXISTS 子查询写成 SELECT 1;IN 子查询里的 DISTINCT、不取行时的 ORDER BY + 去掉;SELECT DISTINCT 写成按选出来的列 GROUP BY,已经按选出来的列分组的 DISTINCT 去掉;原样选出一个子查询 + 全部列的一段换成那个子查询;两张表的 RIGHT JOIN 写成 LEFT JOIN;WHERE 把外连接可空一侧没关联上的行筛掉了的 + 写成内连接;内连接的条件挪进 WHERE、来源写成逗号连接;最外层 ORDER BY 引用的别名换成表达式,最外层和条件里 + 的子查询选出来的列名去掉(列名不影响取值)。 + 2. 化简条件(_simplify:sqlglot 的 simplify 去掉和类型、排序规则、方言有关的几种化简)。 + 3. 起规范的名字、排好交换律成立的地方(见 _named、_Namer、_sort_own)。 + """ + _unquote(tree, read) + _strip_parens(tree) + _strip_asc(tree) + _count_star(tree) + _in_lists(tree, read) + _exists_projection(tree) + _in_subquery_distinct(tree) + _distinct_to_group(tree, read) + _redundant_distinct(tree, read) + _zero_offset(tree) + tree = _unwrap_passthrough(tree, read) + _right_to_left(tree) + _reject_nulls(tree) + _flatten_inner(tree) + _order_by_aliases(tree) + _drop_output_aliases(tree) + tree = _simplify(tree, read) + pre = tree.copy() + tree = _named(tree, read, qualified) + return _Form(level, tree.sql(dialect=read, comments=False), tree, pre, qualified) + + +def _named(tree: Any, read: str | None, qualified: bool, hint: "_Hint | None" = None) -> Any: + """起规范的名字、排好交换律成立的地方,最后统一引号(只给需要的标识符加)。""" + from sqlglot import exp + from sqlglot.optimizer.qualify_columns import quote_identifiers + + _Namer(tree, read, qualified, hint).canon(tree, 0, {}) + tree = _simplify(tree, read) + for select in list(tree.find_all(exp.Select)): + _sort_own(select, read) + return quote_identifiers(tree, dialect=read, identify=False) + + +# ---- 守门:哪些结构不走某一级 + +def _volatile(tree: Any) -> bool: + from sqlglot import exp + + for node in tree.walk(): + if isinstance(node, (exp.Rand, exp.Randn)) or type(node).__name__.lower() in _VOLATILE: + return True + if isinstance(node, exp.Anonymous) and str(node.name).lower() in _VOLATILE: + return True + return False + + +def _unsupported(tree: Any) -> str | None: + """递归 CTE、LATERAL、UNNEST、PIVOT、表函数、VALUES、SEMI / ANTI / ASOF 这些连接:起名、合并都不处理。""" + from sqlglot import exp + + for node in tree.walk(): + if isinstance(node, exp.With) and node.args.get("recursive"): + return "递归 CTE" + if isinstance(node, (exp.Lateral, exp.Unnest, exp.Pivot, exp.TableSample, exp.Values)): + return type(node).__name__ + if isinstance(node, exp.Table) and not isinstance(node.this, exp.Identifier): + return "表函数" + if isinstance(node, exp.Join) and (node.args.get("method") or str(node.kind or "").upper() in + ("SEMI", "ANTI", "ASOF", "POSITIONAL", "STRAIGHT_JOIN")): + return "连接方式" + return None + + +def _starry(select: Any) -> bool: + from sqlglot import exp + + return any(isinstance(p, exp.Star) or (isinstance(p, exp.Column) and p.is_star) for p in select.expressions) + + +def _takes_any(select: Any) -> bool: + """没有 ORDER BY 的 LIMIT / OFFSET:取哪几行 SQL 没规定,库里取先出来的几行。出行的先后会变的改写(调换来源、 + DISTINCT 写成 GROUP BY、RIGHT JOIN 写成 LEFT JOIN)这一段都不做,免得取到的行不同。""" + return (select.args.get("limit") is not None or select.args.get("offset") is not None) and \ + select.args.get("order") is None + + +def _unstable(select: Any) -> bool: + """这一段取到哪几行、取到哪一行的值 SQL 本来就没规定、库里看扫描的先后:没有 ORDER BY 的 LIMIT / OFFSET;聚合的 + 一段(有 GROUP BY 或聚合函数)里既不在聚合函数里、也不是分组键(的一部分)的列(SQLite 取组里任意一行的值)。 + 这样的一段来源不调换先后:调换之后扫描的先后跟着变,实际取到的行就不同了。""" + from sqlglot import exp + + if _takes_any(select): + return True + group = select.args.get("group") + keys = {_fingerprint(k) for k in (group.expressions if group is not None else [])} + + def aggregates(node: Any) -> bool: + return any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) and not _nested(n, node) + for n in node.walk()) + + clauses = [c for c in (*select.expressions, select.args.get("having"), select.args.get("order")) if c is not None] + if group is None and not any(aggregates(p) for p in select.expressions): + return False + + def loose(node: Any) -> bool: + if isinstance(node, (exp.Query, exp.Subquery)) or _fingerprint(node) in keys: + return False + if isinstance(node, exp.AggFunc) and not isinstance(node.parent, exp.Window): + return False + if isinstance(node, exp.Column): + return not node.is_star + return any(loose(c) for c in node.iter_expressions()) + + return any(loose(c) for c in clauses) + + +def _has_star(tree: Any) -> bool: + from sqlglot import exp + + return any(_starry(s) for s in tree.find_all(exp.Select)) + + +def _correlated_derived(tree: Any) -> bool: + """FROM 里的子查询、CTE 引用了外层的列(相关派生表):写成顶层的 CTE 就断了关联,不改写。""" + from sqlglot.optimizer.scope import traverse_scope + + return any((s.is_derived_table or s.is_cte) and s.external_columns for s in traverse_scope(tree)) + + +def _merge_safe(tree: Any) -> bool: + """外连接可空一侧(LEFT JOIN 关联进来的、RIGHT JOIN 之前的、FULL JOIN 两侧)的子查询、CTE,只有「直接读实表、 + 原样选列」的才让 merge_subqueries 并:选出来的是常量、COALESCE、CASE 这类表达式时,没关联上的行本该整行是空值, + 并进外层后成了算出来的值;里面再套着子查询的,里层先并上来也会变成这样。""" + from sqlglot import exp + + ctes = {c.alias_or_name: c.this for c in tree.find_all(exp.CTE)} + for select in tree.find_all(exp.Select): + joins = select.args.get("joins") or [] + items = _items_of(select) + nullable: list[Any] = [] + for i, join in enumerate(joins): + side = str(join.side or "").upper() + if side in ("LEFT", "FULL"): + nullable.append(join.this) + if side in ("RIGHT", "FULL"): + nullable += items[:i + 1] + for item in nullable: + body = item.this if isinstance(item, exp.Subquery) else ( + ctes.get(item.name) if isinstance(item, exp.Table) and not item.args.get("db") else None) + if body is None: + continue + if not isinstance(body, exp.Select): + return False + if not all(isinstance(p.unalias(), exp.Column) for p in body.expressions): + return False + if any(not isinstance(s, exp.Table) or s.name in ctes for s in _items_of(body)): + return False + return True + + +def _alias_or_column(tree: Any, read: str | None, schema: dict[str, dict[str, str]] | None) -> bool: + """有不带表名、又和这一段的列别名同名的名字,qualify 会把它当成别名展开,库里认的却可能是表的列,这一遍不认列: + + - WHERE、GROUP BY、HAVING 里:库里认的是列还是别名各方言不一样(SQLite 的 GROUP BY、HAVING 都先认列:HAVING + amount > 10 里的 amount 是组里任意一行的 amount,不是 SUM(amount) AS amount),qualify 却一律当成别名(GROUP BY + created_at 成了按 DATE(created_at) 分组)。表结构里有这一段的每张表、这个名字又不是其中哪张表的列时,它只能是 + 别名,照常认。 + - SELECT 列表里引用前面列的别名:SQLite、PostgreSQL、MySQL 都不认,名字指的是表的列(SELECT user_id AS amount, + amount * 2 里的 amount 是表的 amount 列),不是表的列就报错;qualify 没有表结构时却当成别名展开成 user_id * 2, + 有表结构时不是表的列的也展开。表结构里有这一段的每张表、这个名字又是其中的列时,qualify 先认列,照常认。""" + from sqlglot import exp + from sqlglot.optimizer.normalize_identifiers import normalize_identifiers + + columns = {name.lower(): {c.lower() for c in cols} for name, cols in (schema or {}).items()} + copy = normalize_identifiers(tree.copy(), dialect=read) + for select in copy.find_all(exp.Select): + aliases = {p.alias.lower() for p in select.expressions if isinstance(p, exp.Alias) and p.alias} + if not aliases: + continue + items = _items_of(select) + known = all(isinstance(i, exp.Table) and i.name.lower() in columns for i in items) + real = set().union(*(columns[i.name.lower()] for i in items)) if known and items else set() + for key in ("where", "group", "having"): + node = select.args.get(key) + if node is None: + continue + names = {c.name.lower() for c in node.find_all(exp.Column) if not c.table and not _nested(c, node)} + if (names & aliases) - (set() if not known else (aliases - real)): + return True + earlier: set[str] = set() + for projection in select.expressions: + names = {c.name.lower() for c in projection.find_all(exp.Column) + if not c.table and (c is projection or not _nested(c, projection))} + if (names & earlier) - real: + return True + if isinstance(projection, exp.Alias) and projection.alias: + earlier.add(projection.alias.lower()) + return False + + +def _remember_names(tree: Any) -> None: + """CTE、FROM 里的子查询原来的名字记在里面那段查询上(见 _ORIGINAL)。""" + from sqlglot import exp + + for cte in tree.find_all(exp.CTE): + cte.this.meta.setdefault(_ORIGINAL, cte.alias_or_name) + for sub in tree.find_all(exp.Subquery): + if isinstance(sub.parent, (exp.From, exp.Join)) and sub.alias: + sub.this.meta.setdefault(_ORIGINAL, sub.alias) + + +def _inline_ctes(tree: Any) -> None: + """(轻量一级)只引用一次、在 FROM / JOIN 里引用的非递归顶层 CTE 写回引用它的地方,成为 FROM 里的子查询:写成 + CTE 还是子查询是同一个意思。CTE 自己写了列名的、语句里还有别的 WITH 的(名字可能被遮住)不动。""" + from sqlglot import exp + + with_ = tree.args.get("with_") + if with_ is None or with_.args.get("recursive"): + return + if any(isinstance(n, exp.With) for n in tree.walk() if n is not with_): + return + changed = True + while changed: + changed = False + for cte in list(with_.expressions): + name = cte.alias_or_name + if cte.args["alias"].columns: + continue + refs = [t for t in tree.find_all(exp.Table) if t.name == name and not t.args.get("db") + and isinstance(t.this, exp.Identifier)] + if len(refs) != 1 or refs[0].find_ancestor(exp.CTE) is cte or \ + not isinstance(refs[0].parent, (exp.From, exp.Join)): + continue + ref = refs[0] + body = cte.this.copy() + sub = exp.Subquery(this=body, alias=exp.TableAlias(this=exp.to_identifier(ref.alias or name))) + ref.replace(sub) + cte.pop() + changed = True + break + if not with_.expressions: + tree.set("with_", None) + + +def _unquote(tree: Any, read: str | None) -> None: + """不需要的引号去掉:按方言的规则,不加引号时认成的还是同一个名字(PostgreSQL 的 "orders" 就是 orders,"Orders" + 不是)。最后统一引号时需要的再加回来(见 _named)。""" + from sqlglot import exp + from sqlglot.dialects.dialect import Dialect + + dialect = Dialect.get_or_raise(read) + for ident in tree.find_all(exp.Identifier): + if ident.quoted and isinstance(ident.this, str): + bare = exp.Identifier(this=ident.this, quoted=False) + if dialect.normalize_identifier(bare).this == ident.this: + ident.set("quoted", False) + + +#: 括号里是这些(运算符的一部分)时,外面是比较运算的话括号可以去掉:算术比比较结合得紧 +_ARITHMETIC = ("Add", "Sub", "Mul", "Div", "Mod") +_COMPARISON = ("EQ", "NEQ", "GT", "GTE", "LT", "LTE") + + +def _strip_parens(tree: Any) -> None: + """不影响结合的括号去掉:括号外面不是运算符(函数参数、别名、SELECT 列表、ORDER BY、GROUP BY、WHERE 整个条件、 + IN 列表的一项……)、括号里是一列一个常量一次函数调用,或者括号里是算术、外面是比较。""" + from sqlglot import exp + + for paren in list(tree.find_all(exp.Paren)): + inner, parent = paren.this, paren.parent + if inner is None or parent is None or isinstance(inner, (exp.Query, exp.Subquery, exp.Tuple)): + continue + atomic = isinstance(inner, (exp.Column, exp.Literal, exp.Null, exp.Boolean, exp.Paren)) or ( + isinstance(inner, exp.Func) and not isinstance(inner, (exp.Binary, exp.Unary, exp.Predicate))) + outside = isinstance(parent, (exp.Alias, exp.Distinct, exp.Select, exp.Ordered, exp.Where, exp.Having, + exp.Group, exp.Window)) or ( + isinstance(parent, exp.Func) and not isinstance(parent, (exp.Binary, exp.Unary, exp.Predicate))) + loose = type(inner).__name__ in _ARITHMETIC and type(parent).__name__ in _COMPARISON + if isinstance(parent, exp.In) and paren.arg_key == "expressions": + outside = True + if atomic or outside or loose: + paren.replace(inner) + + +def _zero_offset(tree: Any) -> None: + from sqlglot import exp + + for offset in list(tree.find_all(exp.Offset)): + value = offset.expression + if isinstance(value, exp.Literal) and value.is_int and int(value.this) == 0: + offset.pop() + + +def _redundant_distinct(tree: Any, read: str | None) -> None: + """已经按选出来的列分组(每个分组键都原样选出来了)的 SELECT DISTINCT:每组一行、各行的分组键都不同,去重 + 去不掉什么。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + distinct, group = select.args.get("distinct"), select.args.get("group") + if distinct is None or distinct.args.get("on") or group is None or not group.expressions or \ + any(group.args.get(k) for k in ("rollup", "cube", "grouping_sets")): + continue + if any(p.find(exp.Window) for p in select.expressions): + continue + selected = {_fingerprint(p.unalias()) for p in select.expressions} + if all(_fingerprint(k) in selected for k in group.expressions): + select.set("distinct", None) + + +def _narrow_stars(tree: Any, read: str | None) -> None: + """没给表结构时 SELECT * 展不开:从一张实表 SELECT * 的 CTE、FROM 里的子查询(不去重、不分组、不取行、没有窗口 + 函数),外面只按名字引用其中几列的,* 换成用到的那几列(外面有 SELECT * 或 t.* 读它的不换)。不去重时一段出几行 + 和选几列无关,外面用不到的列不影响结果;用到的列那张表里没有的话,原文本来就出错。""" + from sqlglot import exp + + namer = _Namer(tree, read, qualified=True) + ctes = {c.alias_or_name: c for c in tree.find_all(exp.CTE)} + starred: set[int] = set() + for select in tree.find_all(exp.Select): + for p in select.expressions: + if isinstance(p, exp.Star): + starred |= {id(i) for i in _items_of(select)} + elif isinstance(p, exp.Column) and p.is_star: + starred |= {id(i) for i in _items_of(select) if i.alias_or_name == p.table} + bodies: list[tuple[Any, list[Any]]] = [] + for cte in ctes.values(): + refs = [t for s in tree.find_all(exp.Select) for t in _items_of(s) + if isinstance(t, exp.Table) and namer.cte_of.get(id(t)) is cte] + bodies.append((cte.this, refs)) + for sub in tree.find_all(exp.Subquery): + if isinstance(sub.parent, (exp.From, exp.Join)): + bodies.append((sub.this, [sub])) + for body, refs in bodies: + if not isinstance(body, exp.Select) or len(body.expressions) != 1 or \ + not isinstance(body.expressions[0], exp.Star) or not _simple_body(body): + continue + items = _items_of(body) + if len(items) != 1 or not isinstance(items[0], exp.Table) or id(items[0]) in namer.cte_of or \ + any(id(r) in starred for r in refs): + continue + names = sorted({col.name for r in refs for col in namer.refs.get(id(r), [])}) + alias = items[0].alias_or_name + body.set("expressions", [exp.alias_(exp.column(n, table=alias), n, copy=False) for n in names] or + [exp.alias_(exp.Literal.number(1), "_one", copy=False)]) + + +def _unique_aliases(tree: Any, read: str | None) -> None: + """整条语句里每个来源(实表、CTE 的引用)的别名改成互不相同的(表名__序号),引用它的列跟着改:按原来的别名 + 认清每一列属于哪个来源之后再改(见 _Namer),相关子查询里引用外层的列不会认错。""" + from sqlglot import exp + + namer = _Namer(tree, read, qualified=True) + taken = {t.name for t in tree.find_all(exp.Table)} | {c.alias_or_name for c in tree.find_all(exp.CTE)} + n = 0 + for item in [i for s in tree.find_all(exp.Select) for i in _items_of(s)]: + n += 1 + name = f"{item.name if isinstance(item, exp.Table) else 'sub'}__{n}" + while name in taken: + name += "_" + alias = item.args.get("alias") + if isinstance(alias, exp.TableAlias): + alias.set("this", exp.to_identifier(name)) + else: + item.set("alias", exp.TableAlias(this=exp.to_identifier(name))) + for col in namer.refs.get(id(item), []): + col.set("table", exp.to_identifier(name)) + + +def _cte_refs(tree: Any, name: str) -> list[Any]: + """引用顶层 CTE name 的表(没有被里层同名的 CTE 遮住的)。""" + from sqlglot import exp + + out = [] + for table in tree.find_all(exp.Table): + if table.name != name or table.args.get("db") or not isinstance(table.this, exp.Identifier): + continue + shadowed = False + node = table.parent + while node is not None and node is not tree: + with_ = node.args.get("with_") if isinstance(node, exp.Query) else None + if with_ is not None and any(c.alias_or_name == name for c in with_.expressions): + shadowed = True + break + node = node.parent + if not shadowed: + out.append(table) + return out + + +def _simple_body(body: Any) -> bool: + """只是从来源里筛行、算列的一段(没有分组、聚合、去重、取行、窗口函数、UNION):在哪儿算、算几遍结果都一样。""" + from sqlglot import exp + + if not isinstance(body, exp.Select) or body.args.get("with_") is not None: + return False + if any(body.args.get(k) for k in ("group", "having", "distinct", "limit", "offset", "qualify", "windows")): + return False + return not any(isinstance(n, (exp.AggFunc, exp.Window)) for p in body.expressions for n in p.walk()) + + +def _split_simple_ctes(tree: Any) -> None: + """引用了好几次的「简单」CTE(见 _simple_body)每处引用各复制一份:merge_subqueries 只并只引用一次的 CTE, + 复制之后每处都能并进去。CTE 是确定的(有不确定函数的语句不走这一级),算几遍结果一样。并不进去的几份定义 + 一模一样,起名时又合回一个(见 _Namer._name_ctes)。""" + from sqlglot import exp + + with_ = tree.args.get("with_") + if with_ is None: + return + taken = {c.alias_or_name for c in with_.expressions} | {t.name for t in tree.find_all(exp.Table)} + for cte in list(with_.expressions): + name = cte.alias_or_name + refs = _cte_refs(tree, name) + if len(refs) < 2 or cte.args["alias"].columns or not _simple_body(cte.this): + continue + position = with_.expressions.index(cte) + for i, ref in enumerate(refs[1:], 2): + copy_name = f"{name}_{i}" + while copy_name in taken: + copy_name += "_" + taken.add(copy_name) + twin = exp.CTE(this=cte.this.copy(), alias=exp.TableAlias(this=exp.to_identifier(copy_name))) + with_.expressions.insert(position + i - 1, twin) + twin.parent = with_ + alias = ref.alias_or_name + ref.set("this", exp.to_identifier(copy_name)) + if not ref.alias: + ref.set("alias", exp.TableAlias(this=exp.to_identifier(alias))) + with_.set("expressions", list(with_.expressions)) + + +def _push_into_groups(tree: Any) -> None: + """外层 WHERE 里只涉及一个分组 CTE(只引用一次、在开头那组内连接里)的条件,换成 CTE 里对应的表达式,挪进它的 + HAVING:每一行就是一组,筛行和筛组是一回事。CTE 有窗口函数、取行、去重、没写成分组键也没聚合的列时不挪。 + 然后(各段都做)HAVING 里只涉及分组键、不含聚合的条件挪进 WHERE:同一组的分组键都一样,先筛后筛一样。两步 + 之后,这类条件写在里层 WHERE、HAVING 还是外层 WHERE 都落到同一处。""" + from sqlglot import exp + + with_ = tree.args.get("with_") + ctes = {c.alias_or_name: c for c in (with_.expressions if with_ is not None else [])} + for select in list(tree.find_all(exp.Select)): + where = select.args.get("where") + joins = select.args.get("joins") or [] + if where is None or select.args.get("from_") is None or any(not _inner(j) for j in joins): + continue + for item in _items_of(select): + cte = ctes.get(item.name) if isinstance(item, exp.Table) and not item.args.get("db") else None + if cte is None or len(_cte_refs(tree, item.name)) != 1: + continue + body = cte.this + if not isinstance(body, exp.Select) or body.args.get("group") is None or \ + any(body.args.get(k) for k in ("limit", "offset", "distinct", "qualify", "windows")) or \ + any(n.find(exp.Window) for n in body.expressions): + continue + outputs = {p.alias_or_name: p.unalias() for p in body.expressions} + keys = {_fingerprint(k) for k in body.args["group"].expressions} + alias = item.alias_or_name + moved, kept = [], [] + for cond in _conjuncts(where.this): + cols = list(cond.find_all(exp.Column)) + if not cols or cond.find(exp.Select, exp.Subquery) is not None or \ + any(c.table != alias or c.name not in outputs for c in cols) or \ + not all(_grouped(outputs[c.name], keys) for c in cols): + kept.append(cond) + continue + new = cond.copy() + for col in list(new.find_all(exp.Column)): + col.replace(exp.paren(outputs[col.name].copy(), copy=False)) + moved.append(new) + if moved: + having = body.args.get("having") + body.set("having", exp.Having(this=_and_all(([having.this] if having else []) + moved))) + select.set("where", exp.Where(this=_and_all(kept)) if kept else None) + where = select.args.get("where") + if where is None: + break + for select in list(tree.find_all(exp.Select)): + having, group = select.args.get("having"), select.args.get("group") + if having is None or group is None or any(group.args.get(k) for k in ("rollup", "cube", "grouping_sets")): + continue + keys = {_fingerprint(k) for k in group.expressions if isinstance(k, exp.Column)} + moved, kept = [], [] + for cond in _conjuncts(having.this): + cols = list(cond.find_all(exp.Column)) + if cols and all(_fingerprint(c) in keys for c in cols) and not any( + isinstance(n, (exp.AggFunc, exp.Window, exp.Select, exp.Subquery)) for n in cond.walk()): + moved.append(cond) + else: + kept.append(cond) + if moved: + where = select.args.get("where") + select.set("where", exp.Where(this=_and_all(([where.this] if where else []) + moved))) + select.set("having", exp.Having(this=_and_all(kept)) if kept else None) + + +def _grouped(expression: Any, keys: set[str]) -> bool: + """分组 CTE 选出来的一列在每组里只有一个值:整个是分组键,或者每一列都在聚合函数里、或者本身是分组键。""" + from sqlglot import exp + + if _fingerprint(expression) in keys: + return True + for col in expression.find_all(exp.Column): + if _fingerprint(col) not in keys and col.find_ancestor(exp.AggFunc) is None: + return False + return not any(isinstance(n, exp.Window) for n in expression.walk()) + + +def _unwrap_passthrough(tree: Any, read: str | None) -> Any: + """原样选出一个子查询(FROM 里的子查询,或只引用一次的 CTE)全部列、什么别的都不做的一段(SELECT t.a, t.b + FROM (…) t),换成那个子查询:出的行一样,列的先后、名字也一样。最外层和条件里的子查询列名不影响取值,列名 + 可以不同;别的地方(FROM 里的子查询、CTE)外面按名字引用,名字也得一样。 + + 子查询是 UNION 这类合并的,SQLite(和不知道是哪种库时)不换:SQLite 从子查询里读合并的结果时,按第一支那一列的 + 类型亲和转换别的支的值(REAL 列在前,另一支的整数 2 读出来成了 2.0,像数字的文本也会被转成数),套不套这一层 + 结果不一样。""" + from sqlglot import exp + + def outputs(query: Any) -> list[str] | None: + if isinstance(query, exp.SetOperation) and read not in (None, "sqlite"): + leaves = _leaves(query) + return outputs(leaves[0]) if leaves else None + if not isinstance(query, exp.Select) or _starry(query): + return None + names = [p.alias_or_name for p in query.expressions] + return names if all(names) and len(set(names)) == len(names) else None + + changed = True + while changed: + changed = False + for select in list(tree.find_all(exp.Select)): + if select.args.get("joins") or select.args.get("from_") is None or any( + select.args.get(k) for k in ("where", "group", "having", "order", "limit", "offset", "distinct", + "qualify", "windows", "hint")): + continue + item = select.args["from_"].this + with_ = tree.args.get("with_") + cte = None + if isinstance(item, exp.Subquery): + body = item.this + elif isinstance(item, exp.Table) and with_ is not None and not item.args.get("db"): + cte = next((c for c in with_.expressions if c.alias_or_name == item.name), None) + if cte is None or len(_cte_refs(tree, item.name)) != 1 or select.find_ancestor(exp.CTE) is cte: + continue + body = cte.this + else: + continue + names = outputs(body) + alias = item.alias_or_name + cols = [p.unalias() for p in select.expressions] + if names is None or len(cols) != len(names) or body.args.get("with_") is not None or any( + not isinstance(c, exp.Column) or c.name != n or (c.table and c.table != alias) + for c, n in zip(cols, names)): + continue + renamed = any(p.alias_or_name != n for p, n in zip(select.expressions, names)) + parent = select.parent + loose = select is tree or isinstance(parent, (exp.In, exp.Exists)) or ( + isinstance(parent, exp.Subquery) and not isinstance(parent.parent, (exp.From, exp.Join, exp.CTE))) + if renamed and not loose: + continue + new = body.copy() + if select is tree: + if select.args.get("with_") is not None: + new.set("with_", select.args["with_"]) + tree = new + else: + select.replace(new) + if cte is not None: + cte.pop() + with_ = tree.args.get("with_") + if with_ is not None and not with_.expressions: + tree.set("with_", None) + changed = True + break + return tree + + +# ---- 和名字无关的改写 + +def _items_of(select: Any) -> list[Any]: + first = select.args.get("from_") + return ([first.this] if first is not None else []) + [j.this for j in select.args.get("joins") or []] + + +def _and_all(items: list[Any]) -> Any: + from sqlglot import exp + + if not items: + return None + return exp.and_(*items, copy=False) if len(items) > 1 else items[0] + + +def _plain_join(join: Any) -> bool: + return not (join.args.get("using") or join.args.get("method") or join.args.get("natural")) + + +def _comma(join: Any) -> bool: + """开头那组内连接里的一项:逗号连接(或者没有条件的 CROSS JOIN,sqlglot 把逗号连接也读成 CROSS),条件都在 WHERE + 里(见 _flatten_inner)。规范形式里写成逗号连接:SQLite 不会调换 CROSS JOIN 的先后,按它执行规范形式会很慢。""" + return _plain_join(join) and not join.side and str(join.kind or "").upper() in ("", "CROSS") and \ + join.args.get("on") is None + + +def _inner(join: Any) -> bool: + return _plain_join(join) and not join.side and str(join.kind or "").upper() in ("", "INNER", "CROSS") + + +def _constant(node: Any) -> bool: + from sqlglot import exp + + return isinstance(node, (exp.Literal, exp.Null, exp.Boolean)) or ( + isinstance(node, exp.Neg) and isinstance(node.this, exp.Literal)) + + +def _strip_asc(tree: Any) -> None: + from sqlglot import exp + + for o in tree.find_all(exp.Ordered): + if o.args.get("desc") is False: + o.set("desc", None) + + +def _count_star(tree: Any) -> None: + """COUNT(1)、COUNT('x'):数的是不为空的常量,就是行数。COUNT(DISTINCT 1) 不是(有行就是 1)。""" + from sqlglot import exp + + for c in list(tree.find_all(exp.Count)): + if isinstance(c.this, exp.Literal) and not c.args.get("distinct"): + c.set("this", exp.Star()) + + +def _in_lists(tree: Any, read: str | None) -> None: + """IN 列表是个集合:去重、按写法排序;只有一项的写成 =(NOT IN 跟着成了 <>,空值的结果一样)。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.In)): + if not node.expressions or node.args.get("query") or node.args.get("unnest") or node.args.get("field"): + continue + uniq = {_key(i, read): i for i in node.expressions} + ordered = [uniq[k] for k in sorted(uniq)] + if len(ordered) == 1: + node.replace(exp.EQ(this=node.this, expression=ordered[0])) + else: + node.set("expressions", ordered) + + +def _aggregate_free(select: Any) -> bool: + from sqlglot import exp + + return not any(isinstance(n, exp.AggFunc) and not n.find_ancestor(exp.Window) + for p in select.expressions for n in p.walk()) + + +def _exists_projection(tree: Any) -> None: + """EXISTS 只看有没有行:选什么、去不去重、怎么排(取行也只看行数)都不影响,统一写成 SELECT 1。没有 GROUP BY + 却有聚合(总出一行)的、有 HAVING 的(HAVING、GROUP BY 1 可能引用选出来的列)不动。""" + from sqlglot import exp + + for ex in list(tree.find_all(exp.Exists)): + sub = ex.this.this if isinstance(ex.this, exp.Subquery) else ex.this + if not isinstance(sub, exp.Select) or sub.args.get("having") or \ + (not sub.args.get("group") and not _aggregate_free(sub)): + continue + distinct = sub.args.get("distinct") + if distinct is not None and not distinct.args.get("on"): + sub.set("distinct", None) + sub.set("expressions", [exp.Literal.number(1)]) + sub.set("order", None) + + +def _in_subquery_distinct(tree: Any) -> None: + """IN (子查询) 只看取值的集合:里面的 DISTINCT、不取行时的 ORDER BY 不影响(NOT IN 也一样,空值还在)。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.In)): + query = node.args.get("query") + sub = query.this if isinstance(query, exp.Subquery) else query + if not isinstance(sub, exp.Select) or any(sub.args.get(k) for k in ("limit", "offset", "fetch")): + continue + sub.set("order", None) + distinct = sub.args.get("distinct") + if distinct is not None and not distinct.args.get("on") and not sub.args.get("group") and _aggregate_free(sub): + sub.set("distinct", None) + + +def _distinct_to_group(tree: Any, read: str | None) -> None: + """SELECT DISTINCT a, b(没有分组、聚合、窗口函数)和 GROUP BY a, b 出的行一样(空值也算一组)。ORDER BY 用到 + 没选出来的列时不改(两种写法对这种排序的规矩不一样);没有 ORDER BY 的 LIMIT 不改(DISTINCT 按扫到的先后出行, + GROUP BY 按分组键排好了出,取到的行不同)。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + distinct = select.args.get("distinct") + if distinct is None or distinct.args.get("on") or select.args.get("group") or select.args.get("having"): + continue + if _starry(select) or _takes_any(select) or not _aggregate_free(select) or \ + any(p.find(exp.Window, exp.Select, exp.Subquery) for p in select.expressions): + continue + keys = [p.unalias().copy() for p in select.expressions] + if any(_constant(k) for k in keys): + continue + order = select.args.get("order") + if order is not None: + allowed = {_fingerprint(k) for k in keys} + aliases = {p.alias for p in select.expressions if p.alias} + if any(_fingerprint(o.this) not in allowed and not ( + isinstance(o.this, exp.Column) and not o.this.table and o.this.name in aliases) + for o in order.expressions): + continue + group = exp.Group(expressions=keys) + group.meta[_FROM_DISTINCT] = True + select.set("distinct", None) + select.set("group", group) + + +def _right_to_left(tree: Any) -> None: + """只有两个来源的 a RIGHT JOIN b ON p 就是 b LEFT JOIN a ON p。选的是 * 时列的先后跟着变、没有 ORDER BY 的 LIMIT + 取先扫到的几行,不改。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + if len(joins) != 1 or str(joins[0].side or "").upper() != "RIGHT" or not _plain_join(joins[0]) or \ + _starry(select) or _takes_any(select): + continue + first = select.args["from_"] + left, right = first.this, joins[0].this + first.set("this", right.copy()) + joins[0].set("this", left.copy()) + joins[0].set("side", "LEFT") + joins[0].set("kind", None) + joins[0].meta[_FLIPPED] = True + for join in tree.find_all(exp.Join): + if join.side and str(join.kind or "").upper() == "OUTER": + join.set("kind", None) + + +def _null_propagating(node: Any, alias: str) -> bool: + """alias 那一侧整行是空值时,node 一定是空值:那一侧的列,或者对它做比较、算术、取大小写、转类型。""" + from sqlglot import exp + + if isinstance(node, exp.Column): + return bool(node.table) and node.table == alias + if type(node).__name__ in _STRICT_UNARY: + return _null_propagating(node.this, alias) + if type(node).__name__ in _STRICT_BINARY: + return _null_propagating(node.left, alias) or _null_propagating(node.right, alias) + return False + + +def _rejects(cond: Any, alias: str) -> bool: + """alias 那一侧整行是空值时 cond 不成立(是空值或假):比较、LIKE、IN、BETWEEN 的一边一定是空值,IS NOT NULL, + 或者 AND 里有一项不成立、OR 里每一项都不成立。IS NULL、COALESCE、CASE 不算。""" + from sqlglot import exp + + if isinstance(cond, exp.Paren): + return _rejects(cond.this, alias) + if isinstance(cond, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.Like, exp.ILike)): + return _null_propagating(cond.left, alias) or _null_propagating(cond.right, alias) + if isinstance(cond, (exp.In, exp.Between)): + return _null_propagating(cond.this, alias) + if isinstance(cond, exp.Not): + inner = cond.this.unnest() if isinstance(cond.this, exp.Paren) else cond.this + if isinstance(inner, exp.Is) and isinstance(inner.expression, exp.Null): + return _null_propagating(inner.this, alias) + if isinstance(inner, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.Like, exp.ILike)): + return _null_propagating(inner.left, alias) or _null_propagating(inner.right, alias) + return False + if isinstance(cond, exp.And): + return _rejects(cond.left, alias) or _rejects(cond.right, alias) + if isinstance(cond, exp.Or): + return _rejects(cond.left, alias) and _rejects(cond.right, alias) + return False + + +def _reject_nulls(tree: Any) -> None: + """LEFT JOIN 关联进来的那一侧,同一段 WHERE 里有把它整行为空的行筛掉的条件(见 _rejects):没关联上的行反正留 + 不下来,这次外连接就是内连接。有 RIGHT / FULL JOIN 的段不动。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + where = select.args.get("where") + if where is None or any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + conds = _conjuncts(where.this) + for join in joins: + if str(join.side or "").upper() != "LEFT" or not _plain_join(join): + continue + alias = join.this.alias_or_name + if alias and any(_rejects(c, alias) for c in conds): + join.set("side", None) + join.set("kind", None) + join.this.meta[_REJECTED] = "RIGHT JOIN" if join.meta.get(_FLIPPED) else "LEFT JOIN" + + +def _flatten_inner(tree: Any) -> None: + """内连接的 ON 条件挪进 WHERE:开头连续的内连接(和后面只引用它们的内连接)是一组来源,写成逗号连接,条件写在 + ON 还是 WHERE 都一样,来源的先后也不影响结果(后面的 LEFT JOIN 只保留左边的行,左边先筛后筛一样)。有 RIGHT / + FULL JOIN、USING、NATURAL 的段不动;引用了 LEFT JOIN 进来的表的内连接留在原处。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + if not joins or select.args.get("from_") is None: + continue + if any(not _plain_join(j) or str(j.side or "").upper() in ("RIGHT", "FULL") or + str(j.kind or "").upper() in ("SEMI", "ANTI", "ASOF") for j in joins): + continue + prefix = [select.args["from_"].this] + lefts: list[Any] = [] + tail: list[Any] = [] + moved: list[Any] = [] + for join in joins: + if tail: + tail.append(join) + continue + if _inner(join): + on = join.args.get("on") + refs = {c.table for c in (on.find_all(exp.Column) if on is not None else []) if c.table} + if refs & {j.this.alias_or_name for j in lefts}: + tail.append(join) + continue + prefix.append(join.this) + moved += _conjuncts(on) if on is not None else [] + else: + lefts.append(join) + select.args["from_"].set("this", prefix[0]) + select.set("joins", [exp.Join(this=s) for s in prefix[1:]] + lefts + tail) + if moved: + where = select.args.get("where") + select.set("where", exp.Where(this=_and_all((_conjuncts(where.this) if where else []) + moved))) + + +def _order_by_aliases(tree: Any) -> None: + """最外层 ORDER BY 引用 SELECT 的别名的,换成那个表达式:别名接下来要去掉。不换的:别名指的是数字常量(ORDER BY 1 + 是按第一列排);别名指的是一个不带表名、又和某个别名同名的列(ORDER BY 里单独一个名字先认别名:SELECT user_id AS + amount, amount AS user_id … ORDER BY amount 换成 ORDER BY user_id 就成了按第二列排)。""" + from sqlglot import exp + + if not isinstance(tree, exp.Select) or tree.args.get("order") is None: + return + aliases = {p.alias: p for p in tree.expressions if p.alias} + for o in tree.args["order"].expressions: + key = o.this + if not (isinstance(key, exp.Column) and not key.table and key.name in aliases): + continue + target = aliases[key.name].unalias() + if target.is_number or (isinstance(target, exp.Column) and not target.table and target.name in aliases): + continue + o.set("this", target.copy()) + + +def _drop_aliases(query: Any) -> None: + from sqlglot import exp + + if not isinstance(query, exp.Select): + return + inside = {id(n) for p in query.expressions for n in p.walk()} + used = {n.name for n in _block_nodes(query) if isinstance(n, exp.Column) and not n.table and id(n) not in inside} + query.set("expressions", [p.unalias() if isinstance(p, exp.Alias) and p.alias not in used else p + for p in query.expressions]) + + +def _drop_output_aliases(tree: Any) -> None: + """最外层(不是 UNION 时)和条件里的子查询选出来的列名不影响任何取值,去掉;本段里还按名字引用的留着。""" + from sqlglot import exp + + if isinstance(tree, exp.SetOperation): + if tree.args.get("order") is None: + for leaf in _leaves(tree): + _drop_aliases(leaf) + else: + _drop_aliases(tree) + for node in tree.find_all(exp.Exists, exp.In): + query = node.this if isinstance(node, exp.Exists) else node.args.get("query") + _drop_aliases(query.this if isinstance(query, exp.Subquery) else query) + for sub in tree.find_all(exp.Subquery): + if not isinstance(sub.parent, (exp.From, exp.Join, exp.Exists, exp.In, exp.CTE)) and not sub.alias: + _drop_aliases(sub.this) + + +def _leaves(query: Any) -> list[Any]: + """一棵 UNION / INTERSECT / EXCEPT 树的各支(套着括号的也拆开)。""" + from sqlglot import exp + + if isinstance(query, exp.Subquery): + return _leaves(query.this) + if isinstance(query, exp.SetOperation): + return _leaves(query.left) + _leaves(query.right) + return [query] + + +def _block_nodes(select: Any) -> Iterator[Any]: + """这一段自己的节点:不进套着的查询;FROM / JOIN 的来源不算(ON 条件算)。""" + from sqlglot import exp + + stack = [v for k, value in select.args.items() if k not in ("with_", "from_") + for v in (value if isinstance(value, list) else [value]) if isinstance(v, exp.Expr)] + while stack: + node = stack.pop() + if isinstance(node, (exp.Query, exp.Subquery)): + continue + if isinstance(node, exp.Join): + if node.args.get("on") is not None: + stack.append(node.args["on"]) + continue + yield node + stack += [v for value in node.args.values() for v in (value if isinstance(value, list) else [value]) + if isinstance(v, exp.Expr)] + + +def _nested_queries(select: Any) -> list[Any]: + """这一段自己的表达式里直接套着的查询(条件、SELECT 列表、ON 里的子查询),不含 FROM 里的子查询。""" + from sqlglot import exp + + out: list[Any] = [] + stack = [v for k, value in select.args.items() if k not in ("with_", "from_") + for v in (value if isinstance(value, list) else [value]) if isinstance(v, exp.Expr)] + while stack: + node = stack.pop() + if isinstance(node, exp.Subquery): + out.append(node.this) + elif isinstance(node, exp.Query): + out.append(node) + elif isinstance(node, exp.Join): + if node.args.get("on") is not None: + stack.append(node.args["on"]) + else: + stack += [v for value in node.args.values() for v in (value if isinstance(value, list) else [value]) + if isinstance(v, exp.Expr)] + return out + + +# ---- 化简:sqlglot 的 simplify 去掉和类型、排序规则、方言有关的几种 + +def _simplify(tree: Any, read: str | None) -> Any: + """sqlglot 的 simplify(AND / OR 去重排序、吸收律、德摩根、BETWEEN 拆开、常量折叠、比较的方向……),去掉三种: + + - 同一列上的几个比较合成一个(x > 5 AND x > 3 → x > 5)、字符串常量之间的比较:结果取决于列的类型和排序规则 + (SQLite 里 TEXT 列和数字比是按文本比,PostgreSQL 的语言排序规则下 'a' < 'B'); + - 等式两边移项(x + 1 = 3 → x = 2):TEXT 列在 SQLite 里不成立; + - DATE_TRUNC 比较改成区间:取决于方言的日期语义。 + + 另外两边都写了 COLLATE 的比较不对调两边(SQLite 用左边那个排序规则)。 + """ + from sqlglot import exp + from sqlglot.optimizer.simplify import Simplifier + + class _Safe(Simplifier): + def _simplify_comparison(self, expression, left, right, or_=False): # noqa: ANN001 + return None + + def sort_comparison(self, expression): # noqa: ANN001 + if isinstance(expression, exp.Binary) and expression.this.find(exp.Collate) is not None and \ + expression.expression.find(exp.Collate) is not None: + return expression + return super().sort_comparison(expression) + + def simplify_equality(self, expression): # noqa: ANN001 + return expression + + def simplify_datetrunc(self, expression): # noqa: ANN001 + return expression + + def _simplify_binary(self, expression, a, b): # noqa: ANN001 + if isinstance(expression, self.COMPARISONS) and not isinstance(expression, exp.Is) and \ + not (a.is_number and b.is_number): + return None + return super()._simplify_binary(expression, a, b) + + return _Safe(dialect=read).simplify(tree) + + +# ---- 起规范的名字 + +@dataclass +class _Hint: + """按另一边起名(差异说明用,见 _Explainer.statement):定义一样的 CTE 叫另一边的名字;names 是这一边原来的 + CTE 名 → 另一边的名字(两边各剩一个配不上的 CTE 时配成一对);taken 是另一边用过的名字,配不上的另起名字时 + 避开。""" + + bodies: dict[str, str] = field(default_factory=dict) + names: dict[str, str] = field(default_factory=dict) + taken: set[str] = field(default_factory=set) + + +class _Namer: + """给一条语句起规范的名字(见 _finish 第 3 步)。 + + 先按原来的别名认清每一列属于哪个来源(相关子查询里的列到外层找),记成「来源 → 引用它的列」;之后改来源的别名 + 时把这些列一起改掉,不再按名字找,换名不会把列认错。别名带层数:内外层的别名不会同名,相关子查询里引用外层的 + 列不会被里层遮住;同一层的兄弟子查询各自看不见对方,可以重名。""" + + def __init__(self, tree: Any, read: str | None, qualified: bool, hint: _Hint | None = None) -> None: + from sqlglot import exp + + self.tree = tree + self.read = read + self.qualified = qualified + self.hint = hint + self.refs: dict[int, list[Any]] = {} + self.cte_of: dict[int, Any] = {} + self.base_names = {t.name.lower() for t in tree.find_all(exp.Table) if t.name} + self.budget = _ORDERING_BUDGET + self._resolve(tree, [], {}) + + def _text(self, node: Any) -> tuple[str, str]: + return _key(node, self.read) + + def _resolve(self, query: Any, frames: list[dict[str, Any]], ctes: dict[str, Any]) -> None: + from sqlglot import exp + + if isinstance(query, exp.Subquery): + self._resolve(query.this, frames, ctes) + return + with_ = query.args.get("with_") + if with_ is not None: + ctes = dict(ctes) + for cte in with_.expressions: + self._resolve(cte.this, frames, ctes) + ctes[cte.alias_or_name] = cte + if isinstance(query, exp.SetOperation): + self._resolve(query.left, frames, ctes) + self._resolve(query.right, frames, ctes) + return + if not isinstance(query, exp.Select): + raise _Unsupported(type(query).__name__) + frame: dict[str, Any] = {} + for item in _items_of(query): + if isinstance(item, exp.Subquery): + self._resolve(item.this, frames, ctes) + elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier): + if not item.args.get("db") and item.name in ctes: + self.cte_of[id(item)] = ctes[item.name] + else: + raise _Unsupported(type(item).__name__) + name = item.alias_or_name + if not name or name in frame: + raise _Unsupported("来源没有名字或重名") + frame[name] = item + stack = frames + [frame] + for node in _block_nodes(query): + if isinstance(node, exp.Column) and node.table: + for f in reversed(stack): + if node.table in f: + self.refs.setdefault(id(f[node.table]), []).append(node) + break + for sub in _nested_queries(query): + self._resolve(sub, stack, ctes) + + def _alias(self, base: str, depth: int, k: int) -> str: + name = base if depth == 0 else f"{base}@{depth}" + if k > 1: + name = f"{name}#{k}" + while (depth or k > 1) and name.lower() in self.base_names: + name += "#" + return name + + def canon(self, query: Any, depth: int, cte_names: dict[int, str]) -> tuple[str, str]: + from sqlglot import exp + + if isinstance(query, exp.Subquery): + return self.canon(query.this, depth, cte_names) + with_ = query.args.get("with_") + if with_ is not None: + cte_names = dict(cte_names) + self._name_ctes(with_, depth, cte_names) + if isinstance(query, exp.SetOperation): + self._canon_setop(query, depth, cte_names) + return self._text(query) + return self._canon_select(query, depth, cte_names) + + def _name_ctes(self, with_: Any, depth: int, cte_names: dict[int, str]) -> None: + """CTE 按依赖分层,同一层按规范化后的定义排序起名;定义一模一样的合成一个(引用它们的都改叫同一个名字)。""" + from sqlglot import exp + + hint = self.hint if depth == 0 else None + pending = list(with_.expressions) + kept: list[Any] = [] + by_text: dict[tuple[str, str], str] = {} + prefix = "cte" if depth == 0 else f"cte@{depth}_" + n = 0 + while pending: + names = {c.alias_or_name for c in pending} + ready = [c for c in pending if not any(t.name in names and t.name != c.alias_or_name and not t.args.get("db") + for t in c.this.find_all(exp.Table))] + if not ready: + raise _Unsupported("CTE 互相引用") + texts = [] + for cte in ready: + original = cte.alias_or_name + body = self.canon(cte.this, depth + 1 if depth else 0, cte_names) + if self.qualified and self._rename_columns(cte): + body = self._text(cte.this) + texts.append((body, original, cte)) + texts.sort(key=lambda x: x[0]) + for body, original, cte in texts: + pending.remove(cte) + if body in by_text: + cte_names[id(cte)] = by_text[body] + continue + name = (hint.names.get(original) or hint.bodies.get(body[1])) if hint else None + while name is None or name.lower() in self.base_names or name in by_text.values(): + n += 1 + name = f"{prefix}{n}" + if hint and name in hint.taken: + name = None + cte_names[id(cte)] = name + by_text[body] = name + cte.set("alias", exp.TableAlias(this=exp.to_identifier(name))) + kept.append(cte) + with_.set("expressions", kept) + + def _rename_columns(self, cte: Any) -> bool: + """CTE 选出来的列按定义排序后叫 c1、c2…,引用它们的列跟着改:外面按名字引用,名字是各自起的(total 还是 + sum_amt),先后也不影响。CTE 自己写了列名、是 UNION、列名重复、本段里还按名字引用自己的列的不改。""" + from sqlglot import exp + + body = cte.this + if not isinstance(body, exp.Select) or cte.args["alias"].columns: + return False + projections = body.expressions + names = [p.alias_or_name for p in projections] + if not projections or any(not n or isinstance(p, exp.Star) for n, p in zip(names, projections)) or \ + len(set(names)) != len(names): + return False + inside = {id(n) for p in projections for n in p.walk()} + if {n.name for n in _block_nodes(body) if isinstance(n, exp.Column) and not n.table and id(n) not in inside} & \ + set(names): + return False + # GROUP BY 2、ORDER BY 1 按位置引用选出来的列:重排之后就指到别的列上了 + for clause in ("group", "order"): + node = body.args.get(clause) + keys = [k.this if isinstance(k, exp.Ordered) else k for k in (node.expressions if node is not None else [])] + if any(k.is_number for k in keys): + return False + refs = [t for t in self.tree.find_all(exp.Table) if self.cte_of.get(id(t)) is cte] + ordered = sorted(projections, key=lambda p: self._text(p.unalias())) + mapping = {p.alias_or_name: f"c{i}" for i, p in enumerate(ordered, 1)} + body.set("expressions", [exp.alias_(p.unalias().copy(), mapping[p.alias_or_name], copy=False) + for p in ordered]) + for table in refs: + for col in self.refs.get(id(table), []): + if col.name in mapping: + col.set("this", exp.to_identifier(mapping[col.name])) + return True + + def _canon_setop(self, query: Any, depth: int, cte_names: dict[int, str]) -> None: + """连着同一种合并写的几支各自规范化;调换先后不改变结果的(见 _reorderable)再按规范化后的写法排序。""" + kind = (type(query), bool(query.args.get("distinct"))) + branches: list[Any] = [] + + def collect(node: Any) -> None: + from sqlglot import exp + + if isinstance(node, exp.SetOperation) and (type(node), bool(node.args.get("distinct"))) == kind and \ + not any(node.args.get(k) for k in ("order", "limit", "offset", "with_")): + collect(node.left) + collect(node.right) + else: + branches.append(node) + + collect(query.left) + collect(query.right) + texts = [(self.canon(b, depth, cte_names), i, b) for i, b in enumerate(branches)] + if len(branches) < 2 or not self._reorderable(query): + return + ordered = [b for _, _, b in sorted(texts, key=lambda x: (x[0], x[1]))] + node = ordered[0] + for b in ordered[1:-1]: + node = type(query)(this=node, expression=b, distinct=query.args.get("distinct")) + query.set("this", node) + query.set("expression", ordered[-1]) + + def _reorderable(self, query: Any) -> bool: + """这一处合并的各支调换先后,结果(行、值、排序)一样不一样。只有 UNION ALL 可能一样: + + - UNION(去重)、INTERSECT 留先出现的那个值,相等不等于一样:SQLite 里 REAL 列的 2.0 和 INTEGER 列的 2、MySQL + 默认不分大小写的排序规则下的 'SH' 和 'sh'、PostgreSQL numeric 的 2 和 2.00;EXCEPT 本来就分左右。 + - 这一层自己带 LIMIT / OFFSET 的,取的是先出来的几行(没有 ORDER BY 时就看哪支在前)。 + - ORDER BY 里有按名字排的:合并结果的列名取第一支的,调换之后 ORDER BY id 排的可能是另一列。 + - SQLite(没说方言的也按它)合并结果的列的排序规则取最左一支那列声明的(排序、从子查询读出来再比较都用它), + 表结构里看不到列声明的排序规则:只排整条语句最外层、没有 ORDER BY 的。""" + from sqlglot import exp + + if not isinstance(query, exp.Union) or query.args.get("distinct"): + return False + if query.args.get("limit") is not None or query.args.get("offset") is not None: + return False + order = query.args.get("order") + keys = [o.this if isinstance(o, exp.Ordered) else o for o in (order.expressions if order is not None else [])] + if any(not k.is_number for k in keys): + return False + if self.read in (None, "sqlite"): + if keys: + return False + top = query + while isinstance(top.parent, exp.Subquery): + top = top.parent + if top is not self.tree: + return False + return True + + def _signature(self, item: Any, cte_names: dict[int, str], subs: dict[int, str]) -> tuple[str, str]: + """来源的 (排序用的标识, 起别名用的名字):实表是表名,CTE 是它的规范名,FROM 里的子查询是规范化后的写法。""" + from sqlglot import exp + + if isinstance(item, exp.Table): + cte = self.cte_of.get(id(item)) + if cte is not None: + name = cte_names.get(id(cte), cte.alias_or_name) + return f"cte:{name}", name + return "table:" + ".".join(p for p in (item.catalog, item.db, item.name) if p), item.name + return f"sub:{subs[id(item)][1]}", "sub" + + def _canon_select(self, select: Any, depth: int, cte_names: dict[int, str]) -> tuple[str, str]: + from sqlglot import exp + + items = _items_of(select) + subs: dict[int, tuple[str, str]] = {} + for item in items: + if isinstance(item, exp.Subquery): + subs[id(item)] = self.canon(item.this, depth + 1, cte_names) + elif id(item) in self.cte_of: + cte = self.cte_of[id(item)] + item.set("this", exp.to_identifier(cte_names.get(id(cte), cte.alias_or_name))) + keys = {id(i): self._signature(i, cte_names, subs) for i in items} + joins = select.args.get("joins") or [] + n_prefix = 0 + if items: + n_prefix = 1 + for join in joins: + if _comma(join): + n_prefix += 1 + else: + break + prefix, rest = items[:n_prefix], joins[max(n_prefix - 1, 0):] if items else joins + movable = not _starry(select) and not _unstable(select) + free_lefts = movable and bool(rest) and self._independent_lefts(prefix, rest) + + def groups(xs: list[Any], key: Any) -> list[list[Any]]: + xs = sorted(xs, key=key) + return [list(g) for _, g in itertools.groupby(xs, key=key)] + + prefix_groups = groups(prefix, lambda s: keys[id(s)][0]) if movable else [[s] for s in prefix] + rest_groups = groups(rest, lambda j: keys[id(j.this)][0]) if free_lefts else [[j] for j in rest] + options = [[list(p) for p in itertools.permutations(g)] if len(g) > 1 else [g] + for g in prefix_groups + rest_groups] + total = 1 + for o in options: + total *= len(o) + if total > _MAX_ORDERINGS or total > self.budget: + options = [[o[0]] for o in options] + total = 1 + self.budget -= total + combos = list(itertools.product(*options)) + best: tuple[tuple[str, str], int] | None = None + for index, combo in enumerate(combos): + text = self._apply(select, depth, cte_names, keys, combo, len(prefix_groups)) + if best is None or text < best[0]: + best = (text, index) + if best is None: + return self._text(select) + if len(combos) > 1: + self._apply(select, depth, cte_names, keys, combos[best[1]], len(prefix_groups)) + return best[0] + + @staticmethod + def _independent_lefts(prefix: list[Any], rest: list[Any]) -> bool: + """跟在一组内连接后面的几个 LEFT JOIN,各自的 ON 只引用这组来源和自己(不引用别的 LEFT JOIN 进来的表、没有 + 子查询):它们之间谁先谁后结果一样。""" + from sqlglot import exp + + names = {s.alias_or_name for s in prefix} + for join in rest: + if str(join.side or "").upper() != "LEFT" or not _plain_join(join): + return False + on = join.args.get("on") + if on is None or on.find(exp.Select, exp.Subquery) is not None: + return False + if not {c.table for c in on.find_all(exp.Column)} <= names | {join.this.alias_or_name}: + return False + return True + + def _apply(self, select: Any, depth: int, cte_names: dict[int, str], keys: dict[int, tuple[str, str]], + combo: tuple[list[Any], ...], n_prefix_groups: int) -> tuple[str, str]: + from sqlglot import exp + + prefix = [s for g in combo[:n_prefix_groups] for s in g] + rest = [j for g in combo[n_prefix_groups:] for j in g] + if prefix: + select.args["from_"].set("this", prefix[0]) + select.set("joins", [exp.Join(this=s) for s in prefix[1:]] + rest or None) + counts: dict[str, int] = {} + for item in _items_of(select): + _, base = keys[id(item)] + counts[base] = counts.get(base, 0) + 1 + name = self._alias(base, depth, counts[base]) + item.set("alias", exp.TableAlias(this=exp.to_identifier(name))) + for col in self.refs.get(id(item), []): + col.set("table", exp.to_identifier(name)) + for sub in _nested_queries(select): + self.canon(sub, depth + 1, cte_names) + _sort_own(select, self.read) + return self._text(select) + + +_FLIP = {"GT": "LT", "LT": "GT", "GTE": "LTE", "LTE": "GTE", "EQ": "EQ", "NEQ": "NEQ"} + + +def _fixed(node: Any) -> bool: + """不涉及任何列、子查询的表达式(常量、常量算出来的日期):排比较、加乘的两边时放在右边,好读。""" + from sqlglot import exp + + return node.find(exp.Column, exp.Select, exp.Subquery) is None + + +def _sort_own(select: Any, read: str | None) -> None: + """这一段自己的表达式里交换律成立的地方按写出来的文本排(里层先排):AND、OR(去重)、比较(不涉及列的一边在 + 右,两边都涉及列时按文本排、大于小于跟着换方向)、两项的加和乘(不涉及列的一项在右,三项以上的不跨项挪动: + 浮点数加法不满足结合律)、IN 列表、GROUP BY、PARTITION BY。""" + from sqlglot import exp + + def key(node: Any) -> tuple[str, str]: + return _key(node, read) + + def depth(node: Any) -> int: + d = 0 + while node is not select and node.parent is not None: + node = node.parent + d += 1 + return d + + for node in sorted(_block_nodes(select), key=depth, reverse=True): + if node is not select and node.parent is None: + continue + if isinstance(node, (exp.And, exp.Or)): + if isinstance(node.parent, type(node)): + continue + uniq: dict[str, Any] = {} + for part in node.flatten(): + uniq.setdefault(key(part), part) + ordered = [uniq[k] for k in sorted(uniq)] + new = ordered[0] if len(ordered) == 1 else ( + exp.and_(*ordered, copy=False) if isinstance(node, exp.And) else exp.or_(*ordered, copy=False)) + node.replace(new) + elif type(node).__name__ in _FLIP: + left, right = node.this, node.expression + if isinstance(left, (exp.Any, exp.All)) or isinstance(right, (exp.Any, exp.All)): + continue + # 两边都写了 COLLATE 时 SQLite 用左边那个:不对调 + if left.find(exp.Collate) is not None and right.find(exp.Collate) is not None: + continue + if (_fixed(left) and not _fixed(right)) or (_fixed(left) == _fixed(right) and key(left) > key(right)): + node.replace(getattr(exp, _FLIP[type(node).__name__])(this=right, expression=left)) + elif isinstance(node, (exp.Add, exp.Mul)): + left, right = node.this, node.expression + if (_fixed(left) and not _fixed(right)) or (_fixed(left) == _fixed(right) and key(left) > key(right)): + node.set("this", right) + node.set("expression", left) + elif isinstance(node, exp.Group): + if node.expressions and not any(node.args.get(k) for k in ("rollup", "cube", "grouping_sets")): + node.set("expressions", sorted(node.expressions, key=key)) + elif isinstance(node, exp.Window): + if node.args.get("partition_by"): + node.set("partition_by", sorted(node.args["partition_by"], key=key)) + elif isinstance(node, exp.In) and node.expressions: + uniq = {key(i): i for i in node.expressions} + node.set("expressions", [uniq[k] for k in sorted(uniq)]) + + +# ========================================================================== +# 差异说明 # ========================================================================== @dataclass(frozen=True) class FacetDiff: - """一处不同。aspect:time(时间列)/ time_range(同一时间列、范围不同,或者同一个时间条件筛的先后不同)/ filter / - aggregate / expression(算出来的列:最外层的算法、CTE 和子查询里被用到的列的定义,见 Computed)/ join(类型、 - 外连接的主表、关联条件、只有一边关联了)/ tables / set_op(几支查询怎么合并:UNION 还是 UNION ALL、几支)/ - group_by(分组键、去重,以及最后出来的行按什么分)/ having / limit(取哪几行:行数、跳过的行数、取前几行时的 - 排序)/ parse。a、b 是两边各自的写法(给人看,没有就是 None),text 是一句中文说明。 - detail 只在 time 上有:两边的时间列 [{table, column}](一直换到实表),调用方据此对照数据目录(业务日期)。""" + """一处不同。aspect:time(时间列)/ time_range(同一时间列、范围不同)/ filter(筛选条件)/ aggregate(聚合函数、 + 去重计数)/ expression(选出来的列怎么算)/ join(关联的表、关联类型、关联条件、外连接的主表)/ tables(查的是 + 哪张表)/ set_op(几支查询怎么合并)/ group_by(分组键、去重)/ having / limit(取哪几行)/ order(只是排序不同)/ + parse(解析不了)/ other(兜底:未能归纳成具体差异,见 explain_sql_diff)。a、b 是两边各自的写法(给人看,没有 + 就是 None),text 是一句中文说明。detail 只在 time 上有:两边的时间列 [{table, column}],调用方据此对照数据目录 + (业务日期);在 filter 上有 {"columns": [{table, column}]}:条件只涉及的实表列,对照有效行过滤用。""" aspect: str a: Any @@ -2484,13 +3843,40 @@ class FacetDiff: return out +@dataclass(frozen=True) +class SqlDiff: + """两条 SQL 差在哪(explain_sql_diff 的结果)。 + + equivalent:两边经过只保持语义的变换之后,规范形式一字不差(level 是哪一级的规范形式,见 _canonical_forms)。 + items:具体差异,等价时为空;不等价时一定不为空——归纳不出具体差异时是一条 aspect="other" 的兜底说明。 + fallback:用了兜底说明(一条具体说法都没有,或者具体说法只覆盖了一部分)。 + text_diff:规范化之后两条 SQL 不同的行(difflib 比两边 sqlglot pretty 的输出,「- 」开头是 A 的、「+ 」开头是 + B 的,最多 40 行),给界面高亮;等价时为空。""" + + equivalent: bool + items: list[FacetDiff] = field(default_factory=list) + fallback: bool = False + text_diff: tuple[str, ...] = () + level: str | None = None + + def as_dict(self) -> dict[str, Any]: + return {"equivalent": self.equivalent, "items": [d.as_dict() for d in self.items], "fallback": self.fallback, + "text_diff": list(self.text_diff), "level": self.level} + + +#: 兜底说明 +_FALLBACK = "两条 SQL 的写法不同,未能归纳成具体差异,请对照原文" +_FALLBACK_REST = "另有未能归纳的差异,请对照原文" +#: text_diff 最多列几行 +_MAX_DIFF_LINES = 40 + _CJK_PUNCT = frozenset(",。;:、!?「」『』()《》【】…—") #: 聚合函数的说法 _VERBS = {"sum": "求和", "count": "计数", "avg": "求平均", "min": "取最小值", "max": "取最大值"} def _wide(ch: str) -> bool: - return "⺀" <= ch <= "鿿" or "豈" <= ch <= "﫿" or "＀" <= ch <= "￯" + return "⺀" <= ch <= "鿿" or "豈" <= ch <= "﫿" or "＀" <= ch <= "￯" def _say(*parts: str) -> str: @@ -2511,8 +3897,8 @@ def _say(*parts: str) -> str: def catalog_labels(notes: Mapping[str, Mapping[str, Any]] | None) -> dict[str, str]: - """数据目录({表名: notes})里列的中文名 → diff_facets 的 names:{「表.列」: 中文名};只在一张表里有、或者各表 - 叫法一致的列另记 {列: 中文名}。被驳回的项不算。""" + """数据目录({表名: notes})里列的中文名 → diff_facets / explain_sql_diff 的 names:{「表.列」: 中文名};只在一张 + 表里有、或者各表叫法一致的列另记 {列: 中文名}。被驳回的项不算。""" out: dict[str, str] = {} bare: dict[str, tuple[str, set[str]]] = {} for table, raw in (notes or {}).items(): @@ -2540,20 +3926,881 @@ def _labeler(names: Mapping[str, str] | None) -> Any: return label -def _single_column(part: Part, *, loose: bool) -> str | None: - """只涉及一列的条件:那一列(Part.column_keys:实表.列,CTE、子查询的列写「编号.列」),用来把两边对同一列的 - 不同写法配成一对。loose 时实表的列只看列名(表换了一张),派生来源的列照样带编号:两个 CTE 的同名列不是一列。""" - if len(part.columns) != 1 or len(part.column_keys) != 1: +@dataclass +class _Rel: + """一段里的一个来源:alias 是规范别名,name 是实表名或 CTE 的规范名,table 表示是不是实表,shown 是给人看的 + 名字(实表是表名,CTE 是原来起的名字),rejected 表示写的是 LEFT JOIN、被 WHERE 变成了内连接。""" + + alias: str + name: str + table: bool + shown: str + rejected: str | None = None + + +@dataclass +class _Block: + """一段 SELECT 拆开来比:开头那组内连接的来源(rels)、后面的连接(joins:LEFT JOIN 等,按先后)、WHERE、 + GROUP BY、HAVING 的各项,选出来的列,排序和取行。""" + + select: Any + rels: list[_Rel] + joins: list[Any] + where: list[Any] + group: list[Any] + having: list[Any] + exprs: list[Any] + order: list[Any] + limit: str | None + offset: str | None + distinct: bool + deduped: bool + ctes: dict[str, str] = field(default_factory=dict) + + @classmethod + def of(cls, select: Any, read: str | None, ctes: Mapping[str, str]) -> "_Block | None": + from sqlglot import exp + + if not isinstance(select, exp.Select) or select.args.get("with_") is not None: + return None + items = _items_of(select) + if any(not isinstance(i, exp.Table) for i in items): + return None + joins = select.args.get("joins") or [] + n = 1 + for join in joins: + if _comma(join): + n += 1 + else: + break + ctes = dict(ctes) + rels = [_Rel(i.alias_or_name, i.name, i.name not in ctes, ctes.get(i.name, i.name), + i.meta.get(_REJECTED)) for i in items[:n]] + where, group, having = select.args.get("where"), select.args.get("group"), select.args.get("having") + order, limit, offset = select.args.get("order"), select.args.get("limit"), select.args.get("offset") + + def number(node: Any) -> str | None: + return node.expression.sql(dialect=read) if node is not None and node.expression is not None else None + + return cls(select, rels, joins[n - 1:] if items else joins, + _conjuncts(where.this) if where is not None else [], + list(group.expressions) if group is not None else [], + _conjuncts(having.this) if having is not None else [], + list(select.expressions), list(order.expressions) if order is not None else [], + number(limit), number(offset), select.args.get("distinct") is not None, + bool(group is not None and group.meta.get(_FROM_DISTINCT)), ctes) + + def rel(self, alias: str) -> _Rel | None: + for r in self.rels: + if r.alias == alias: + return r + for join in self.joins: + if join.this.alias_or_name == alias: + name = join.this.name + return _Rel(alias, name, name not in self.ctes, self.ctes.get(name, name)) return None - key = part.column_keys[0] - if not loose or key.startswith("@"): - return key - # 外层的列(相关子查询里引用外层,见 _Statement._depth)不写表名也带着标记 - return name_key(part.columns[0][1]) + "^" * (len(key) - len(key.rstrip("^"))) + + def names(self) -> list[str]: + return [r.name for r in self.rels] + [j.this.name for j in self.joins] -def _verb(g: Aggregate) -> str: - return ("去重" if g.distinct else "") + _VERBS.get(g.func, f"用 {g.func.upper()} 汇总") +class _Explainer: + """在两边的规范形式上逐段比,只把有把握的不同写成具体的中文句子,同时记下是不是所有不同都说到了。 + + 有把握的门槛:两段读的来源对得上(一模一样,或者只差一处说得清的:换了一张表、一边多关联了一张表、同一张表的 + 关联类型或关联条件不同、外连接的主表对调)之后,才逐项比 WHERE、选出来的列、分组、HAVING、取行;每一项只在它 + 只涉及这一段的实表列(不涉及 CTE 算出来的列、外层的列、子查询,同名的表在这一段里只出现一次)时才说具体的, + 说的都是「这一段的这一项两边各是什么」,不推断对结果的影响。来源对不上、两段结构不同、条件里套着子查询又配不成 + 一对的,宁可不说具体的,交给兜底说明。CTE 两边定义一样的按另一边的名字起名后对齐;各剩一个对不上的配成一对, + 往里比。CTE 里选出来的列不比:规范化时去掉了没用到的列、按定义重排过,和原文的列对不上。""" + + def __init__(self, read: str | None, labels: tuple[str, str], names: Mapping[str, str] | None) -> None: + self.read = read + self.la, self.lb = labels + self.label = _labeler(names) + self.items: list[FacetDiff] = [] + self.ctes: dict[str, str] = {} + #: 有一句话没说出口(两边给人看的写法一样,说出来自相矛盾):这一处不同就没说到 + self.withheld = False + + def sql(self, node: Any) -> str: + """给人看的写法:按方言生成;生成的写法读回来不是同一棵树时(SQLite 把 CAST(x AS DATE) 生成 DATE(x),两者 + 在 SQLite 里结果不同)改用 sqlglot 的通用写法,免得给人看的和实际的不是一回事。""" + import sqlglot + + text = node.sql(dialect=self.read, comments=False) + try: + if _fingerprint(sqlglot.parse_one(text, read=self.read)) == _fingerprint(node): + return text + except Exception: # noqa: BLE001 - 片段单独读不回来:用通用写法 + pass + return node.sql(comments=False) + + @staticmethod + def fp(node: Any) -> str: + """比「是不是同一个」用的写法(见 _fingerprint):生成的 SQL 可能把不同的写法写成一样的。""" + return _fingerprint(node) + + # ---- 入口:整条语句 + + def statement(self, form_a: _Form, form_b: _Form) -> bool: + """比两条语句的规范形式,返回是不是全都说到了。B 先按 A 的 CTE 名重新起名(定义一样的叫同一个名字,各剩一个 + 对不上的也配成一对),两边才能逐段对齐。""" + ta, tb = form_a.tree, form_b.tree + wa = ta.args.get("with_") + names_a = {c.alias_or_name: c for c in (wa.expressions if wa is not None else [])} + if form_b.pre is not None and names_a: + tb = self._renamed(form_a, form_b, names_a) + wb = tb.args.get("with_") + names_b = {c.alias_or_name: c for c in (wb.expressions if wb is not None else [])} + shown_a = {n: c.this.meta.get(_ORIGINAL) or n for n, c in names_a.items()} + shown_b = {n: c.this.meta.get(_ORIGINAL) or n for n, c in names_b.items()} + covered = set(names_a) == set(names_b) + main_a, main_b = ta.copy(), tb.copy() + main_a.set("with_", None) + main_b.set("with_", None) + self.ctes = {**shown_b, **shown_a} + covered &= self.query(main_a, main_b, ("", ""), outputs=True) + for name in sorted(set(names_a) & set(names_b)): + body_a, body_b = names_a[name].this, names_b[name].this + if self.fp(body_a) != self.fp(body_b): + where = (_say("在", shown_a[name], "里"), _say("在", shown_b[name], "里")) + covered &= self.query(body_a, body_b, where, outputs=False) + return covered and not self.withheld + + def _renamed(self, form_a: _Form, form_b: _Form, names_a: dict[str, Any]) -> Any: + """B 起名之前的树按 A 的 CTE 名重新起名:定义一样的叫 A 的名字;两边各剩一个对不上的,B 的那个也叫 A 的。""" + from sqlglot import exp + + hint = _Hint(bodies={_fingerprint(c.this): name for name, c in names_a.items()}, taken=set(names_a)) + first = _named(form_b.pre.copy(), self.read, form_b.qualified, hint) + wb = first.args.get("with_") + left_b = [c for c in (wb.expressions if wb is not None else []) if c.alias_or_name not in names_a] + used = {c.alias_or_name for c in (wb.expressions if wb is not None else [])} + left_a = [name for name in names_a if name not in used] + if len(left_a) == 1 and len(left_b) == 1: + pre = form_b.pre.copy() + original = left_b[0].this.meta.get(_ORIGINAL) + matches = [c.alias_or_name for c in pre.find_all(exp.CTE) if c.this.meta.get(_ORIGINAL) == original] + if original is not None and len(matches) == 1: + hint.names[matches[0]] = left_a[0] + return _named(pre, self.read, form_b.qualified, hint) + return first + + def query(self, qa: Any, qb: Any, where: tuple[str, str], *, outputs: bool) -> bool: + from sqlglot import exp + + if self.fp(qa) == self.fp(qb): + return True + if isinstance(qa, exp.Select) and isinstance(qb, exp.Select): + return self.select(qa, qb, where, outputs) + if isinstance(qa, exp.SetOperation) and isinstance(qb, exp.SetOperation): + return self.setop(qa, qb, where, outputs) + return False + + # ---- 说法的拼法 + + def sides(self, where: tuple[str, str], first: int = 0) -> tuple[str, str]: + """两边的叫法,带上在哪一段(「A 在 paid 里」),按 A、B 的先后返回。两边都说在哪:只说一边,「另一边没有」 + 会读成整条语句都没有。""" + return _say(self.la, where[0]), _say(self.lb, where[1]) + + def add(self, aspect: str, a: Any, b: Any, text: str, detail: dict[str, Any] | None = None) -> None: + """记一句话。两边给人看的写法一样(「A 算的是 1,B 算的是 1」)就不说:读起来自相矛盾,这一处算没说到。""" + if isinstance(a, str) and isinstance(b, str) and a == b: + self.withheld = True + return + self.items.append(FacetDiff(aspect, a, b, text, detail)) + + def base_columns(self, node: Any, block: _Block) -> list[tuple[_Rel, Any]] | None: + """node 只涉及这一段的实表列(同名的表在这一段只出现一次)、不套子查询时,返回各列和它的来源;否则 None。""" + from sqlglot import exp + + if node.find(exp.Select, exp.Subquery) is not None: + return None + names = block.names() + out = [] + for col in node.find_all(exp.Column): + rel = block.rel(col.table) if col.table else None + if rel is None or not rel.table or names.count(rel.name) != 1: + return None + out.append((rel, col)) + return out + + def show(self, node: Any, block: _Block, *, full: bool = False) -> str: + """给人看的写法:只有一个来源时列不写表名,否则写实表名(规范别名换回表名,CTE 换回原来的名字);选出来的 + 列去掉规范化起的别名。full 时一律写表名。""" + from sqlglot import exp + + copy = node.unalias().copy() if isinstance(node, exp.Alias) else node.copy() + single = len(block.rels) + len(block.joins) == 1 and not full + for col in copy.find_all(exp.Column): + rel = block.rel(col.table) if col.table else None + if rel is not None: + col.set("table", None if single else exp.to_identifier(rel.shown)) + return self.sql(copy) + + def show_all(self, conds: list[Any], block: _Block) -> str: + """几个条件用 AND 连起来给人看:OR 套上括号(按 sqlglot 的结合规则生成)。""" + from sqlglot import exp + + if len(conds) == 1: + return self.show(conds[0], block) + return self.show(exp.and_(*[c.copy() for c in conds], copy=False), block) + + def named(self, rel: _Rel, column: str) -> str: + """列带中文名(有的话):「支付时间 paid_at」。""" + label = self.label((rel.name, column)) + return _say(label, column) if label else column + + def what(self, rel: _Rel) -> str: + return _say(rel.shown, "表") if rel.table else _say("子查询", rel.shown) + + # ---- 一段 SELECT + + def select(self, a: Any, b: Any, where: tuple[str, str], outputs: bool) -> bool: + A, B = _Block.of(a, self.read, self.ctes), _Block.of(b, self.read, self.ctes) + if A is None or B is None: + return False + matched = self.relations(A, B, where) + if matched is None: + return False + covered, skip_a, skip_b, B = matched + covered &= self.conditions(A, B, where, skip_a, skip_b) + if outputs: + covered &= self.outputs(A, B, where) + else: + # CTE 里选出来的列规范化时去掉过没用到的、重排过,和原文的列对不上:不同就不说具体的 + covered &= self.groups(A, B, where) + covered &= [self.fp(e) for e in A.exprs] == [self.fp(e) for e in B.exprs] + covered &= self.having(A, B, where) + covered &= self.rows(A, B, where) + return covered + + def relations(self, A: _Block, B: _Block, where: tuple[str, str] + ) -> tuple[bool, set[str], set[str], _Block] | None: + """两段读的来源对不对得上。对得上返回 (说到了没有, A 已经说过的 WHERE 项, B 已经说过的 WHERE 项, B);B 可能换了 + 一张表(换成 A 的表名之后再比别的)。对不上返回 None。""" + la, lb = self.sides(where) + ra, rb = [r.alias for r in A.rels], [r.alias for r in B.rels] + ja, jb = [self.fp(j) for j in A.joins], [self.fp(j) for j in B.joins] + if ra == rb and ja == jb: + return True, set(), set(), B + # 换了一张表:开头那组来源只差一张实表,其余一样 + if len(ra) == len(rb) and ja == jb and not A.joins: + only_a = [r for r in A.rels if r.alias not in rb] + only_b = [r for r in B.rels if r.alias not in ra] + if len(only_a) == 1 and len(only_b) == 1 and only_a[0].table and only_b[0].table and \ + only_a[0].name == only_a[0].alias and only_b[0].name == only_b[0].alias: + x, y = only_a[0], only_b[0] + self.add("tables", x.name, y.name, _say(la, "查的是", x.name, "表,", lb, "查的是", y.name, "表")) + return True, set(), set(), self._swap(B, y, x) + # 一边多关联了一张表(内连接) + if ja == jb and not A.joins: + if set(rb) < set(ra) and len(ra) == len(rb) + 1: + return self._extra(A, B, la, lb, 0) + if set(ra) < set(rb) and len(rb) == len(ra) + 1: + first_b = self.sides(where, 1) + return self._extra(B, A, first_b[1], first_b[0], 1) + # 同一张表:LEFT JOIN 对 INNER JOIN、关联条件不同、只有一边关联了(LEFT JOIN)、外连接的主表对调 + return self._outer(A, B, where) + + def _swap(self, B: _Block, old: _Rel, new: _Rel) -> _Block | None: + from sqlglot import exp + + select = B.select.copy() + for col in select.find_all(exp.Column): + if col.table == old.alias: + col.set("table", exp.to_identifier(new.alias)) + for table in select.find_all(exp.Table): + if table.alias_or_name == old.alias: + table.set("this", exp.to_identifier(new.name)) + table.set("alias", exp.TableAlias(this=exp.to_identifier(new.alias))) + for s in list(select.find_all(exp.Select)): + _sort_own(s, self.read) + return _Block.of(select, self.read, self.ctes) + + def _uses(self, block: _Block, name: str) -> bool: + """这一段在子查询里(不是开头的来源、连接)用到了 name 这张表。""" + from sqlglot import exp + + own = {id(i) for i in _items_of(block.select)} + return any(t.name == name and id(t) not in own for t in block.select.find_all(exp.Table)) + + def _extra(self, big: _Block, small: _Block, lbig: str, lsmall: str, side: int + ) -> tuple[bool, set[str], set[str], _Block] | None: + from sqlglot import exp + + [rel] = [r for r in big.rels if r.alias not in {s.alias for s in small.rels}] + if big.names().count(rel.name) != 1: + return None + conds = [c for c in big.where if rel.alias in {col.table for col in c.find_all(exp.Column)}] + what = self.what(rel) + shown = _say("(关联的条件是", self.show_all(conds, big), ")") if conds and \ + all(self.base_columns(c, big) is not None for c in conds) else "" + tail = ("没有关联它(只在子查询里用到了", rel.shown, ")") if self._uses(small, rel.name) else ("没有",) + text = _say(lbig, "关联了", what, shown, ",", lsmall, *tail) + self.add("join", what if side == 0 else None, what if side == 1 else None, text) + skip = {self.fp(c) for c in conds} + return (True, skip, set(), small) if side == 0 else (True, set(), skip, big) + + def _outer(self, A: _Block, B: _Block, where: tuple[str, str]) -> tuple[bool, set[str], set[str], _Block] | None: + from sqlglot import exp + + la, lb = self.sides(where) + first_b = self.sides(where, 1) + + def kind(join: Any) -> str: + return str(join.side or "").lower() or "inner" + + ra, rb = [r.alias for r in A.rels], [r.alias for r in B.rels] + ja = {j.this.alias_or_name: j for j in A.joins} + jb = {j.this.alias_or_name: j for j in B.joins} + if len(A.joins) != len(ja) or len(B.joins) != len(jb): + return None + # 外连接的主表对调:两个来源,一边 x LEFT JOIN y,一边 y LEFT JOIN x,条件一样 + if len(ra) == len(rb) == 1 and len(ja) == len(jb) == 1 and ra != rb and set(ra) == set(jb) and \ + set(rb) == set(ja): + x, y = A.joins[0], B.joins[0] + if kind(x) == kind(y) == "left" and self.fp(x.args["on"]) == self.fp(y.args["on"]): + wa = _say("以", A.rels[0].shown, "为主表外连接", B.rels[0].shown) + wb = _say("以", B.rels[0].shown, "为主表外连接", A.rels[0].shown) + self.add("join", wa, wb, _say(la, wa, ",", lb, wb)) + return True, set(), set(), B + return None + if [r for r in ra if r not in rb and r not in jb] or [r for r in rb if r not in ra and r not in ja]: + return None + one_sided_a = [k for k in ja if k not in jb and k not in rb] + one_sided_b = [k for k in jb if k not in ja and k not in ra] + if one_sided_a and one_sided_b: + return None + skip_a: set[str] = set() + skip_b: set[str] = set() + for alias in sorted(set(ja) | set(jb)): + x, y = ja.get(alias), jb.get(alias) + if x is not None and y is not None: + if kind(x) != kind(y): + return None + if self.fp(x) != self.fp(y): + on_x, on_y = self._on(x, A), self._on(y, B) + if on_x is None or on_y is None: + return None + rel = A.rel(alias) + self.add("join", on_x, on_y, _say(la, "关联", self.what(rel), "的条件是", on_x, ",", lb, "是", on_y)) + continue + # 一边 LEFT JOIN,另一边内连接(在开头那组来源里,关联条件在 WHERE 里) + if x is not None and alias in rb and kind(x) == "left": + done = self._left_inner(x, A, B, alias, la, lb, 0) + if done is None: + return None + skip_b |= done + continue + if y is not None and alias in ra and kind(y) == "left": + done = self._left_inner(y, B, A, alias, la, lb, 1) + if done is None: + return None + skip_a |= done + continue + # 只有一边关联了 + join = x if x is not None else y + if join is None or kind(join) != "left": + return None + block, me, other = (A, la, lb) if x is not None else (B, first_b[1], first_b[0]) + rel = block.rel(alias) + conds = [c for c in block.where if alias in {col.table for col in c.find_all(exp.Column)}] + text = _say(me, *self._outer_words(join, block), self.what(rel), ",", other, "没有") + if conds: + if not all(self.base_columns(c, block) is not None for c in conds): + return None + text += _say("(WHERE 里还有", self.show_all(conds, block), ")") + self.add("join", rel.shown if x is not None else None, rel.shown if y is not None else None, text) + (skip_a if x is not None else skip_b).update(self.fp(c) for c in conds) + return True, skip_a, skip_b, B + + def _on(self, join: Any, block: _Block) -> str | None: + on = join.args.get("on") + if on is None or self.base_columns(on, block) is None: + return None + return self.show(on, block) + + def _left_inner(self, join: Any, left: _Block, inner: _Block, alias: str, la: str, lb: str, + side: int) -> set[str] | None: + """left 这边 LEFT JOIN alias,inner 那边同一张表是内连接:inner 的 WHERE 里有 left 的 ON 的每一项才算关联条件 + 一样,只差连接类型。返回 inner 那边已经说过的 WHERE 项。照原文说:left 那边原来写的是 RIGHT JOIN 的说以哪一侧 + 为主表;inner 那边写的是 LEFT / RIGHT JOIN、被 WHERE 里的条件变成了内连接的,说写的是什么、实际是什么。side 是 + left 在哪一边(0 是 A);la、lb 是 A、B 的叫法。""" + on = join.args.get("on") + conds = [self.fp(c) for c in _conjuncts(on)] if on is not None else [] + if not conds or not set(conds) <= {self.fp(c) for c in inner.where}: + return None + rel = inner.rel(alias) + what = self.what(rel) + outer = self._outer_words(join, left) + turned = [r for r in inner.rels if r.rejected] + if turned: + words = "、".join(dict.fromkeys(r.rejected for r in turned)) + names = "、".join(r.shown for r in turned) + inner_says = ("写的是", words, ",但 WHERE 里的条件把关联不上", names, "的行筛掉了,实际是 INNER JOIN") + if side == 0: + text = _say(la, *outer, what, ",", lb, *(inner_says if turned else ("用 INNER JOIN",))) + self.add("join", "OUTER JOIN", "INNER JOIN", text) + else: + first = (*inner_says, ";") if turned else ("用 INNER JOIN 关联", what, ",") + second = (*outer, what) if turned or join.meta.get(_FLIPPED) else ("用 LEFT JOIN",) + text = _say(la, *first, lb, *second) + self.add("join", "INNER JOIN", "OUTER JOIN", text) + return set(conds) + + def _outer_words(self, join: Any, block: _Block) -> tuple[str, ...]: + """一次外连接的说法:原文就是 LEFT JOIN 的说「用 LEFT JOIN 关联」;从 RIGHT JOIN 改写来的说「以谁为主表外连接」 + (照写 LEFT 会和原文对不上)。""" + if join.meta.get(_FLIPPED): + return ("以", "、".join(r.shown for r in block.rels), "为主表外连接") + return ("用 LEFT JOIN 关联",) + + # ---- WHERE + + def timed(self, cond: Any, block: _Block) -> tuple[_Rel, Any] | None: + """时间条件:一列(可以套着日期函数、CAST)和常量比较,列名像时间或者常量像日期。返回 (来源, 列)。""" + from sqlglot import exp + + if not isinstance(cond, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.In)): + return None + if isinstance(cond, exp.In): + column_side, others = cond.this, list(cond.expressions) + elif _fixed(cond.expression): + column_side, others = cond.this, [cond.expression] + elif _fixed(cond.this): + column_side, others = cond.expression, [cond.this] + else: + return None + if not others or not all(_fixed(o) for o in others): + return None + cols = self.base_columns(column_side, block) + if not cols or len({(r.alias, c.name) for r, c in cols}) != 1 or _value_column(column_side) is None: + return None + rel, col = cols[0] + if _temporal_name(col.name) or any(_datey(o) for o in others) or _datey(column_side): + return rel, col + return None + + def conditions(self, A: _Block, B: _Block, where: tuple[str, str], skip_a: set[str], skip_b: set[str]) -> bool: + la, lb = self.sides(where) + only_a, only_b = self._unmatched([c for c in A.where if self.fp(c) not in skip_a], + [c for c in B.where if self.fp(c) not in skip_b]) + if not only_a and not only_b: + return True + covered = True + only_a, only_b = self._times(A, B, la, lb, only_a, only_b) + # 同一列上的两个条件配成一对 + for c in list(only_a): + col_a = self._single_column(c, A) + if col_a is None: + continue + mine = [x for x in only_a if self._single_column(x, A) == col_a] + theirs = [y for y in only_b if self._single_column(y, B) == col_a] + if len(mine) == 1 and len(theirs) == 1: + d = theirs[0] + wa, wb = self.show(c, A), self.show(d, B) + if wa == wb: + wa, wb = self.show(c, A, full=True), self.show(d, B, full=True) + self.add("filter", wa, wb, _say(la, "按", wa, "筛选,", lb, "按", wb, "筛选"), + {"columns": self._columns(c, A)}) + only_a.remove(c) + only_b.remove(d) + # 两边各剩一个、涉及的列一样的条件,也配成一对说 + if len(only_a) == 1 and len(only_b) == 1: + ca, cb = self.base_columns(only_a[0], A), self.base_columns(only_b[0], B) + if ca and cb and {(r.name, c.name) for r, c in ca} == {(r.name, c.name) for r, c in cb}: + wa, wb = self.show(only_a[0], A), self.show(only_b[0], B) + self.add("filter", wa, wb, _say(la, "按", wa, "筛选,", lb, "按", wb, "筛选"), + {"columns": self._columns(only_a[0], A)}) + only_a, only_b = [], [] + # 只在一边的条件:「另一边没有」带上在哪一段,免得读成整条语句都没有 + fa, fb = _say(self.la, where[0]), _say(self.lb, where[1]) + for side, rest, block, me, other in ((0, only_a, A, fa, fb), (1, only_b, B, fb, fa)): + for c in list(rest): + if self.base_columns(c, block) is None: + continue + shown = self.show(c, block) + excluded = self._excludes(c, block) + text = _say(me, "排除了", excluded, "的行,", other, "没有") if excluded else \ + _say(me, "只保留", shown, "的行,", other, "没有这个条件") + self.add("filter", shown if side == 0 else None, shown if side == 1 else None, text, + {"columns": self._columns(c, block)}) + rest.remove(c) + # 条件里套着的子查询:两边各一个、套子查询之外写法一样的配成一对,往里比 + for c in list(only_a): + template = self._template(c) + if template is None: + continue + mine = [x for x in only_a if self._template(x) == template] + theirs = [d for d in only_b if self._template(d) == template] + if len(mine) == 1 and len(theirs) == 1: + d = theirs[0] + place = self._place(c) + inner = (_deeper(where[0], place), _deeper(where[1], place)) + covered &= self.query(self._subquery(c), self._subquery(d), inner, outputs=True) + only_a.remove(c) + only_b.remove(d) + return covered and not only_a and not only_b + + def _times(self, A: _Block, B: _Block, la: str, lb: str, only_a: list[Any], only_b: list[Any] + ) -> tuple[list[Any], list[Any]]: + """时间条件:两边差在按哪一列筛时间、同一列上的范围,或者只有一边按时间筛。说到了的从两边拿掉。""" + ta = [(c, t) for c in only_a if (t := self.timed(c, A)) is not None] + tb = [(c, t) for c in only_b if (t := self.timed(c, B)) is not None] + cols_a = {(t[0].name, t[1].name) for _, t in ta} + cols_b = {(t[0].name, t[1].name) for _, t in tb} + if not (ta or tb) or len(cols_a) > 1 or len(cols_b) > 1: + return only_a, only_b + all_a = [(c, (t[0].name, t[1].name)) for c in A.where if (t := self.timed(c, A)) is not None] + all_b = [(c, (t[0].name, t[1].name)) for c in B.where if (t := self.timed(c, B)) is not None] + detail = {"a": [{"table": t, "column": c} for t, c in sorted(cols_a)], + "b": [{"table": t, "column": c} for t, c in sorted(cols_b)]} + if cols_a and cols_b and cols_a != cols_b: + (_, ca), (_, cb) = next(iter(cols_a)), next(iter(cols_b)) + text = _say(la, "按", self.named(ta[0][1][0], ca), "筛时间,", lb, "按", self.named(tb[0][1][0], cb)) + if self._shape([c for c, _ in ta]) != self._shape([c for c, _ in tb]): + text += _say("(", la, "是", self.show_all(_by_bound([c for c, _ in ta]), A), ",", lb, "是", + self.show_all(_by_bound([c for c, _ in tb]), B), ")") + self.add("time", ca, cb, text, detail) + else: + [col] = list(cols_a | cols_b) + ra = _by_bound([c for c, k in all_a if k == col]) + rb = _by_bound([c for c, k in all_b if k == col]) + rel = (ta or tb)[0][1][0] + if ra and rb: + wa, wb = self.show_all(ra, A), self.show_all(rb, B) + self.add("time_range", wa, wb, _say(la, "的时间范围是", wa, ",", lb, "是", wb)) + elif ra and not all_b: + self.add("time", col[1], None, _say(la, "按", self.named(rel, col[1]), "筛时间,", lb, "没有按时间筛选"), + detail) + elif rb and not all_a: + self.add("time", None, col[1], _say(la, "没有按时间筛选,", lb, "按", self.named(rel, col[1]), "筛时间"), + detail) + else: + return only_a, only_b + return [c for c in only_a if all(c is not x for x, _ in ta)], [c for c in only_b if all(c is not x for x, _ in tb)] + + def _unmatched(self, xs: list[Any], ys: list[Any]) -> tuple[list[Any], list[Any]]: + rest = list(ys) + only_x = [] + for x in xs: + found = next((y for y in rest if self.fp(y) == self.fp(x)), None) + if found is None: + only_x.append(x) + else: + rest.remove(found) + return only_x, rest + + def _shape(self, conds: list[Any]) -> list[str]: + """时间条件把那一列换成占位符之后的写法:两边只差时间列时一样。""" + from sqlglot import exp + + out = [] + for c in conds: + copy = c.copy() + for col in list(copy.find_all(exp.Column)): + col.replace(exp.column("_time_")) + out.append(self.fp(copy)) + return sorted(out) + + def _single_column(self, cond: Any, block: _Block) -> tuple[str, str] | None: + cols = self.base_columns(cond, block) + if not cols or len({(r.name, c.name) for r, c in cols}) != 1: + return None + return cols[0][0].name, cols[0][1].name + + def _columns(self, cond: Any, block: _Block) -> list[dict[str, str]]: + cols = self.base_columns(cond, block) or [] + return [{"table": t, "column": c} for t, c in sorted({(r.name, col.name) for r, col in cols})] + + def _excludes(self, cond: Any, block: _Block) -> str | None: + """「排除」的写法去掉否定之后排除的是什么:<> 写成 =,NOT x 写成 x,IS NOT NULL 写成「列 为空」。""" + from sqlglot import exp + + if isinstance(cond, exp.NEQ): + return self.show(exp.EQ(this=cond.this.copy(), expression=cond.expression.copy()), block) + if isinstance(cond, exp.Not): + inner = cond.this.unnest() if isinstance(cond.this, exp.Paren) else cond.this + if isinstance(inner, exp.Is) and isinstance(inner.expression, exp.Null): + return _say(self.show(inner.this, block), "为空") + return self.show(inner, block) + return None + + def _template(self, cond: Any) -> str | None: + """条件里只套着一个子查询时,子查询换成占位符之后的写法;不然 None。""" + from sqlglot import exp + + tops = [n for n in cond.walk() if isinstance(n, (exp.Select, exp.SetOperation)) and + not any(isinstance(p, (exp.Select, exp.SetOperation)) for p in _ancestors(n, cond))] + if len(tops) != 1: + return None + copy = cond.copy() + for node in copy.walk(): + if isinstance(node, (exp.Select, exp.SetOperation)): + node.replace(exp.Literal.string("")) + break + return self.fp(copy) + + def _subquery(self, cond: Any) -> Any: + from sqlglot import exp + + return next(n for n in cond.walk() if isinstance(n, (exp.Select, exp.SetOperation))) + + def _place(self, cond: Any) -> str: + from sqlglot import exp + + if cond.find(exp.Exists) is not None: + return "EXISTS 子查询" + if cond.find(exp.In) is not None: + return "IN 子查询" + return "条件的子查询" + + # ---- 选出来的列、分组 + + def groups(self, A: _Block, B: _Block, where: tuple[str, str]) -> bool: + """分组键:两边不同、又都只涉及实表列时说各按什么分组(从 SELECT DISTINCT 改写来的说「去重」)。""" + la, lb = self.sides(where) + if sorted(self.fp(g) for g in A.group) == sorted(self.fp(g) for g in B.group): + return True + if not all(self.base_columns(g, A) is not None for g in A.group) or \ + not all(self.base_columns(g, B) is not None for g in B.group): + return False + + def say(block: _Block, other: _Block) -> tuple[str, ...]: + if not block.group: + return ("没有去重",) if other.deduped else ("没有分组",) + shown = "、".join(self.show(k, block) for k in block.group) + return ("对", shown, "去重") if block.deduped else ("按", shown, "分组") + + wa = "、".join(self.show(k, A) for k in A.group) or None + wb = "、".join(self.show(k, B) for k in B.group) or None + self.add("group_by", wa, wb, _say(la, *say(A, B), ",", lb, *say(B, A))) + return True + + def outputs(self, A: _Block, B: _Block, where: tuple[str, str]) -> bool: + """分组键、选出来的列。分组键不同时,选出来的列只差在分组键那几列上(都排在前面、各是各的分组键)的,算在 + 分组键的不同里,不另说;其余逐列比,列数不同、只是先后不同各说一句。""" + la, lb = self.sides(where) + ga, gb = [self.fp(g) for g in A.group], [self.fp(g) for g in B.group] + regrouped = sorted(ga) != sorted(gb) + covered = self.groups(A, B, where) + ea, eb = A.exprs, B.exprs + ta, tb = [self.fp(e) for e in ea], [self.fp(e) for e in eb] + if ta == tb: + return covered + if regrouped and covered: + ka = next((i for i, t in enumerate(ta) if t not in ga), len(ta)) + kb = next((i for i, t in enumerate(tb) if t not in gb), len(tb)) + if set(ta[:ka]) == set(ga) and set(tb[:kb]) == set(gb) and ta[ka:] == tb[kb:]: + return covered + confident = all(self.base_columns(e, A) is not None for e in ea) and \ + all(self.base_columns(e, B) is not None for e in eb) + if len(ea) != len(eb) or sorted(ta) == sorted(tb): + if not confident: + return False + wa = "、".join(self.show(e, A) for e in ea) + wb = "、".join(self.show(e, B) for e in eb) + if len(ea) != len(eb): + text = _say(la, "选出", str(len(ea)), "列(", wa, "),", lb, "选出", str(len(eb)), "列(", wb, ")") + else: + text = _say("两边选出来的列一样,先后不同:", la, "是", wa, ",", lb, "是", wb) + self.add("expression", wa, wb, text) + return covered + for i, (x, y) in enumerate(zip(ea, eb), 1): + if self.fp(x) == self.fp(y): + continue + # 只差在列名上(最外层还按名字排序时别名留着):取值一样,没什么可说的,也说不清排序那边,交给别的项 + if self.fp(x.unalias()) == self.fp(y.unalias()) or not self.output(x, y, A, B, la, lb, i, len(ea)): + covered = False + return covered + + def output(self, x: Any, y: Any, A: _Block, B: _Block, la: str, lb: str, i: int, n: int) -> bool: + from sqlglot import exp + + if self.base_columns(x, A) is None or self.base_columns(y, B) is None: + return False + ga, gb = _aggregate(x.unalias()), _aggregate(y.unalias()) + if ga is not None and gb is not None: + (fa, arg_a, da), (fb, arg_b, db) = ga, gb + wa = self.show(arg_a, A) if arg_a is not None else "*" + wb = self.show(arg_b, B) if arg_b is not None else "*" + same_arg = (arg_a is None and arg_b is None) or ( + arg_a is not None and arg_b is not None and self.fp(arg_a) == self.fp(arg_b)) + if wa == wb and not same_arg: + wa, wb = self.show(arg_a, A, full=True), self.show(arg_b, B, full=True) + if same_arg and fa == fb: + text = _say(la, "对", wa, _verb(fa, da) + ",", lb, "没去重") if da else \ + _say(la, "对", wa, _verb(fa, da) + "时没去重,", lb, _verb(fb, db)) + elif same_arg and arg_a is not None: + text = _say(la, "对", wa, _verb(fa, da) + ",", lb, _verb(fb, db)) + else: + what_a = ("数的是行数",) if fa == "count" and arg_a is None else ("对", wa, _verb(fa, da)) + what_b = ("数的是行数",) if fb == "count" and arg_b is None else ("对", wb, _verb(fb, db)) + text = _say(la, *what_a, ",", lb, *what_b) + self.add("aggregate", self.show(x, A), self.show(y, B), text) + return True + wa, wb = self.show(x, A), self.show(y, B) + if wa == wb: + wa, wb = self.show(x, A, full=True), self.show(y, B, full=True) + verb = "选的是" if isinstance(x.unalias(), exp.Column) and isinstance(y.unalias(), exp.Column) else "算的是" + where_col = _say("第", str(i), "列") if n > 1 else "" + self.add("expression", wa, wb, _say(la, where_col, verb, wa, ",", lb, verb, wb)) + return True + + # ---- HAVING、取行 + + def having(self, A: _Block, B: _Block, where: tuple[str, str]) -> bool: + la, lb = self.sides(where) + only_a, only_b = self._unmatched(A.having, B.having) + if not only_a and not only_b: + return True + if len(only_a) == 1 and len(only_b) == 1 and self.base_columns(only_a[0], A) is not None and \ + self.base_columns(only_b[0], B) is not None: + wa, wb = self.show(only_a[0], A), self.show(only_b[0], B) + self.add("having", wa, wb, _say(la, "分组后只保留", wa, "的组,", lb, "只保留", wb, "的组")) + return True + covered = True + fa, fb = _say(self.la, where[0]), _say(self.lb, where[1]) + for side, rest, block, me, other in ((0, only_a, A, fa, fb), (1, only_b, B, fb, fa)): + for c in rest: + if self.base_columns(c, block) is None: + covered = False + continue + shown = self.show(c, block) + self.add("having", shown if side == 0 else None, shown if side == 1 else None, + _say(me, "分组后只保留", shown, "的组,", other, "没有这个条件")) + return covered + + def rows(self, A: _Block, B: _Block, where: tuple[str, str]) -> bool: + la, lb = self.sides(where) + if A.limit != B.limit: + self.add("limit", A.limit, B.limit, _say(la, _rows(A.limit), ",", lb, _rows(B.limit))) + if A.offset != B.offset: + self.add("limit", A.offset, B.offset, _say(la, _skip(A.offset), ",", lb, _skip(B.offset))) + if [self.fp(o) for o in A.order] != [self.fp(o) for o in B.order]: + if not all(self.base_columns(o, A) is not None for o in A.order) or \ + not all(self.base_columns(o, B) is not None for o in B.order): + return False + wa, wb = self._sorting(A.order, A), self._sorting(B.order, B) + if A.limit is not None or B.limit is not None: + self.add("limit", wa, wb, _say(la, "取行前", wa, ",", lb, wb)) + else: + self.add("order", wa, wb, _say(la, wa, ",", lb, wb)) + if A.distinct != B.distinct: + me, other = (la, lb) if A.distinct else (lb, la) + self.add("group_by", "DISTINCT" if A.distinct else None, "DISTINCT" if B.distinct else None, + _say(me, "用 SELECT DISTINCT 去掉了重复的行,", other, "没有")) + return True + + def _sorting(self, order: list[Any], block: _Block) -> str: + if not order: + return "没有排序" + keys = "、".join(_say(self.show(o.this, block), "降序" if o.args.get("desc") else "升序") for o in order) + return _say("按", keys, "排") + + # ---- UNION + + def setop(self, a: Any, b: Any, where: tuple[str, str], outputs: bool) -> bool: + from sqlglot import exp + + la, lb = self.sides(where) + if type(a) is not type(b): + return False + xa, xb = _set_branches(a), _set_branches(b) + if len(xa) != len(xb): + if isinstance(a, exp.Union) and bool(a.args.get("distinct")) == bool(b.args.get("distinct")): + self.add("set_op", str(len(xa)), str(len(xb)), + _say(la, "合并了", str(len(xa)), "支查询,", lb, "合并了", str(len(xb)), "支")) + return False + covered = True + if bool(a.args.get("distinct")) != bool(b.args.get("distinct")) and isinstance(a, exp.Union): + wa, wb = _merging(a), _merging(b) + self.add("set_op", wa, wb, _say(la, "用", wa, ",", lb, "用", wb)) + only_a, only_b = self._unmatched(xa, xb) + if len(only_a) == 1 and len(only_b) == 1: + branch = (_deeper(where[0], "合并的一支"), _deeper(where[1], "合并的一支")) + covered &= self.query(only_a[0], only_b[0], branch, outputs=outputs) + elif only_a or only_b: + covered = False + for key in ("order", "limit", "offset"): + va = self.fp(a.args[key]) if a.args.get(key) else "" + vb = self.fp(b.args[key]) if b.args.get(key) else "" + if va != vb: + covered = False + return covered + + +def _deeper(where: str, place: str) -> str: + """在哪一段里再往里一层:「在 t 里」+「IN 子查询」→「在 t 的 IN 子查询里」。""" + if not where: + return _say("在", place, "里") + return _say(where[:-1] if where.endswith("里") else where, "的", place, "里") + + +def _by_bound(conds: list[Any]) -> list[Any]: + """时间条件给人看时下界在前:>=、> 在前,<、<= 在后,其余照原来的先后。""" + from sqlglot import exp + + def rank(c: Any) -> int: + return 0 if isinstance(c, (exp.GTE, exp.GT)) else 1 if isinstance(c, (exp.LT, exp.LTE)) else 2 + + return sorted(conds, key=rank) + + +def _ancestors(node: Any, stop: Any) -> Iterator[Any]: + node = node.parent + while node is not None and node is not stop: + yield node + node = node.parent + + +def _set_branches(query: Any) -> list[Any]: + """连着同一种合并方式的几支(规范形式里已经排好序)。""" + kind = (type(query), bool(query.args.get("distinct"))) + out: list[Any] = [] + + def collect(node: Any) -> None: + from sqlglot import exp + + if isinstance(node, exp.SetOperation) and (type(node), bool(node.args.get("distinct"))) == kind and \ + (node is query or not any(node.args.get(k) for k in ("order", "limit", "offset"))): + collect(node.left) + collect(node.right) + else: + out.append(node) + + collect(query) + return out + + +def _merging(query: Any) -> str: + return "UNION 合并(去掉重复的行)" if query.args.get("distinct") else "UNION ALL(重复的行都留着)" + + +def _aggregate(node: Any) -> tuple[str, Any, bool] | None: + """光一个聚合(SUM / AVG / COUNT / MIN / MAX):(函数, 参数(COUNT(*) 是 None), 是否去重)。""" + from sqlglot import exp + + kinds = {exp.Sum: "sum", exp.Avg: "avg", exp.Count: "count", exp.Min: "min", exp.Max: "max"} + func = kinds.get(type(node)) + if func is None: + return None + arg = node.this + distinct = isinstance(arg, exp.Distinct) + if distinct: + if len(arg.expressions) != 1: + return None + arg = arg.expressions[0] + if isinstance(arg, exp.Star): + arg = None + return func, arg, distinct + + +def _verb(func: str, distinct: bool) -> str: + return ("去重" if distinct else "") + _VERBS.get(func, f"用 {func.upper()} 汇总") def _rows(limit: str | None) -> str: @@ -2568,741 +4815,91 @@ def _skip(offset: str | None) -> str: return f"跳过前 {offset} 行" if offset.isdigit() else f"按 OFFSET {offset} 跳过行" -class _Compare: - """一次比较:两边的叫法、列的中文名,以及按哪种写法比。 +def _text_diff(a: _Form | None, b: _Form | None, read: str | None) -> tuple[str, ...]: + """规范化之后两条 SQL 不同的行:difflib 比两边 sqlglot pretty 的输出,「- 」是 A 的、「+ 」是 B 的。""" + import difflib - 两边各有一张只在自己这边出现、又不是关联进来的表(orders 换成了 orders_archive)时,其余各方面按不写表名的 - loose 比:表换了这件事由一条「tables」说清,同一个条件、同一个聚合不再因为表名不同被重复报一遍。 + def lines(form: _Form | None) -> list[str]: + if form is None: + return [] + try: + return form.tree.sql(dialect=read, pretty=True, comments=False).splitlines() + except Exception: # noqa: BLE001 + return form.text.splitlines() - 派生来源的编号由 diff_facets 先按结构配好(两边配成一对的写同一个编号,见模块说明),这里各方面都只按 k 比: - 编号不同就是不同的来源,不再拿「不写编号」的写法补配(那样会把 p.total 和 r.total 当成同一列)。 + out = [line for line in difflib.ndiff(lines(a), lines(b)) if line[:2] in ("- ", "+ ")] + return tuple(out[:_MAX_DIFF_LINES]) - 条件、分组键、连接、聚合、HAVING 按层计数(见 leftovers):同一项在一边出现在几层就算几个,另一边也有、只是在的 - 层少了的,说清多出来的是哪一层(layered)。 - 几个方面之间互相交代,同一处不同只报一遍:聚合那一面报过的聚合(reported)、时间那一面报过的派生来源的列 - (explained),算出来的列那一面不再重复报;合并了几支、怎么合并的那一面(set_ops)报了的,另一边没有的那几支里 - 多出来的条件不再单说;分组键那一面报了的,最后出来的行按什么分(grain)不再说。 +def explain_sql_diff(a: str, b: str, dialect: str | None = None, *, schema: Mapping[str, Any] | None = None, + names: Mapping[str, str] | None = None, labels: tuple[str, str] = ("A", "B")) -> SqlDiff: + """两条 SQL 是不是等价、不是的话差在哪。从不抛异常。 + + 等价怎么判:两边各自规范化(见 _canonical_forms:sqlglot 优化器那一级、轻量那一级、只重新生成的那一级),每一步都 + 只做保持语义的变换;同一级的两个规范形式完全一样(比语法树,见 _Form.key)才算等价,否则一律不等价,items 一定 + 不为空。schema 是数据源的 schema_cache,给了就按它认列、展开 SELECT *(表结构要和库对得上)。 + + 不等价时怎么说:在两边的规范形式上逐段比(见 _Explainer),只把有把握的不同写成具体的中文句子——实表上的时间列 + 不同、某个只涉及实表列的筛选条件一边有一边没有(或者同一列上两边的条件不同)、聚合函数或去重计数不同、选出来的 + 某一列算法不同、分组键不同、关联的表或关联类型、关联条件、外连接的主表不同、换了一张表、取行和排序不同、UNION + 还是 UNION ALL;两段读的来源对不上、条件里套着说不准的子查询、牵涉 CTE 算出来的列的,不说具体的。具体说法一条都 + 没有时给一条 aspect="other" 的兜底说明(「两条 SQL 的写法不同,未能归纳成具体差异,请对照原文」),只覆盖了一部分 + 时追加一条「另有未能归纳的差异」,fallback 为真;text_diff 给出规范化之后不同的行。 + + labels 是两边的叫法(缺省 A / B,双盲复算传「原流程」「盲算」);names 是列的中文名({「表.列」或「列」: 中文名}, + 可以用 catalog_labels 从数据目录取),时间列写成「支付时间 paid_at」。任一边解析不了:只给一条 aspect="parse"。 """ - - def __init__(self, a: Facets, b: Facets, labels: tuple[str, str], names: Mapping[str, str] | None) -> None: - self.a, self.b = a, b - self.la, self.lb = labels - self.label = _labeler(names) - joined = {j.table.key for j in (*a.joins, *b.joins)} - self.ta = [t for t in a.tables if t.key not in {u.key for u in b.tables} and t.key not in joined] - self.tb = [t for t in b.tables if t.key not in {u.key for u in a.tables} and t.key not in joined] - self.swapped = len(self.ta) == 1 and len(self.tb) == 1 - #: 两边的派生来源 {编号: 名字}(同名却不是同一个时拿来说清对应关系)、各段来源 {key: 名字}(外连接的主表) - self.names = ({p.key: p.text for p in a.derived}, {p.key: p.text for p in b.derived}) - self.called = ({p.key: p.text for p in a.sources}, {p.key: p.text for p in b.sources}) - #: 聚合那一面报过的两边独有的聚合、时间那一面已经说了的派生来源的列(「编号.列」) - self.reported: tuple[set[Any], set[Any]] = (set(), set()) - self.explained: set[str] = set() - #: 两边各有哪些层 {层的标记: 说法}(见 Part.scope、Facets.scopes);另一边也有、只是在的层少了的项多出来的 - #: 那几个 {id(项): 另一边有它的那几层}(见 leftovers) - self.layers = ({p.key: p.text for p in a.scopes}, {p.key: p.text for p in b.scopes}) - self.elsewhere: dict[int, tuple[str, ...]] = {} - - def k(self, part: Part | None) -> str | None: - if part is None: - return None - return part.loose if self.swapped else part.key - - def side(self, key: str) -> str: - """比较用的来源 key;换了一张表时,换的那两张记成同一个。""" - return "?" if self.swapped and key in (self.ta[0].key, self.tb[0].key) else key - - def sides(self, j: JoinFacet) -> tuple[str, ...]: - """连接涉及的来源(见 JoinFacet.sides)。""" - return self.keys(j.sides) - - def keys(self, keys: Iterable[str]) -> tuple[str, ...]: - return tuple(sorted(self.side(s) for s in keys)) - - def agg_key(self, g: Aggregate) -> tuple[str, str | None, bool]: - return g.func, self.k(g.arg), g.distinct - - def added(self, j: JoinFacet, mine: Facets, theirs: Facets) -> tuple[str, set[str]]: - """一边独有的一次连接,多出来的是哪个来源:(给人看的名字, 比较用的 key)。 - - 一般是关联进来的那一侧;可内连接、CROSS JOIN 关联进来的那一侧另一边也有时,多出来的其实是这次连接接上的 - 另一头(FROM stores JOIN orders,另一边只查 orders:多出来的是 stores,不是 orders),说成「关联了 orders - 表」就错了。这时报涉及的来源里另一边没有的那几个。""" - if j.kind in ("inner", "cross"): - have = {self.side(s.key) for s in theirs.sources} - new = [k for k in j.sides if self.side(k) not in have] - if new and self.side(j.table.key) in have: - names = {s.key: s.text for s in mine.sources} - return "、".join(names.get(k, k) for k in new), set(new) - return j.table.text, {j.table.key} - - @staticmethod - def unmatched(xs: Iterable[Any], ys: Iterable[Any], key: Any) -> tuple[list[Any], list[Any]]: - """两边各自独有的项(按 key 比,不管在哪一层),各按原来的先后。派生来源在 key 里已经是两边按结构配好的共同 - 编号(见 diff_facets),这里不再拿不写编号的写法补配一遍:那样两个 CTE 的同名列(p.total、r.total)会被当成 - 一列,换了用哪一个的列也报不出来。配不上的派生来源(一边多出来的 CTE)编号和另一边的都不一样,按它写的项照样 - 各算各的。时间列、算出来的列按它比;条件、分组键、连接、聚合、HAVING 按 leftovers 比(还看在哪一层)。""" - xs, ys = list(xs), list(ys) - kx, ky = {key(x) for x in xs}, {key(y) for y in ys} - return [x for x in xs if key(x) not in ky], [y for y in ys if key(y) not in kx] - - def leftovers(self, xs: Iterable[Any], ys: Iterable[Any], key: Any) -> tuple[list[Any], list[Any]]: - """两边各自多出来的项,各按原来的先后。项都记着在哪一层(scope,见 Part.scope),同一项(key 一样)在一边 - 出现在几层就算几个,一对一地配: - - - 同一层的先配掉;剩下的不管在哪一层也配(条件从 CTE 挪到外层、从 IN 子查询换成关联进来的 CTE 筛,常常是同一个 - 口径),要单说的层先配(不单说的见下,配不完剩下的尽量是它们)。配不完的才是多出来的:外层的 status = 'paid' - 和 IN 子查询里的同一个条件原来按整条语句去重成了一个,外层少了它报不出来;现在一边两层、一边一层,多出来外层 - 那一个。这种另一边也有、只是在的层少了的,记进 elsewhere(另一边有它的那几层),说的时候带上是哪一层。 - - 另一边根本没有这一项的,只报一个(在好几层也只报一次,挑要单说的那一层的),照旧不说哪一层。 - - 条件里的子查询那一层(「?」)多出来的不报:它整个写在外层那个条件、那一列里,那边已经报了,再说一句「只保留 - state = 'done' 的行」,读的人会以为外层也这么筛。另一边没有的 UNION 分支里多出来的也不报:合并了几支、怎么合并 - 的那一面(set_ops)已经报了。""" - xs, ys = list(xs), list(ys) - by_x: dict[Any, list[Any]] = {} - by_y: dict[Any, list[Any]] = {} - for x in xs: - by_x.setdefault(key(x), []).append(x) - for y in ys: - by_y.setdefault(key(y), []).append(y) - left_x: list[Any] = [] - left_y: list[Any] = [] - for k in dict.fromkeys([*by_x, *by_y]): - ax, ay = by_x.get(k, []), by_y.get(k, []) - if not ax or not ay: - # 另一边根本没有:挑一个要单说的报(全在不单说的层里就不报) - mine, other = (ax, 1) if ax else (ay, 0) - if (first := next((m for m in mine if self.visible(m.scope, other)), None)) is not None: - (left_x if ax else left_y).append(first) - continue - sx, sy = {x.scope for x in ax}, {y.scope for y in ay} - # 不单说的层(条件子查询里、另一边没有的 UNION 分支里)排在后面:配不完剩下的尽量是它们 - rx = sorted((x for x in ax if x.scope not in sy), key=lambda x: not self.visible(x.scope, 1)) - ry = sorted((y for y in ay if y.scope not in sx), key=lambda y: not self.visible(y.scope, 0)) - n = min(len(rx), len(ry)) - for x in rx[n:]: - left_x.append(x) - self.elsewhere[id(x)] = tuple(sorted(sy)) - for y in ry[n:]: - left_y.append(y) - self.elsewhere[id(y)] = tuple(sorted(sx)) - order_x = {id(x): i for i, x in enumerate(xs)} - order_y = {id(y): i for i, y in enumerate(ys)} - return ([x for x in sorted(left_x, key=lambda x: order_x[id(x)]) if self.visible(x.scope, 1)], - [y for y in sorted(left_y, key=lambda y: order_y[id(y)]) if self.visible(y.scope, 0)]) - - def visible(self, scope: str, other: int) -> bool: - """这一层多出来的项要不要单说(见 leftovers):条件里的子查询那一层不说,另一边(other)没有的 UNION 分支里的 - 不说。""" - head, bar, _ = scope.partition("|") - return head != "?" and (not bar or not self.layers[other] or scope in self.layers[other]) - - def place(self, scope: str, side: int) -> str: - """一层在那一边的说法(见 Facets.scopes),接在「在」后面:「外层」「t 里」「t 的 UNION 第 2 支里」「条件的 - 子查询里」。""" - text = self.layers[side].get(scope) or ("条件的子查询" if scope.startswith("?") else - "外层" if not scope.partition("|")[0] else "子查询") - return text if text == "外层" else _say(text, "里") - - def layered(self, side: int, item: Any, did: tuple[str, ...], lack: str, have: str) -> str: - """另一边也有、只是在的层少了的一项(item,见 leftovers)多出来的那一个怎么说:「原流程在外层也只保留 X 的行, - 盲算在外层没有这个条件」。did 是这一边做了什么(「也只保留」X「的行」);lack 是另一边在这一层没有时怎么说, - have 是另一边根本没有这一层时、说它在哪几层有的说法(「有这个条件」)。""" - me, other = (self.la, self.lb) if side == 0 else (self.lb, self.la) - if item.scope in self.layers[1 - side]: - tail: tuple[str, ...] = ("在", self.place(item.scope, 1 - side), lack) + la, lb = labels + read = _read(dialect) + ra, ea = _parse(a, read) + rb, eb = _parse(b, read) + if ea or eb: + if ea and eb: + text = _say(la, ":", ea, ";", lb, ":", eb, ";没有逐项比较") else: - seen = self.elsewhere.get(id(item), ()) - tail = ("只在", "、".join(self.place(s, 1 - side) for s in seen), have) if seen else (lack,) - return _say(me, "在", self.place(item.scope, side), *did, ",", other, *tail) - - @staticmethod - def lookalikes(only_a: list[Part], only_b: list[Part]) -> list[tuple[Part, Part]]: - """两边各自独有、给人看的写法却一样的条件配成一对,就地从两边拿掉:同一个条件挂在不同的来源上(p.total > 100 - 和 r.total > 100,各自只读得出 total > 100),分开各说一句「只保留 total > 100 的行」读起来自相矛盾。""" - pairs = [] - for p in list(only_a): - q = next((q for q in only_b if q.text == p.text), None) - if q is not None: - pairs.append((p, q)) - only_a.remove(p) - only_b.remove(q) - return pairs - - @staticmethod - def staged(only_a: list[Part], only_b: list[Part]) -> list[tuple[Part, Part]]: - """两边各自独有、一路透到底却是同一个条件的配成一对,就地从两边拿掉(见 Part.pierce):只有一边被取前几行、 - 算窗口函数的派生来源挡住,是同一个条件筛的先后不同,不是两个条件。""" - pairs = [] - for p in list(only_a): - q = next((q for q in only_b - if bool(p.stage) != bool(q.stage) and (p.pierce or p.key) == (q.pierce or q.key)), None) - if q is not None: - pairs.append((p, q)) - only_a.remove(p) - only_b.remove(q) - return pairs - - def stage_text(self, p: Part, q: Part) -> str: - """同一个条件筛的先后不同:哪一边先取前几行 / 算窗口函数、再筛,哪一边先筛。条件照没被挡住的那一边写:它 - 写的是里面那一列,读起来和原 SQL 一样。""" - cond, stage = (q if p.stage else p).text, p.stage or q.stage - if p.stage: - return _say(self.la, "先", stage, "、再按", cond, "筛选,", self.lb, "先按", cond, "筛选、再", stage) - return _say(self.la, "先按", cond, "筛选、再", stage, ",", self.lb, "先", stage, "、再按", cond, "筛选") - - def shown(self, xs: list[Part], ys: list[Part], *, text: Any = None, full: Any = None) -> tuple[str, str]: - """两边给人看的写法。读起来一样却不是同一个东西时,依次改成:一律写表名的写法(各自只写列名时两张表的 - amount 读起来一样);带上派生来源各列的来历(两边同名的 t1 一个算付款合计、一个算退款合计);两边同名的派生 - 来源各对应另一边的哪一个(两个 CTE 只差里面的条件,来历也一样)。""" - text = text or (lambda ps: "、".join(p.text for p in ps)) - full = full or (lambda ps: "、".join(p.full for p in ps)) - wa, wb = text(xs), text(ys) - if wa != wb: - return wa, wb - wa, wb = full(xs), full(ys) - return (wa, wb) if wa != wb else self.annotated(wa, wb, xs, ys) - - def annotated(self, wa: str, wb: str, xs: list[Part], ys: list[Part]) -> tuple[str, str]: - """两边一律写表名的写法 wa、wb 也一样时(见 shown):带上派生来源各列的来历;来历也一样、用到了自己关联自己的 - 别名时说清各自的别名是哪一个(关联的方向反了,两边都写 a.amount);再不然说清对应关系。""" - ga, gb = self.gloss(xs), self.gloss(ys) - if ga != gb: - return _say(wa, ga), _say(wb, gb) - ra, rb = self.role_note(xs), self.role_note(ys) - if ra != rb: - return _say(wa, ra), _say(wb, rb) - return wa, _say(wb, self.correspondence(xs, ys)) - - @staticmethod - def role_note(parts: list[Part]) -> str: - """用到的自己关联自己的别名各是哪一个(见 Part.roles、_Statement._roles):「(a 是关联条件里出 referrer_id 的 - 那个 t)」。没有返回空串。""" - notes: list[str] = [] - for p in parts: - for alias, role, name in p.roles: - note = _say(alias, "是关联条件里出", "、".join(role.split("+")), "的那个", name) - if note not in notes: - notes.append(note) - return f"({'、'.join(notes)})" if notes else "" - - @staticmethod - def gloss(parts: list[Part]) -> str: - """用到的派生来源的列各自的来历(见 Part.origins):「(即 SUM(orders.amount))」,好几列时「(p.total 即 …、 - r.total 即 …)」。没有返回空串。""" - notes: list[str] = [] - for p in parts: - for column, origin in p.origins: - note = _say("即", origin) if len(p.origins) == 1 and p.full == column else _say(column, "即", origin) - if note not in notes: - notes.append(note) - return f"({'、'.join(notes)})" if notes else "" - - def correspondence(self, xs: list[Part], ys: list[Part]) -> str: - """用到的派生来源两边同名、却各自对应另一边别的派生来源时,说清对应关系:「(盲算的 paid 对应原流程的 open)」; - 一边的在另一边配不上的也说。没有返回空串。""" - keys = dict.fromkeys(ck.split(".", 1)[0].split("#", 1)[0] for p in (*ys, *xs) for ck in p.column_keys - if ck.startswith("@")) - na, nb = self.names - notes = [] - for key in keys: - a, b = na.get(key), nb.get(key) - if a and b and a != b: - notes.append(_say(self.lb, "的", b, "对应", self.la, "的", a)) - elif b and not a: - notes.append(_say(self.lb, "的", b, "在", self.la, "里没有对应的")) - elif a and not b: - notes.append(_say(self.la, "的", a, "在", self.lb, "里没有对应的")) - return f"({'、'.join(notes)})" if notes else "" - - def named(self, parts: list[Part], *, full: bool = False) -> str: - """时间列:有中文名的写「支付时间 paid_at」,几列用顿号隔开。""" - out = [] - for p in parts: - text = p.full if full else p.text - name = self.label(p.columns[0]) if p.columns else None - out.append(_say(name, text) if name else text) - return "、".join(out) - - def one_sided(self, index: int, part: Part, *, group: bool = False) -> str: - """一边独有的一个条件(index 是哪一边:0 是第一条)。另一边也有、只是在的层少了的(见 leftovers),带上是哪一层。""" - side, other = (self.la, self.lb) if index == 0 else (self.lb, self.la) - if id(part) in self.elsewhere: - did = ("分组后也只保留", part.text, "的组") if group else ("也排除了", part.excludes, "的行") \ - if part.excludes else ("也只保留", part.text, "的行") - return self.layered(index, part, did, "没有这个条件", "有这个条件") - if group: - return _say(side, "分组后只保留", part.text, "的组,", other, "没有这个条件") - if part.excludes: - return _say(side, "排除了", part.excludes, "的行,", other, "没有") - return _say(side, "只保留", part.text, "的行,", other, "没有这个条件") - - def at(self, side: int, item: Any, other: Any) -> tuple[str, ...]: - """配成一对的两项不在同一层时,这一边说「在哪一层」(见 place);在同一层不说。""" - return ("在", self.place(item.scope, side)) if item.scope != other.scope else () - - def paired(self, p: Part, q: Part, verb: str = "筛选") -> str: - """两边各一个、配成一对的条件:「原流程按 X 筛选,盲算按 Y 筛选」。""" - wa, wb = self.shown([p], [q]) - return _say(self.la, *self.at(0, p, q), "按", wa, verb + ",", self.lb, *self.at(1, q, p), "按", wb, verb) - - # ---- 各个方面 - - def time(self) -> list[FacetDiff]: - a, b, la, lb = self.a, self.b, self.la, self.lb - ca = list(_dedupe((t.column for t in a.time), self.k)) - cb = list(_dedupe((t.column for t in b.time), self.k)) - if any(self.unmatched(ca, cb, self.k)): - wa, wb = self.shown(ca, cb, text=self.named, full=lambda ps: self.named(ps, full=True)) - if ca and cb: - text = _say(la, "按", wa, "筛时间,", lb, "按", wb) - elif ca: - text = _say(la, "按", wa, "筛时间,", lb, "没有按时间筛选") - else: - text = _say(la, "没有按时间筛选,", lb, "按", wb, "筛时间") - # 对照数据目录用的表、列一直换到实表(TimePredicate.table):不透过去的派生来源比较时照写它自己 - tables = {self.k(t.column): t.table for t in (*a.time, *b.time) if t.table} - detail = {side: [{"table": real[0], "column": real[1]} if (real := tables.get(self.k(p))) - else {"table": p.columns[0][0], "column": p.columns[0][1]} for p in parts if p.columns] - for side, parts in (("a", ca), ("b", cb))} - # 时间列是从 CTE、子查询算出来的列透过去的(见 TimePredicate.column):那一列的定义不同,这里已经说了 - self.explained |= {v for t in (*a.time, *b.time) for v in t.via} - return [FacetDiff("time", "、".join(p.text for p in ca) or None, "、".join(p.text for p in cb) or None, - text, detail)] - only_a, only_b = self.unmatched(a.time, b.time, lambda t: self.k(t.condition)) - conds_a, conds_b = [t.condition for t in only_a], [t.condition for t in only_b] - pairs = self.staged(conds_a, conds_b) - out = [FacetDiff("time_range", p.text, q.text, self.stage_text(p, q)) for p, q in pairs] - if conds_a or conds_b: - staged = {id(p) for pair in pairs for p in pair} - # 同一个时间条件在好几层都有的,范围里只写一次 - ra = list(_dedupe((t.condition for t in a.time if id(t.condition) not in staged), self.k)) - rb = list(_dedupe((t.condition for t in b.time if id(t.condition) not in staged), self.k)) - wa, wb = self.shown(ra, rb, text=lambda ps: " AND ".join(p.text for p in ps), - full=lambda ps: " AND ".join(p.full for p in ps)) - out.append(FacetDiff("time_range", wa, wb, _say(la, "的时间范围是", wa, ",", lb, "是", wb))) - return out - # 两边的时间条件一样,只是一边在更多的层里筛了(外层筛了一遍、IN 子查询里又筛一遍,另一边只在子查询里筛) - more_a, more_b = self.leftovers([t.condition for t in a.time], [t.condition for t in b.time], self.k) - out += [FacetDiff("time_range", p.text, None, - self.layered(0, p, ("也按", p.text, "筛了时间"), "没有这个条件", "有这个条件")) - for p in more_a if id(p) in self.elsewhere] - out += [FacetDiff("time_range", None, p.text, - self.layered(1, p, ("也按", p.text, "筛了时间"), "没有这个条件", "有这个条件")) - for p in more_b if id(p) in self.elsewhere] - return out - - def filters(self) -> list[FacetDiff]: - timed = {self.k(t.condition) for t in (*self.a.time, *self.b.time)} - fa = [p for p in self.a.where if self.k(p) not in timed] - fb = [p for p in self.b.where if self.k(p) not in timed] - only_a, only_b = self.leftovers(fa, fb, self.k) - # 先配同一个条件筛的先后不同的,再配同一列的不同写法(status = 'paid' 和 status IN (…)),再配读起来一样、 - # 挂在不同来源上的同一个条件 - out = [FacetDiff("filter", p.text, q.text, self.stage_text(p, q)) for p, q in self.staged(only_a, only_b)] - for p in list(only_a): - column = _single_column(p, loose=self.swapped) - mine = [q for q in only_a if column and _single_column(q, loose=self.swapped) == column] - theirs = [q for q in only_b if column and _single_column(q, loose=self.swapped) == column] - if len(mine) == 1 and len(theirs) == 1: - q = theirs[0] - out.append(FacetDiff("filter", p.text, q.text, self.paired(p, q))) - only_a.remove(p) - only_b.remove(q) - for p, q in self.lookalikes(only_a, only_b): - wa, wb = self.shown([p], [q]) - out.append(FacetDiff("filter", wa, wb, self.paired(p, q))) - # 两边各剩一个 EXISTS(或 NOT EXISTS):差在子查询里(有子单还是有父单),外层没有列可配,配成一对说 - for kind in ("EXISTS(", "NOT EXISTS("): - mine = [p for p in only_a if (self.k(p) or "").startswith(kind)] - theirs = [q for q in only_b if (self.k(q) or "").startswith(kind)] - if len(mine) == 1 and len(theirs) == 1: - p, q = mine[0], theirs[0] - wa, wb = self.shown([p], [q]) - out.append(FacetDiff("filter", wa, wb, self.paired(p, q))) - only_a.remove(p) - only_b.remove(q) - out += [FacetDiff("filter", p.text, None, self.one_sided(0, p)) for p in only_a] - out += [FacetDiff("filter", None, p.text, self.one_sided(1, p)) for p in only_b] - return out - - def aggregates(self) -> list[FacetDiff]: - la, lb = self.la, self.lb - - def pair(xs: list[Aggregate], ys: list[Aggregate], same: Any) -> list[tuple[Aggregate, Aggregate]]: - pairs = [] - for x in list(xs): - y = next((y for y in ys if same(x, y)), None) - if y is not None: - pairs.append((x, y)) - xs.remove(x) - ys.remove(y) - return pairs - - ga, gb = self.leftovers(self.a.aggregates, self.b.aggregates, self.agg_key) - self.reported = ({self.agg_key(g) for g in ga}, {self.agg_key(g) for g in gb}) - # 先配参数相同的(同一列:函数或去重不同),再配函数相同的(同一种算法:列不同);各剩一个的也配成一对 - pairs = pair(ga, gb, lambda x, y: self.k(x.arg) == self.k(y.arg)) - pairs += pair(ga, gb, lambda x, y: x.func == y.func) - if len(ga) == 1 and len(gb) == 1: - pairs.append((ga.pop(), gb.pop())) - out = [] - for x, y in pairs: - same_arg = self.k(x.arg) == self.k(y.arg) - wa, wb = self.shown([x.arg], [y.arg]) if not same_arg else (x.arg.text, y.arg.text) - ha, hb = self.at(0, x, y), self.at(1, y, x) - if same_arg and x.func == y.func: - # 只差去重:COUNT(DISTINCT order_id) 和 COUNT(order_id) - text = _say(la, *ha, "对", wa, _verb(x) + ",", lb, *hb, "没去重") if x.distinct \ - else _say(la, *ha, "对", wa, _verb(x) + "时没去重,", lb, *hb, _verb(y)) - elif same_arg and x.arg.key != "*": - # 同一列、算法不同:后半句不再重复列名 - text = _say(la, *ha, "对", wa, _verb(x) + ",", lb, *hb, _verb(y)) - else: - what_a = ["数的是行数"] if x.func == "count" and x.arg.key == "*" else ["对", wa, _verb(x)] - what_b = ["数的是行数"] if y.func == "count" and y.arg.key == "*" else ["对", wb, _verb(y)] - text = _say(la, *ha, *what_a, ",", lb, *hb, *what_b) - out.append(FacetDiff("aggregate", x.text, y.text, text)) - for side, rest in ((0, ga), (1, gb)): - me, other = (la, lb) if side == 0 else (lb, la) - for g in rest: - text = self.layered(side, g, ("也算了", g.text), "没有", "算了它") if id(g) in self.elsewhere \ - else _say(me, "多算了", g.text, ",", other, "没有") - out.append(FacetDiff("aggregate", g.text if side == 0 else None, g.text if side == 1 else None, text)) - return out - - def expressions(self) -> list[FacetDiff]: - """算出来的列(见 Computed)。最外层的按算法比,两边各自独有的按先后配成一对说「算的是什么」;派生来源里被 - 用到的列按两边配好对的编号和列名配,定义不同就报。只差在聚合上、聚合那一面已经报了的,按透过去的那一列 - 筛时间、时间那一面已经报了的,不再报一遍。""" - la, lb = self.la, self.lb - out: list[FacetDiff] = [] - - def aggs(c: Computed) -> list[str]: - return sorted(repr(self.agg_key(g)) for g in c.aggregates) - - def pair(xs: list[Computed], ys: list[Computed], same: Any) -> list[tuple[Computed, Computed]]: - pairs = [] - for x in list(xs): - y = next((y for y in ys if same(x, y)), None) - if y is not None: - pairs.append((x, y)) - xs.remove(x) - ys.remove(y) - return pairs - - xa = [c for c in self.a.expressions if c.scope_key is None] - xb = [c for c in self.b.expressions if c.scope_key is None] - only_a, only_b = self.unmatched(xa, xb, lambda c: self.k(c.expr)) - # 先配用到的聚合一样的(同样的合计,算法不同:SUM(amount) 对 SUM(amount) / 100),再配算法一样的(聚合不同), - # 各剩一个的也配成一对 - pairs = pair(only_a, only_b, lambda x, y: aggs(x) == aggs(y)) - pairs += pair(only_a, only_b, lambda x, y: self.k(x.template) == self.k(y.template)) - if len(only_a) == 1 and len(only_b) == 1: - pairs.append((only_a.pop(), only_b.pop())) - for x, y in pairs: - if not self.explained_by_aggregates(x, y): - wa, wb = self.shown([x.expr], [y.expr]) - out.append(FacetDiff("expression", x.expr.text, y.expr.text, - _say(la, "算的是", wa, ",", lb, "算的是", wb))) - # 一边独有的光一个聚合不说:聚合那一面按整条语句比过了(另一边可能在子查询里算了同一个合计、外面只选那一列) - out += [FacetDiff("expression", c.expr.text, None, _say(la, "多算了", c.expr.text, ",", lb, "没有")) - for c in only_a if c.template.key != PLACEHOLDER] - out += [FacetDiff("expression", None, c.expr.text, _say(lb, "多算了", c.expr.text, ",", la, "没有")) - for c in only_b if c.template.key != PLACEHOLDER] - ib = {(c.scope_key, c.name): c for c in self.b.expressions if c.scope_key is not None} - for x in self.a.expressions: - y = ib.get((x.scope_key, x.name)) - if x.scope_key is None or y is None or self.k(x.expr) == self.k(y.expr) or \ - f"{x.scope_key}.{x.name}" in self.explained: - continue - # 两边起名不同、按位置配成同一列的(见 _renamed)照报:聚合那一面只说了合计不同,没说外面用的是这一列 - if name_key(y.shown) == y.name and self.explained_by_aggregates(x, y): - continue - wa, wb = self.shown([x.expr], [y.expr]) - ca, cb = f"{x.scope}.{x.shown}", f"{y.scope}.{y.shown}" - out.append(FacetDiff("expression", wa, wb, - _say(la, "的", ca, "是", wa, ",", lb, *(("是",) if ca == cb else ("的", cb, "是")), wb))) - return out - - def explained_by_aggregates(self, x: Computed, y: Computed) -> bool: - """两边的算法只差在里面的聚合上(聚合换成占位符之后一样),差的那几个聚合聚合那一面又都报了:不再报一遍。 - 聚合的集合一样、只是换了位置(SUM(a) - SUM(b) 对 SUM(b) - SUM(a)),聚合那一面什么也没报,照报。""" - if self.k(x.template) != self.k(y.template) or len(x.aggregates) != len(y.aggregates): - return False - ka, kb = self.reported - differing = [(g, h) for g, h in zip(x.aggregates, y.aggregates) if self.agg_key(g) != self.agg_key(h)] - return bool(differing) and all(self.agg_key(g) in ka or self.agg_key(h) in kb for g, h in differing) - - def joins(self) -> list[FacetDiff]: - """连接:先把完全一样的两边配掉(按层计数,见 leftovers:外层关联了 refunds、EXISTS 子查询里又关联了一次, - 另一边只在子查询里关联,多出来外层那一次),剩下的按关联的是哪张表(派生来源按配好的编号)、再按涉及的来源配成 - 一对,说类型、主表或条件哪里不同,再剩下的是只有一边关联了(多出来的是哪个来源见 added)。 - - 完全一样:内连接、CROSS JOIN 比类型、涉及的来源和条件(FROM paid JOIN refunded 和 FROM refunded JOIN paid 是 - 同一次连接);LEFT / RIGHT JOIN 比主表、涉及的来源和条件,不比写的是 LEFT 还是 RIGHT(orders LEFT JOIN refunds - 和 refunds RIGHT JOIN orders 是同一次连接,和 refunds LEFT JOIN orders 不是);FULL JOIN 比涉及的来源和条件; - 其余(SEMI、ANTI、NATURAL)比类型、关联进来的表和条件。外连接被同一段 WHERE 里的条件筛掉了没关联上的行 - (JoinFacet.rejects)按内连接比:写的是 LEFT JOIN、实际是内连接,和另一边真的 LEFT JOIN 要报出来,和另一边写的 - INNER JOIN 是一回事。先配完全一样的,同一张表关联了两次、先后不同也不算不同。配成一对的两边名字不同(两边给 - CTE 起的名字不同、内连接换了先后)时两个名字都说。派生来源已经按结构配好了编号:一边多出来的 CTE 编号和 - 另一边的都不一样,不会被配给另一边不相干的 CTE、说成「关联条件不同」。 - - 做不到的:内连接里只涉及一张表的条件写在 ON 里还是 WHERE 里意思一样,仍算两处不同;三张以上不带关联条件的 - 表(逗号连接、CROSS JOIN)换了先后,涉及的来源按「它之前的各个来源」算,仍可能算不同;外连接链(a LEFT JOIN b - LEFT JOIN c)的主表按关联条件里出现的来源算,不按整个左边算;WHERE 筛掉了 c 没关联上的行时,c 是按 b 的列关联的, - b 没关联上的行连带也没了,前面那次 LEFT JOIN 其实也成了内连接,这里只认后面那次;FULL JOIN 被 WHERE 筛掉一边的 - 不认。""" - la, lb = self.la, self.lb - - def effective(j: JoinFacet) -> str: - return "inner" if j.rejects is not None and j.kind in ("left", "right") else j.kind - - def same(j: JoinFacet) -> tuple[Any, ...]: - kind = effective(j) - if kind in ("inner", "cross"): - return kind, self.sides(j), self.k(j.on) - if kind in ("left", "right"): - return "outer", self.keys(j.keep), self.sides(j), self.k(j.on) - if kind == "full": - return kind, self.sides(j), self.k(j.on) - return kind, j.table.key, self.k(j.on) - - rest_a, rest_b = self.leftovers(self.a.joins, self.b.joins, same) - pairs: list[tuple[JoinFacet, JoinFacet]] = [] - for table in (lambda j: j.table.key, self.sides): - for x in list(rest_a): - y = next((y for y in rest_b if table(y) == table(x)), None) - if y is not None: - pairs.append((x, y)) - rest_a.remove(x) - rest_b.remove(y) - out: list[FacetDiff] = [] - for x, y in pairs: - tx, ty = x.table.text, y.table.text - ex, ey = effective(x), effective(y) - outer_x, outer_y = ex in ("left", "right"), ey in ("left", "right") - if outer_x and outer_y: - if self.keys(x.keep) != self.keys(y.keep): - wa, wb = self.outer(x, 0), self.outer(y, 1) - out.append(FacetDiff("join", wa, wb, _say(la, wa, ",", lb, wb))) - elif (outer_x and y.rejects is not None) or (outer_y and x.rejects is not None): - wa = self.outer(x, 0) if outer_x else self.rejected(x, 0) - wb = self.outer(y, 1) if outer_y else self.rejected(y, 1) - out.append(FacetDiff("join", self.outer(x, 0) if outer_x else _JOIN_WORDS[x.kind] + "(实际是 INNER JOIN)", - self.outer(y, 1) if outer_y else _JOIN_WORDS[y.kind] + "(实际是 INNER JOIN)", - _say(la, wa, ",", lb, wb))) - elif ex != ey: - kx, ky = _JOIN_WORDS.get(x.kind, x.kind), _JOIN_WORDS.get(y.kind, y.kind) - tail = () if ty == tx else ("关联", ty, "表") - out.append(FacetDiff("join", kx, ky, _say(la, "用", kx, "关联", tx, "表,", lb, "用", ky, *tail))) - if self.k(x.on) != self.k(y.on): - xa, ya = (x.on.text if x.on else "(无条件)"), (y.on.text if y.on else "(无条件)") - tail = ("是", ya) if ty == tx else ("关联", ty, "表的条件是", ya) - out.append(FacetDiff("join", xa, ya, _say(la, "关联", tx, "表的条件是", xa, ",", lb, *tail))) - said: dict[str, set[str]] = {"a": set(), "b": set()} - for side, rest, mine, theirs in (("a", rest_a, self.a, self.b), ("b", rest_b, self.b, self.a)): - me, other = (la, lb) if side == "a" else (lb, la) - for j in rest: - name, keys = self.added(j, mine, theirs) - said[side] |= keys - # 另一边也有这次连接、只是在的层少了(外层关联了 refunds,另一边只在 EXISTS 子查询里关联):说是哪一层 - text = self.layered(0 if side == "a" else 1, j, ("也关联了", name, "表"), "没有", "关联了它") \ - if id(j) in self.elsewhere else _say(me, "关联了", name, "表,", other, "没有") - out.append(FacetDiff("join", name if side == "a" else None, name if side == "b" else None, text)) - # 关联之外的表(FROM 的表、子查询里的表)只在一边出现的;上面已经说过「关联了它」的不再说一遍 - if self.swapped: - ta, tb = self.ta[0], self.tb[0] - return out + [FacetDiff("tables", ta.text, tb.text, _say(la, "查的是", ta.text, "表,", lb, "查的是", tb.text, - "表"))] - out += [FacetDiff("tables", t.text, None, _say(la, "用到了", t.text, "表,", lb, "没有")) - for t in self.ta if t.key not in said["a"]] - out += [FacetDiff("tables", None, t.text, _say(lb, "用到了", t.text, "表,", la, "没有")) - for t in self.tb if t.key not in said["b"]] - return out - - def called_as(self, keys: Iterable[str], side: int) -> str: - """来源 key → 给人看的名字(那一边的叫法),几个用顿号隔开。""" - return "、".join(self.called[side].get(k, k) for k in keys) - - def outer(self, j: JoinFacet, side: int) -> str: - """一次 LEFT / RIGHT JOIN 保留哪一侧:「以 orders 为主表外连接 refunds」。不说 LEFT 还是 RIGHT:orders RIGHT - JOIN refunds 保留的是 refunds,照写关键字反倒容易读反。""" - return _say("以", self.called_as(j.keep, side), "为主表外连接", self.called_as(j.nullable, side)) - - def rejected(self, j: JoinFacet, side: int) -> str: - """写的是外连接、WHERE 里的条件又把没关联上的行筛掉了(见 JoinFacet.rejects)。""" - word = _JOIN_WORDS.get(j.kind, j.kind) - cond = j.rejects.full if j.rejects is not None else "" - return _say("的 WHERE 里有", cond, ",把关联不上", self.called_as(j.nullable, side), "的行筛掉了,", word, - "实际成了 INNER JOIN") - - def group(self) -> list[FacetDiff]: - """分组键按层比(见 leftovers):有不同的那一层,说两边在这一层各按什么分组(当分组用的 DISTINCT 说「去重」, - 见 Facets.distinct)。最外层之外的层、或者另一边在别的层也有这个分组键的,带上是哪一层。分组键都配得上(挪了 - 一层也算配上)时再看最后出来的行按什么分(见 grain)。""" - left_a, left_b = self.leftovers(self.a.group_by, self.b.group_by, self.k) - if not left_a and not left_b: - return self.grain() - distinct = (set(self.a.distinct), set(self.b.distinct)) - out = [] - for scope in dict.fromkeys(p.scope for p in (*left_a, *left_b)): - keys = ([p for p in self.a.group_by if p.scope == scope], [p for p in self.b.group_by if p.scope == scope]) - noted = scope != "" or any(id(p) in self.elsewhere for p in (*left_a, *left_b) if p.scope == scope) - shown = self.shown(*keys) - - def say(side: int) -> tuple[str, ...]: - here = ("在", self.place(scope, side)) if noted and scope in self.layers[side] else () - if keys[side]: - return (*here, "对", shown[side], "去重") if scope in distinct[side] else \ - (*here, "按", shown[side], "分组") - return (*here, "没有去重" if scope in distinct[1 - side] else "没有分组") - - out.append(FacetDiff("group_by", shown[0] or None, shown[1] or None, - _say(self.la, *say(0), ",", self.lb, *say(1)))) - return out - - def grain(self) -> list[FacetDiff]: - """最后出来的行按什么分(见 Facets.grain):分组键挪了一层,按层配得上,可一边外层只出一行(先在子查询里按用户 - 分组、外层数用户数),另一边每个用户一行。只在两边都汇总了(分组或者算了聚合)时比:一边逐行出、一边汇总的, - 聚合那一面已经报了。""" - ga, gb = self.a.grain, self.b.grain - if ga is None or gb is None or "rows" in (ga.key, gb.key) or self.k(ga) == self.k(gb): - return [] - wa, wb = self.shown([ga], [gb]) - - def say(g: Part, written: str) -> tuple[str, ...]: - return ("最后只出一行",) if g.key == "one" else ("按", written, "分组、每组一行") - - return [FacetDiff("group_by", ga.text or None, gb.text or None, - _say(self.la, *say(ga, wa), ",", self.lb, *say(gb, wb)))] - - def set_ops(self) -> list[FacetDiff]: - """各段的 UNION、INTERSECT、EXCEPT(见 SetOpFacet):同一层的配成一对,剩下的合并方式、支数都一样的不管在哪 - 一层也配掉(最外层直接 UNION,和把同一个 UNION 包进子查询再 SELECT * 是一回事)。配成一对的报合并方式(UNION - 去掉重复的行、UNION ALL 都留着)或者支数不同,配不上的报一边合并了、另一边没有。""" - la, lb = self.la, self.lb - rest_a, rest_b = list(self.a.set_ops), list(self.b.set_ops) - pairs: list[tuple[SetOpFacet, SetOpFacet]] = [] - for same in (lambda x, y: x.scope_key == y.scope_key, lambda x, y: (x.ops, x.branches) == (y.ops, y.branches)): - for x in list(rest_a): - y = next((y for y in rest_b if same(x, y)), None) - if y is not None: - pairs.append((x, y)) - rest_a.remove(x) - rest_b.remove(y) - out = [] - for x, y in [*pairs, *((x, None) for x in rest_a), *((None, y) for y in rest_b)]: - if x is not None and y is not None and (x.ops, x.branches) == (y.ops, y.branches): - continue - hx = ("在", x.scope, "里") if x is not None and x.scope else () - hy = ("在", y.scope, "里") if y is not None and y.scope and (x is None or y.scope != x.scope) else () - if x is not None and y is not None and x.branches == y.branches: - text = _say(la, *hx, "用", self.merging(x), ",", lb, *hy, "用", self.merging(y)) - elif x is not None and y is not None: - text = _say(la, *hx, "合并了", str(x.branches), "支查询,", lb, *hy, "合并了", str(y.branches), "支") - elif x is not None: - text = _say(la, *hx, "把", str(x.branches), "支查询用", "、".join(dict.fromkeys(x.ops)), "合起来,", - lb, "没有") - else: - text = _say(lb, *hy, "把", str(y.branches), "支查询用", "、".join(dict.fromkeys(y.ops)), "合起来,", - la, "没有") - out.append(FacetDiff("set_op", " / ".join(x.ops) if x else None, " / ".join(y.ops) if y else None, text)) - return out - - @staticmethod - def merging(x: SetOpFacet) -> str: - """合并的方式:只有 UNION 的说去掉重复的行,只有 UNION ALL 的说重复的行都留着,别的照写。""" - ops = set(x.ops) - if ops == {"UNION"}: - return "UNION 合并(去掉重复的行)" - if ops == {"UNION ALL"}: - return "UNION ALL(重复的行都留着)" - return "、".join(x.ops) - - def having(self) -> list[FacetDiff]: - only_a, only_b = self.leftovers(self.a.having, self.b.having, self.k) - out = [] - for p, q in self.lookalikes(only_a, only_b): - wa, wb = self.shown([p], [q]) - out.append(FacetDiff("having", wa, wb, _say(self.la, *self.at(0, p, q), "分组后只保留", wa, "的组,", self.lb, - *self.at(1, q, p), "只保留", wb, "的组"))) - return out + \ - [FacetDiff("having", p.text, None, self.one_sided(0, p, group=True)) for p in only_a] + \ - [FacetDiff("having", None, p.text, self.one_sided(1, p, group=True)) for p in only_b] - - def limit(self) -> list[FacetDiff]: - """取哪几行:最外层两边比;CTE、子查询里的按所属派生来源配好的编号(LimitFacet.scope_key)配成一对,两边给 - 它起的名字可以不同。配不上的派生来源各说各的,不按先后凑对:一边多一个限了行数的 CTE,不能拿它和另一边的 - CTE 比。不属于哪个派生来源整段的(UNION 的某一支这类)按出现的先后配。""" - def outer(f: Facets) -> LimitFacet | None: - return next((x for x in f.limits if x.scope is None), None) - - out = self.rows(outer(self.a), outer(self.b)) - ia = [x for x in self.a.limits if x.scope is not None] - ib = [x for x in self.b.limits if x.scope is not None] - pairs: list[tuple[LimitFacet | None, LimitFacet | None]] = [] - for x in list(ia): - y = next((y for y in ib if x.scope_key is not None and y.scope_key == x.scope_key), None) - if y is not None: - pairs.append((x, y)) - ia.remove(x) - ib.remove(y) - pairs += zip_longest([x for x in ia if x.scope_key is None], [y for y in ib if y.scope_key is None]) - pairs += [(x, None) for x in ia if x.scope_key is not None] - pairs += [(None, y) for y in ib if y.scope_key is not None] - for x, y in pairs: - out += self.rows(x, y) - return out - - def lk(self, x: LimitFacet | None) -> tuple[Any, ...] | None: - """一段取哪几行的比较用写法:行数、跳过的行数、排序键。""" - if x is None: - return None - return x.count, x.offset, tuple((self.k(p), desc) for p, desc in x.order) - - def sorting(self, x: LimitFacet, *, full: bool = False) -> str: - if not x.order: - return "没有排序" - keys = "、".join(_say(p.full if full else p.text, "降序" if desc else "升序") for p, desc in x.order) - return _say("按", keys, "排") - - def rows(self, x: LimitFacet | None, y: LimitFacet | None) -> list[FacetDiff]: - """一对段(一边可以没有)取的行有什么不同:行数、跳过的行数,两边都限了行时再比排序。""" - if self.lk(x) == self.lk(y): - return [] - sa, sb = (x.scope if x else None), (y.scope if y else None) - la = _say(self.la, "在", sa, "里") if sa else self.la - lb = _say(self.lb, "在", sb, "里") if sb and sb != sa else self.lb - out = [] - ca, cb = (x.count if x else None), (y.count if y else None) - if ca != cb: - out.append(FacetDiff("limit", ca, cb, _say(la, _rows(ca), ",", lb, _rows(cb)))) - oa, ob = (x.offset if x else None), (y.offset if y else None) - if oa != ob: - out.append(FacetDiff("limit", oa, ob, _say(la, _skip(oa), ",", lb, _skip(ob)))) - if x is not None and y is not None and self.lk(x)[2] != self.lk(y)[2]: - wa, wb = self.sorting(x), self.sorting(y) - if wa == wb: - wa, wb = self.sorting(x, full=True), self.sorting(y, full=True) - if wa == wb: - wa, wb = self.annotated(wa, wb, [p for p, _ in x.order], [p for p, _ in y.order]) - out.append(FacetDiff("limit", wa, wb, _say(la, "取行前", wa, ",", lb, wb))) - return out + side, error = (la, ea) if ea else (lb, eb) + text = _say(side, ":", error or "", ",没有逐项比较") + return SqlDiff(False, [FacetDiff("parse", ea, eb, text)]) + mapping = _schema_mapping(schema) + forms_a = {} if _lost_in_parsing(a, read) else _canonical_forms(ra, read, mapping) + forms_b = {} if _lost_in_parsing(b, read) else _canonical_forms(rb, read, mapping) + for level in (_OPTIMIZER, _LIGHT, _PARSED): + fa, fb = forms_a.get(level), forms_b.get(level) + if fa is not None and fb is not None and fa.key == fb.key: + return SqlDiff(True, level=level) + level = next((lv for lv in (_OPTIMIZER, _LIGHT) if lv in forms_a and lv in forms_b), None) + items: list[FacetDiff] = [] + covered = False + if level is not None: + explainer = _Explainer(read, (la, lb), names) + try: + covered = explainer.statement(forms_a[level], forms_b[level]) + items = explainer.items + except Exception: # noqa: BLE001 - 说明本身的缺陷不能连累调用方:当作说不清,记下来修 + logger.warning("SQL 差异说明失败", exc_info=True) + items, covered = [], False + shown = level or _PARSED + if not items: + items = [FacetDiff("other", None, None, _FALLBACK)] + fallback = True + elif not covered: + items = items + [FacetDiff("other", None, None, _FALLBACK_REST)] + fallback = True + else: + fallback = False + text_diff = _text_diff(forms_a.get(shown), forms_b.get(shown), read) + if not text_diff: + text_diff = tuple(line for line in (f"- {a.strip()}", f"+ {b.strip()}")) + return SqlDiff(False, items, fallback, text_diff, level) def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, names: Mapping[str, str] | None = None) -> list[FacetDiff]: - """逐个方面比两份 facets,每处不同一条,按对结果影响大致从大到小排:时间、筛选、聚合、算出来的列、连接和表、 - 分组、HAVING、取哪几行。完全一致返回空列表。 + """两份 facets 记下的原文交给 explain_sql_diff 比,返回它的 items(含兜底条目):完全一致(规范形式一字不差) + 才返回空列表,不等价时一定不为空。 - labels 是两边的叫法(缺省 A / B,双盲复算可以传「原流程」「盲算」);names 是列的中文名({「表.列」或「列」: - 中文名},可以用 catalog_labels 从数据目录取),给了的话时间列写成「支付时间 paid_at」。任一边解析不了就只返回 - 一条 parse,不逐项比较:拿半份结构去比,只会报出一堆不存在的不同。 - - 比之前先把两边的派生来源按结构配好对,按共同的编号重新分析两边(见模块说明、_aligned);给人看的写法不受影响。 + labels、names 见 explain_sql_diff。任一边解析不了:只返回一条 parse。不是 facets 算出来的 Facets(没记原文) + 比不了原文:同一个对象才返回空列表,否则返回一条兜底说明(facets 记的结构不全,两份一样不说明 SQL 一样)。 """ la, lb = labels if a.parse_error or b.parse_error: @@ -3312,53 +4909,13 @@ def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, side, error = (la, a.parse_error) if a.parse_error else (lb, b.parse_error) text = _say(side, ":", error or "", ",没有逐项比较") return [FacetDiff("parse", a.parse_error, b.parse_error, text)] - cmp = _Compare(*_aligned(a, b), labels, names) - return cmp.time() + cmp.filters() + cmp.aggregates() + cmp.expressions() + cmp.joins() + cmp.set_ops() + \ - cmp.group() + cmp.having() + cmp.limit() - - -def _aligned(a: Facets, b: Facets) -> tuple[Facets, Facets]: - """b 的派生来源按结构配给 a 的(见 _align),配上的改写成 a 里对应的编号、配不上的另起 a 里没有的编号,按这套 - 编号重新分析两边的原文(a 的编号就是它的位置编号)。 - - 没用到的 CTE 不收进 facets(见 _Statement._enliven),只有一种例外:两边都定义了同一个 CTE(配成了一对),一边 - 用到、一边没用到,而用到的那边是把它关联进来的。连接那一面会说「X 关联了它,Y 没有」;它里面的条件、聚合两边的 - 定义里都有,只收用到的那一边,就会再被说成一边独有的一堆条件、聚合(两边都定义了付款、退款两个 CTE,一边关联 - 付款的、一边关联退款的)。这时没用到的那一边也照样收它。用到的那边是从它取数(FROM 它)而不是关联它时不收: - 盲算定义了只筛已付的 CTE 却直接查 orders,少的那个条件正要报出来。 - - 任一边不是 facets 算出来的(没记原文)、重新解析不了,或者分析时出了意外,照原样返回:两边仍按各自的位置编号 - 比,一边多出排在前面的 CTE 时会多报。 - """ - try: - sa, sb = _Statement.reparse(a), _Statement.reparse(b) - if sa is None or sb is None: - return a, b - da, db = sa.derived(), sb.derived() - pairs = _align(da, db) - fresh = iter(range(max((d.number for d in da), default=0) + 1, 1 << 30)) - sb.relabel({d.number: f"@{pairs[d.number]}" if d.number in pairs else f"@{next(fresh)}" for d in db}, - _renamed(da, db, pairs)) - fa, fb = sa.facets(), sb.facets() - extra_a = _switched(sb, fb, sa) - extra_b = _switched(sa, fa, sb) - if extra_a or extra_b: - fa, fb = sa.facets(extra_a), sb.facets(extra_b) - return replace(fa, _origin=a._origin), replace(fb, _origin=b._origin) - except Exception: # noqa: BLE001 - 配对本身的缺陷不能连累比较:退回按各自的编号比,记下来修 - logger.warning("SQL 派生来源配对失败,按各自的位置编号比较", exc_info=True) - return a, b - - -def _switched(user: "_Statement", facets_of_user: Facets, other: "_Statement") -> set[str]: - """user 用到、而且关联进来了的派生来源里,other 也定义了(配成了一对)却没用到的 CTE:other 照样收它(见 - _aligned)。返回 other 里这些 CTE 的 name_key。""" - joined = {k for j in facets_of_user.joins for k in (*j.sides, j.table.key)} - used, defined = other.used_labels(), other.defined_labels() - return {cte for label in user.used_labels() if label in joined and label not in used - and (cte := defined.get(label)) is not None} + if a._origin is None or b._origin is None: + return [] if a is b else [FacetDiff("other", None, None, _FALLBACK)] + (sql_a, schema_a), (sql_b, schema_b) = a._origin, b._origin + return explain_sql_diff(sql_a, sql_b, a.dialect or b.dialect, schema=schema_a or schema_b, names=names, + labels=labels).items __all__ = ["MAX_SQL_CHARS", "PLACEHOLDER", "Aggregate", "ColumnRef", "Computed", "FacetDiff", "Facets", "JoinFacet", - "LimitFacet", "NormalizedSql", "Part", "TimePredicate", "catalog_labels", "diff_facets", "facets", - "normalize_sql"] + "LimitFacet", "NormalizedSql", "Part", "SqlDiff", "TimePredicate", "catalog_labels", "diff_facets", + "explain_sql_diff", "facets", "normalize_sql"] diff --git a/backend/tests/fixtures/sql_pairs.py b/backend/tests/fixtures/sql_pairs.py new file mode 100644 index 00000000..46fb6fd6 --- /dev/null +++ b/backend/tests/fixtures/sql_pairs.py @@ -0,0 +1,1865 @@ +"""SQL 差异说明的执行比对夹具:虚构业务库的表结构、按固定种子生成的几组数据、SQL 对的语料。 + +为什么要执行:explain_sql_diff 判「等价」靠的是两条 SQL 规范化之后的形式一字不差,规范化的每一步都必须保持语义。 +光看规范化的代码保证不了这一点(sqlglot 的 merge_subqueries 把外连接可空一侧的常量列并进外层、unnest_subqueries 把 +OR 里的标量子查询改成 CROSS JOIN,都是复验时真踩到的),所以拿真实的库来守门:判等价的两条在每组数据上结果都得 +一样,结果不一样的绝不能判等价。 + +数据刻意带上会让写法差异暴露出来的东西:空值(金额、状态、时间、关联键)、重复的行、边界值(金额正好 100、时间 +正好在月初零点、月末 23:59:59)、已删除的行、退款(一单多退、退款单关联不上订单)、悬空的外键、空表。金额都是 +2 的负幂次的倍数,不同的求和先后得到的浮点数一样;比较结果时浮点数另按 6 位小数取整(乘 0.9 这类每行都有舍入)。 + +语料(PAIRS,SQLite,拿去执行)每一项 (编号, A, B, 预期):same 是已知等价的改写(别名、CTE 改名、关联先后、条件 +左右、子查询和 CTE 互换……),diff 是不许判等价的(换时间列、少一个条件、去重与否、外连接与内连接,以及只在某个方言、 +某种排序规则下才等价的)。LIMIT 都带能定出唯一顺序的排序,没有 ORDER BY 的 LIMIT 不进语料。过去复验报过的反例都 +收进来了:第 5 轮回退的「只在 IN 子查询里用到的 CTE」在 PAIRS 里;复验者留在 /private/tmp 的脚本里、在这几组数据上 +结果不同的对是 VALIDATOR_DIFF(也进 PAIRS 执行);旧测试(五轮复核修正按复验报告补的用例,各方言,用到的列这个库里 +不全有)里报过不同的对是 REGRESSIONS、意思一样的对是 DIALECT_SAME,这两份不执行,只查判没判等价。 +""" +from __future__ import annotations + +import random +import sqlite3 +from dataclasses import dataclass + +DDL = { + "users": "id INTEGER PRIMARY KEY, name TEXT, city TEXT, vip INTEGER, grade INTEGER, referrer_id INTEGER, " + "created_at TEXT, is_deleted INTEGER", + "stores": "id INTEGER PRIMARY KEY, name TEXT, city TEXT, region TEXT, owner_id INTEGER", + "orders": "id INTEGER PRIMARY KEY, user_id INTEGER, store_id INTEGER, parent_id INTEGER, referrer_id INTEGER, " + "amount REAL, discount REAL, tax REAL, status TEXT, region TEXT, created_at TEXT, paid_at TEXT, " + "is_deleted INTEGER", + "orders_archive": "id INTEGER PRIMARY KEY, user_id INTEGER, store_id INTEGER, parent_id INTEGER, " + "referrer_id INTEGER, amount REAL, discount REAL, tax REAL, status TEXT, region TEXT, " + "created_at TEXT, paid_at TEXT, is_deleted INTEGER", + "order_items": "id INTEGER PRIMARY KEY, order_id INTEGER, product_id INTEGER, qty INTEGER, price REAL", + "refunds": "id INTEGER PRIMARY KEY, order_id INTEGER, user_id INTEGER, amount REAL, state TEXT, refunded_at TEXT", + "payments": "id INTEGER PRIMARY KEY, order_id INTEGER, user_id INTEGER, amount REAL, method TEXT, paid_at TEXT", + "shipments": "id INTEGER PRIMARY KEY, order_id INTEGER, carrier TEXT, shipped_at TEXT", +} + + +def schema_cache() -> dict: + """数据源 schema_cache 的形状({"tables": {表: {"columns": [{"name", "type"}]}}}),explain_sql_diff 的 schema。""" + tables = {} + for name, ddl in DDL.items(): + cols = [] + for part in ddl.split(", "): + col, typ = part.split(" ")[:2] + cols.append({"name": col, "type": typ}) + tables[name] = {"columns": cols} + return {"tables": tables} + + +AMOUNTS = [0, 0.5, 10, 12.5, 99.5, 100, 100.25, 250, -5, None] +DATES = ["2026-08-31", "2026-08-31 23:59:59", "2026-09-01", "2026-09-01 08:00:00", "2026-09-15", "2026-09-30", + "2026-09-30 23:59:59", "2026-10-01", "2026-10-01 00:00:00", None] +STATUSES = ["paid", "paid", "open", "done", "void", "shipped", None] +CITIES = ["SH", "BJ", "虚构市", None] + + +def _rows(seed: int) -> dict[str, list[tuple]]: + rng = random.Random(seed) + pick = rng.choice + n_users = rng.randint(4, 12) + users = [(i, f"u{i}", pick(CITIES), pick([0, 1, 1, None]), pick([1, 2, 3, 4, 5, None]), + pick([None, None] + list(range(1, n_users + 1))), pick(DATES), pick([0, 0, 0, 1, None])) + for i in range(1, n_users + 1)] + n_stores = rng.randint(2, 5) + stores = [(i, f"s{i}", pick(CITIES), pick(["east", "west", None]), pick([None] + list(range(1, n_users + 1)))) + for i in range(1, n_stores + 1)] + user_ids = list(range(1, n_users + 1)) + [99, None] + orders: list[tuple] = [] + for i in range(1, rng.randint(8, 36) + 1): + parent = pick([None, None, None] + [o[0] for o in orders[-5:]]) + orders.append((i, pick(user_ids), pick(list(range(1, n_stores + 1)) + [None]), parent, + pick([None] + list(range(1, n_users + 1))), pick(AMOUNTS), pick([0, 0.5, 2, None]), + pick([0, 1, 2.5, None]), pick(STATUSES), pick(["east", "west", None]), pick(DATES), + pick(DATES), pick([0, 0, 1, None]))) + # 重复的行:同一个用户同一笔金额再来一单 + for o in rng.sample(orders, min(3, len(orders))): + orders.append((len(orders) + 1, *o[1:])) + archive = [o for o in orders if rng.random() < 0.5] + items, refunds, payments, shipments = [], [], [], [] + for o in orders: + for _ in range(rng.randint(0, 3)): + items.append((len(items) + 1, o[0], rng.randint(1, 4), pick([0, 1, 2, 5, None]), pick([0.5, 10, 12.5, None]))) + for _ in range(pick([0, 0, 1, 1, 2])): + refunds.append((len(refunds) + 1, o[0], pick([o[1], o[1], None]), pick(AMOUNTS), + pick(["done", "pending", "rejected", None]), pick(DATES))) + for _ in range(rng.randint(0, 2)): + payments.append((len(payments) + 1, o[0], pick([o[1], None]), pick(AMOUNTS), pick(["card", "cash", None]), + pick(DATES))) + if rng.random() < 0.6: + shipments.append((len(shipments) + 1, o[0], pick(["sf", "ems", None]), pick(DATES))) + # 关联不上订单的退款、付款 + refunds.append((len(refunds) + 1, pick([None, 999]), pick(user_ids), pick(AMOUNTS), "done", pick(DATES))) + payments.append((len(payments) + 1, None, pick(user_ids), pick(AMOUNTS), "cash", pick(DATES))) + return {"users": users, "stores": stores, "orders": orders, "orders_archive": archive, "order_items": items, + "refunds": refunds, "payments": payments, "shipments": shipments} + + +def _edge_rows() -> dict[str, list[tuple]]: + """一组极端数据:退款、付款、发货都是空表,订单只有几行,状态、时间全是空值或边界值。""" + users = [(1, "u1", None, None, None, None, None, None), (2, "u2", "SH", 1, 3, 1, "2026-09-01", 0)] + stores = [(1, "s1", None, None, None)] + orders = [(1, 1, 1, None, None, 100, None, None, None, None, "2026-09-01", "2026-10-01", None), + (2, 1, None, 1, 2, None, 0, 0, "paid", "east", None, None, 0), + (3, None, 1, None, None, 100, 0, 1, "paid", None, "2026-09-30 23:59:59", "2026-09-01", 1)] + return {"users": users, "stores": stores, "orders": orders, "orders_archive": [], "order_items": [], + "refunds": [], "payments": [], "shipments": []} + + +#: 生成数据用的种子(每个种子一组),外加一组极端数据 +SEEDS = (7, 11, 23, 42, 2026) + + +def build_databases() -> list[sqlite3.Connection]: + """每组数据一个内存库。""" + out = [] + for rows in [_rows(seed) for seed in SEEDS] + [_edge_rows()]: + conn = sqlite3.connect(":memory:") + for name, ddl in DDL.items(): + conn.execute(f"CREATE TABLE {name} ({ddl})") + if rows[name]: + marks = ", ".join("?" * len(rows[name][0])) + conn.executemany(f"INSERT INTO {name} VALUES ({marks})", rows[name]) + out.append(conn) + return out + + +def _norm(value): + return round(value, 6) if isinstance(value, float) else value + + +def run(conn: sqlite3.Connection, sql: str) -> list[tuple]: + """执行结果按行排序(行的先后不算结果的一部分),浮点数按 6 位小数取整。""" + rows = [tuple(_norm(v) for v in row) for row in conn.execute(sql).fetchall()] + return sorted(rows, key=lambda r: tuple((v is None, str(type(v)), v if v is not None else 0) for v in r)) + + +@dataclass(frozen=True) +class Pair: + id: str + a: str + b: str + #: same:已知等价的改写;diff:不许判等价——语义不同,或者只在某个方言、某种排序规则、某种列类型下才等价 + #: (比如 SQLite 里 status > 'b' AND status > 'o' 和 status > 'o' 一样,PostgreSQL 的语言排序规则下不一定) + expect: str + + +PR = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) ") + +_SAME = [ + # ---- 别名、大小写、空白、注释 + ("alias", "SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid'", + "SELECT SUM(x.amount) FROM orders AS x WHERE x.status = 'paid'"), + ("bare-vs-qualified", "SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid' AND o.paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND status = 'paid'"), + ("case-comments", "SELECT store_id, SUM(amount) FROM orders WHERE status = 'paid' GROUP BY store_id", + "select STORE_ID , -- 门店\n sum( Amount ) /* 金额 */ from ORDERS\n where status='paid' group by store_id"), + ("quoted-names", 'SELECT SUM("amount") FROM "orders" WHERE "status" = \'paid\'', + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("output-alias", "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id", + "SELECT o.user_id AS uid, SUM(o.amount) AS s FROM orders o GROUP BY o.user_id"), + # ---- 条件左右、方向、先后、括号、等价写法 + ("flip-compare", "SELECT COUNT(*) FROM orders WHERE amount > 100 AND paid_at >= '2026-09-01' AND store_id = 3", + "SELECT COUNT(*) FROM orders WHERE 3 = store_id AND '2026-09-01' <= paid_at AND 100 < amount"), + ("and-order-parens", "SELECT COUNT(*) FROM orders WHERE (status = 'paid') AND ((amount > 0)) AND is_deleted = 0", + "SELECT COUNT(*) FROM orders WHERE is_deleted = 0 AND amount > 0 AND status = 'paid'"), + ("or-order", "SELECT COUNT(*) FROM orders WHERE status = 'paid' OR status = 'done'", + "SELECT COUNT(*) FROM orders WHERE status = 'done' OR status = 'paid'"), + ("between", "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at <= '2026-09-30'"), + ("not-eq", "SELECT COUNT(*) FROM orders WHERE NOT status = 'void'", + "SELECT COUNT(*) FROM orders WHERE status <> 'void'"), + ("bang-eq", "SELECT COUNT(*) FROM orders WHERE status != 'void'", + "SELECT COUNT(*) FROM orders WHERE status <> 'void'"), + ("not-null", "SELECT COUNT(*) FROM orders WHERE paid_at IS NOT NULL", + "SELECT COUNT(*) FROM orders WHERE NOT (paid_at IS NULL)"), + ("not-gt", "SELECT COUNT(*) FROM orders WHERE NOT amount > 100", + "SELECT COUNT(*) FROM orders WHERE amount <= 100"), + ("in-order", "SELECT COUNT(*) FROM orders WHERE status IN ('paid', 'done')", + "SELECT COUNT(*) FROM orders WHERE status IN ('done', 'paid', 'done')"), + ("in-single", "SELECT COUNT(*) FROM orders WHERE status IN ('paid')", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("mul-commute", "SELECT SUM(price * qty) FROM order_items WHERE price + 1 > 10", + "SELECT SUM(qty * price) FROM order_items WHERE 1 + price > 10"), + ("paren-arith", "SELECT COUNT(*) FROM orders WHERE (amount + tax) > 10", + "SELECT COUNT(*) FROM orders WHERE amount + tax > 10"), + ("count-1", "SELECT COUNT(1) FROM orders WHERE status = 'paid'", + "SELECT count(*) FROM orders WHERE status = 'paid'"), + ("count-1-having", "SELECT store_id FROM orders GROUP BY store_id HAVING COUNT(1) > 2", + "SELECT store_id FROM orders GROUP BY store_id HAVING 2 < COUNT(*)"), + ("having-flip", "SELECT store_id FROM orders GROUP BY store_id HAVING COUNT(*) >= 5", + "SELECT store_id FROM orders GROUP BY store_id HAVING 5 <= COUNT(*)"), + ("scalar-flip", "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders WHERE is_deleted = 0)", + "SELECT SUM(amount) FROM orders WHERE (SELECT AVG(amount) FROM orders WHERE 0 = is_deleted) < amount"), + ("count-distinct-paren", "SELECT COUNT(DISTINCT(user_id)) FROM orders", + "SELECT COUNT(DISTINCT user_id) FROM orders"), + ("expr-commute", "SELECT store_id, COUNT(*) * 100 AS x FROM orders GROUP BY store_id", + "SELECT store_id, 100 * COUNT(*) AS y FROM orders GROUP BY store_id"), + # ---- 分组、排序、取行 + ("group-ordinal", "SELECT store_id, SUM(amount) AS s FROM orders GROUP BY 1", + "SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id"), + ("group-order", "SELECT region, store_id, SUM(amount) FROM orders GROUP BY region, store_id HAVING SUM(amount) > 10", + "SELECT region, store_id, SUM(amount) AS s FROM orders GROUP BY store_id, region HAVING s > 10"), + ("group-alias", "SELECT o.region, SUM(o.amount) FROM orders o GROUP BY o.region HAVING SUM(o.amount) > 100", + "SELECT region AS r, SUM(amount) AS s FROM orders GROUP BY r HAVING s > 100"), + ("group-expr-ordinal", "SELECT DATE(paid_at) AS d, SUM(amount) FROM orders GROUP BY DATE(paid_at)", + "SELECT DATE(paid_at) AS d, SUM(amount) FROM orders GROUP BY 1"), + ("limit-offset0", "SELECT id, amount FROM orders ORDER BY amount DESC, id LIMIT 3 OFFSET 0", + "SELECT o.id, o.amount FROM orders o ORDER BY o.amount DESC, o.id ASC LIMIT 3"), + ("order-alias", "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id ORDER BY total DESC, user_id LIMIT 3", + "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id ORDER BY SUM(amount) DESC, user_id LIMIT 3"), + ("order-ordinal", "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id ORDER BY 2 DESC, 1 LIMIT 3", + "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id ORDER BY total DESC, user_id LIMIT 3"), + ("distinct-vs-group", "SELECT user_id, store_id FROM orders GROUP BY user_id, store_id", + "SELECT DISTINCT user_id, store_id FROM orders"), + ("distinct-group-redundant", "SELECT DISTINCT user_id FROM orders GROUP BY user_id", + "SELECT user_id FROM orders GROUP BY user_id"), + ("distinct-subq-vs-group", "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' GROUP BY user_id) AS u"), + ("window-alias", "SELECT o.id, ROW_NUMBER() OVER (PARTITION BY o.user_id ORDER BY o.paid_at, o.id) AS rn FROM orders o", + "SELECT x.id, ROW_NUMBER() OVER (PARTITION BY x.user_id ORDER BY x.paid_at, x.id) AS k FROM orders x"), + ("window-partition-order", "SELECT id, SUM(amount) OVER (PARTITION BY user_id, store_id) AS s FROM orders", + "SELECT id, SUM(amount) OVER (PARTITION BY store_id, user_id) AS s FROM orders"), + ("case-alias", "SELECT SUM(CASE WHEN o.status = 'paid' THEN o.amount ELSE 0 END) FROM orders o", + "SELECT SUM(CASE WHEN x.status = 'paid' THEN x.amount ELSE 0 END) FROM orders AS x"), + # ---- 连接 + ("join-swap", "SELECT s.name, SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id GROUP BY s.name", + "SELECT s.name, SUM(o.amount) FROM stores s INNER JOIN orders o ON o.store_id = s.id GROUP BY s.name"), + ("comma-join", "SELECT COUNT(*) FROM orders o, stores s WHERE o.store_id = s.id AND s.city = 'SH'", + "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'"), + ("on-vs-where", "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'", + "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id AND s.city = 'SH'"), + ("cross-where", "SELECT COUNT(*) FROM orders o CROSS JOIN stores s WHERE s.id = o.store_id", + "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id"), + ("three-way", "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id JOIN users u ON u.id = o.user_id " + "WHERE u.vip = 1 AND s.city = 'SH'", + "SELECT COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id JOIN stores s ON o.store_id = s.id " + "WHERE s.city = 'SH' AND u.vip = 1"), + ("multi-on", "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id AND r.user_id = o.user_id", + "SELECT COUNT(*) FROM refunds r JOIN orders o ON o.user_id = r.user_id AND o.id = r.order_id"), + ("left-outer-word", "SELECT o.id, r.amount FROM orders o LEFT OUTER JOIN refunds r ON r.order_id = o.id", + "SELECT o.id, r.amount FROM orders o LEFT JOIN refunds r ON o.id = r.order_id"), + ("left-right-mirror", "SELECT o.id, r.amount FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT o.id, r.amount FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("left-chain-reorder", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id " + "LEFT JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM orders o LEFT JOIN users u ON u.id = o.user_id LEFT JOIN refunds r ON r.order_id = o.id"), + ("left-rejected-vs-inner", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.amount > 10", + "SELECT COUNT(*) FROM refunds r JOIN orders o ON o.id = r.order_id WHERE r.amount > 10"), + ("left-not-null-vs-inner", "SELECT SUM(o.amount) FROM orders o LEFT JOIN users u ON u.id = o.user_id " + "WHERE u.city IS NOT NULL", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id WHERE u.city IS NOT NULL"), + ("anti-join-alias", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.id IS NULL", + "SELECT COUNT(*) FROM orders x LEFT JOIN refunds y ON x.id = y.order_id WHERE y.id IS NULL"), + ("using-vs-on", "SELECT COUNT(*) FROM orders o JOIN refunds r USING (user_id)", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.user_id = o.user_id"), + ("count-distinct-join", "SELECT COUNT(DISTINCT o.user_id) FROM orders o JOIN stores s ON s.id = o.store_id " + "WHERE s.city = 'SH'", + "SELECT COUNT(DISTINCT a.user_id) FROM stores b JOIN orders a ON a.store_id = b.id WHERE b.city = 'SH'"), + # ---- 自连接 + ("self-join-swap", "SELECT COUNT(*) FROM orders o JOIN orders c ON c.parent_id = o.id WHERE c.amount > o.amount", + "SELECT COUNT(*) FROM orders x JOIN orders y ON x.id = y.parent_id WHERE x.amount < y.amount"), + ("self-join-names-exchanged", "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id " + "WHERE a.vip = 1 AND b.vip = 0", + "SELECT COUNT(*) FROM users b JOIN users a ON a.id = b.referrer_id WHERE b.vip = 1 AND a.vip = 0"), + ("self-join-flip-op", "SELECT COUNT(*) FROM orders a JOIN orders b ON a.user_id = b.user_id WHERE a.id < b.id", + "SELECT COUNT(*) FROM orders a JOIN orders b ON a.user_id = b.user_id WHERE b.id > a.id"), + ("self-join-symmetric", "SELECT COUNT(*) FROM orders a JOIN orders b ON a.user_id = b.user_id " + "WHERE a.status = 'paid' AND b.status = 'void'", + "SELECT COUNT(*) FROM orders b JOIN orders a ON b.user_id = a.user_id WHERE b.status = 'paid' AND a.status = 'void'"), + ("self-join-3way", "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id JOIN users c " + "ON c.id = b.referrer_id WHERE c.vip = 1 AND a.vip = 0", + "SELECT COUNT(*) FROM users x JOIN users y ON y.id = x.referrer_id JOIN users z ON z.id = y.referrer_id " + "WHERE x.vip = 0 AND z.vip = 1"), + # ---- CTE、子查询 + ("cte-rename", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT store_id, SUM(amount) FROM paid GROUP BY store_id", + "WITH p2 AS (SELECT * FROM orders WHERE status = 'paid') SELECT store_id, SUM(amount) FROM p2 GROUP BY store_id"), + ("cte-inline", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT store_id, SUM(amount) FROM paid GROUP BY store_id", + "SELECT store_id, SUM(amount) FROM orders WHERE status = 'paid' GROUP BY store_id"), + ("cte-vs-subquery", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE t.total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id) AS q WHERE q.s > 100"), + ("cte-def-order", "WITH a AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "b AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " + "SELECT SUM(a.total - b.total) FROM a JOIN b ON a.user_id = b.user_id", + "WITH b AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "a AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(a.total - b.total) FROM b JOIN a ON a.user_id = b.user_id"), + ("cte-vs-subq-swap", PR + "SELECT SUM(p.total) - SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "SELECT SUM(a.total) - SUM(b.total) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) b " + "JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) a ON a.user_id = b.user_id"), + ("cte-names-swapped", PR + "SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "refunded AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(r.total - p.total) FROM refunded r JOIN paid p ON p.user_id = r.user_id"), + ("cte-condition-flipped", PR + "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id " + "WHERE p.total - r.total > 100", + "WITH rf AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "pd AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM rf JOIN pd ON pd.user_id = rf.user_id WHERE 100 < pd.total - rf.total"), + ("cte-dead", "WITH unused AS (SELECT * FROM refunds), t AS (SELECT user_id, SUM(amount) AS s FROM orders " + "GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 10", + "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 10"), + ("cte-chain-vs-nested", "WITH a AS (SELECT * FROM orders WHERE status = 'paid'), b AS (SELECT user_id, " + "SUM(amount) AS s FROM a GROUP BY user_id) SELECT COUNT(*) FROM b WHERE s > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS s FROM (SELECT * FROM orders WHERE status = 'paid') AS a " + "GROUP BY user_id) AS b WHERE b.s > 100"), + ("cte-filter-then-group", "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), per_user AS (SELECT user_id, " + "SUM(amount) AS total FROM base GROUP BY user_id) " + "SELECT COUNT(*) FROM per_user WHERE total > 100", + "WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " + "SELECT COUNT(*) FROM totals WHERE total > 100"), + ("cte-filter-inner-vs-outer", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT store_id, SUM(amount) FROM p GROUP BY store_id", + "WITH p AS (SELECT * FROM orders) SELECT store_id, SUM(amount) FROM p WHERE status = 'paid' GROUP BY store_id"), + ("cte-join-inline", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT SUM(p.amount) FROM users u JOIN p ON p.user_id = u.id WHERE u.vip = 1", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON o.user_id = u.id WHERE o.status = 'paid' AND u.vip = 1"), + ("subq-join-inline", "SELECT COUNT(*) FROM users u JOIN (SELECT * FROM orders WHERE status = 'paid') o " + "ON o.user_id = u.id", + "SELECT COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id WHERE o.status = 'paid'"), + ("cte-group-city", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT s.city, SUM(p.amount) FROM p JOIN stores s ON s.id = p.store_id GROUP BY s.city", + "SELECT s.city, SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE o.status = 'paid' " + "GROUP BY s.city"), + ("cte-join-in-cte", "WITH t AS (SELECT o.amount, s.city FROM orders o JOIN stores s ON s.id = o.store_id) " + "SELECT city, SUM(amount) FROM t GROUP BY city", + "SELECT s.city, SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id GROUP BY s.city"), + ("cte-passthrough-group", "SELECT store_id, SUM(amount) FROM orders GROUP BY store_id", + "WITH t AS (SELECT store_id, amount FROM orders) SELECT store_id, SUM(amount) FROM t GROUP BY store_id"), + ("cte-computed-col", "WITH t AS (SELECT id, (amount * 0.5) AS net FROM orders) SELECT SUM(net) FROM t", + "WITH t AS (SELECT id, amount * 0.5 AS net FROM orders) SELECT SUM(t.net) FROM t"), + ("cte-date-inline", "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) " + "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "SELECT SUM(amount) FROM orders WHERE DATE(paid_at) >= '2026-09-01'"), + ("cte-grouped-wrapped", "SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id", + "SELECT t.store_id, t.s FROM (SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id) t"), + ("cte-grouped-star", "WITH t AS (SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id) SELECT * FROM t", + "SELECT * FROM (SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id) x"), + ("cte-having-vs-where", "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id " + "HAVING SUM(amount) > 100) SELECT COUNT(*) FROM t", + "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 100"), + ("cte-join-subq-mix", "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(p.total - r.total) FROM paid p JOIN (SELECT user_id, SUM(amount) AS total " + "FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "WITH refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " + "SELECT SUM(p.total - r.total) FROM refunded r JOIN (SELECT user_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id) p ON p.user_id = r.user_id"), + ("cte-three-reorder", "WITH p AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "r AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "v AS (SELECT id FROM users WHERE vip = 1) " + "SELECT SUM(p.total - r.total) FROM p JOIN r ON r.user_id = p.user_id JOIN v ON v.id = p.user_id", + "WITH v AS (SELECT id FROM users WHERE vip = 1), r AS (SELECT user_id, SUM(amount) AS total FROM refunds " + "GROUP BY user_id), p AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(p.total - r.total) FROM v JOIN p ON p.user_id = v.id JOIN r ON p.user_id = r.user_id"), + ("cte-self-join-swapped", "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM t a JOIN t b " + "ON b.user_id = a.referrer_id WHERE a.total > b.total", + "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) " + "SELECT COUNT(*) FROM t b JOIN t a ON a.referrer_id = b.user_id WHERE b.total < a.total"), + ("two-passthrough-ctes", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders " + "WHERE status = 'open') SELECT SUM(p.amount) - SUM(o.amount) FROM paid p JOIN open o " + "ON o.user_id = p.user_id", + "WITH op AS (SELECT * FROM orders WHERE status = 'open'), pd AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT SUM(x.amount) - SUM(y.amount) FROM op y JOIN pd x ON x.user_id = y.user_id"), + ("passthrough-cte-own-table", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) " + "FROM orders o JOIN paid p ON p.parent_id = o.id WHERE o.amount > p.amount", + "WITH pp AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM pp c JOIN orders m " + "ON c.parent_id = m.id WHERE c.amount < m.amount"), + # ---- 条件里的子查询:IN、EXISTS、标量 + ("in-cte", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)", + "WITH v AS (SELECT id FROM users WHERE vip = 1) SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM v)"), + ("in-cte-regression", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM p)", + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + ("in-alias", "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id AS uid FROM users WHERE vip = 1)"), + ("in-and-order", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1 AND city = 'SH')", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users WHERE city = 'SH' AND vip = 1)"), + ("in-join-swap", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT u.id FROM users u JOIN stores s " + "ON s.owner_id = u.id)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT u.id FROM stores s JOIN users u ON u.id = s.owner_id)"), + ("in-distinct", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT DISTINCT user_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM refunds)"), + ("exists-alias", "SELECT o.id FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT orders.id FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), + ("exists-star", "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT * FROM refunds r WHERE r.order_id = o.id)"), + ("exists-and-order", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id " + "AND c.status = 'paid')", + "SELECT COUNT(*) FROM orders AS m WHERE EXISTS (SELECT 1 FROM orders AS k WHERE k.status = 'paid' " + "AND m.id = k.parent_id)"), + ("exists-cte-vs-subq", "WITH v AS (SELECT id FROM users WHERE vip = 1) SELECT COUNT(*) FROM orders o " + "WHERE EXISTS (SELECT 1 FROM v WHERE v.id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM (SELECT id FROM users WHERE vip = 1) v " + "WHERE v.id = o.user_id)"), + ("exists-names-exchanged", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders o2 " + "WHERE o2.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o2 WHERE EXISTS (SELECT 1 FROM orders o WHERE o.parent_id = o2.id)"), + ("exists-outer-by-name", "SELECT COUNT(*) FROM orders WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = orders.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)"), + ("correlated-scalar", "SELECT COUNT(*) FROM orders o WHERE o.amount > (SELECT AVG(i.amount) FROM orders i " + "WHERE i.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders x WHERE (SELECT AVG(y.amount) FROM orders y WHERE x.user_id = y.user_id) < x.amount"), + ("correlated-select-list", "SELECT o.id, (SELECT COUNT(*) FROM refunds r WHERE r.order_id = o.id) AS n FROM orders o", + "SELECT x.id, (SELECT COUNT(*) FROM refunds y WHERE x.id = y.order_id) AS cnt FROM orders x"), + ("scalar-vs-cte", "SELECT COUNT(*) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders)", + "WITH a AS (SELECT AVG(amount) AS m FROM orders) SELECT COUNT(*) FROM orders WHERE amount > (SELECT m FROM a)"), + ("in-outer-and-sub", "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND o.user_id IN " + "(SELECT p.user_id FROM orders p WHERE p.status = 'paid' AND p.amount > 50)", + "SELECT COUNT(*) FROM orders a WHERE a.user_id IN (SELECT b.user_id FROM orders b WHERE b.amount > 50 " + "AND b.status = 'paid') AND a.status = 'paid'"), + ("cte-exists-renamed", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p " + "WHERE EXISTS (SELECT 1 FROM p p2 WHERE p2.parent_id = p.id)", + "WITH q AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM q AS m WHERE EXISTS " + "(SELECT 1 FROM q AS c WHERE c.parent_id = m.id)"), + ("cte-correlated-vs-subq", "WITH t AS (SELECT o.* FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r " + "WHERE r.order_id = o.id)) SELECT COUNT(*) FROM t", + "SELECT COUNT(*) FROM (SELECT x.* FROM orders x WHERE EXISTS (SELECT 1 FROM refunds y WHERE y.order_id = x.id)) AS q"), + ("cte-in-and-in", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM p " + "WHERE p.user_id IN (SELECT user_id FROM orders WHERE amount > 50)", + "SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid' AND o.user_id IN (SELECT user_id FROM orders " + "WHERE amount > 50)"), + ("subq-from-subq-alias", "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT s.user_id FROM " + "(SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) s WHERE s.total > 100)", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT agg.user_id FROM " + "(SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) agg WHERE agg.total > 100)"), + # ---- UNION + ("union-branch-order", "SELECT user_id, amount FROM orders WHERE amount > 1 UNION ALL SELECT user_id, amount FROM refunds", + "SELECT user_id, amount FROM refunds UNION ALL SELECT user_id, amount FROM orders WHERE 1 < amount"), + ("union-wrapped", "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT * FROM (SELECT user_id FROM orders UNION SELECT user_id FROM refunds) t"), + ("union-cte-vs-subq", "WITH t AS (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) " + "SELECT COUNT(*) FROM t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) AS q"), + ("union-aliases", "SELECT o.user_id, o.amount FROM orders o WHERE o.status = 'paid' UNION ALL " + "SELECT r.user_id, -r.amount FROM refunds r", + "SELECT x.user_id, x.amount FROM orders x WHERE x.status = 'paid' UNION ALL SELECT y.user_id, -y.amount FROM refunds y"), + ("union-output-alias", "SELECT user_id AS u FROM orders UNION SELECT user_id FROM refunds", + "SELECT user_id AS uid FROM orders UNION SELECT user_id FROM refunds"), + ("union-cte-filter", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT user_id FROM p WHERE amount > 10 " + "UNION ALL SELECT user_id FROM p WHERE amount < 0", + "SELECT user_id FROM orders WHERE status = 'paid' AND amount > 10 UNION ALL " + "SELECT user_id FROM orders WHERE status = 'paid' AND amount < 0"), + ("archive-union-reorder", "SELECT id, amount FROM orders UNION SELECT id, amount FROM orders_archive", + "SELECT id, amount FROM orders_archive UNION SELECT id, amount FROM orders"), + # ---- 过滤挪了一层、分组键上的条件 + ("group-key-filter-moved", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 3 " + "GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t AS z WHERE 100 < z.total AND z.user_id > 3"), + ("preserved-side-constant", "SELECT d.flag, s.id FROM (SELECT id, 1 AS flag FROM users) d LEFT JOIN stores s " + "ON s.owner_id = d.id", + "SELECT 1 AS flag, s.id FROM users u LEFT JOIN stores s ON s.owner_id = u.id"), + ("orders-items-join", "SELECT o.id, SUM(i.qty * i.price) FROM orders o JOIN order_items i ON i.order_id = o.id " + "GROUP BY o.id", + "SELECT x.id, SUM(y.price * y.qty) FROM order_items y JOIN orders x ON x.id = y.order_id GROUP BY x.id"), + ("payments-left", "SELECT o.id, COALESCE(SUM(p.amount), 0) FROM orders o LEFT JOIN payments p ON p.order_id = o.id " + "GROUP BY o.id", + "SELECT a.id, COALESCE(SUM(b.amount), 0) FROM orders a LEFT JOIN payments b ON a.id = b.order_id GROUP BY a.id"), + ("shipments-exists", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM shipments s WHERE s.order_id = o.id " + "AND s.carrier = 'sf')", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT * FROM shipments AS t WHERE 'sf' = t.carrier " + "AND o.id = t.order_id)"), +] + +_DIFF = [ + # ---- 时间列、时间范围 + ("time-column", "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' AND created_at < '2026-10-01'"), + ("time-one-side", "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders"), + ("time-range-end", "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'"), + ("time-range-op", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at > '2026-09-01'"), + ("time-in-cte", "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT SUM(amount) FROM t " + "WHERE d >= '2026-09-01'", + "WITH t AS (SELECT DATE(created_at) AS d, amount FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("time-date-fn", "SELECT COUNT(*) FROM orders WHERE DATE(paid_at) = '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at = '2026-09-01'"), + # ---- 有效行过滤 + ("filter-deleted", "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01'"), + ("filter-exclusion", "SELECT SUM(amount) FROM orders WHERE is_deleted <> 1", "SELECT SUM(amount) FROM orders"), + ("filter-status-value", "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status = 'done'"), + ("filter-status-in", "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')"), + ("filter-not-vs-neq-null", "SELECT COUNT(*) FROM orders WHERE status <> 'void'", + "SELECT COUNT(*) FROM orders WHERE status <> 'void' OR status IS NULL"), + ("filter-threshold", "SELECT COUNT(*) FROM orders WHERE amount > 100", "SELECT COUNT(*) FROM orders WHERE amount >= 100"), + ("filter-and-vs-or", "SELECT COUNT(*) FROM orders WHERE status = 'paid' AND amount > 10", + "SELECT COUNT(*) FROM orders WHERE status = 'paid' OR amount > 10"), + ("filter-in-cte-dropped", "WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT v.user_id) FROM v", + "SELECT COUNT(DISTINCT user_id) FROM orders"), + ("filter-cte-unused", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("filter-cte-status", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT store_id, SUM(amount) FROM p GROUP BY store_id", + "WITH p AS (SELECT * FROM orders WHERE status = 'open') SELECT store_id, SUM(amount) FROM p GROUP BY store_id"), + ("in-cte-regression", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM p)", + "WITH p AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM p)"), + ("exists-cte-regression", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM p WHERE p.user_id = u.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM p WHERE p.user_id = u.id)"), + ("scalar-cte-regression", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM users u WHERE (SELECT COUNT(*) FROM p WHERE p.user_id = u.id) > 0", + "WITH p AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT COUNT(*) FROM users u WHERE (SELECT COUNT(*) FROM p WHERE p.user_id = u.id) > 0"), + ("in-subquery-column", "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.referrer_id FROM users u WHERE u.vip = 1)"), + ("in-subquery-filter", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users)"), + ("not-in-vs-not-exists", "SELECT COUNT(*) FROM orders o WHERE o.id NOT IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)"), + ("in-vs-not-in", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE user_id NOT IN (SELECT user_id FROM refunds)"), + ("exists-child-vs-parent", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)"), + ("correlated-earlier", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p " + "WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id " + "AND p.created_at > o.created_at)"), + ("scalar-subquery-filter", "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders " + "WHERE is_deleted = 0)", + "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders)"), + ("unnest-or-trap", "SELECT id FROM orders WHERE status = 'paid' OR amount > (SELECT amount FROM refunds WHERE id = 999999)", + "SELECT orders.id FROM orders CROSS JOIN (SELECT amount FROM refunds WHERE id = 999999) AS q " + "WHERE orders.status = 'paid' OR orders.amount > q.amount"), + ("unnest-scalar-trap", "SELECT id FROM orders WHERE amount > (SELECT amount FROM refunds ORDER BY id LIMIT 1)", + "SELECT orders.id FROM orders CROSS JOIN (SELECT amount FROM refunds) AS q WHERE orders.amount > q.amount"), + # ---- 聚合、去重 + ("count-distinct", "SELECT COUNT(DISTINCT user_id) FROM orders", "SELECT COUNT(user_id) FROM orders"), + ("count-star-vs-col", "SELECT COUNT(*) FROM orders", "SELECT COUNT(user_id) FROM orders"), + ("count-star-vs-distinct", "SELECT COUNT(*) FROM orders", "SELECT COUNT(DISTINCT id) FROM orders WHERE amount > 0"), + ("sum-vs-avg", "SELECT SUM(amount) FROM orders", "SELECT AVG(amount) FROM orders"), + ("sum-column", "SELECT SUM(amount) FROM orders", "SELECT SUM(discount) FROM orders"), + ("sum-vs-coalesce", "SELECT SUM(amount) FROM refunds", "SELECT COALESCE(SUM(amount), 0) FROM refunds"), + ("max-vs-min", "SELECT user_id, MAX(paid_at) FROM orders GROUP BY user_id", + "SELECT user_id, MIN(paid_at) FROM orders GROUP BY user_id"), + ("expression-factor", "WITH t AS (SELECT id, amount * 0.5 AS net FROM orders) SELECT SUM(net) FROM t", + "WITH t AS (SELECT id, amount * 0.25 AS net FROM orders) SELECT SUM(net) FROM t"), + ("expression-swap-sub", PR + "SELECT SUM(p.total) - SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + PR + "SELECT SUM(r.total) - SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("cte-which-total", PR + "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > 100", + PR + "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE r.total > 100"), + ("cte-swapped-defs", PR + "SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "refunded AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("distinct-dropped", "SELECT DISTINCT user_id FROM orders", "SELECT user_id FROM orders"), + ("distinct-in-subq", "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders) t"), + ("count-distinct-vs-subq", "SELECT COUNT(DISTINCT user_id) FROM orders", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders) t"), + ("window-partition", "SELECT id, SUM(amount) OVER (PARTITION BY user_id) AS s FROM orders", + "SELECT id, SUM(amount) OVER (PARTITION BY store_id) AS s FROM orders"), + ("window-rank-vs-rownum", "SELECT id, RANK() OVER (ORDER BY amount) AS r FROM orders", + "SELECT id, ROW_NUMBER() OVER (ORDER BY amount, id) AS r FROM orders"), + ("window-filter-stage", "SELECT COUNT(*) FROM (SELECT id, status, ROW_NUMBER() OVER (PARTITION BY user_id " + "ORDER BY paid_at, id) AS rn FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at, id) AS rn " + "FROM orders WHERE status = 'paid') t WHERE t.rn = 1"), + ("limit-filter-stage", "SELECT COUNT(*) FROM (SELECT * FROM orders ORDER BY id LIMIT 5) t WHERE t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid' ORDER BY id LIMIT 5) t"), + ("case-default", "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE 0 END) FROM orders", + "SELECT SUM(CASE WHEN status = 'paid' THEN amount END) FROM orders"), + ("output-columns-order", "SELECT user_id, SUM(amount) FROM orders GROUP BY user_id", + "SELECT SUM(amount), user_id FROM orders GROUP BY user_id"), + ("output-extra-column", "SELECT user_id, SUM(amount) FROM orders GROUP BY user_id", + "SELECT user_id, SUM(amount), COUNT(*) FROM orders GROUP BY user_id"), + # ---- 分组、HAVING、取行 + ("group-key", "SELECT region, SUM(amount) FROM orders GROUP BY region", + "SELECT region, status, SUM(amount) FROM orders GROUP BY region, status"), + ("group-none", "SELECT SUM(amount) FROM orders", "SELECT region, SUM(amount) FROM orders GROUP BY region"), + ("group-grain", "SELECT COUNT(*) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id"), + ("having-dropped", "SELECT region, SUM(amount) FROM orders GROUP BY region HAVING SUM(amount) > 100", + "SELECT region, SUM(amount) FROM orders GROUP BY region"), + ("having-threshold", "SELECT store_id FROM orders GROUP BY store_id HAVING COUNT(*) > 2", + "SELECT store_id FROM orders GROUP BY store_id HAVING COUNT(*) > 3"), + ("limit-count", "SELECT id FROM orders ORDER BY id LIMIT 3", "SELECT id FROM orders ORDER BY id LIMIT 5"), + ("limit-vs-none", "SELECT id FROM orders ORDER BY id LIMIT 3", "SELECT id FROM orders ORDER BY id"), + ("limit-offset", "SELECT id FROM orders ORDER BY id LIMIT 3", "SELECT id FROM orders ORDER BY id LIMIT 3 OFFSET 2"), + ("limit-order-dir", "SELECT id FROM orders ORDER BY amount DESC, id LIMIT 3", + "SELECT id FROM orders ORDER BY amount, id LIMIT 3"), + ("limit-in-cte", "WITH t AS (SELECT * FROM orders ORDER BY id LIMIT 5) SELECT SUM(amount) FROM t", + "WITH t AS (SELECT * FROM orders ORDER BY id LIMIT 8) SELECT SUM(amount) FROM t"), + ("order-only", "SELECT id, amount FROM orders ORDER BY amount DESC, id", "SELECT id, amount FROM orders"), + # ---- 连接 + ("left-vs-inner", "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("left-join-key", "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.user_id = o.user_id"), + ("join-dropped", "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(amount) FROM orders"), + ("inner-join-added", "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id", + "SELECT SUM(amount) FROM orders"), + ("join-filter-city", "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'", + "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'BJ'"), + ("left-keep-side", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON r.order_id = o.id"), + ("right-keep-side", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id"), + ("left-on-vs-where", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id AND r.state = 'done'", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'"), + ("left-is-null-not-rejected", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id " + "WHERE r.amount > 5 OR r.id IS NULL", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.amount > 5 OR r.id IS NULL"), + ("left-coalesce-not-rejected", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id " + "WHERE COALESCE(r.amount, 0) = 0", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE COALESCE(r.amount, 0) = 0"), + ("nullable-side-constant", "SELECT o.id, r.flag FROM orders o LEFT JOIN (SELECT order_id, 1 AS flag FROM refunds) r " + "ON r.order_id = o.id", + "SELECT o.id, 1 AS flag FROM orders o LEFT JOIN refunds r ON r.order_id = o.id"), + ("nullable-side-coalesce", "SELECT o.id, r.a FROM orders o LEFT JOIN (SELECT order_id, COALESCE(amount, 0) AS a " + "FROM refunds) r ON r.order_id = o.id", + "SELECT o.id, COALESCE(r.amount, 0) AS a FROM orders o LEFT JOIN refunds r ON r.order_id = o.id"), + ("nullable-nested-constant", "SELECT o.id, d.flag FROM orders o LEFT JOIN (SELECT x.order_id, x.flag FROM " + "(SELECT order_id, 1 AS flag FROM refunds) x) d ON d.order_id = o.id", + "SELECT o.id, 1 AS flag FROM orders o LEFT JOIN refunds r ON r.order_id = o.id"), + ("right-side-constant", "SELECT d.flag, s.id FROM (SELECT id, 1 AS flag FROM users) d RIGHT JOIN stores s " + "ON s.owner_id = d.id", + "SELECT 1 AS flag, s.id FROM users u RIGHT JOIN stores s ON s.owner_id = u.id"), + ("full-side-constant", "SELECT d.flag, s.id FROM (SELECT id, 1 AS flag FROM users) d FULL JOIN stores s " + "ON s.owner_id = d.id", + "SELECT 1 AS flag, s.id FROM users u FULL JOIN stores s ON s.owner_id = u.id"), + ("preserved-side-constant", "SELECT d.flag, s.id FROM (SELECT id, 1 AS flag FROM users) d LEFT JOIN stores s " + "ON s.owner_id = d.id", + "SELECT d.flag, s.id FROM (SELECT id, 2 AS flag FROM users) d LEFT JOIN stores s ON s.owner_id = d.id"), + ("self-join-direction", "SELECT COUNT(*) FROM orders o JOIN orders c ON c.parent_id = o.id WHERE c.amount > o.amount", + "SELECT COUNT(*) FROM orders x JOIN orders y ON x.id = y.parent_id WHERE x.amount > y.amount"), + ("self-join-roles", "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE a.vip = 1", + "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE b.vip = 1"), + ("table-swap", "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 0 AND paid_at >= '2026-09-01'"), + ("two-columns-same-name", "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("join-multiplies", "SELECT SUM(o.amount) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("passthrough-which-cte", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders " + "WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT SUM(o.amount) FROM paid p JOIN open o ON o.user_id = p.user_id"), + ("cte-self-join-direction", "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM t a JOIN t b " + "ON b.user_id = a.referrer_id WHERE a.total > b.total", + "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) " + "SELECT COUNT(*) FROM t a JOIN t b ON b.user_id = a.referrer_id WHERE b.total > a.total"), + ("extra-cte-join", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE total > 100", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id) SELECT COUNT(*) FROM vip JOIN t ON t.user_id = vip.id WHERE t.total > 100"), + # ---- UNION + ("union-vs-union-all", "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds"), + ("union-branch-filter", "SELECT user_id FROM orders WHERE status = 'paid' UNION ALL SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds"), + ("union-branch-dup", "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds UNION ALL SELECT user_id FROM refunds"), + ("union-distinct-branch", "SELECT DISTINCT user_id FROM orders UNION ALL SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds"), + ("except-order", "SELECT user_id FROM orders EXCEPT SELECT user_id FROM refunds", + "SELECT user_id FROM refunds EXCEPT SELECT user_id FROM orders"), + ("intersect-vs-union", "SELECT user_id FROM orders INTERSECT SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION SELECT user_id FROM refunds"), + # ---- 写法上看着一样 + ("in-vs-exists-null", "SELECT COUNT(*) FROM users u WHERE u.id NOT IN (SELECT user_id FROM orders)", + "SELECT COUNT(*) FROM users u WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"), + ("coalesce-filter", "SELECT COUNT(*) FROM orders WHERE COALESCE(is_deleted, 0) = 0", + "SELECT COUNT(*) FROM orders WHERE is_deleted = 0"), + ("integer-division", "SELECT SUM(amount) / COUNT(*) FROM orders", "SELECT AVG(amount) FROM orders"), + ("cast-date-sqlite", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= CAST('2026-09-01' AS DATE)"), + ("string-compare-redundant", "SELECT COUNT(*) FROM orders WHERE status > 'b' AND status > 'o'", + "SELECT COUNT(*) FROM orders WHERE status > 'o'"), + ("text-number-compare", "SELECT COUNT(*) FROM orders WHERE region > 10 AND region > 9", + "SELECT COUNT(*) FROM orders WHERE region > 10"), + ("arith-equality", "SELECT COUNT(*) FROM orders WHERE amount + 1 = 101", "SELECT COUNT(*) FROM orders WHERE amount = 100"), + # ---- 别名和列同名(压测撞到的):SELECT 列表里的名字指表的列,不指前面列的别名;去重的 UNION 留先出现的值 + ("select-alias-shadows-column", "SELECT user_id AS amount, amount * 2 AS twice FROM orders", + "SELECT user_id AS amount, user_id * 2 AS twice FROM orders"), + ("select-alias-shadows-column-2", "SELECT store_id AS region, region FROM orders WHERE status = 'paid'", + "SELECT store_id AS region, store_id FROM orders WHERE status = 'paid'"), + ("having-alias-shadows-column", "SELECT user_id, SUM(amount) AS amount FROM orders GROUP BY user_id " + "HAVING SUM(amount) > 10", + "SELECT user_id, SUM(amount) AS amount FROM orders GROUP BY user_id HAVING amount > 10"), + ("union-dedup-branch-order", "SELECT tax FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", + "SELECT referrer_id FROM orders UNION SELECT tax FROM orders WHERE status = 'paid'"), +] + + +# ---------------------------------------------------------------------------- 过去的反例 +# 下面三份是从旧的 tests/test_sql_semantics.py(五轮复核修正里按复验报告补的用例)和复验者留在 /private/tmp 的 +# 脚本里收集来的 SQL 对,原样收进来:名字是出处(旧测试的函数名、复验脚本:用例名)。 + +#: 旧测试里两边写法不同、意思一样的对(各方言),新做法认得出是等价的 +DIALECT_SAME: list[tuple[str, str, str, str]] = [ + ("names_given_to_derived_tables_are_no_difference", "mysql", + "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) r ON r.order_id = o.id WHERE r.n > 1", + "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) `退款` ON o.id = `退款`.order_id WHERE `退款`.n > 1"), + ("the_same_derived_column_written_differently_is_no_difference", "mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x", + "SELECT SUM(y.total) FROM (SELECT a.user_id, a.total FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) b JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) a ON a.user_id = b.user_id) y"), + ("derived_tables_reading_the_same_table_written_differently_ar", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH settled AS (SELECT * FROM orders WHERE status = 'paid'), pending AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(s.amount) FROM settled s JOIN pending q ON q.user_id = s.user_id"), + ("names_given_to_derived_tables_are_no_difference", "postgres", + "SELECT SUM(amount) FROM orders WHERE user_id IN (WITH t AS (SELECT id FROM users WHERE grade >= 3) SELECT id FROM t)", + "SELECT SUM(amount) FROM orders WHERE user_id IN (WITH g AS (SELECT id FROM users WHERE grade >= 3) SELECT g.id FROM g)"), + ("derived_tables_reading_the_same_table_written_differently_ar", "postgres", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(x.amount) FROM (SELECT * FROM orders WHERE status = 'open') y JOIN (SELECT * FROM orders WHERE status = 'paid') x ON x.user_id = y.user_id"), + ("the_same_output_computation_written_differently_is_no_differ", "postgres", + "SELECT order_id, RANK() OVER (PARTITION BY user_id ORDER BY amount DESC) AS k FROM orders", + "SELECT o.order_id, RANK() OVER (PARTITION BY o.user_id ORDER BY o.amount DESC) FROM orders o"), + ("a_self_join_tells_its_two_aliases_apart", "postgres", + "WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id WHERE b1.total > b2.total", + "WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM base y JOIN base x ON x.referrer_id = y.user_id WHERE y.total < x.total"), + ("the_same_outer_join_written_differently_is_no_difference", "postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM refunded r RIGHT JOIN paid p ON p.user_id = r.user_id"), + ("names_given_to_derived_tables_are_no_difference", "postgres", + "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM user_totals WHERE total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) AS agg WHERE agg.total > 100"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION ALL SELECT referrer_id FROM orders WHERE status = 'paid') t", + "WITH ids AS (SELECT user_id FROM orders WHERE status = 'paid' UNION ALL SELECT referrer_id FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM ids"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE p.total > r.total", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id WHERE p.total > r.total"), + ("the_same_outer_join_written_differently_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("the_same_outer_join_written_differently_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id LEFT JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM orders o LEFT JOIN users u ON u.id = o.user_id LEFT JOIN refunds r ON r.order_id = o.id"), + ("the_same_outer_join_written_differently_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + "SELECT COUNT(*) FROM orders AS o LEFT OUTER JOIN refunds AS r ON o.id = r.order_id WHERE o.status = 'paid'"), + ("the_same_outer_join_written_differently_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'"), + ("the_same_correlated_subquery_written_differently_is_no_diffe", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders x WHERE EXISTS (SELECT 1 FROM orders y WHERE x.id = y.parent_id)"), + ("the_same_correlated_subquery_written_differently_is_no_diffe", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), + ("a_self_join_tells_its_two_aliases_apart", "sqlite", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o1.amount > o2.amount", + "SELECT COUNT(*) FROM orders c JOIN orders p ON c.parent_id = p.id WHERE c.amount > p.amount"), + ("the_same_item_on_equivalent_layers_is_no_difference", "sqlite", + "SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT x.user_id FROM orders x WHERE x.status = 'paid') AND o.status = 'paid'"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.amount > r.amount", + "SELECT SUM(o.amount) FROM refunds r JOIN orders o ON o.id = r.order_id WHERE o.amount > r.amount"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(o2.amount) AS avg_amount FROM orders o2 WHERE o2.is_deleted = 0)", + "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders WHERE is_deleted = 0)"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted = 0", + "select sum(amount) from orders where 0 = is_deleted and '2026-09-01' <= paid_at"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT s.user_id FROM (SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) s WHERE s.total > 100)", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT agg.user_id FROM (SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) agg WHERE agg.total > 100)"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id AS uid FROM users u WHERE u.vip = 1)", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o, refunds r WHERE r.order_id = o.id", + "SELECT SUM(o.amount) FROM refunds r, orders o WHERE r.order_id = o.id"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o, stores s WHERE o.store_id = s.id AND s.city = '虚构市'", + "SELECT SUM(orders.amount) FROM orders INNER JOIN stores ON stores.id = orders.store_id WHERE stores.city = '虚构市'"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10 OFFSET 0", + "SELECT o.amount FROM orders o ORDER BY o.amount DESC LIMIT 10"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT o.id FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT orders.id FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), + ("the_same_output_computation_written_differently_is_no_differ", "sqlite", + "SELECT region, SUM(amount) * 1.0 / COUNT(*) AS avg_amt FROM orders GROUP BY region", + "SELECT o.region, 1.0 * SUM(o.amount) / COUNT(*) FROM orders o GROUP BY 1"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT region, SUM(amount) FROM orders GROUP BY region", + "WITH t AS (SELECT region, SUM(amount) AS s FROM orders GROUP BY region) SELECT region, s FROM t"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT user_id FROM orders GROUP BY user_id", + "SELECT DISTINCT user_id FROM orders"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id", + "SELECT user_id, n FROM (SELECT user_id, COUNT(*) AS n FROM orders GROUP BY user_id) t"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), per_user AS (SELECT user_id, SUM(amount) AS total FROM base GROUP BY user_id) SELECT COUNT(*) FROM per_user WHERE total > 100", + "WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM totals WHERE total > 100"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH c AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), d AS (SELECT user_id, COUNT(*) AS n FROM refunds GROUP BY user_id), a AS (SELECT * FROM c WHERE total > 100), b AS (SELECT * FROM d WHERE n > 1) SELECT COUNT(*) FROM b JOIN a ON a.user_id = b.user_id", + "SELECT COUNT(*) FROM (SELECT * FROM (SELECT user_id, COUNT(*) AS n FROM refunds GROUP BY user_id) x WHERE n > 1) b JOIN (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) y WHERE total > 100) a ON a.user_id = b.user_id"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH c AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), d AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), a AS (SELECT * FROM c), b AS (SELECT * FROM d) SELECT COUNT(*) FROM b JOIN a ON a.user_id = b.user_id WHERE a.total > b.total", + "SELECT COUNT(*) FROM (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) x) b JOIN (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) y) a ON a.user_id = b.user_id WHERE a.total > b.total"), + ("derived_tables_reading_the_same_table_written_differently_ar", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM paid p JOIN refunds r ON r.order_id = p.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'"), + ("derived_tables_reading_the_same_table_written_differently_ar", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > o.amount", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM open o JOIN paid p ON p.user_id = o.user_id WHERE o.amount < p.amount"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM refunded r JOIN paid p ON p.user_id = r.user_id WHERE p.total > r.total"), + ("the_same_output_computation_written_differently_is_no_differ", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) - SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) - SUM(r.total) FROM refunded r JOIN paid p ON p.user_id = r.user_id"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > o.total", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) SELECT COUNT(*) FROM open o JOIN paid p ON p.user_id = o.user_id WHERE o.total < p.total"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", + "WITH r AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), p AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT COUNT(*) FROM p JOIN r ON p.user_id = r.user_id WHERE p.total > r.total"), + ("filtering_at_an_equivalent_stage_is_no_difference", "sqlite", + "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t WHERE status = 'paid'", + "WITH top10 AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(t.amount) FROM top10 t WHERE t.status = 'paid'"), + ("derived_tables_reading_the_same_table_written_differently_ar", "sqlite", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(y.amount) FROM t x JOIN t y ON y.referrer_id = x.user_id"), + ("derived_columns_renamed_on_one_side_are_paired", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100", + "WITH x AS (SELECT user_id, SUM(amount) AS sum_amt FROM orders GROUP BY user_id) SELECT COUNT(*) FROM x WHERE sum_amt > 100"), + ("a_shifted_number_inside_a_condition_subquery_is_no_differenc", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u WHERE u.vip = 1 AND u.id IN (SELECT user_id FROM t WHERE total > 100)", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM vip WHERE vip.id IN (SELECT user_id FROM t WHERE total > 100)"), + ("swapped_join_order_of_two_derived_tables_is_no_difference", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM t JOIN u ON u.user_id = t.user_id WHERE t.total > u.cnt", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM u JOIN t ON t.user_id = u.user_id WHERE t.total > u.cnt"), + ("the_same_item_on_equivalent_layers_is_no_difference", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 100 GROUP BY user_id) SELECT SUM(total) FROM t", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(total) FROM t WHERE user_id > 100"), + ("derived_columns_renamed_on_one_side_are_paired", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100 AND n > 1", + "WITH t AS (SELECT user_id, SUM(amount) AS s, COUNT(*) AS c FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 100 AND c > 1"), + ("the_same_derived_column_written_differently_is_no_difference", "sqlite", + "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT SUM(net) FROM t", + "WITH x AS (SELECT user_id, 0.9 * amount AS net FROM orders) SELECT SUM(x.net) FROM x"), + ("derived_columns_renamed_on_one_side_are_paired", "sqlite", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS day FROM orders) SELECT SUM(amount) FROM t WHERE day >= '2026-09-01'"), + ("the_same_derived_column_written_differently_is_no_difference", "sqlite", + "WITH t AS (SELECT user_id, amount, paid_at AS ts FROM orders) SELECT SUM(amount) FROM t", + "WITH t AS (SELECT user_id, amount, created_at AS ts FROM orders) SELECT SUM(amount) FROM t"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM user_totals WHERE total > 100", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM t WHERE t.total > 100"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT v.order_id) FROM v", + "SELECT COUNT(DISTINCT order_id) FROM orders WHERE is_deleted = 0"), + ("the_same_item_on_equivalent_layers_is_no_difference", "sqlite", + "WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT SUM(amount) FROM v WHERE is_deleted = 0", + "SELECT SUM(amount) FROM orders WHERE is_deleted = 0"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH vip AS (SELECT id FROM users WHERE grade >= 3) SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM vip)", + "WITH good_users AS (SELECT id FROM users WHERE grade >= 3) SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT g.id FROM good_users g)"), +] + +#: 过去五轮复验、旧测试里报过的不同(各方言):绝不能判等价 +REGRESSIONS: list[tuple[str, str, str, str]] = [ + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY p.store_id", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid') t"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT id FROM orders WHERE status = 'paid' UNION ALL SELECT id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100) x", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100) x"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT referrer_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT referrer_id FROM orders) t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + ("which_side_an_outer_join_keeps_is_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "mysql", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "mysql", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND o.created_at < p.created_at)"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)"), + ("each_dialect_gives_the_same_three_findings", "mysql", + "SELECT COUNT(DISTINCT o.order_id) AS `订单数` FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted <> 1", + "SELECT COUNT(o.order_id) AS `订单数` FROM orders o WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(t.user_id) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT COUNT(user_id) FROM orders GROUP BY user_id"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT DISTINCT region FROM orders", + "SELECT region FROM orders"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.referrer_id = a.user_id"), + ("the_same_output_computation_written_differently_is_no_differ", "mysql", + "SELECT SUM(amount) FROM orders", + "SELECT IFNULL(SUM(amount), 0) FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", + "SELECT AVG(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(o.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id"), + ("how_a_derived_column_is_computed_is_reported", "mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x", + "SELECT SUM(x.total) FROM (SELECT p.user_id, r.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "mysql", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT user_id, COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1) GROUP BY user_id", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1)"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id"), + ("time_columns_inside_derived_tables", "mysql", + "WITH a AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id), b AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'", + "WITH a AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id), b AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id) SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("an_outer_join_turned_inner_by_where_is_reported", "mysql", + "WITH done AS (SELECT * FROM refunds WHERE state = 'done') SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id", + "WITH done AS (SELECT * FROM refunds) SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id WHERE d.state = 'done'"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "mysql", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = p.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = p.parent_id)"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY p.store_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT p.amount - o.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT o.amount - p.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(o.amount) FROM paid p JOIN open o ON o.user_id = p.user_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.user_id = o.referrer_id", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.referrer_id = o.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id WHERE o.status = 'paid'", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id"), + ("how_a_derived_column_is_computed_is_reported", "mysql", + "WITH r AS (SELECT user_id, amount, RANK() OVER (ORDER BY amount DESC) AS k FROM orders) SELECT SUM(amount) FROM r WHERE k <= 10", + "WITH r AS (SELECT user_id, amount, ROW_NUMBER() OVER (ORDER BY amount DESC) AS k FROM orders) SELECT SUM(amount) FROM r WHERE k <= 10"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.referrer_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(b.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM t p JOIN orders c ON c.parent_id = p.id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(c.amount) FROM t p JOIN orders c ON c.parent_id = p.id"), + ("how_a_derived_column_is_computed_is_reported", "mysql", + "WITH t AS (SELECT DATE_FORMAT(paid_at, '%Y-%m') AS m, amount FROM orders) SELECT m, SUM(amount) FROM t GROUP BY m", + "WITH t AS (SELECT DATE_FORMAT(created_at, '%Y-%m') AS m, amount FROM orders) SELECT m, SUM(amount) FROM t GROUP BY m"), + ("how_a_derived_column_is_computed_is_reported", "mysql", + "WITH t AS (SELECT user_id, CASE WHEN status = 'paid' THEN amount ELSE 0 END AS paid_amt FROM orders) SELECT SUM(paid_amt) FROM t", + "WITH t AS (SELECT user_id, CASE WHEN status IN ('paid', 'shipped') THEN amount ELSE 0 END AS paid_amt FROM orders) SELECT SUM(paid_amt) FROM t"), + ("a_time_column_computed_in_a_cte_is_seen_through", "mysql", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(created_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY p.store_id", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid') t"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(*) FROM (SELECT id FROM orders WHERE status = 'paid' UNION ALL SELECT id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100) x", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100) x"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT referrer_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT referrer_id FROM orders) t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + ("filtering_at_an_equivalent_stage_is_no_difference", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE t.rn = 1 AND t.user_id > 100", + "SELECT COUNT(*) FROM (SELECT user_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders WHERE user_id > 100) t WHERE t.rn = 1"), + ("how_a_derived_column_is_computed_is_reported", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id, amount - fee AS net FROM orders) a WHERE a.net > 100", + "SELECT COUNT(*) FROM (SELECT user_id, amount + fee AS net FROM orders) b WHERE b.net > 100"), + ("filtering_before_or_after_a_window_or_limit_is_reported", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders WHERE status = 'paid') t WHERE t.rn = 1"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)"), + ("an_outer_join_turned_inner_by_where_is_reported", "postgres", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'"), + ("which_side_an_outer_join_keeps_is_compared", "postgres", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "postgres", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "postgres", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND o.created_at < p.created_at)"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)"), + ("each_dialect_gives_the_same_three_findings", "postgres", + "SELECT COUNT(DISTINCT o.order_id) AS \"订单数\" FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted <> 1", + "SELECT COUNT(o.order_id) AS \"订单数\" FROM orders o WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(t.user_id) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT COUNT(user_id) FROM orders GROUP BY user_id"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT DISTINCT region FROM orders", + "SELECT region FROM orders"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.referrer_id = a.user_id"), + ("how_a_derived_column_is_computed_is_reported", "postgres", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE rn = 1", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at DESC) AS rn FROM orders) t WHERE rn = 1"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + ("an_outer_join_turned_inner_by_where_is_reported", "postgres", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE o.status = 'paid'"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", + "SELECT AVG(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)"), + ("names_given_to_derived_tables_are_no_difference", "postgres", + "SELECT SUM(o.amount) FROM orders o, LATERAL (SELECT COUNT(*) AS n FROM refunds r WHERE r.order_id = o.id) l WHERE l.n > 0", + "SELECT SUM(o.amount) FROM orders o CROSS JOIN LATERAL (SELECT COUNT(*) AS n FROM refunds r WHERE r.order_id = o.id) AS x WHERE x.n > 0"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(o.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id"), + ("how_an_output_column_is_computed_is_compared", "postgres", + "SELECT SUM(r.total) / SUM(p.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT SUM(p.total) / SUM(r.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id"), + ("time_columns_inside_derived_tables", "postgres", + "SELECT SUM(x.amount) FROM (SELECT amount, CAST(paid_at AS DATE) AS day FROM orders) x WHERE x.day BETWEEN '2026-09-01' AND '2026-09-30'", + "SELECT SUM(y.amount) FROM (SELECT amount, CAST(created_at AS DATE) AS day FROM orders) y WHERE y.day BETWEEN '2026-09-01' AND '2026-09-30'"), + ("a_lateral_subquery_is_named_by_its_alias", "postgres", + "SELECT o.id FROM orders o, LATERAL (SELECT amount FROM refunds r WHERE r.order_id = o.id ORDER BY amount DESC LIMIT 1) l", + "SELECT o.id FROM orders o CROSS JOIN LATERAL (SELECT amount FROM refunds r WHERE r.order_id = o.id ORDER BY amount DESC LIMIT 2) AS x"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "postgres", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o"), + ("a_correlated_lateral_subquery_on_the_outer_table_is_reported", "postgres", + "SELECT o.id, x.s FROM orders o CROSS JOIN LATERAL (SELECT SUM(c.amount) AS s FROM orders c WHERE c.parent_id = o.id) x", + "SELECT o.id, x.s FROM orders o CROSS JOIN LATERAL (SELECT SUM(c.amount) AS s FROM orders c WHERE c.id = o.parent_id) x"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT user_id, COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1) GROUP BY user_id", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1)"), + ("how_a_derived_column_is_computed_is_reported", "postgres", + "WITH RECURSIVE t(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM t WHERE n < 10) SELECT SUM(n) FROM t", + "WITH RECURSIVE t(n) AS (SELECT 1 UNION ALL SELECT n + 2 FROM t WHERE n < 10) SELECT SUM(n) FROM t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("a_self_join_tells_its_two_aliases_apart", "postgres", + "WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id WHERE b1.total > b2.total", + "WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id WHERE b2.total > b1.total"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "postgres", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = p.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = p.parent_id)"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY p.store_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("a_time_column_of_a_pass_through_derived_table_names_the_real", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.created_at::date >= '2026-09-01'", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.created_at::date >= '2026-09-01'"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT p.amount - o.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT o.amount - p.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(o.amount) FROM paid p JOIN open o ON o.user_id = p.user_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("how_a_derived_column_is_computed_is_reported", "postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, p.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT COUNT(*) FROM net WHERE v > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, r.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT COUNT(*) FROM net WHERE v > 100"), + ("which_side_an_outer_join_keeps_is_compared", "postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM refunded r LEFT JOIN paid p ON p.user_id = r.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.user_id = o.referrer_id", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.referrer_id = o.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id WHERE o.status = 'paid'", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.referrer_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(b.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM t p JOIN orders c ON c.parent_id = p.id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(c.amount) FROM t p JOIN orders c ON c.parent_id = p.id"), + ("time_columns_inside_derived_tables", "postgres", + "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT CAST(paid_at AS DATE) AS d, amount FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("the_same_derived_column_written_differently_is_no_difference", "postgres", + "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT d, SUM(amount) FROM t GROUP BY d", + "WITH t AS (SELECT CAST(paid_at AS DATE) AS d, amount FROM orders) SELECT d, SUM(amount) FROM t GROUP BY d"), + ("a_time_column_computed_in_a_cte_is_seen_through", "postgres", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(created_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("same_named_derived_tables_with_swapped_definitions_are_told_", "postgres", + "WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), t2 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100", + "WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), t2 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY p.store_id", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid') t"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(*) FROM (SELECT id FROM orders WHERE status = 'paid' UNION ALL SELECT id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100) x", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100) x"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT referrer_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT referrer_id FROM orders) t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE p.total - r.total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE r.total - p.total > 100"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE p.total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE r.total > 100"), + ("other_aggregate_differences", "sqlite", + "SELECT COUNT(*) FROM orders", + "SELECT COUNT(DISTINCT order_id) FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)"), + ("an_outer_join_turned_inner_by_where_is_reported", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN (SELECT * FROM refunds WHERE state = 'done') r ON r.order_id = o.id", + "SELECT COUNT(*) FROM orders o LEFT JOIN (SELECT * FROM refunds) r ON r.order_id = o.id WHERE r.state = 'done'"), + ("which_side_an_outer_join_keeps_is_compared", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id"), + ("only_null_rejecting_where_conditions_turn_an_outer_join_inne", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE COALESCE(r.amount, 0) < 100", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id"), + ("which_side_an_outer_join_keeps_is_compared", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.order_id IS NULL", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE r.order_id IS NULL"), + ("only_null_rejecting_where_conditions_turn_an_outer_join_inne", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done' OR r.id IS NULL", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done' OR r.id IS NULL"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND o.created_at < p.created_at)"), + ("a_difference_inside_a_condition_subquery_is_said_once", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND r.state = 'done')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)"), + ("a_self_join_tells_its_two_aliases_apart", "sqlite", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o1.amount > o2.amount", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o2.amount > o1.amount"), + ("each_dialect_gives_the_same_three_findings", "sqlite", + "SELECT COUNT(DISTINCT o.order_id) AS \"订单数\" FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted <> 1", + "SELECT COUNT(o.order_id) AS \"订单数\" FROM orders o WHERE o.created_at >= '2026-09-01'"), + ("distinct_count_versus_plain_count", "sqlite", + "SELECT COUNT(DISTINCT order_id) FROM orders", + "SELECT COUNT(order_id) FROM orders"), + ("side_labels_in_chinese_have_no_stray_spaces", "sqlite", + "SELECT COUNT(DISTINCT order_id) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(order_id) FROM orders WHERE created_at >= '2026-09-01'"), + ("distinct_count_versus_plain_count", "sqlite", + "SELECT COUNT(order_id) FROM orders", + "SELECT COUNT(DISTINCT order_id) FROM orders"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(t.user_id) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT COUNT(user_id) FROM orders GROUP BY user_id"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT DISTINCT region FROM orders", + "SELECT region FROM orders"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.referrer_id = a.user_id"), + ("how_an_output_column_is_computed_is_compared", "sqlite", + "SELECT SUM(amount) - SUM(fee) FROM orders", + "SELECT SUM(amount), SUM(fee) FROM orders"), + ("other_aggregate_differences", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT AVG(amount) FROM orders"), + ("the_same_output_computation_written_differently_is_no_differ", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT ROUND(SUM(amount), 2) FROM orders"), + ("how_an_output_column_is_computed_is_compared", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) / 100 FROM orders"), + ("time_filter_on_one_side_only", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'"), + ("an_exclusion_on_one_side", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) FROM orders WHERE refund_id IS NOT NULL"), + ("a_kept_condition_on_one_side", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("which_rows_are_taken_is_compared", "sqlite", + "SELECT SUM(amount) FROM orders ORDER BY 1 DESC", + "SELECT SUM(amount) FROM orders"), + ("an_exclusion_on_one_side", "sqlite", + "SELECT SUM(amount) FROM orders WHERE is_deleted <> 1", + "SELECT SUM(amount) FROM orders"), + ("a_different_source_table_is_said_once", "sqlite", + "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 0 AND paid_at >= '2026-09-01'"), + ("a_different_source_table_is_said_once", "sqlite", + "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 1 AND paid_at >= '2026-09-01'"), + ("time_filter_on_one_side_only", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders"), + ("time_column_differs_with_catalog_names", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' AND created_at < '2026-10-01'"), + ("only_literals_differ_is_still_a_different_range", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-08' AND paid_at < '2026-10-08'"), + ("same_time_column_different_range", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')"), + ("two_conditions_on_the_same_column_are_paired", "sqlite", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + ("an_extra_source_written_in_from_is_named_correctly", "sqlite", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM users u JOIN refunds r ON r.user_id = u.id JOIN orders o ON o.id = r.order_id"), + ("two_columns_that_read_the_same_are_told_apart", "sqlite", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("join_differences", "sqlite", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(amount) FROM orders"), + ("join_differences", "sqlite", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("join_differences", "sqlite", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.user_id = o.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", + "SELECT AVG(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)"), + ("an_extra_source_written_in_from_is_named_correctly", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE o.status = 'paid'"), + ("an_extra_source_written_in_from_is_named_correctly", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT id FROM users WHERE vip = 1)", + "SELECT SUM(o.amount) FROM users u JOIN orders o ON o.user_id = u.id WHERE u.vip = 1"), + ("a_real_difference_inside_a_subquery_is_still_reported", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.referrer_id FROM users u WHERE u.vip = 1)"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(o.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT SUM(p.total - r.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT SUM(r.total - p.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT SUM(p.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT SUM(r.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id"), + ("other_aggregate_differences", "sqlite", + "SELECT SUM(paid_amount) FROM orders", + "SELECT SUM(amount) FROM orders"), + ("limit_difference", "sqlite", + "SELECT amount FROM orders LIMIT 10", + "SELECT amount FROM orders"), + ("limit_difference", "sqlite", + "SELECT amount FROM orders LIMIT 10", + "SELECT amount FROM orders LIMIT 100"), + ("which_rows_are_taken_is_compared", "sqlite", + "SELECT amount FROM orders LIMIT 10 OFFSET 0", + "SELECT amount FROM orders LIMIT 10 OFFSET 10"), + ("which_rows_are_taken_is_compared", "sqlite", + "SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10", + "SELECT amount FROM orders ORDER BY 1 ASC LIMIT 10"), + ("which_rows_are_taken_is_compared", "sqlite", + "SELECT amount FROM orders ORDER BY amount DESC LIMIT 10", + "SELECT amount FROM orders LIMIT 10"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "sqlite", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o"), + ("how_an_output_column_is_computed_is_compared", "sqlite", + "SELECT order_id, SUM(amount) OVER (PARTITION BY user_id) AS s FROM orders", + "SELECT order_id, SUM(amount) OVER (PARTITION BY store_id) AS s FROM orders"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT p.user_id FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id GROUP BY p.user_id HAVING MAX(p.total) > 10", + "SELECT p.user_id FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id GROUP BY p.user_id HAVING MAX(r.total) > 10"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT p.user_id FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id ORDER BY p.total DESC LIMIT 10", + "SELECT p.user_id FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id ORDER BY r.total DESC LIMIT 10"), + ("group_by_difference", "sqlite", + "SELECT region, SUM(amount) FROM orders GROUP BY region", + "SELECT region, city, SUM(amount) FROM orders GROUP BY region, city"), + ("having_difference", "sqlite", + "SELECT region, SUM(amount) FROM orders GROUP BY region HAVING SUM(amount) > 100", + "SELECT region, SUM(amount) FROM orders GROUP BY region"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", + "SELECT * FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT user_id, COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1) GROUP BY user_id", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1)"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "sqlite", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = p.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = p.parent_id)"), + ("a_cte_defined_but_not_used_does_not_count", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM paid", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM orders"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY p.store_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT p.amount - o.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT o.amount - p.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(o.amount) FROM paid p JOIN open o ON o.user_id = p.user_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("how_a_derived_column_is_computed_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, p.total - r.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT SUM(v) FROM net", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, r.total - p.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT SUM(v) FROM net"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total - r.total > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE r.total - p.total > 100"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE r.total > 100"), + ("a_reversed_comparison_between_two_derived_tables_is_still_re", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE r.total > p.total"), + ("an_extra_source_written_in_from_is_named_correctly", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM vip v JOIN refunded r ON r.user_id = v.id JOIN paid p ON p.user_id = r.user_id WHERE p.total > r.total"), + ("unused_ctes_take_part_in_pairing", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN paid p ON p.user_id = u.id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN refunded r ON r.user_id = u.id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(r.total - p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("how_an_output_column_is_computed_is_compared", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) - SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(r.total) - SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("renamed_and_reordered_derived_tables_still_tell_their_column", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH ref_sum AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), pay_sum AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(r.total) FROM ref_sum r JOIN pay_sum p ON p.user_id = r.user_id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT p.user_id FROM paid p JOIN refunded r ON r.user_id = p.user_id GROUP BY p.user_id HAVING MAX(p.total) > 10", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT p.user_id FROM paid p JOIN refunded r ON r.user_id = p.user_id GROUP BY p.user_id HAVING MAX(r.total) > 10"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT p.user_id FROM paid p JOIN refunded r ON r.user_id = p.user_id ORDER BY p.total DESC LIMIT 10", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT p.user_id FROM paid p JOIN refunded r ON r.user_id = p.user_id ORDER BY r.total DESC LIMIT 10"), + ("same_named_derived_tables_that_differ_only_in_conditions_say", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' AND is_deleted = 0 GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' AND is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > 100"), + ("two_same_shape_derived_tables_are_paired_by_their_conditions", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > o.total", + "WITH unpaid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' AND is_deleted = 0 GROUP BY user_id), settled AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' AND is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM unpaid u JOIN settled s ON s.user_id = u.user_id WHERE u.total < s.total"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.user_id = o.referrer_id", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.referrer_id = o.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id WHERE o.status = 'paid'", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id"), + ("filtering_before_or_after_a_window_or_limit_is_reported", "sqlite", + "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t WHERE paid_at >= '2026-09-01'", + "WITH t AS (SELECT * FROM orders WHERE paid_at >= '2026-09-01' ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t"), + ("filtering_before_or_after_a_window_or_limit_is_reported", "sqlite", + "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t WHERE status = 'paid'", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid' ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.referrer_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(b.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM t p JOIN orders c ON c.parent_id = p.id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(c.amount) FROM t p JOIN orders c ON c.parent_id = p.id"), + ("time_columns_inside_derived_tables", "sqlite", + "WITH t AS (SELECT amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT amount, DATE(paid_at, '+8 hours') AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT store_id, user_id, SUM(amount) AS total FROM orders GROUP BY store_id, user_id) SELECT store_id FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) GROUP BY store_id HAVING MAX(total) > 100", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT store_id, user_id, SUM(amount) AS total FROM orders GROUP BY store_id, user_id) SELECT t.store_id FROM vip JOIN t ON t.user_id = vip.id GROUP BY t.store_id HAVING MAX(t.total) > 100"), + ("how_a_derived_column_is_computed_is_reported", "sqlite", + "WITH t AS (SELECT strftime('%Y-%m', paid_at) AS m, amount FROM orders) SELECT m, SUM(amount) FROM t GROUP BY m", + "WITH t AS (SELECT strftime('%Y-%m', created_at) AS m, amount FROM orders) SELECT m, SUM(amount) FROM t GROUP BY m"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT user_id, DATE(paid_at) AS day, amount FROM orders) SELECT SUM(amount) FROM t WHERE day >= '2026-09-01' AND user_id IN (SELECT id FROM users WHERE vip = 1)", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, DATE(paid_at) AS day, amount FROM orders) SELECT SUM(t.amount) FROM vip JOIN t ON t.user_id = vip.id WHERE t.day >= '2026-09-01'"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT AVG(total) FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1)", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT AVG(t.total) FROM vip JOIN t ON t.user_id = vip.id"), + ("a_changed_column_inside_a_renamed_derived_table_is_said_once", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), x AS (SELECT user_id, SUM(paid_amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM vip JOIN x ON x.user_id = vip.id WHERE total > 100"), + ("derived_columns_renamed_on_one_side_are_paired", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100", + "WITH x AS (SELECT user_id, SUM(amount - discount) AS net_total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM x WHERE net_total > 100"), + ("a_shifted_derived_number_does_not_split_one_condition_in_two", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100 AND user_id IN (SELECT id FROM users WHERE vip = 1)", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM vip JOIN t ON t.user_id = vip.id WHERE t.total > 100"), + ("a_paired_derived_join_with_another_name_says_both_names", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id", + "WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN totals ON totals.user_id = u.referrer_id"), + ("an_extra_derived_table_on_one_side_is_not_paired_with_anothe", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id JOIN t ON t.user_id = u.referrer_id"), + ("a_shifted_derived_join_is_paired_with_its_counterpart", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id WHERE t.total > 100", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id JOIN t ON t.user_id = u.id WHERE t.total > 100"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT user_id FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) ORDER BY total DESC LIMIT 10", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT t.user_id FROM vip JOIN t ON t.user_id = vip.id ORDER BY t.total DESC LIMIT 10"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) SELECT n, COUNT(*) FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) GROUP BY n", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) SELECT t.n, COUNT(*) FROM vip JOIN t ON t.user_id = vip.id GROUP BY t.n"), + ("how_a_derived_column_is_computed_is_reported", "sqlite", + "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT * FROM t", + "WITH t AS (SELECT user_id, amount * 0.8 AS net FROM orders) SELECT * FROM t"), + ("how_a_derived_column_is_computed_is_reported", "sqlite", + "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT SUM(net) FROM t", + "WITH t AS (SELECT user_id, amount * 0.8 AS net FROM orders) SELECT SUM(net) FROM t"), + ("a_time_column_computed_in_a_cte_is_seen_through", "sqlite", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(created_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("which_rows_are_taken_is_compared", "sqlite", + "WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM top", + "WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 100) SELECT SUM(amount) FROM top"), + ("limits_inside_derived_tables_are_paired_by_structure", "sqlite", + "WITH top AS (SELECT user_id, amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM top", + "WITH vip AS (SELECT id FROM users WHERE vip = 1 LIMIT 5), top AS (SELECT user_id, amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(top.amount) FROM top JOIN vip ON vip.id = top.user_id"), + ("a_real_difference_next_to_renamed_derived_tables_is_reported", "sqlite", + "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM user_totals WHERE total > 100", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100"), +] + +#: 过去复验者自己造的 SQL 对里,在这几组数据上结果不同的(SQLite,进执行比对) +VALIDATOR_DIFF: list[tuple[str, str, str]] = [ + ("cases.py:A29a", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(DISTINCT user_id) FROM orders WHERE status = 'paid'"), + ("cases.py:A29b", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(user_id) FROM orders WHERE status = 'paid'"), + ("real1.py:W08 distinct in subq vs none", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders) t"), + ("exec_check3.py:E4", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id, store_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders) t"), + ("batch4.py:N1 nested 3-level DISTINCT in middle", + "SELECT COUNT(*) FROM (SELECT user_id FROM (SELECT DISTINCT user_id, store_id FROM orders) a) b", + "SELECT COUNT(*) FROM (SELECT user_id FROM (SELECT user_id, store_id FROM orders) a) b"), + ("exec_check.py:F2", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders EXCEPT SELECT user_id FROM refunds) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM refunds EXCEPT SELECT user_id FROM orders) t"), + ("real1.py:W09 grain", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id"), + ("exec_check3.py:F3", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders INTERSECT SELECT user_id FROM refunds) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders EXCEPT SELECT user_id FROM refunds) t"), + ("exec_check.py:F1", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT user_id FROM refunds) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) t"), + ("cases.py:mysql", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id"), + ("cases.py:postgres", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders WHERE status = 'paid') t WHERE t.rn = 1"), + ("batch3.py:K2 extra time col on one side; shared paid_at uppe", + "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT COUNT(*) FROM orders WHERE created_at >= '2026-01-01' AND paid_at >= '2026-09-01' AND paid_at < '2026-11-01'"), + ("p123c.py:Z08 real diff in 3-table chain same order (wording)", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM orders o JOIN users u ON u.id = o.user_id JOIN refunds r ON r.user_id = u.id"), + ("p123c.py:Z07 real diff in 3-table chain (wording)", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id JOIN refunds r ON r.user_id = u.id"), + ("p123d.py:Z10 real diff + permutation (tables)", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM users u JOIN refunds r ON r.user_id = u.id JOIN orders o ON o.id = r.order_id"), + ("real1.py:W03 join swap + LEFT vs INNER", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r JOIN orders o ON o.id = r.order_id"), + ("cases.py:postgres", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("batch2.py:C4 amount above own-user avg vs below (correlated ", + "SELECT COUNT(*) FROM orders o WHERE o.amount > (SELECT AVG(x.amount) FROM orders x WHERE x.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE (SELECT AVG(x.amount) FROM orders x WHERE x.user_id = o.user_id) > o.amount"), + ("real1.py:W01 alias + value", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid'", + "SELECT COUNT(*) FROM orders x WHERE x.status = 'done'"), + ("real1.py:W10 extra cond in outer vs IN subq", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND o.user_id IN (SELECT p.user_id FROM orders p WHERE p.status = 'paid')", + "SELECT COUNT(*) FROM orders o WHERE o.user_id IN (SELECT p.user_id FROM orders p WHERE p.status = 'paid')"), + ("exec_check3.py:SJ2", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.user_id = o1.user_id AND o2.created_at > o1.created_at WHERE o1.amount > 100", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.user_id = o1.user_id AND o2.created_at > o1.created_at WHERE o2.amount > 100"), + ("real2.py:X02 table self-join role reversed, WHERE", + "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE a.vip = 1", + "SELECT COUNT(*) FROM users a JOIN users b ON a.id = b.referrer_id WHERE a.vip = 1"), + ("real1.py:W04 self-join direction reversed", + "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE a.vip = 1", + "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE b.vip = 1"), + ("batch4.py:V1 COUNT(DISTINCT user_id) vs DISTINCT COUNT(user_", + "SELECT COUNT(DISTINCT user_id) FROM orders", + "SELECT DISTINCT COUNT(user_id) FROM orders"), + ("compare.py:count-distinct-vs-sub", + "SELECT COUNT(DISTINCT user_id) FROM orders WHERE status='paid'", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status='paid') t"), + ("batch2.py:E5 COUNT(t.user_id) over DISTINCT subquery vs COUN", + "SELECT COUNT(t.user_id) FROM (SELECT DISTINCT user_id FROM orders) t", + "SELECT COUNT(user_id) FROM orders"), + ("batch6.py:Q3s GROUP BY alias shadowing a column (sqlite)", + "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY created_at", + "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY DATE(created_at)"), + ("exec_check2.py:SQ3 sqlite", + "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY created_at ORDER BY 1", + "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY DATE(created_at) ORDER BY 1"), + ("fp4.py:outer-date-vs-cast", + "SELECT DATE(paid_at) AS d, COUNT(*) FROM orders GROUP BY DATE(paid_at)", + "SELECT CAST(paid_at AS DATE) AS d, COUNT(*) FROM orders GROUP BY CAST(paid_at AS DATE)"), + ("regress2.py:group-date-forms2", + "SELECT DATE(paid_at) AS d, SUM(amount) FROM orders GROUP BY DATE(paid_at)", + "SELECT CAST(paid_at AS DATE) AS d, SUM(amount) FROM orders GROUP BY CAST(paid_at AS DATE)"), + ("regress2.py:group-date-forms", + "SELECT DATE(paid_at), SUM(amount) FROM orders GROUP BY DATE(paid_at)", + "SELECT CAST(paid_at AS DATE) AS d, SUM(amount) FROM orders GROUP BY 1"), + ("batch2.py:E6 outer SELECT DISTINCT vs no DISTINCT", + "SELECT DISTINCT user_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid'"), + ("eq2.py:T01 distinct col order", + "SELECT DISTINCT user_id, store_id FROM orders", + "SELECT DISTINCT store_id, user_id FROM orders"), + ("exec_check4.py:F3 (sets)", + "SELECT GROUP_CONCAT(user_id) FROM (SELECT user_id FROM orders INTERSECT SELECT user_id FROM refunds) t", + "SELECT GROUP_CONCAT(user_id) FROM (SELECT user_id FROM orders EXCEPT SELECT user_id FROM refunds) t"), + ("real1.py:W14 self-join direction reversed renamed", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.id = a.parent_id", + "SELECT SUM(y.amount) FROM orders x JOIN orders y ON y.id = x.parent_id"), + ("batch1.py:B4 real-table self-join control (should report)", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(b.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id"), + ("batch4.py:N3 LIMIT two levels down, filter at top vs inside ", + "SELECT SUM(amount) FROM (SELECT * FROM (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) a) b WHERE status = 'paid'", + "SELECT SUM(amount) FROM (SELECT * FROM (SELECT * FROM orders WHERE status = 'paid' ORDER BY amount DESC LIMIT 10) a) b"), + ("cases.py:postgres", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE rn = 1", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at DESC) AS rn FROM orders) t WHERE rn = 1"), + ("fp2.py:cast-date-where", + "SELECT SUM(amount) FROM orders WHERE CAST(paid_at AS DATE) >= '2026-09-01'", + "SELECT SUM(amount) FROM orders WHERE DATE(paid_at) >= '2026-09-01'"), + ("batch3.py:L2 table swapped AND SUM moved from order amount t", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(r.amount) FROM orders_archive o JOIN refunds r ON r.order_id = o.id"), + ("exec_check.py:L1", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.amount > 100", + "SELECT SUM(o.amount) FROM orders_archive o JOIN refunds r ON r.order_id = o.id WHERE r.amount > 100"), + ("cases.py:postgres", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE o.status = 'paid'"), + ("exec_check2.py:C2 (sum of qualifying orders)", + "SELECT SUM(o.amount), GROUP_CONCAT(o.id) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT SUM(o.amount), GROUP_CONCAT(o.id) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND o.created_at < p.created_at)"), + ("exec_check3.py:SJ1", + "SELECT SUM(o1.amount) FROM orders o1 JOIN orders o2 ON o2.user_id = o1.user_id AND o2.created_at > o1.created_at", + "SELECT SUM(o2.amount) FROM orders o1 JOIN orders o2 ON o2.user_id = o1.user_id AND o2.created_at > o1.created_at"), + ("cases.py:postgres", + "SELECT SUM(r.total) / SUM(p.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT SUM(p.total) / SUM(r.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id"), + ("batch4.py:U1 UNION branch signs swapped (control)", + "SELECT SUM(x) FROM (SELECT amount AS x FROM orders UNION ALL SELECT -amount FROM refunds) t", + "SELECT SUM(x) FROM (SELECT -amount AS x FROM orders UNION ALL SELECT amount FROM refunds) t"), + ("fp6.py:time-derived-renamed", + "SELECT SUM(x.amount) FROM (SELECT amount, DATE(paid_at) AS day FROM orders) x WHERE x.day BETWEEN '2026-09-01' AND '2026-09-30'", + "WITH y AS (SELECT amount, CAST(paid_at AS DATE) AS d FROM orders) SELECT SUM(amount) FROM y WHERE d BETWEEN '2026-09-01' AND '2026-09-30'"), + ("cases.py:mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x", + "SELECT SUM(x.total) FROM (SELECT p.user_id, r.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x"), + ("batch3.py:J1 ORDER BY DESC NULLS LAST vs NULLS FIRST LIMIT 1", + "SELECT id FROM orders ORDER BY amount DESC NULLS LAST LIMIT 10", + "SELECT id FROM orders ORDER BY amount DESC NULLS FIRST LIMIT 10"), + ("exec_check.py:J1", + "SELECT id FROM orders ORDER BY amount DESC NULLS LAST LIMIT 2", + "SELECT id FROM orders ORDER BY amount DESC NULLS FIRST LIMIT 2"), + ("batch3.py:H3 window frame 6 vs 2 preceding (control)", + "SELECT id, SUM(amount) OVER (ORDER BY created_at ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS s FROM orders", + "SELECT id, SUM(amount) OVER (ORDER BY created_at ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS s FROM orders"), + ("batch3.py:H1 named WINDOW partition user_id vs store_id", + "SELECT id, SUM(amount) OVER w AS s FROM orders WINDOW w AS (PARTITION BY user_id)", + "SELECT id, SUM(amount) OVER w AS s FROM orders WINDOW w AS (PARTITION BY store_id)"), + ("exec_check.py:H1", + "SELECT id, SUM(amount) OVER w AS s FROM orders WINDOW w AS (PARTITION BY user_id) ORDER BY id", + "SELECT id, SUM(amount) OVER w AS s FROM orders WINDOW w AS (PARTITION BY store_id) ORDER BY id"), + ("exec_check3.py:C3", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o ORDER BY o.id", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o ORDER BY o.id"), + ("eq1.py:E05 column order", + "SELECT store_id, user_id, SUM(amount) FROM orders GROUP BY store_id, user_id", + "SELECT SUM(amount), user_id, store_id FROM orders GROUP BY user_id, store_id"), + ("real1.py:W07 union vs union all wrapped", + "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT * FROM (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) t"), + ("batch2.py:F4 outer UNION vs UNION ALL", + "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds"), + ("compare.py:union-root-extra-branch-cond", + "SELECT user_id FROM orders WHERE status='paid' UNION SELECT referrer_id FROM orders WHERE status='paid'", + "SELECT user_id FROM orders WHERE status='paid' UNION SELECT referrer_id FROM orders"), + ("fp1.py:select-order", + "SELECT user_id, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY user_id", + "SELECT COUNT(*) AS cnt, o.user_id, SUM(o.amount) AS total FROM orders o GROUP BY o.user_id"), + ("eq2.py:T02 distinct vs group by", + "SELECT user_id, store_id FROM orders GROUP BY user_id, store_id", + "SELECT DISTINCT store_id, user_id FROM orders"), + ("compare.py:multi-cte-same-cond", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status='paid'), b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status='open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status='paid'), b AS (SELECT * FROM orders WHERE status='open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id"), + ("cases.py:mysql", + "WITH a AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id), b AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'", + "WITH a AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id), b AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id) SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'"), + ("cases.py:sqlite", + "WITH a AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), b AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(a.total) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), b AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(a.total) FROM a JOIN b ON b.user_id = a.user_id"), + ("cases.py:mysql", + "WITH done AS (SELECT * FROM refunds WHERE state = 'done') SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id", + "WITH done AS (SELECT * FROM refunds) SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id WHERE d.state = 'done'"), + ("real2.py:X05 condition moved into CTE + extra in outer", + "WITH p AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT SUM(amount) FROM p WHERE status = 'paid'", + "WITH p AS (SELECT * FROM orders) SELECT SUM(amount) FROM p WHERE status = 'paid'"), + ("real1.py:W02 CTE renamed + agg func", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM p", + "WITH q AS (SELECT * FROM orders WHERE status = 'paid') SELECT AVG(q.amount) FROM q"), + ("batch4.py:O1 LEFT JOIN keep side swapped (control)", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p LEFT JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM open o LEFT JOIN paid p ON p.user_id = o.user_id"), + ("batch4.py:O2 LEFT JOIN, WHERE on keep side vs nullable side", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p LEFT JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p LEFT JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("real1.py:W12 sealed renamed CTEs, used the other one", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH a1 AS (SELECT * FROM orders WHERE status = 'open'), b1 AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(x.amount) FROM b1 y JOIN a1 x ON x.user_id = y.user_id"), + ("exec_check3.py:A5", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.store_id, COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY p.store_id ORDER BY 1", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT o.store_id, COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY o.store_id ORDER BY 1"), + ("batch4.py:T2 DATE(created_at) AS d in paid/open CTEs, p.d vs", + "WITH paid AS (SELECT user_id, DATE(created_at) AS d FROM orders WHERE status = 'paid'), open AS (SELECT user_id, DATE(created_at) AS d FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.d >= '2026-09-01'", + "WITH paid AS (SELECT user_id, DATE(created_at) AS d FROM orders WHERE status = 'paid'), open AS (SELECT user_id, DATE(created_at) AS d FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.d >= '2026-09-01'"), + ("cases.py:postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, p.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT COUNT(*) FROM net WHERE v > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, r.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT COUNT(*) FROM net WHERE v > 100"), + ("p123.py:1k renamed WHERE", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > 100", + "WITH a1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), b1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM a1 x JOIN b1 y ON y.user_id = x.user_id WHERE y.total > 100"), + ("cases.py:postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM refunded r LEFT JOIN paid p ON p.user_id = r.user_id"), + ("p123.py:1l renamed diff", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH a1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), b1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(y.total - x.total) FROM a1 x JOIN b1 y ON y.user_id = x.user_id"), + ("p123.py:1j renamed SUM", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH a1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), b1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(y.total) FROM a1 x JOIN b1 y ON y.user_id = x.user_id"), + ("p123b.py:Z04 diff survives join swap", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), vip AS (SELECT id FROM users WHERE vip = 1) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), vip AS (SELECT id FROM users WHERE vip = 1) SELECT SUM(r.total) FROM refunded r JOIN paid p ON p.user_id = r.user_id"), + ("p123b.py:Z05 diff survives names swapped", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), vip AS (SELECT id FROM users WHERE vip = 1) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH refunded AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), paid AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), v AS (SELECT id FROM users WHERE vip = 1) SELECT SUM(x.total) FROM paid x JOIN refunded y ON y.user_id = x.user_id"), + ("cases.py:mysql", + "WITH r AS (SELECT user_id, amount, RANK() OVER (ORDER BY amount DESC) AS k FROM orders) SELECT SUM(amount) FROM r WHERE k <= 10", + "WITH r AS (SELECT user_id, amount, ROW_NUMBER() OVER (ORDER BY amount DESC) AS k FROM orders) SELECT SUM(amount) FROM r WHERE k <= 10"), + ("real1.py:W13 same name CTE different meaning", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(t.amount) FROM t", + "WITH t AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(t.amount) FROM t"), + ("real1.py:W06 time col via renamed CTE", + "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH x AS (SELECT DATE(created_at) AS d, amount FROM orders) SELECT SUM(amount) FROM x WHERE d >= '2026-09-01'"), + ("exec_check4.py:E3", + "WITH t AS (SELECT DISTINCT user_id, amount FROM orders) SELECT SUM(amount) FROM t", + "SELECT SUM(amount) FROM orders"), + ("batch3.py:H2 named WINDOW in CTE + outer filter", + "WITH t AS (SELECT id, amount, ROW_NUMBER() OVER w AS rn FROM orders WINDOW w AS (PARTITION BY user_id ORDER BY created_at)) SELECT SUM(amount) FROM t WHERE rn = 1", + "WITH t AS (SELECT id, amount, ROW_NUMBER() OVER w AS rn FROM orders WINDOW w AS (PARTITION BY user_id ORDER BY created_at DESC)) SELECT SUM(amount) FROM t WHERE rn = 1"), + ("cases.py:mysql", + "WITH t AS (SELECT user_id, CASE WHEN status = 'paid' THEN amount ELSE 0 END AS paid_amt FROM orders) SELECT SUM(paid_amt) FROM t", + "WITH t AS (SELECT user_id, CASE WHEN status IN ('paid', 'shipped') THEN amount ELSE 0 END AS paid_amt FROM orders) SELECT SUM(paid_amt) FROM t"), + ("real2.py:X03 extra outer cond also in CTE layer", + "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT COUNT(*) FROM t JOIN orders o ON o.user_id = t.user_id WHERE o.status = 'paid'", + "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT COUNT(*) FROM t JOIN orders o ON o.user_id = t.user_id"), + ("p123b.py:Z06 problem2 variant: extra CTE after t, used via I", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), vip AS (SELECT id FROM users WHERE vip = 1) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id WHERE u.id IN (SELECT id FROM vip)"), + ("real2.py:X01 CTE self-join role reversed", + "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT SUM(a.total) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT SUM(a.total) FROM t a JOIN t b ON a.user_id = b.referrer_id"), + ("cases.py:postgres", + "WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), t2 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100", + "WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), t2 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100"), + ("batch5.py:S1 filter on LIMIT-CTE column vs on another CTE's ", + "WITH top AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10), allo AS (SELECT * FROM orders) SELECT COUNT(*) FROM top t JOIN allo a ON a.user_id = t.user_id WHERE t.amount > 100", + "WITH top AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10), allo AS (SELECT * FROM orders) SELECT COUNT(*) FROM top t JOIN allo a ON a.user_id = t.user_id WHERE a.amount > 100"), + ("exec_check.py:S1", + "WITH top AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 2), allo AS (SELECT * FROM orders) SELECT COUNT(*) FROM top t JOIN allo a ON a.user_id = t.user_id WHERE t.amount > 100", + "WITH top AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 2), allo AS (SELECT * FROM orders) SELECT COUNT(*) FROM top t JOIN allo a ON a.user_id = t.user_id WHERE a.amount > 100"), + ("exec_check.py:E7", + "WITH u AS (SELECT DISTINCT user_id FROM refunds) SELECT SUM(o.amount) FROM orders o JOIN u ON u.user_id = o.user_id", + "WITH u AS (SELECT user_id FROM refunds) SELECT SUM(o.amount) FROM orders o JOIN u ON u.user_id = o.user_id"), +] + + +PAIRS: list[Pair] = [Pair(f"same:{i}", a, b, "same") for i, a, b in _SAME] + \ + [Pair(f"diff:{i}", a, b, "diff") for i, a, b in _DIFF] + \ + [Pair(f"validator:{n}:{k}", a, b, "diff") for k, (n, a, b) in enumerate(VALIDATOR_DIFF)] diff --git a/backend/tests/test_sql_equivalence.py b/backend/tests/test_sql_equivalence.py new file mode 100644 index 00000000..b2f8e817 --- /dev/null +++ b/backend/tests/test_sql_equivalence.py @@ -0,0 +1,82 @@ +"""SQL 差异说明的「等价」结论拿真实执行守门(data/sql_semantics.py 的 explain_sql_diff)。 + +explain_sql_diff 判两条 SQL 等价,当且仅当两边经过只保持语义的变换之后规范形式一字不差。规范化的每一步都必须 +保持语义,可光看代码保证不了(sqlglot 自带的优化器里就有不保持语义的改写),所以这里在虚构的业务库上执行: + +- 判了等价的两条,在每组数据上结果都必须一样(结果集按行排序比较); +- 在某组数据上结果不一样的两条,绝不能判等价; +- 语料里标 diff 的(语义不同,或者只在某个方言、某种排序规则下才等价),一律不能判等价; +- 标 same 的(已知等价的改写),记下认出来的比例:目标是大多数都认得出,但不强求全部。 + +库和语料见 tests/fixtures/sql_pairs.py:5 组按固定种子生成的数据加一组极端数据,带空值、重复、边界值、已删除的 +行、退款;语料里有手写的等价改写和语义不同的对,也有复验者以前造的、在这几组数据上结果不同的对。每一对都在不给表 +结构和给表结构(数据源的 schema_cache)两种情况下各判一次,两种都要守住上面的规矩。 +""" +from __future__ import annotations + +import pytest + +from app.data.sql_semantics import explain_sql_diff +from tests.fixtures import sql_pairs + +SCHEMA = sql_pairs.schema_cache() + + +@pytest.fixture(scope="module") +def databases(): + conns = sql_pairs.build_databases() + yield conns + for c in conns: + c.close() + + +def _results(databases, sql: str) -> list[list[tuple]]: + return [sql_pairs.run(c, sql) for c in databases] + + +def _check_shape(diff, a: str, b: str) -> None: + """SqlDiff 自己的承诺:等价才没有条目;不等价一定有条目,兜底时一定有 aspect=other 的那条,并给出不同的行。""" + if diff.equivalent: + assert diff.items == [] and not diff.fallback and not diff.text_diff, (a, b) + return + assert diff.items, (a, b) + if diff.fallback: + assert any(d.aspect == "other" for d in diff.items), (a, b, diff.items) + assert diff.text_diff, (a, b) + assert len(diff.text_diff) <= 40 + + +@pytest.mark.parametrize("pair", sql_pairs.PAIRS, ids=[p.id for p in sql_pairs.PAIRS]) +def test_equivalence_is_guarded_by_execution(databases, pair): + ra, rb = _results(databases, pair.a), _results(databases, pair.b) + differs = [i for i, (x, y) in enumerate(zip(ra, rb)) if x != y] + if pair.expect == "same": + assert not differs, f"语料标错了:{pair.id} 在第 {differs} 组数据上结果不同" + for schema in (None, SCHEMA): + diff = explain_sql_diff(pair.a, pair.b, "sqlite", schema=schema) + _check_shape(diff, pair.a, pair.b) + if differs: + assert not diff.equivalent, f"{pair.id}:第 {differs} 组数据上结果不同,却判了等价(schema={bool(schema)})" + if pair.expect == "diff": + assert not diff.equivalent, f"{pair.id} 不该判等价(schema={bool(schema)})" + # 换个先后问,结论一样 + assert explain_sql_diff(pair.b, pair.a, "sqlite", schema=schema).equivalent == diff.equivalent, pair.id + + +def test_most_known_equivalent_rewrites_are_recognized(): + """已知等价的改写里认出来的比例(给表结构时)。不强求全部认出来:认不出的给的是兜底说明,不是错误的结论。""" + same = [p for p in sql_pairs.PAIRS if p.expect == "same"] + hits = [p.id for p in same if explain_sql_diff(p.a, p.b, "sqlite", schema=SCHEMA).equivalent] + bare = [p.id for p in same if explain_sql_diff(p.a, p.b, "sqlite").equivalent] + ratio, bare_ratio = len(hits) / len(same), len(bare) / len(same) + print(f"\n已知等价的 {len(same)} 对里认出 {len(hits)} 对({ratio:.0%}),不给表结构时 {len(bare)} 对({bare_ratio:.0%})") + print("没认出来的:", sorted(set(p.id for p in same) - set(hits))) + assert ratio >= 0.9 and bare_ratio >= 0.9 + + +def test_the_corpus_is_large_and_varied_enough(): + assert len(sql_pairs.PAIRS) >= 150 and len(sql_pairs.SEEDS) >= 5 + text = " ".join(p.a + " " + p.b for p in sql_pairs.PAIRS).upper() + for construct in ("WITH ", "EXISTS", " IN (SELECT", "UNION", "OVER (", "DISTINCT", "LEFT JOIN", "HAVING", + "LIMIT", "ORDER BY", "RIGHT JOIN", "FULL JOIN", "NOT IN", "CROSS JOIN"): + assert construct in text, construct diff --git a/backend/tests/test_sql_semantics.py b/backend/tests/test_sql_semantics.py index eb57882b..95199efa 100644 --- a/backend/tests/test_sql_semantics.py +++ b/backend/tests/test_sql_semantics.py @@ -1,20 +1,12 @@ -"""SQL 语义(data/sql_semantics.py):两条 SQL 算的是不是一回事,不是的话差在哪。 +"""SQL 语义(data/sql_semantics.py):两条 SQL 是不是等价,不是的话差在哪。 - normalize_sql:字面量换成占位符后的规范文本 + 字面量。只有字面量不同的两条 SQL 规范文本相同(定时运行每期的日期 - 不同,不能算模板变了);关键字大小写、空白、注释也不算不同;解析不了的退回逐词规范化,照样认得出只改了字面量 -- facets:表、连接(类型 / 关联进来的表 / ON 条件)、WHERE 合取式、时间条件(列 / 运算符 / 取值)、分组、聚合 - (函数 / 参数 / 去重)、HAVING、取哪几行(各段的 LIMIT / OFFSET / 排序)。别名换成真实表名(条件里套着的子查询、 - HAVING 里的别名也是),CTE、子查询不按起的名字比(单看一条按位置编号),条件里的子查询选出来的列名不算;比较的 - 两边排好顺序;解析不了带 parse_error,不抛 -- diff_facets:每处不同一句中文:「A 按支付时间 paid_at 筛时间,B 按下单时间 created_at」「A 排除了 - is_deleted = 1 的行,B 没有」「A 对 order_id 去重计数,B 没去重」……写法不同、口径相同的不报。两边的派生来源先按 - 结构(读的表、各列的定义、名字)配好对再比:一边多出一个排在前面的 CTE、内连接换了先后不报,两个 CTE 算出同名 - 的列时外层用的是哪一个照报;外连接按主表比(LEFT / RIGHT 对调不报,主表换了、被 WHERE 变成内连接照报);算出来 - 的列按算法比(最外层的算式、窗口函数,CTE 里被外面用到的列的定义);时间列透过 CTE 里的日期表达式认;取前几行、 - 窗口函数前后筛的不同照报;没用到的 CTE 不算;两边同名却不是同一个的派生来源说清各是什么;自己关联自己的两个 - 别名分得开;读同一张表的几个派生来源同在一段时不透成那张表的列(用的是已付那份还是未付那份照报);相关子查询里 - 外层的列另带标记(子单、父单分得开);条件、分组、连接、聚合按层计数(外层少了一个和子查询里一样的条件照报); - 去重、UNION 还是 UNION ALL、最后出一行还是每组一行照报 + 不同,不能算模板变了);关键字大小写、空白、注释也不算不同;结构变了的一定不同;解析不了的退回逐词规范化 +- facets:把一条查询拆成表、连接、WHERE 合取式、时间条件、分组、聚合、HAVING、取哪几行(给调用方取结构用) +- explain_sql_diff:两边规范化后的形式一字不差才算等价(规范化的每一步都只做保持语义的变换);不等价时只把有把握 + 的不同写成具体的中文句子(时间列、筛选条件、聚合与去重、关联、分组、HAVING、取行、UNION),说不准的给兜底说明, + 并给出规范化后不同的行。等价的结论另有执行比对守门:见 test_sql_equivalence.py +- diff_facets:照旧收两份 facets,交给 explain_sql_diff 比原文,返回它的条目(含兜底条目) 夹具全是虚构的表:订单 orders、退款 refunds、门店 stores、用户 users、付款 payments,以及一张中文表「订单」。 """ @@ -25,24 +17,27 @@ import json import pytest from app.data.sql_semantics import ( + FacetDiff, Facets, + SqlDiff, catalog_labels, diff_facets, + explain_sql_diff, facets, normalize_sql, ) +from tests.fixtures import sql_pairs DIALECTS = ("sqlite", "mysql", "postgres") +LABELS = ("原流程", "盲算") +FALLBACK = "两条 SQL 的写法不同,未能归纳成具体差异,请对照原文" +FALLBACK_REST = "另有未能归纳的差异,请对照原文" def texts(a: str, b: str, dialect: str = "sqlite", **kwargs) -> list[str]: return [d.text for d in diff_facets(facets(a, dialect), facets(b, dialect), **kwargs)] -def aspects(a: str, b: str, dialect: str = "sqlite") -> list[str]: - return [d.aspect for d in diff_facets(facets(a, dialect), facets(b, dialect))] - - # -------------------------------------------------------------------------- # normalize_sql # -------------------------------------------------------------------------- @@ -282,1064 +277,435 @@ def test_facets_are_json_ready(): assert not [k for k in data if k.startswith("_")], "拿来重新分析的原文不进 as_dict" + # -------------------------------------------------------------------------- -# diff_facets:每个方面一对 +# normalize_sql:「只改了参数」的判断不受差异说明改动影响 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b", [ + ("SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-10-01' AND paid_at < '2026-11-01'"), + ("WITH p AS (SELECT * FROM orders WHERE status = 'paid' AND amount > 100) SELECT store_id, SUM(amount) FROM p " + "GROUP BY store_id HAVING SUM(amount) > 1000 ORDER BY 2 DESC LIMIT 5", + "WITH p AS (SELECT * FROM orders WHERE status = 'done' AND amount > 50) SELECT store_id, SUM(amount) FROM p " + "GROUP BY store_id HAVING SUM(amount) > 10 ORDER BY 2 DESC LIMIT 20"), + ("SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM orders WHERE status IN ('paid', 'done'))", + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM orders WHERE status IN ('open', 'void'))"), + ("SELECT o.id, CASE WHEN o.amount > 100 THEN 'big' ELSE 'small' END FROM orders o LEFT JOIN refunds r " + "ON r.order_id = o.id AND r.amount > 0 WHERE o.created_at BETWEEN '2026-09-01' AND '2026-09-30'", + "SELECT o.id, CASE WHEN o.amount > 200 THEN 'huge' ELSE 'tiny' END FROM orders o LEFT JOIN refunds r " + "ON r.order_id = o.id AND r.amount > 5 WHERE o.created_at BETWEEN '2026-10-01' AND '2026-10-31'"), +], ids=["dates", "cte-having-limit", "in-list", "case-join-between"]) +def test_only_parameters_changed_is_still_the_same_template(dialect, a, b): + """定时运行每期只换日期、阈值:规范文本相同(同一个模板),字面量各是各的;同时 explain_sql_diff 照样说两条 + 不等价(取值不同就是不同的口径)。""" + na, nb = normalize_sql(a, dialect), normalize_sql(b, dialect) + assert na.parse_error is None and na.same_shape(nb) and na.literals != nb.literals + assert not explain_sql_diff(a, b, dialect).equivalent + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("b", [ + "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' GROUP BY store_id", + "SELECT SUM(amount) FROM orders WHERE paid_at > '2026-09-01' GROUP BY store_id", + "SELECT AVG(amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY store_id", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND is_deleted = 0 GROUP BY store_id", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY user_id", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE paid_at >= '2026-09-01' GROUP BY store_id", + "SELECT SUM(DISTINCT amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY store_id", + "SELECT SUM(amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE paid_at >= '2026-09-01' " + "GROUP BY store_id", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY store_id ORDER BY 1", + "SELECT SUM(amount) FROM orders WHERE paid_at IN ('2026-09-01') GROUP BY store_id", + "WITH t AS (SELECT * FROM orders) SELECT SUM(amount) FROM t WHERE paid_at >= '2026-09-01' GROUP BY store_id", +]) +def test_a_structural_change_is_never_the_same_template(dialect, b): + a = "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY store_id" + assert not normalize_sql(a, dialect).same_shape(normalize_sql(b, dialect)), b + + +# -------------------------------------------------------------------------- +# explain_sql_diff:等价怎么判 +# -------------------------------------------------------------------------- + + +def check_shape(diff: SqlDiff) -> None: + """SqlDiff 自己的承诺:等价才没有条目;不等价一定有条目,兜底时最后一条是 aspect=other,并给出不同的行。""" + assert isinstance(diff, SqlDiff) + if diff.equivalent: + assert diff.items == [] and not diff.fallback and diff.text_diff == () + return + assert diff.items and diff.text_diff and len(diff.text_diff) <= 40 + assert all(line[:2] in ("- ", "+ ") for line in diff.text_diff) + if diff.fallback: + assert diff.items[-1].aspect == "other" and diff.items[-1].text in (FALLBACK, FALLBACK_REST) + else: + assert all(d.aspect != "other" for d in diff.items) + # 不自相矛盾:同一句话不说两遍,配成一对的两边写法不会一样 + assert len({d.text for d in diff.items}) == len(diff.items) + assert all(d.a != d.b for d in diff.items if isinstance(d.a, str) and isinstance(d.b, str)) + + +@pytest.mark.parametrize("dialect,a,b", [ + ("sqlite", "SELECT SUM(o.amount) FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted = 0", + "select sum(amount) from orders where 0 = is_deleted and '2026-09-01' <= paid_at"), + ("sqlite", "WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT v.order_id) FROM v", + "SELECT COUNT(DISTINCT order_id) FROM orders WHERE is_deleted = 0"), + ("sqlite", "SELECT SUM(o.amount) FROM orders o, stores s WHERE o.store_id = s.id AND s.city = '虚构市'", + "SELECT SUM(orders.amount) FROM orders INNER JOIN stores ON stores.id = orders.store_id " + "WHERE stores.city = '虚构市'"), + ("sqlite", "SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10 OFFSET 0", + "SELECT o.amount FROM orders o ORDER BY o.amount DESC LIMIT 10"), + ("postgres", "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM user_totals WHERE total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) AS agg " + "WHERE agg.total > 100"), + ("mysql", "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) r " + "ON r.order_id = o.id WHERE r.n > 1", + "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) " + "`退款` ON o.id = `退款`.order_id WHERE `退款`.n > 1"), + ("postgres", "SELECT SUM(amount) FROM orders WHERE user_id IN " + "(WITH t AS (SELECT id FROM users WHERE grade >= 3) SELECT id FROM t)", + "SELECT SUM(amount) FROM orders WHERE user_id IN " + "(WITH g AS (SELECT id FROM users WHERE grade >= 3) SELECT g.id FROM g)"), + ("postgres", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("mysql", "SELECT SUM(`amount`) FROM `orders` WHERE `status` = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("postgres", 'SELECT SUM("amount") FROM "orders" WHERE "status" = \'paid\'', + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("postgres", "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT * FROM (SELECT user_id FROM orders UNION SELECT user_id FROM refunds) t"), + # UNION ALL 调换各支:行一样多、一样的值,各方言都认得出(SQLite 只限整条语句最外层、不排序的,见下面的反例) + ("sqlite", "SELECT user_id FROM orders WHERE amount > 1 UNION ALL SELECT user_id FROM refunds", + "SELECT user_id FROM refunds UNION ALL SELECT user_id FROM orders WHERE amount > 1"), + ("postgres", "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds ORDER BY 1", + "SELECT user_id FROM refunds UNION ALL SELECT user_id FROM orders ORDER BY 1"), + ("mysql", "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM refunds UNION ALL SELECT user_id FROM orders) t"), +], ids=["aliases-and-flips", "filter-in-cte", "comma-join", "offset-0", "cte-vs-subquery", + "mysql-chinese-alias", "cte-inside-in", "left-vs-right", "mysql-backticks", "pg-quotes", "pg-union-wrapped", + "sqlite-union-all-branches", "pg-union-all-order-by-position", "mysql-union-all-in-subquery"]) +def test_rewrites_that_keep_the_meaning_are_equivalent(dialect, a, b): + diff = explain_sql_diff(a, b, dialect) + check_shape(diff) + assert diff.equivalent and diff.level in ("optimizer", "light", "parsed") + assert texts(a, b, dialect) == [] + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.DIALECT_SAME, ids=[f"{n}-{d}-{i}" for i, (n, d, _, _) + in enumerate(sql_pairs.DIALECT_SAME)]) +def test_rewrites_from_earlier_rounds_are_recognized(name, dialect, a, b): + """旧测试里「写法不同、意思一样」的对(各方言,SQLite 以外的不能执行,只在这里查):都认得出是等价的。""" + diff = explain_sql_diff(a, b, dialect) + check_shape(diff) + assert diff.equivalent, name + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.REGRESSIONS, ids=[f"{n}-{d}-{i}" for i, (n, d, _, _) + in enumerate(sql_pairs.REGRESSIONS)]) +def test_differences_reported_in_earlier_rounds_are_never_called_equivalent(name, dialect, a, b): + """过去五轮复验、旧测试报过的每一处不同:一律不判等价,条目不为空、不自相矛盾;换个先后问也一样。第 5 轮回退的 + 「只在 IN / EXISTS / 标量子查询里用到的 CTE 里的不同」也在里面。""" + for x, y in ((a, b), (b, a)): + diff = explain_sql_diff(x, y, dialect, labels=LABELS) + check_shape(diff) + assert not diff.equivalent, name + + +@pytest.mark.parametrize("dialect,a,b,why", [ + ("sqlite", "SELECT o.id, r.flag FROM orders o LEFT JOIN (SELECT order_id, 1 AS flag FROM refunds) r " + "ON r.order_id = o.id", + "SELECT o.id, 1 AS flag FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "merge_subqueries 会把外连接可空一侧的常量列并进外层:没关联上的行本该是空值"), + ("sqlite", "SELECT id FROM orders WHERE status = 'paid' OR amount > (SELECT amount FROM refunds WHERE id = 999)", + "SELECT orders.id FROM orders CROSS JOIN (SELECT amount FROM refunds WHERE id = 999) AS q " + "WHERE orders.status = 'paid' OR orders.amount > q.amount", + "unnest_subqueries 把 OR 里的标量子查询改成 CROSS JOIN:子查询没有行时整条都没了"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= CAST('2026-09-01' AS DATE)", + "canonicalize 给日期比较加 CAST;SQLite 里 CAST(… AS DATE) 取的是数字"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE CAST(paid_at AS DATE) >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE DATE(paid_at) >= '2026-09-01'", + "sqlglot 生成 SQLite 时把 CAST(x AS DATE) 写成 DATE(x):比规范形式不能比生成的文本"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE +amount = '10'", "SELECT COUNT(*) FROM orders WHERE amount = '10'", + "一元加号在 SQLite 里去掉了类型亲和,解析时却被丢掉了"), + ("postgres", "SELECT COUNT(*) FROM orders WHERE status > 'b' AND status > 'O'", + "SELECT COUNT(*) FROM orders WHERE status > 'b'", + "字符串比较的强弱取决于排序规则:simplify 的这条化简不做"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE region > 10 AND region > 9", + "SELECT COUNT(*) FROM orders WHERE region > 10", "TEXT 列和数字比是按文本比:不合并同一列上的比较"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE amount + 1 = 101", "SELECT COUNT(*) FROM orders WHERE amount = 100", + "等式两边移项对 TEXT 列不成立"), + ("sqlite", "SELECT user_id, 1 AS total FROM orders GROUP BY user_id ORDER BY total DESC, user_id LIMIT 3", + "SELECT user_id, 1 AS total FROM orders GROUP BY user_id ORDER BY 1 DESC, user_id LIMIT 3", + "ORDER BY 别名换成常量 1 就成了按第一列排"), + ("sqlite", "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT * FROM (SELECT user_id FROM orders UNION SELECT user_id FROM refunds) t", + "SQLite 从子查询读 UNION 时按第一支的类型亲和转换别的支的值"), + ("sqlite", "SELECT COUNT(*) FROM orders o WHERE o.id NOT IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "NOT IN 遇到空值整个不成立,NOT EXISTS 不会"), + ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.id IS NULL OR r.amount > 5", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.id IS NULL OR r.amount > 5", + "IS NULL 留得下没关联上的行:这次外连接不是内连接"), + # 合并的各支调换先后:下面这些都会改变结果(复验压测时撞到的是第一种,SQLite 里留下的是 2 还是 2.0) + ("sqlite", "SELECT tax FROM orders UNION SELECT referrer_id FROM orders", + "SELECT referrer_id FROM orders UNION SELECT tax FROM orders", + "UNION 去重时留先出现的那个值:SQLite 里 REAL 列的 2.0 和 INTEGER 列的 2 算重复,留下哪个取决于哪支在前"), + ("mysql", "SELECT city FROM stores UNION SELECT city FROM users", + "SELECT city FROM users UNION SELECT city FROM stores", + "MySQL 默认的排序规则不分大小写,'SH' 和 'sh' 去重时留先出现的那个"), + ("postgres", "SELECT amount FROM orders INTERSECT SELECT amount FROM refunds", + "SELECT amount FROM refunds INTERSECT SELECT amount FROM orders", + "INTERSECT 留左边那支的值:numeric 的 2 和 2.00 相等,写出来却不一样"), + ("postgres", "SELECT id FROM orders UNION ALL SELECT id FROM refunds LIMIT 3", + "SELECT id FROM refunds UNION ALL SELECT id FROM orders LIMIT 3", + "没有 ORDER BY 的 LIMIT 取先出来的几行:各支调换先后,取到的就是另一张表的行"), + ("postgres", "SELECT id, user_id FROM orders UNION ALL SELECT user_id, id FROM refunds ORDER BY id", + "SELECT user_id, id FROM refunds UNION ALL SELECT id, user_id FROM orders ORDER BY id", + "合并结果的列名取第一支的:调换之后 ORDER BY id 排的是另一列"), + ("sqlite", "SELECT name FROM users UNION ALL SELECT city FROM stores ORDER BY 1", + "SELECT city FROM stores UNION ALL SELECT name FROM users ORDER BY 1", + "SQLite 给合并的结果排序时用最左一支那列声明的排序规则(COLLATE NOCASE 的列在前就不分大小写)"), + ("sqlite", "SELECT COUNT(*) FROM (SELECT name AS v FROM users UNION ALL SELECT city AS v FROM stores) t " + "WHERE v = 'sh'", + "SELECT COUNT(*) FROM (SELECT city AS v FROM stores UNION ALL SELECT name AS v FROM users) t WHERE v = 'sh'", + "SQLite 从子查询读合并的结果再比较时,列的排序规则也取最左一支的"), + ("sqlite", "SELECT COUNT(*) FROM users u JOIN stores s ON u.name COLLATE NOCASE = s.city COLLATE BINARY", + "SELECT COUNT(*) FROM users u JOIN stores s ON s.city COLLATE BINARY = u.name COLLATE NOCASE", + "比较的两边都写了 COLLATE 时 SQLite 用左边那个:两边不能对调"), + # 别名和名字:qualify 没有表结构时把 SELECT 列表里和前面别名同名的名字当成别名展开,库里却不认 + ("sqlite", "SELECT amount * 2 AS x, x + 1 AS y FROM orders", + "SELECT amount * 2 AS x, amount * 2 + 1 AS y FROM orders", + "SELECT 列表里不能引用前面列的别名:x 不是表的列,前一条在库里报错"), + ("sqlite", "SELECT user_id AS amount, amount AS user_id FROM orders ORDER BY amount", + "SELECT user_id AS amount, amount AS user_id FROM orders ORDER BY user_id", + "ORDER BY 里单独一个名字先认别名:两条排的是不同的列(第二列的 amount 是表的列)"), + # 取到哪几行、哪一行的值 SQL 本来就没规定的:库里看扫描的先后,调换关联先后就变了(压测撞到的是第一种) + ("sqlite", "SELECT o.created_at, COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id GROUP BY o.store_id", + "SELECT o.created_at, COUNT(*) FROM refunds r JOIN orders o ON r.order_id = o.id GROUP BY o.store_id", + "聚合查询里既不分组也不聚合的列,SQLite 取组里任意一行的值"), + ("sqlite", "SELECT o.id, r.id FROM orders o JOIN refunds r ON r.order_id = o.id LIMIT 5", + "SELECT o.id, r.id FROM refunds r JOIN orders o ON r.order_id = o.id LIMIT 5", + "没有 ORDER BY 的 LIMIT 取先扫到的几行"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT DISTINCT user_id FROM refunds LIMIT 2)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM refunds GROUP BY user_id LIMIT 2)", + "没有 ORDER BY 的 LIMIT:DISTINCT 按扫到的先后出行,GROUP BY 按分组键排好了出,取到的两个不同"), + ("sqlite", "SELECT r.id, o.id FROM refunds r RIGHT JOIN orders o ON r.order_id = o.id LIMIT 5", + "SELECT r.id, o.id FROM orders o LEFT JOIN refunds r ON r.order_id = o.id LIMIT 5", + "没有 ORDER BY 的 LIMIT:RIGHT JOIN 改写成 LEFT JOIN,扫描的先后就变了"), +], ids=["nullable-merge", "unnest-or", "cast-date", "cast-vs-date", "unary-plus", "collation", "text-number", + "equality-move", "order-by-constant", "sqlite-union-affinity", "not-in-null", "outer-join-is-null", + "union-dedup-storage-class", "union-dedup-collation", "intersect-numeric-scale", "union-all-limit", + "union-all-order-by-name", "sqlite-union-all-order-collation", "sqlite-nested-union-all", "collate-sides", + "select-list-alias-ref", "order-by-swapped-aliases", "loose-column-join-order", "limit-join-order", + "limit-distinct-vs-group", "limit-right-vs-left"]) +def test_rewrites_that_change_the_meaning_are_not_equivalent(dialect, a, b, why): + """这些写法看着能改写成同一个样子,其实结果可能不同(why):sqlglot 优化器里有的改写不保持语义,规范化不用它们, + 或者守住了用法。""" + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, dialect, schema=schema) + check_shape(diff) + assert not diff.equivalent, why + + +def test_the_schema_expands_select_star_and_qualifies_columns(): + """给了表结构(数据源的 schema_cache)才认得出 SELECT * 和列全写出来是一回事、几张表时不写表名的列是哪张表的。""" + schema = sql_pairs.schema_cache() + star = "SELECT * FROM stores" + listed = "SELECT id, name, city, region, owner_id FROM stores" + assert explain_sql_diff(star, listed, "sqlite", schema=schema).equivalent + assert not explain_sql_diff(star, listed, "sqlite").equivalent + a = "SELECT SUM(amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE city = 'SH'" + b = "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE s.city = 'SH'" + assert explain_sql_diff(a, b, "sqlite", schema=schema).equivalent + + +def test_an_alias_used_in_group_by_needs_the_schema_to_be_told_from_a_column(): + """WHERE、GROUP BY、HAVING 里的名字库里先认表的列、没有这一列才认别名:GROUP BY r 里的 r 是不是 orders 的列, + 给了表结构才知道。给了就认得出和按 region 分组是一回事;没给不猜(DATE(created_at) AS created_at 再 GROUP BY + created_at 按的是原来的时间列,当成别名就成了按日期分组)。""" + a = "SELECT o.region, SUM(o.amount) FROM orders o GROUP BY o.region HAVING SUM(o.amount) > 100" + b = "SELECT region AS r, SUM(amount) AS s FROM orders GROUP BY r HAVING s > 100" + assert explain_sql_diff(a, b, "sqlite", schema=sql_pairs.schema_cache()).equivalent + assert not explain_sql_diff(a, b, "sqlite").equivalent + shadow = "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY created_at" + by_day = "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY DATE(created_at)" + for schema in (None, sql_pairs.schema_cache()): + assert not explain_sql_diff(shadow, by_day, "sqlite", schema=schema).equivalent + + +def test_equivalence_comes_with_the_level_it_was_decided_at(): + diff = explain_sql_diff("SELECT COUNT(1) FROM orders", "select count(*) from ORDERS", "sqlite") + assert diff.equivalent and diff.level == "optimizer" + data = json.loads(json.dumps(diff.as_dict(), ensure_ascii=False)) + assert data == {"equivalent": True, "items": [], "fallback": False, "text_diff": [], "level": "optimizer"} + + +# -------------------------------------------------------------------------- +# explain_sql_diff:有把握的具体说法 # -------------------------------------------------------------------------- BASE = "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'" def test_time_column_differs_with_catalog_names(): - a = "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'" + a = BASE b = "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' AND created_at < '2026-10-01'" - [diff] = diff_facets(facets(a), facets(b), names={"orders.paid_at": "支付时间", "created_at": "下单时间"}) - assert diff.aspect == "time" - assert diff.text == "A 按支付时间 paid_at 筛时间,B 按下单时间 created_at" - assert (diff.a, diff.b) == ("paid_at", "created_at") - assert diff.detail == {"a": [{"table": "orders", "column": "paid_at"}], + diff = explain_sql_diff(a, b, names={"orders.paid_at": "支付时间", "created_at": "下单时间"}) + check_shape(diff) + [item] = diff.items + assert item.aspect == "time" and item.text == "A 按支付时间 paid_at 筛时间,B 按下单时间 created_at" + assert (item.a, item.b) == ("paid_at", "created_at") + assert item.detail == {"a": [{"table": "orders", "column": "paid_at"}], "b": [{"table": "orders", "column": "created_at"}]} assert texts(a, b) == ["A 按 paid_at 筛时间,B 按 created_at"], "没有中文名就只写列名" -def test_time_filter_on_one_side_only(): - assert texts(BASE, "SELECT SUM(amount) FROM orders") == ["A 按 paid_at 筛时间,B 没有按时间筛选"] - assert texts("SELECT SUM(amount) FROM orders", BASE) == ["A 没有按时间筛选,B 按 paid_at 筛时间"] - - -def test_same_time_column_different_range(): - other = "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'" - [diff] = diff_facets(facets(BASE), facets(other)) - assert diff.aspect == "time_range" - assert diff.text == ("A 的时间范围是 paid_at >= '2026-09-01' AND paid_at < '2026-10-01'," - "B 是 paid_at BETWEEN '2026-09-01' AND '2026-09-30'") - - -def test_only_literals_differ_is_still_a_different_range(): - """规范文本相同(同一个模板),但比口径时日期不同就是不同的范围:两个函数回答的是两个问题。""" - later = BASE.replace("2026-09-01", "2026-09-08").replace("2026-10-01", "2026-10-08") - assert normalize_sql(BASE).same_shape(normalize_sql(later)) - assert aspects(BASE, later) == ["time_range"] - - -def test_an_exclusion_on_one_side(): - assert texts("SELECT SUM(amount) FROM orders WHERE is_deleted <> 1", "SELECT SUM(amount) FROM orders") == \ - ["A 排除了 is_deleted = 1 的行,B 没有"] - assert texts("SELECT SUM(amount) FROM orders", "SELECT SUM(amount) FROM orders WHERE refund_id IS NOT NULL") == \ - ["B 排除了 refund_id 为空的行,A 没有"] - - -def test_a_kept_condition_on_one_side(): - assert texts("SELECT SUM(amount) FROM orders", "SELECT SUM(amount) FROM orders WHERE status = 'paid'") == \ - ["B 只保留 status = 'paid' 的行,A 没有这个条件"] - - -def test_two_conditions_on_the_same_column_are_paired(): - [diff] = diff_facets(facets("SELECT SUM(amount) FROM orders WHERE status = 'paid'"), - facets("SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')")) - assert (diff.aspect, diff.text) == ("filter", "A 按 status = 'paid' 筛选,B 按 status IN ('paid', 'shipped') 筛选") - assert (diff.a, diff.b) == ("status = 'paid'", "status IN ('paid', 'shipped')") - - -def test_distinct_count_versus_plain_count(): - assert texts("SELECT COUNT(DISTINCT order_id) FROM orders", "SELECT COUNT(order_id) FROM orders") == \ - ["A 对 order_id 去重计数,B 没去重"] - assert texts("SELECT COUNT(order_id) FROM orders", "SELECT COUNT(DISTINCT order_id) FROM orders") == \ - ["A 对 order_id 计数时没去重,B 去重计数"] - - -@pytest.mark.parametrize("a,b,text", [ - ("SELECT SUM(amount) FROM orders", "SELECT AVG(amount) FROM orders", "A 对 amount 求和,B 求平均"), - ("SELECT SUM(paid_amount) FROM orders", "SELECT SUM(amount) FROM orders", "A 对 paid_amount 求和,B 对 amount 求和"), +@pytest.mark.parametrize("a,b,expected", [ + # 时间 + (BASE, "SELECT SUM(amount) FROM orders", [("time", "A 按 paid_at 筛时间,B 没有按时间筛选")]), + ("SELECT SUM(amount) FROM orders", BASE, [("time", "A 没有按时间筛选,B 按 paid_at 筛时间")]), + (BASE, "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'", + [("time_range", "A 的时间范围是 paid_at >= '2026-09-01' AND paid_at < '2026-10-01'," + "B 是 paid_at >= '2026-09-01' AND paid_at <= '2026-09-30'")]), + ("WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(created_at) AS d FROM orders) SELECT SUM(amount) FROM t " + "WHERE d >= '2026-09-01'", + [("time", "A 按 paid_at 筛时间,B 按 created_at")]), + # 筛选 + ("SELECT SUM(amount) FROM orders WHERE is_deleted <> 1", "SELECT SUM(amount) FROM orders", + [("filter", "A 排除了 is_deleted = 1 的行,B 没有")]), + ("SELECT SUM(amount) FROM orders", "SELECT SUM(amount) FROM orders WHERE refund_id IS NOT NULL", + [("filter", "B 排除了 refund_id 为空的行,A 没有")]), + ("SELECT SUM(amount) FROM orders", "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + [("filter", "B 只保留 status = 'paid' 的行,A 没有这个条件")]), + ("SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')", + [("filter", "A 按 status = 'paid' 筛选,B 按 status IN ('paid', 'shipped') 筛选")]), + ("WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM paid", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM orders", + [("filter", "A 只保留 status = 'paid' 的行,B 没有这个条件")]), + # 聚合、去重 + ("SELECT COUNT(DISTINCT order_id) FROM orders", "SELECT COUNT(order_id) FROM orders", + [("aggregate", "A 对 order_id 去重计数,B 没去重")]), + ("SELECT COUNT(order_id) FROM orders", "SELECT COUNT(DISTINCT order_id) FROM orders", + [("aggregate", "A 对 order_id 计数时没去重,B 去重计数")]), + ("SELECT SUM(amount) FROM orders", "SELECT AVG(amount) FROM orders", [("aggregate", "A 对 amount 求和,B 求平均")]), + ("SELECT SUM(paid_amount) FROM orders", "SELECT SUM(amount) FROM orders", + [("aggregate", "A 对 paid_amount 求和,B 对 amount 求和")]), ("SELECT COUNT(*) FROM orders", "SELECT COUNT(DISTINCT order_id) FROM orders", - "A 数的是行数,B 对 order_id 去重计数"), -]) -def test_other_aggregate_differences(a, b, text): - assert texts(a, b) == [text] - - -def test_join_differences(): - left = "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id" - inner = "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id" - other_key = "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.user_id = o.user_id" - plain = "SELECT SUM(amount) FROM orders" - assert texts(left, inner) == ["A 用 LEFT JOIN 关联 refunds 表,B 用 INNER JOIN"] - assert texts(left, other_key) == ["A 关联 refunds 表的条件是 orders.id = refunds.order_id," - "B 是 orders.user_id = refunds.user_id"] - assert texts(left, plain) == ["A 关联了 refunds 表,B 没有"] - - -def test_a_different_source_table_is_said_once(): - """表换了一张:说一次「查的是哪张表」就够了,同一个聚合、同一个条件不再因为表名不同各报一遍。""" - a = "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'" - b = "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 0 AND paid_at >= '2026-09-01'" - [diff] = diff_facets(facets(a), facets(b)) - assert (diff.aspect, diff.text) == ("tables", "A 查的是 orders 表,B 查的是 orders_archive 表") - assert aspects(a, b.replace("is_deleted = 0", "is_deleted = 1")) == ["filter", "tables"], "换表之外的不同照报" - - -def test_two_columns_that_read_the_same_are_told_apart(): - """各自只写列名时两边读起来一样,其实是两张表的列:改写成带表名的写法,不说「A 对 amount 求和,B 对 amount 求和」。""" - a = "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id" - b = "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id" - assert texts(a, b) == ["A 对 orders.amount 求和,B 对 refunds.amount 求和"] - - -def test_group_by_difference(): - assert texts("SELECT region, SUM(amount) FROM orders GROUP BY region", - "SELECT region, city, SUM(amount) FROM orders GROUP BY region, city") == \ - ["A 按 region 分组,B 按 region、city 分组"] - assert texts("SELECT SUM(amount) FROM orders", "SELECT region, SUM(amount) FROM orders GROUP BY region") == \ - ["A 没有分组,B 按 region 分组"] - - -def test_having_difference(): - assert texts("SELECT region, SUM(amount) FROM orders GROUP BY region HAVING SUM(amount) > 100", - "SELECT region, SUM(amount) FROM orders GROUP BY region") == \ - ["A 分组后只保留 SUM(amount) > 100 的组,B 没有这个条件"] - - -def test_limit_difference(): - assert texts("SELECT amount FROM orders LIMIT 10", "SELECT amount FROM orders") == ["A 只取前 10 行,B 不限行数"] - assert texts("SELECT amount FROM orders LIMIT 10", "SELECT amount FROM orders LIMIT 100") == \ - ["A 只取前 10 行,B 只取前 100 行"] - - -def test_an_unparsable_side_is_reported_once_without_item_by_item_comparison(): - [diff] = diff_facets(facets("SELECT FROM WHERE"), facets(BASE)) - assert diff.aspect == "parse" and diff.text.startswith("A:SQL 无法解析") and "没有逐项比较" in diff.text - assert diff.b is None - - -@pytest.mark.parametrize("a,b", [ - # 别名、比较两边的顺序、大小写 - ("SELECT SUM(o.amount) FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted = 0", - "select sum(amount) from orders where 0 = is_deleted and '2026-09-01' <= paid_at"), - # 有效记录条件写在 CTE 里、还是写在外层 - ("WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT v.order_id) FROM v", - "SELECT COUNT(DISTINCT order_id) FROM orders WHERE is_deleted = 0"), - # 逗号连接和 INNER JOIN - ("SELECT SUM(o.amount) FROM orders o, stores s WHERE o.store_id = s.id AND s.city = '虚构市'", - "SELECT SUM(orders.amount) FROM orders INNER JOIN stores ON stores.id = orders.store_id " - "WHERE stores.city = '虚构市'"), - # 条件里套着的子查询用了别名:NOT EXISTS、IN (SELECT …) - ("SELECT o.id FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", - "SELECT orders.id FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), - ("SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)"), - # HAVING 里用 SELECT 的别名 - ("SELECT o.region, SUM(o.amount) FROM orders o GROUP BY o.region HAVING SUM(o.amount) > 100", - "SELECT region AS r, SUM(amount) AS s FROM orders GROUP BY r HAVING s > 100"), - # OFFSET 0 等于没写;取前几行时 ORDER BY 1 和写出列名是同一个排序 - ("SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10 OFFSET 0", - "SELECT o.amount FROM orders o ORDER BY o.amount DESC LIMIT 10"), -]) -def test_the_same_meaning_written_differently_is_no_difference(a, b): - assert texts(a, b) == [] - - -def test_a_real_difference_inside_a_subquery_is_still_reported(): - a = "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)" - b = "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.referrer_id FROM users u WHERE u.vip = 1)" - assert texts(a, b) == ["A 按 user_id IN (SELECT id FROM users WHERE vip = 1) 筛选," - "B 按 user_id IN (SELECT referrer_id FROM users WHERE vip = 1) 筛选"] - - -@pytest.mark.parametrize("dialect,a,b", [ - # 条件里的子查询给选出来的列起了名字:IN、标量子查询只看取值,列名不影响口径 - ("sqlite", - "SELECT SUM(o.amount) FROM orders o WHERE o.amount > " - "(SELECT AVG(o2.amount) AS avg_amount FROM orders o2 WHERE o2.is_deleted = 0)", - "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders WHERE is_deleted = 0)"), - ("sqlite", - "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id AS uid FROM users u WHERE u.vip = 1)", - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)"), - # 条件里的子查询从 FROM 里的子查询取数,两边给它起的别名不同 - ("sqlite", - "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT s.user_id FROM " - "(SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) s WHERE s.total > 100)", - "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT agg.user_id FROM " - "(SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) agg WHERE agg.total > 100)"), - # CTE 起的名字不同,外层按 CTE 算出来的列筛选 - ("sqlite", - "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " - "SELECT COUNT(*) FROM user_totals WHERE total > 100", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE t.total > 100"), - # 写成 CTE 还是写成 FROM 里的子查询 - ("postgres", - "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM user_totals WHERE total > 100", - "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) AS agg " - "WHERE agg.total > 100"), - # LATERAL 子查询的别名不同:别名挂在 LATERAL 上,不在子查询上 - ("postgres", - "SELECT SUM(o.amount) FROM orders o, LATERAL (SELECT COUNT(*) AS n FROM refunds r WHERE r.order_id = o.id) l " - "WHERE l.n > 0", - "SELECT SUM(o.amount) FROM orders o CROSS JOIN LATERAL (SELECT COUNT(*) AS n FROM refunds r " - "WHERE r.order_id = o.id) AS x WHERE x.n > 0"), - # 关联进来的是子查询,两边的别名不同(一边是带反引号的中文别名) - ("mysql", - "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) r " - "ON r.order_id = o.id WHERE r.n > 1", - "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) " - "`退款` ON o.id = `退款`.order_id WHERE `退款`.n > 1"), - # 条件里的子查询用到 CTE,两边给 CTE 起的名字不同 - ("sqlite", - "WITH vip AS (SELECT id FROM users WHERE grade >= 3) " - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM vip)", - "WITH good_users AS (SELECT id FROM users WHERE grade >= 3) " - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT g.id FROM good_users g)"), - # CTE 定义在条件里的子查询里:定义连同子查询整个写进外层的条件,定义上的名字也不算 - ("postgres", - "SELECT SUM(amount) FROM orders WHERE user_id IN " - "(WITH t AS (SELECT id FROM users WHERE grade >= 3) SELECT id FROM t)", - "SELECT SUM(amount) FROM orders WHERE user_id IN " - "(WITH g AS (SELECT id FROM users WHERE grade >= 3) SELECT g.id FROM g)"), - # 两个 CTE 定义的先后、起的名字都不同,按用到的先后它们是同一对 - ("sqlite", - "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " - "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " - "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", - "WITH r AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " - "p AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id) " - "SELECT COUNT(*) FROM p JOIN r ON p.user_id = r.user_id WHERE p.total > r.total"), - # CTE 定义的先后和用到的先后不一致,另一边又把它们写成了 FROM 里的子查询:按用到的先后编号才对得上 - ("sqlite", - "WITH c AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "d AS (SELECT user_id, COUNT(*) AS n FROM refunds GROUP BY user_id), " - "a AS (SELECT * FROM c WHERE total > 100), b AS (SELECT * FROM d WHERE n > 1) " - "SELECT COUNT(*) FROM b JOIN a ON a.user_id = b.user_id", - "SELECT COUNT(*) FROM (SELECT * FROM (SELECT user_id, COUNT(*) AS n FROM refunds GROUP BY user_id) x " - "WHERE n > 1) b JOIN (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) y " - "WHERE total > 100) a ON a.user_id = b.user_id"), - # 同上,两个 CTE 还算出了同名的列、外层拿它们互相比:不写编号的写法分不出是哪个的 total,编号得对得上 - ("sqlite", - "WITH c AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "d AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " - "a AS (SELECT * FROM c), b AS (SELECT * FROM d) " - "SELECT COUNT(*) FROM b JOIN a ON a.user_id = b.user_id WHERE a.total > b.total", - "SELECT COUNT(*) FROM (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) x) b " - "JOIN (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) y) a " - "ON a.user_id = b.user_id WHERE a.total > b.total"), - # 先用一个 CTE 筛一遍再汇总,还是直接在汇总的 CTE 里筛:里层多一段,不挪动外层那一段的编号 - ("sqlite", - "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), " - "per_user AS (SELECT user_id, SUM(amount) AS total FROM base GROUP BY user_id) " - "SELECT COUNT(*) FROM per_user WHERE total > 100", - "WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " - "SELECT COUNT(*) FROM totals WHERE total > 100"), -]) -def test_names_given_to_derived_tables_are_no_difference(dialect, a, b): - """CTE 名、子查询的别名、条件里的子查询选出来的列名都是各自起的(双盲复算的盲算更是):只因起的名字不同, - 不算换了口径。""" - assert texts(a, b, dialect) == [] - - -def test_derived_tables_are_compared_by_position_and_shown_by_name(): - """派生来源(CTE、FROM 里的子查询)比较用的写法按它在语句里的位置编号,不写名字;给人看的写法照原样写名字。""" - a = facets("WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM user_totals WHERE total > 100") - b = facets("SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) agg " - "WHERE agg.total > 100") - assert a.where[0].key == b.where[0].key - assert "user_totals" not in a.where[0].key and "agg" not in b.where[0].key - assert (a.where[0].text, a.where[0].full) == ("total > 100", "user_totals.total > 100") - assert (b.where[0].text, b.where[0].full) == ("total > 100", "agg.total > 100") - joined = facets("SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds " - "GROUP BY order_id) r ON r.order_id = o.id") - [join] = joined.joins - assert join.table.text == "r" and join.on.text == "orders.id = refunds.order_id" - - -SHIFTED_A = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 100 AND user_id IN (SELECT id FROM users WHERE vip = 1)") -SHIFTED_B = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM vip JOIN t ON t.user_id = vip.id WHERE t.total > 100") - - -def test_a_shifted_derived_number_does_not_split_one_condition_in_two(): - """位置编号只看先后:一边多出一个排在前面的 CTE(筛 VIP 用户),后面那个 CTE 的位置编号就错开了。两边的派生 - 来源先按结构配好对(t 配 t,vip 配不上),按共同的编号比,同一个条件不会被说成两边各有一个;真正的不同(VIP - 用 IN 筛还是关联进来)照报。""" - found = diff_facets(facets(SHIFTED_A), facets(SHIFTED_B), labels=("原流程", "盲算")) - filters = [d.text for d in found if d.aspect == "filter"] - assert filters == ["原流程只保留 user_id IN (SELECT id FROM users WHERE vip = 1) 的行,盲算没有这个条件"] - # 错开之后的条件真不同,也照样配成一对说 - other = SHIFTED_B.replace("t.total > 100", "t.total > 200") - filters = [d.text for d in diff_facets(facets(SHIFTED_A), facets(other), labels=("原流程", "盲算")) - if d.aspect == "filter"] - assert filters[0] == "原流程按 total > 100 筛选,盲算按 total > 200 筛选" and len(filters) == 2 - - -@pytest.mark.parametrize("aspect,a,b", [ - # 时间条件、分组键、聚合、HAVING、取前几行时的排序都是按 CTE 算出来的列 - ("time", - "WITH t AS (SELECT user_id, DATE(paid_at) AS day, amount FROM orders) " - "SELECT SUM(amount) FROM t WHERE day >= '2026-09-01' AND user_id IN (SELECT id FROM users WHERE vip = 1)", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, DATE(paid_at) AS day, amount FROM orders) " - "SELECT SUM(t.amount) FROM vip JOIN t ON t.user_id = vip.id WHERE t.day >= '2026-09-01'"), - ("group_by", - "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) " - "SELECT n, COUNT(*) FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) GROUP BY n", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) " - "SELECT t.n, COUNT(*) FROM vip JOIN t ON t.user_id = vip.id GROUP BY t.n"), - ("aggregate", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT AVG(total) FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1)", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT AVG(t.total) FROM vip JOIN t ON t.user_id = vip.id"), - ("having", - "WITH t AS (SELECT store_id, user_id, SUM(amount) AS total FROM orders GROUP BY store_id, user_id) " - "SELECT store_id FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) GROUP BY store_id " - "HAVING MAX(total) > 100", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT store_id, user_id, SUM(amount) AS total FROM orders GROUP BY store_id, user_id) " - "SELECT t.store_id FROM vip JOIN t ON t.user_id = vip.id GROUP BY t.store_id HAVING MAX(t.total) > 100"), - ("limit", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT user_id FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) ORDER BY total DESC LIMIT 10", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT t.user_id FROM vip JOIN t ON t.user_id = vip.id ORDER BY t.total DESC LIMIT 10"), -]) -def test_a_shifted_derived_number_is_no_difference_in_any_aspect(aspect, a, b): - """编号错开之后,按 CTE 算出来的列写的时间条件、分组键、聚合、HAVING、排序也不算不同:只剩 VIP 那处真正的 - 不同。""" - assert not [x for x in aspects(a, b) if x.startswith(aspect)] - - -def test_a_shifted_number_inside_a_condition_subquery_is_no_difference(): - """条件里的子查询用到的 CTE 位置编号错开了:两边的 t 按结构配成一对、写同一个编号,子查询整个写进外层条件的 - 那一份照样对得上。""" - a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u WHERE u.vip = 1 AND u.id IN (SELECT user_id FROM t WHERE total > 100)") - b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM vip WHERE vip.id IN (SELECT user_id FROM t WHERE total > 100)") - assert texts(a, b) == [] - - -def test_a_shifted_derived_join_is_paired_with_its_counterpart(): - """关联进来的 CTE 位置编号错开了(盲算多关联了一个排在前面的 CTE):两边的派生来源先按结构配好对,原流程的 t - 配盲算的 t,不会和盲算的 vip 当成同一处关联、说成「关联条件不同」。""" - a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id WHERE t.total > 100") - b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id JOIN t ON t.user_id = u.id WHERE t.total > 100") - assert texts(a, b, labels=("原流程", "盲算")) == ["盲算只保留 vip = 1 的行,原流程没有这个条件", - "盲算关联了 vip 表,原流程没有"] - - -def test_swapped_join_order_of_two_derived_tables_is_no_difference(): - """两个 CTE 关联的先后对调,位置编号也跟着对调:两边按结构配好对、写同一个编号之后,拿它们的列互相比的条件 - 排成同一个方向(只按位置编号排,同一个比较会排成两个方向),关联按涉及的两边比,不按哪一侧是关联进来的。""" - ctes = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) ") - assert texts(ctes + "SELECT COUNT(*) FROM t JOIN u ON u.user_id = t.user_id WHERE t.total > u.cnt", - ctes + "SELECT COUNT(*) FROM u JOIN t ON t.user_id = u.user_id WHERE t.total > u.cnt") == [] - - -def test_a_reversed_comparison_between_two_derived_tables_is_still_reported(): - """两个 CTE 算出了同名的列:按结构配好对之后照样分得清是哪个 CTE 的,比较的方向反了要报出来(只按配好的编号 - 比,不拿不写编号的写法补配,不会把它吞掉)。""" - a = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " - "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total") - b = a.replace("WHERE p.total > r.total", "WHERE r.total > p.total") - assert texts(a, b) == ["A 只保留 paid.total > refunded.total 的行,B 没有这个条件", - "B 只保留 paid.total < refunded.total 的行,A 没有这个条件"] - - -def test_a_real_difference_next_to_renamed_derived_tables_is_reported_once(): - """派生来源起的名字不同、里面又真有不同时,只报真正不同的那一处:CTE 里少一个条件,外层按 CTE 算出来的列 - 筛选的条件不跟着报(不然两边各报一句「只保留 total > 100 的行」,自相矛盾);外层的条件本身不同,两边配成 - 一对说。""" - a = ("WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 " - "GROUP BY user_id) SELECT COUNT(*) FROM user_totals WHERE total > 100") - inner = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 100") - assert texts(a, inner, labels=("原流程", "盲算")) == ["原流程只保留 is_deleted = 0 的行,盲算没有这个条件"] - outer = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 200") - assert texts(a, outer, labels=("原流程", "盲算")) == ["原流程按 total > 100 筛选,盲算按 total > 200 筛选"] - - -# -------------------------------------------------------------------------- -# diff_facets:两边的派生来源按结构配对 -# -------------------------------------------------------------------------- - -#: 两个 CTE 算出了同名的列 total:一个是付款合计,一个是退款合计 -PAID_REFUNDED = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) ") -PR_JOIN = "FROM paid p JOIN refunded r ON r.user_id = p.user_id" -#: 同样两段写成 FROM 里的子查询 -PR_SUBQUERIES = ("FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p " - "JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r " - "ON r.user_id = p.user_id") - - -@pytest.mark.parametrize("form", ["cte", "subquery"]) -@pytest.mark.parametrize("a,b,expected", [ - ("SELECT SUM(p.total) {j}", "SELECT SUM(r.total) {j}", - ["原流程对 paid.total 求和,盲算对 refunded.total 求和"]), - ("SELECT SUM(p.total - r.total) {j}", "SELECT SUM(r.total - p.total) {j}", - ["原流程对 paid.total - refunded.total 求和,盲算对 refunded.total - paid.total 求和"]), - ("SELECT COUNT(*) {j} WHERE p.total > 100", "SELECT COUNT(*) {j} WHERE r.total > 100", - ["原流程按 paid.total > 100 筛选,盲算按 refunded.total > 100 筛选"]), - ("SELECT p.user_id {j} ORDER BY p.total DESC LIMIT 10", "SELECT p.user_id {j} ORDER BY r.total DESC LIMIT 10", - ["原流程取行前按 paid.total 降序排,盲算按 refunded.total 降序排"]), - ("SELECT p.user_id {j} GROUP BY p.user_id HAVING MAX(p.total) > 10", - "SELECT p.user_id {j} GROUP BY p.user_id HAVING MAX(r.total) > 10", - ["原流程对 paid.total 取最大值,盲算对 refunded.total 取最大值", - "原流程分组后只保留 MAX(paid.total) > 10 的组,盲算只保留 MAX(refunded.total) > 10 的组"]), - ("SELECT COUNT(*) {j} WHERE p.total - r.total > 100", "SELECT COUNT(*) {j} WHERE r.total - p.total > 100", - ["原流程只保留 paid.total - refunded.total > 100 的行,盲算没有这个条件", - "盲算只保留 refunded.total - paid.total > 100 的行,原流程没有这个条件"]), -], ids=["sum", "sum-of-difference", "where", "order-by", "having", "where-difference"]) -def test_which_derived_table_a_column_comes_from_is_reported(form, a, b, expected): - """两个派生来源算出了同名的列(付款合计、退款合计都叫 total),编号也对得上:外层用的是哪一个的 total 不同, - 聚合、条件、排序、HAVING 里都得照报,不能因为都是「某个 CTE 的 total」就当成一样。只读得出列名的写法两边 - 一样时,改用带来源名字的写法说。""" - if form == "cte": - head, j = PAID_REFUNDED, PR_JOIN - else: - head, j = "", PR_SUBQUERIES - expected = [t.replace("paid.", "p.").replace("refunded.", "r.") for t in expected] - assert texts(head + a.format(j=j), head + b.format(j=j), labels=("原流程", "盲算")) == expected - - -def test_renamed_and_reordered_derived_tables_still_tell_their_columns_apart(): - """盲算给两个 CTE 另起了名字、定义和关联的先后也反过来:按里面读的表和各列的定义认出谁对谁,外层用的是 - 哪一个的列照样分得清;同一列只是名字、先后不同的不报。""" - b_head = ("WITH ref_sum AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " - "pay_sum AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) ") - b_join = "FROM ref_sum r JOIN pay_sum p ON p.user_id = r.user_id" - labels = ("原流程", "盲算") - assert texts(PAID_REFUNDED + f"SELECT SUM(p.total) {PR_JOIN}", - b_head + f"SELECT SUM(r.total) {b_join}", labels=labels) == \ - ["原流程对 paid.total 求和,盲算对 ref_sum.total 求和"] - assert texts(PAID_REFUNDED + f"SELECT COUNT(*) {PR_JOIN} WHERE p.total > 100", - b_head + f"SELECT COUNT(*) {b_join} WHERE r.total > 100", labels=labels) == \ - ["原流程按 paid.total > 100 筛选,盲算按 ref_sum.total > 100 筛选"] - assert texts(PAID_REFUNDED + f"SELECT SUM(p.total) {PR_JOIN} WHERE p.total > r.total", - b_head + f"SELECT SUM(p.total) {b_join} WHERE r.total < p.total") == [] - - -def test_an_extra_derived_table_on_one_side_is_not_paired_with_another(): - """盲算多一个排在前面的 CTE(vip),位置编号就错开了:原流程的 t 要和盲算的 t 配成一对(里面读的表、各列的 - 定义都一样),不能按编号配给 vip。""" - a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id") - b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id JOIN t ON t.user_id = u.referrer_id") - assert texts(a, b, labels=("原流程", "盲算")) == [ - "盲算只保留 vip = 1 的行,原流程没有这个条件", - "原流程关联 t 表的条件是 orders.user_id = users.id,盲算是 orders.user_id = users.referrer_id", - "盲算关联了 vip 表,原流程没有"] - # 两边各一个、结构毫不相干的派生来源也不配成一对:各说各的 - other = ("WITH vip AS (SELECT id FROM users WHERE vip = 1) " - "SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id") - joins = [d.text for d in diff_facets(facets(a), facets(other), labels=("原流程", "盲算")) if d.aspect == "join"] - assert joins == ["原流程关联了 t 表,盲算没有", "盲算关联了 vip 表,原流程没有"] - - -def test_unused_ctes_take_part_in_pairing(): - """两边都定义了付款、退款两个 CTE,原流程关联付款的、盲算关联退款的:没用到的那个也参加配对,付款配付款、退款 - 配退款,不会把两边各自用到的那个只凭列名都叫 user_id、total 凑成一对,说成「关联条件不同」。""" - a = PAID_REFUNDED + "SELECT COUNT(*) FROM users u JOIN paid p ON p.user_id = u.id" - b = PAID_REFUNDED + "SELECT COUNT(*) FROM users u JOIN refunded r ON r.user_id = u.id" - assert texts(a, b, labels=("原流程", "盲算")) == ["原流程关联了 paid 表,盲算没有", "盲算关联了 refunded 表,原流程没有"] - - -@pytest.mark.parametrize("a,b,expected", [ + [("aggregate", "A 数的是行数,B 对 order_id 去重计数")]), + ("SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + [("aggregate", "A 对 orders.amount 求和,B 对 refunds.amount 求和")]), + ("SELECT DISTINCT region FROM orders", "SELECT region FROM orders", [("group_by", "A 对 region 去重,B 没有去重")]), + ("SELECT SUM(amount) FROM orders", "SELECT ROUND(SUM(amount), 2) FROM orders", + [("expression", "A 算的是 SUM(amount),B 算的是 ROUND(SUM(amount), 2)")]), + ("SELECT user_id, SUM(amount) FROM orders GROUP BY user_id", "SELECT SUM(amount), user_id FROM orders GROUP BY user_id", + [("expression", "两边选出来的列一样,先后不同:A 是 user_id、SUM(amount),B 是 SUM(amount)、user_id")]), + # 分组、HAVING、取行 + ("SELECT region, SUM(amount) FROM orders GROUP BY region", + "SELECT region, city, SUM(amount) FROM orders GROUP BY region, city", + [("group_by", "A 按 region 分组,B 按 city、region 分组")]), + ("SELECT SUM(amount) FROM orders", "SELECT region, SUM(amount) FROM orders GROUP BY region", + [("group_by", "A 没有分组,B 按 region 分组")]), + ("SELECT region, SUM(amount) FROM orders GROUP BY region HAVING SUM(amount) > 100", + "SELECT region, SUM(amount) FROM orders GROUP BY region", + [("having", "A 分组后只保留 SUM(amount) > 100 的组,B 没有这个条件")]), + ("SELECT amount FROM orders LIMIT 10", "SELECT amount FROM orders", [("limit", "A 只取前 10 行,B 不限行数")]), + ("SELECT amount FROM orders LIMIT 10", "SELECT amount FROM orders LIMIT 100", + [("limit", "A 只取前 10 行,B 只取前 100 行")]), + ("SELECT amount FROM orders LIMIT 10 OFFSET 0", "SELECT amount FROM orders LIMIT 10 OFFSET 10", + [("limit", "A 不跳过行,B 跳过前 10 行")]), + ("SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10", "SELECT amount FROM orders ORDER BY 1 ASC LIMIT 10", + [("limit", "A 取行前按 amount 降序排,B 按 amount 升序排")]), + ("SELECT id, amount FROM orders ORDER BY amount DESC", "SELECT id, amount FROM orders", + [("order", "A 按 amount 降序排,B 没有排序")]), + # 连接、表 + ("SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + [("join", "A 用 LEFT JOIN 关联 refunds 表,B 用 INNER JOIN")]), + ("SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.user_id = o.user_id", + [("join", "A 关联 refunds 表的条件是 orders.id = refunds.order_id,B 是 orders.user_id = refunds.user_id")]), + ("SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", "SELECT SUM(amount) FROM orders", + [("join", "A 用 LEFT JOIN 关联 refunds 表,B 没有")]), ("SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid'", "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE o.status = 'paid'", - ["盲算关联了 stores 表,原流程没有"]), - ("SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", - "SELECT SUM(o.amount) FROM users u JOIN refunds r ON r.user_id = u.id JOIN orders o ON o.id = r.order_id", - ["盲算关联了 users 表,原流程没有"]), - (PAID_REFUNDED + f"SELECT COUNT(*) {PR_JOIN} WHERE p.total > r.total", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " - "SELECT COUNT(*) FROM vip v JOIN refunded r ON r.user_id = v.id JOIN paid p ON p.user_id = r.user_id " - "WHERE p.total > r.total", - ["盲算只保留 vip = 1 的行,原流程没有这个条件", "盲算关联了 vip 表,原流程没有", "盲算用到了 users 表,原流程没有"]), - # 原流程只在条件的子查询里用到 users:那不是连接的一头,盲算多关联的还是 users - ("SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT id FROM users WHERE vip = 1)", - "SELECT SUM(o.amount) FROM users u JOIN orders o ON o.user_id = u.id WHERE u.vip = 1", - ["原流程只保留 user_id IN (SELECT id FROM users WHERE vip = 1) 的行,盲算没有这个条件", - "盲算关联了 users 表,原流程没有"]), -], ids=["real-table", "real-table-chain", "derived-shifted-and-swapped", "in-subquery-is-not-a-join"]) -def test_an_extra_source_written_in_from_is_named_correctly(a, b, expected): - """盲算多出来的来源写在 FROM 里、靠内连接接上原流程也有的表(FROM stores JOIN orders):多出来的是 stores, - 不能说成「盲算关联了 orders 表」(原流程查的就是 orders);已经说了「关联了 stores 表」的,不再另说一遍「用到了 - stores 表」。""" - assert texts(a, b, labels=("原流程", "盲算")) == expected - - -def test_a_lateral_subquery_is_named_by_its_alias(): - """LATERAL 子查询的别名挂在 LATERAL 上:取哪几行的说明里照写它,不写「子查询」。""" - a = ("SELECT o.id FROM orders o, LATERAL (SELECT amount FROM refunds r WHERE r.order_id = o.id " - "ORDER BY amount DESC LIMIT 1) l") - b = ("SELECT o.id FROM orders o CROSS JOIN LATERAL (SELECT amount FROM refunds r WHERE r.order_id = o.id " - "ORDER BY amount DESC LIMIT 2) AS x") - assert texts(a, b, "postgres") == ["A 在 l 里只取前 1 行,B 在 x 里只取前 2 行"] - - -def test_a_paired_derived_join_with_another_name_says_both_names(): - """配成一对的派生来源两边名字不同(原流程叫 t、盲算叫 totals):关联条件不同时两个名字都说出来。""" - a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id") - b = ("WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u JOIN totals ON totals.user_id = u.referrer_id") - assert texts(a, b, labels=("原流程", "盲算")) == [ - "原流程关联 t 表的条件是 orders.user_id = users.id,盲算关联 totals 表的条件是 orders.user_id = users.referrer_id"] - left = b.replace("JOIN totals ON totals.user_id = u.referrer_id", "LEFT JOIN totals ON totals.user_id = u.id") - assert texts(a, left, labels=("原流程", "盲算")) == ["原流程用 INNER JOIN 关联 t 表,盲算用 LEFT JOIN 关联 totals 表"] - - -@pytest.mark.parametrize("a,b", [ - (PAID_REFUNDED + f"SELECT COUNT(*) {PR_JOIN} WHERE p.total > r.total", - PAID_REFUNDED + "SELECT COUNT(*) FROM refunded r JOIN paid p ON p.user_id = r.user_id WHERE p.total > r.total"), - (f"SELECT COUNT(*) {PR_SUBQUERIES} WHERE p.total > r.total", - "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r " - "JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id " - "WHERE p.total > r.total"), - # 两个 CTE 读同一张表、各列的定义也一样,只差筛选条件:按整段的写法认得出谁是谁 - ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " - "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) " - "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > o.total", - "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " - "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) " - "SELECT COUNT(*) FROM open o JOIN paid p ON p.user_id = o.user_id WHERE o.total < p.total"), - # 实表也一样:内连接换个先后写,意思不变 - ("SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.amount > r.amount", - "SELECT SUM(o.amount) FROM refunds r JOIN orders o ON o.id = r.order_id WHERE o.amount > r.amount"), - ("SELECT SUM(o.amount) FROM orders o, refunds r WHERE r.order_id = o.id", - "SELECT SUM(o.amount) FROM refunds r, orders o WHERE r.order_id = o.id"), -], ids=["cte", "subquery", "same-shape-ctes", "real-tables", "comma-join"]) -def test_swapping_the_order_of_an_inner_join_is_no_difference(a, b): - """内连接两边换个先后(FROM paid JOIN refunded 换成 FROM refunded JOIN paid)意思不变:派生来源按结构配对, - 比较的两边按配好的编号排,不会排成两个方向、报出两条自相矛盾的话;连接按涉及的两边和条件比,不按哪一侧是 - 关联进来的。""" - assert texts(a, b) == [] - - -def test_two_same_shape_derived_tables_are_paired_by_their_conditions(): - """两个 CTE 读同一张表、各列定义也一样(付款的、未付的),盲算两个都另起了名字、都多加了 is_deleted = 0、关联的 - 先后也反过来:结构上只剩里面的条件分得开,按条件的重合配(status = 'paid' 配 status = 'paid'),只报多出来的那个 - 条件,外层拿两者比的条件不跟着报。""" - a = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " - "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) " - "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > o.total") - b = ("WITH unpaid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' AND is_deleted = 0 " - "GROUP BY user_id), " - "settled AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' AND is_deleted = 0 " - "GROUP BY user_id) " - "SELECT COUNT(*) FROM unpaid u JOIN settled s ON s.user_id = u.user_id WHERE u.total < s.total") - assert texts(a, b, labels=("原流程", "盲算")) == ["盲算只保留 is_deleted = 0 的行,原流程没有这个条件"] - - -def test_limits_inside_derived_tables_are_paired_by_structure(): - """CTE 里取哪几行按配好对的派生来源比,不按出现的先后:盲算多一个排在前面、也限了行数的 CTE,不能拿它和 - 原流程的 top 比。""" - a = ("WITH top AS (SELECT user_id, amount FROM orders ORDER BY amount DESC LIMIT 10) " - "SELECT SUM(amount) FROM top") - b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1 LIMIT 5), " - "top AS (SELECT user_id, amount FROM orders ORDER BY amount DESC LIMIT 10) " - "SELECT SUM(top.amount) FROM top JOIN vip ON vip.id = top.user_id") - found = [d.text for d in diff_facets(facets(a), facets(b), labels=("原流程", "盲算")) if d.aspect == "limit"] - assert found == ["原流程不限行数,盲算在 vip 里只取前 5 行"] - - -def test_a_changed_column_inside_a_renamed_derived_table_is_said_once(): - """CTE 里一列的算法变了(对 amount 求和换成对 paid_amount 求和),名字也另起了:按读的表和选出来的列名仍配成 - 一对,只报里面那处不同,外层按这一列写的条件不跟着报成两边各有一个。""" - a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 100") - b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "x AS (SELECT user_id, SUM(paid_amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM vip JOIN x ON x.user_id = vip.id WHERE total > 100") - found = diff_facets(facets(a), facets(b), labels=("原流程", "盲算")) - assert "原流程对 amount 求和,盲算对 paid_amount 求和" in [d.text for d in found] - assert not [d for d in found if d.aspect == "filter" and "total" in d.text] - - -@pytest.mark.parametrize("a,b,text", [ - ("WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM top", - "WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 100) SELECT SUM(amount) FROM top", - "A 在 top 里只取前 10 行,B 只取前 100 行"), - ("SELECT amount FROM orders LIMIT 10 OFFSET 0", "SELECT amount FROM orders LIMIT 10 OFFSET 10", - "A 不跳过行,B 跳过前 10 行"), - ("SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10", "SELECT amount FROM orders ORDER BY 1 ASC LIMIT 10", - "A 取行前按 amount 降序排,B 按 amount 升序排"), - ("SELECT amount FROM orders ORDER BY amount DESC LIMIT 10", "SELECT amount FROM orders LIMIT 10", - "A 取行前按 amount 降序排,B 没有排序"), -]) -def test_which_rows_are_taken_is_compared(a, b, text): - """取哪几行:每一段的行数、跳过的行数,以及取前几行时的排序。排序只在有 LIMIT / OFFSET 的段里比: - 没有取前几行,排序不影响算出来的数。""" - assert texts(a, b) == [text] - assert texts("SELECT SUM(amount) FROM orders ORDER BY 1 DESC", "SELECT SUM(amount) FROM orders") == [] - - -# -------------------------------------------------------------------------- -# diff_facets:外连接以哪一侧为主表 -# -------------------------------------------------------------------------- - -LABELS = ("原流程", "盲算") - - -@pytest.mark.parametrize("dialect,a,b,expected", [ - ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + [("join", "B 关联了 stores 表(关联的条件是 orders.store_id = stores.id),A 没有")]), + ("SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id", - ["原流程以 orders 为主表外连接 refunds,盲算以 refunds 为主表外连接 orders"]), - # 「没有退款的订单数」对「order_id 为空的退款数」:IS NULL 不会把外连接变成内连接,差的是主表 - ("sqlite", - "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.order_id IS NULL", - "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE r.order_id IS NULL", - ["原流程以 orders 为主表外连接 refunds,盲算以 refunds 为主表外连接 orders"]), - ("postgres", "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", - "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id", - ["原流程以 refunds 为主表外连接 orders,盲算以 orders 为主表外连接 refunds"]), - ("postgres", PAID_REFUNDED + "SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", - PAID_REFUNDED + "SELECT COUNT(*) FROM refunded r LEFT JOIN paid p ON p.user_id = r.user_id", - ["原流程以 paid 为主表外连接 refunded,盲算以 refunded 为主表外连接 paid"]), - ("mysql", - "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p " - "LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", - "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r " - "LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id", - ["原流程以 p 为主表外连接 r,盲算以 r 为主表外连接 p"]), -], ids=["left", "left-anti", "right", "cte", "subquery"]) -def test_which_side_an_outer_join_keeps_is_compared(dialect, a, b, expected): - """外连接两边对调,保留的就是另一张表的全部行(有没有退款的订单都算,还是有没有订单的退款都算):按主表比, - 关联的表、条件一样也要报出来。""" - assert texts(a, b, dialect, labels=LABELS) == expected + [("join", "A 以 orders 为主表外连接 refunds,B 以 refunds 为主表外连接 orders")]), + ("SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 1 AND paid_at >= '2026-09-01'", + [("tables", "A 查的是 orders 表,B 查的是 orders_archive 表"), + ("filter", "A 按 is_deleted = 0 筛选,B 按 is_deleted = 1 筛选")]), + # UNION + ("SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds", + [("set_op", "A 用 UNION 合并(去掉重复的行),B 用 UNION ALL(重复的行都留着)")]), + ("SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", + [("filter", "A 在合并的一支里只保留 status = 'paid' 的行,B 在合并的一支里没有这个条件")]), +], ids=lambda v: None) +def test_specific_differences_are_said_in_one_sentence_each(a, b, expected): + diff = explain_sql_diff(a, b, "sqlite") + check_shape(diff) + assert not diff.fallback and [(d.aspect, d.text) for d in diff.items] == expected -@pytest.mark.parametrize("dialect,a,b", [ - ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", - "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), - ("postgres", PAID_REFUNDED + "SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", - PAID_REFUNDED + "SELECT COUNT(*) FROM refunded r RIGHT JOIN paid p ON p.user_id = r.user_id"), - # 外连接后又在 WHERE 里筛可空那一侧:等于内连接加同一个条件 - ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'", - "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'"), - # 筛的是主表那一侧,外连接照旧 - ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", - "SELECT COUNT(*) FROM orders AS o LEFT OUTER JOIN refunds AS r ON o.id = r.order_id WHERE o.status = 'paid'"), - # 外连接链换了先后,各自的主表不变 - ("sqlite", - "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id LEFT JOIN users u ON u.id = o.user_id", - "SELECT COUNT(*) FROM orders o LEFT JOIN users u ON u.id = o.user_id LEFT JOIN refunds r ON r.order_id = o.id"), -], ids=["left-vs-right", "left-vs-right-cte", "rejected-left-vs-inner", "filter-on-kept-side", "chain-reordered"]) -def test_the_same_outer_join_written_differently_is_no_difference(dialect, a, b): - """orders LEFT JOIN refunds 和 refunds RIGHT JOIN orders 是同一次连接:按主表比,不按写的是 LEFT 还是 RIGHT。""" - assert texts(a, b, dialect) == [] +def test_a_difference_inside_a_cte_used_only_in_a_condition_subquery_is_reported(): + """第 5 轮回退的例子:只在 IN 子查询里用到的 CTE 里的条件不同,以前整个漏报(返回空列表)。现在 CTE 并进了 IN + 子查询,往里比,说得出是哪个条件。EXISTS、标量子查询一样。""" + head = "WITH p AS (SELECT * FROM orders WHERE status = '{}') SELECT COUNT(*) FROM users u WHERE " + for tail, place in (("u.id IN (SELECT user_id FROM p)", "在 IN 子查询里"), + ("EXISTS (SELECT 1 FROM p WHERE p.user_id = u.id)", "在 EXISTS 子查询里"), + ("(SELECT COUNT(*) FROM p WHERE p.user_id = u.id) > 0", "在条件的子查询里")): + a, b = head.format("paid") + tail, head.format("open") + tail + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, "sqlite", schema=schema, labels=LABELS) + check_shape(diff) + assert [(d.aspect, d.text) for d in diff.items] == [ + ("filter", f"原流程{place}按 status = 'paid' 筛选,盲算{place}按 status = 'open' 筛选")] + assert diff_facets(facets(a), facets(b), labels=LABELS) == diff.items -@pytest.mark.parametrize("dialect,a,b,expected", [ - ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN (SELECT * FROM refunds WHERE state = 'done') r " - "ON r.order_id = o.id", - "SELECT COUNT(*) FROM orders o LEFT JOIN (SELECT * FROM refunds) r ON r.order_id = o.id WHERE r.state = 'done'", - ["原流程以 orders 为主表外连接 r,盲算的 WHERE 里有 refunds.state = 'done',把关联不上 r 的行筛掉了," - "LEFT JOIN 实际成了 INNER JOIN"]), - ("mysql", "WITH done AS (SELECT * FROM refunds WHERE state = 'done') " - "SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id", - "WITH done AS (SELECT * FROM refunds) " - "SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id WHERE d.state = 'done'", - ["原流程以 orders 为主表外连接 done,盲算的 WHERE 里有 refunds.state = 'done',把关联不上 done 的行筛掉了," - "LEFT JOIN 实际成了 INNER JOIN"]), - # 盲算以 refunds 为主表,WHERE 又筛掉了没对上订单的行:没有退款的已付订单原流程算、盲算不算 - ("postgres", - "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", - "SELECT SUM(o.amount) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE o.status = 'paid'", - ["原流程以 orders 为主表外连接 refunds,盲算的 WHERE 里有 orders.status = 'paid',把关联不上 orders 的行筛掉了," - "LEFT JOIN 实际成了 INNER JOIN"]), - ("postgres", "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", - "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", - ["盲算只保留 status = 'paid' 的行,原流程没有这个条件", - "原流程以 refunds 为主表外连接 orders,盲算的 WHERE 里有 orders.status = 'paid',把关联不上 orders 的行筛掉了," - "RIGHT JOIN 实际成了 INNER JOIN"]), -], ids=["subquery", "cte", "swapped-and-filtered", "right"]) -def test_an_outer_join_turned_inner_by_where_is_reported(dialect, a, b, expected): - """可空那一侧的条件写在 ON / 子查询里,和写在 WHERE 里不是一回事:WHERE 里拿空值比不成立,没关联上的行被筛掉, - LEFT JOIN 实际成了内连接。条件本身两边一样(透过子查询都是 refunds.state = 'done'),只报连接。""" - assert texts(a, b, dialect, labels=LABELS) == expected +def test_a_difference_inside_an_aggregating_cte_names_the_cte_on_each_side(): + a = ("WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " + "SELECT COUNT(*) FROM user_totals WHERE total > 100") + b = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE total > 100") + diff = explain_sql_diff(a, b, labels=LABELS) + check_shape(diff) + assert [d.text for d in diff.items] == ["原流程在 user_totals 里只保留 is_deleted = 0 的行,盲算在 t 里没有这个条件"] -def test_only_null_rejecting_where_conditions_turn_an_outer_join_inner(): - """IS NULL、和 IS NULL 用 OR 连着的、套在 COALESCE 里的,没关联上的行照样留着:外连接不变(宁可漏报)。""" - left = "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id" - inner = "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id" - either = " WHERE r.state = 'done' OR r.id IS NULL" - assert texts(left + either, inner + either) == ["A 用 LEFT JOIN 关联 refunds 表,B 用 INNER JOIN"] - assert aspects(left + " WHERE COALESCE(r.amount, 0) < 100", left) == ["filter"] - - -# -------------------------------------------------------------------------- -# diff_facets:算出来的列 -# -------------------------------------------------------------------------- - - -@pytest.mark.parametrize("dialect,a,b,expected", [ - ("sqlite", PAID_REFUNDED + f"SELECT SUM(p.total) - SUM(r.total) {PR_JOIN}", - PAID_REFUNDED + f"SELECT SUM(r.total) - SUM(p.total) {PR_JOIN}", - ["原流程算的是 SUM(paid.total) - SUM(refunded.total),盲算算的是 SUM(refunded.total) - SUM(paid.total)"]), - # 退款率的分子分母对调 - ("postgres", f"SELECT SUM(r.total) / SUM(p.total) {PR_SUBQUERIES}", - f"SELECT SUM(p.total) / SUM(r.total) {PR_SUBQUERIES}", - ["原流程算的是 SUM(r.total) / SUM(p.total),盲算算的是 SUM(p.total) / SUM(r.total)"]), - # 窗口:每个用户的合计还是每家门店的合计 - ("sqlite", "SELECT order_id, SUM(amount) OVER (PARTITION BY user_id) AS s FROM orders", - "SELECT order_id, SUM(amount) OVER (PARTITION BY store_id) AS s FROM orders", - ["原流程算的是 SUM(amount) OVER (PARTITION BY user_id),盲算算的是 SUM(amount) OVER (PARTITION BY store_id)"]), - # 同一个合计再算一下(分换成元):和光一个聚合配成一对说 - ("sqlite", "SELECT SUM(amount) FROM orders", "SELECT SUM(amount) / 100 FROM orders", - ["原流程算的是 SUM(amount),盲算算的是 SUM(amount) / 100"]), - ("sqlite", "SELECT SUM(amount) - SUM(fee) FROM orders", "SELECT SUM(amount), SUM(fee) FROM orders", - ["原流程多算了 SUM(amount) - SUM(fee),盲算没有"]), -], ids=["difference", "ratio", "window-partition", "scaled", "one-sided"]) -def test_how_an_output_column_is_computed_is_compared(dialect, a, b, expected): - """最外层算出来的列按算法比:两边用到的聚合一模一样(SUM(paid.total)、SUM(refunded.total)),只是谁减谁、谁除谁 - 换了,聚合那一面看不出来。""" - assert texts(a, b, dialect, labels=LABELS) == expected - - -@pytest.mark.parametrize("dialect,a,b", [ - ("sqlite", PAID_REFUNDED + f"SELECT SUM(p.total) - SUM(r.total) {PR_JOIN}", - PAID_REFUNDED + "SELECT SUM(p.total) - SUM(r.total) FROM refunded r JOIN paid p ON p.user_id = r.user_id"), - ("sqlite", "SELECT region, SUM(amount) * 1.0 / COUNT(*) AS avg_amt FROM orders GROUP BY region", - "SELECT o.region, 1.0 * SUM(o.amount) / COUNT(*) FROM orders o GROUP BY 1"), - ("postgres", "SELECT order_id, RANK() OVER (PARTITION BY user_id ORDER BY amount DESC) AS k FROM orders", - "SELECT o.order_id, RANK() OVER (PARTITION BY o.user_id ORDER BY o.amount DESC) FROM orders o"), - # 最外层取整到分、空值记 0:还是那个合计 - ("sqlite", "SELECT SUM(amount) FROM orders", "SELECT ROUND(SUM(amount), 2) FROM orders"), - ("mysql", "SELECT SUM(amount) FROM orders", "SELECT IFNULL(SUM(amount), 0) FROM orders"), -], ids=["swapped-join", "commutative", "window", "round", "default"]) -def test_the_same_output_computation_written_differently_is_no_difference(dialect, a, b): - """别名、加法乘法的先后、最外层取整、空值记 0 这类写法不同,算的是同一个数。""" - assert texts(a, b, dialect) == [] - - -@pytest.mark.parametrize("dialect,a,b,expected", [ - # 往里挪一层:net 的 v 是谁减谁 - ("sqlite", - PAID_REFUNDED + ", net AS (SELECT p.user_id, p.total - r.total AS v FROM paid p JOIN refunded r " - "ON r.user_id = p.user_id) SELECT SUM(v) FROM net", - PAID_REFUNDED + ", net AS (SELECT p.user_id, r.total - p.total AS v FROM paid p JOIN refunded r " - "ON r.user_id = p.user_id) SELECT SUM(v) FROM net", - ["原流程的 net.v 是 paid.total - refunded.total,盲算是 refunded.total - paid.total"]), - ("postgres", - PAID_REFUNDED + ", net AS (SELECT p.user_id, p.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) " - "SELECT COUNT(*) FROM net WHERE v > 100", - PAID_REFUNDED + ", net AS (SELECT p.user_id, r.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) " - "SELECT COUNT(*) FROM net WHERE v > 100", - ["原流程的 net.v 是 paid.total,盲算是 refunded.total"]), - ("mysql", - "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total " + PR_SUBQUERIES + ") x", - "SELECT SUM(x.total) FROM (SELECT p.user_id, r.total " + PR_SUBQUERIES + ") x", - ["原流程的 x.total 是 p.total,盲算是 r.total"]), - ("sqlite", "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT SUM(net) FROM t", - "WITH t AS (SELECT user_id, amount * 0.8 AS net FROM orders) SELECT SUM(net) FROM t", - ["原流程的 t.net 是 amount * 0.9,盲算是 amount * 0.8"]), - ("mysql", - "WITH t AS (SELECT user_id, CASE WHEN status = 'paid' THEN amount ELSE 0 END AS paid_amt FROM orders) " - "SELECT SUM(paid_amt) FROM t", - "WITH t AS (SELECT user_id, CASE WHEN status IN ('paid', 'shipped') THEN amount ELSE 0 END AS paid_amt " - "FROM orders) SELECT SUM(paid_amt) FROM t", - ["原流程的 t.paid_amt 是 CASE WHEN status = 'paid' THEN amount ELSE 0 END," - "盲算是 CASE WHEN status IN ('paid', 'shipped') THEN amount ELSE 0 END"]), - ("postgres", "SELECT COUNT(*) FROM (SELECT user_id, amount - fee AS net FROM orders) a WHERE a.net > 100", - "SELECT COUNT(*) FROM (SELECT user_id, amount + fee AS net FROM orders) b WHERE b.net > 100", - ["原流程的 a.net 是 amount - fee,盲算的 b.net 是 amount + fee"]), - # 按支付月份还是下单月份分组 - ("mysql", "WITH t AS (SELECT DATE_FORMAT(paid_at, '%Y-%m') AS m, amount FROM orders) " - "SELECT m, SUM(amount) FROM t GROUP BY m", - "WITH t AS (SELECT DATE_FORMAT(created_at, '%Y-%m') AS m, amount FROM orders) " - "SELECT m, SUM(amount) FROM t GROUP BY m", - ["原流程的 t.m 是 DATE_FORMAT(paid_at, '%Y-%m'),盲算是 DATE_FORMAT(created_at, '%Y-%m')"]), - ("sqlite", "WITH t AS (SELECT strftime('%Y-%m', paid_at) AS m, amount FROM orders) " - "SELECT m, SUM(amount) FROM t GROUP BY m", - "WITH t AS (SELECT strftime('%Y-%m', created_at) AS m, amount FROM orders) " - "SELECT m, SUM(amount) FROM t GROUP BY m", - ["原流程的 t.m 是 STRFTIME('%Y-%m', paid_at),盲算是 STRFTIME('%Y-%m', created_at)"]), - # 窗口:每人第一笔还是最后一笔;有并列时 RANK 和 ROW_NUMBER 取到的行数不同 - ("postgres", - "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) " - "AS rn FROM orders) t WHERE rn = 1", - "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at DESC) " - "AS rn FROM orders) t WHERE rn = 1", - ["原流程的 t.rn 是 ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at)," - "盲算是 ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at DESC)"]), - ("mysql", "WITH r AS (SELECT user_id, amount, RANK() OVER (ORDER BY amount DESC) AS k FROM orders) " - "SELECT SUM(amount) FROM r WHERE k <= 10", - "WITH r AS (SELECT user_id, amount, ROW_NUMBER() OVER (ORDER BY amount DESC) AS k FROM orders) " - "SELECT SUM(amount) FROM r WHERE k <= 10", - ["原流程的 r.k 是 RANK() OVER (ORDER BY amount DESC),盲算是 ROW_NUMBER() OVER (ORDER BY amount DESC)"]), - # 外面直接 SELECT *:每一列都用到了 - ("sqlite", "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT * FROM t", - "WITH t AS (SELECT user_id, amount * 0.8 AS net FROM orders) SELECT * FROM t", - ["原流程的 t.net 是 amount * 0.9,盲算是 amount * 0.8"]), - # 递归 CTE 自己写了列名:按列名找各支同一位置的定义 - ("postgres", "WITH RECURSIVE t(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM t WHERE n < 10) SELECT SUM(n) FROM t", - "WITH RECURSIVE t(n) AS (SELECT 1 UNION ALL SELECT n + 2 FROM t WHERE n < 10) SELECT SUM(n) FROM t", - ["原流程的 t.n 是 1 | n + 1,盲算是 1 | n + 2"]), -], ids=["nested-difference", "nested-column", "nested-subquery", "scale", "case", "renamed-subquery", "month", - "month-strftime", "window-order", "window-function", "select-star", "recursive"]) -def test_how_a_derived_column_is_computed_is_reported(dialect, a, b, expected): - """外面写的都是 t.net、net.v、x.total,两边一样;它在 CTE、子查询里怎么算的不一样,要报出来,而且只报这一处: - 外面按它写的条件、聚合两边写法一样,不跟着报。""" - assert texts(a, b, dialect, labels=LABELS) == expected - - -@pytest.mark.parametrize("dialect,a,b", [ - ("sqlite", "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT SUM(net) FROM t", - "WITH x AS (SELECT user_id, 0.9 * amount AS net FROM orders) SELECT SUM(x.net) FROM x"), - # 外面没用到的列算法不同,不影响结果 - ("sqlite", "WITH t AS (SELECT user_id, amount, paid_at AS ts FROM orders) SELECT SUM(amount) FROM t", - "WITH t AS (SELECT user_id, amount, created_at AS ts FROM orders) SELECT SUM(amount) FROM t"), - # 取日期的两种写法 - ("postgres", "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT d, SUM(amount) FROM t GROUP BY d", - "WITH t AS (SELECT CAST(paid_at AS DATE) AS d, amount FROM orders) SELECT d, SUM(amount) FROM t GROUP BY d"), - ("mysql", - "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total " + PR_SUBQUERIES + ") x", - "SELECT SUM(y.total) FROM (SELECT a.user_id, a.total FROM (SELECT user_id, SUM(amount) AS total FROM refunds " - "GROUP BY user_id) b JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) a " - "ON a.user_id = b.user_id) y"), -], ids=["commutative", "unused-column", "date-cast", "nested-renamed"]) -def test_the_same_derived_column_written_differently_is_no_difference(dialect, a, b): - """派生来源里一列的写法不同、算的一样(0.9 * amount 和 amount * 0.9、DATE() 和 CAST(… AS DATE)),或者外面根本 - 没用到那一列:不报。""" - assert texts(a, b, dialect) == [] - - -@pytest.mark.parametrize("dialect,a,b,expected", [ - # 换了名字、算法一样:配成同一列 - ("sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 100", - "WITH x AS (SELECT user_id, SUM(amount) AS sum_amt FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM x WHERE sum_amt > 100", []), - ("sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 100 AND n > 1", - "WITH t AS (SELECT user_id, SUM(amount) AS s, COUNT(*) AS c FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE s > 100 AND c > 1", []), - ("sqlite", "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) " - "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", - "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS day FROM orders) " - "SELECT SUM(amount) FROM t WHERE day >= '2026-09-01'", []), - # 换了名字又改了算法:两边各剩一列,配成同一列,外面的条件不报成两边各有一个 - ("sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 100", - "WITH x AS (SELECT user_id, SUM(amount - discount) AS net_total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM x WHERE net_total > 100", - ["原流程对 amount 求和,盲算对 amount - discount 求和", - "原流程的 t.total 是 SUM(amount),盲算的 x.net_total 是 SUM(amount - discount)"]), -], ids=["renamed", "two-renamed", "renamed-date", "renamed-and-changed"]) -def test_derived_columns_renamed_on_one_side_are_paired(dialect, a, b, expected): - """CTE、子查询选出来的列外面按名字引用,盲算常给它另起名字(total、sum_amt):按定义配成同一列,外面按它写的条件 - 不算两边各有一个。""" - assert texts(a, b, dialect, labels=LABELS) == expected - - -# -------------------------------------------------------------------------- -# diff_facets:时间列写在 CTE、子查询里 -# -------------------------------------------------------------------------- - - -@pytest.mark.parametrize("dialect", DIALECTS) -def test_a_time_column_computed_in_a_cte_is_seen_through(dialect): - """按支付时间还是下单时间筛,日期表达式写在 CTE 里也认得出:时间那一面透过 d = DATE(paid_at) 认出是 paid_at,带上 - 表名、列名给调用方对照数据目录;d 的定义不同不再另报一遍。""" - a = ("WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) " - "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'") - b = a.replace("DATE(paid_at)", "DATE(created_at)") - [diff] = diff_facets(facets(a, dialect), facets(b, dialect), labels=LABELS, - names={"orders.paid_at": "支付时间", "created_at": "下单时间"}) - assert (diff.aspect, diff.text) == ("time", "原流程按支付时间 paid_at 筛时间,盲算按下单时间 created_at") - assert diff.detail == {"a": [{"table": "orders", "column": "paid_at"}], - "b": [{"table": "orders", "column": "created_at"}]} - - -@pytest.mark.parametrize("dialect,a,b,expected", [ - ("postgres", "SELECT SUM(x.amount) FROM (SELECT amount, CAST(paid_at AS DATE) AS day FROM orders) x " - "WHERE x.day BETWEEN '2026-09-01' AND '2026-09-30'", - "SELECT SUM(y.amount) FROM (SELECT amount, CAST(created_at AS DATE) AS day FROM orders) y " - "WHERE y.day BETWEEN '2026-09-01' AND '2026-09-30'", - ["原流程按 paid_at 筛时间,盲算按 created_at"]), - # 两个 CTE 同名、定义对调:原流程筛付款时间,盲算筛退款时间 - ("mysql", "WITH a AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id), " - "b AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id) " - "SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'", - "WITH a AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id), " - "b AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'", - ["原流程按 paid_at 筛时间,盲算按 refunded_at"]), - # 带时区偏移的不是同一天:时间列一样,d 的定义不同照报 - ("sqlite", "WITH t AS (SELECT amount, DATE(paid_at) AS d FROM orders) " - "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", - "WITH t AS (SELECT amount, DATE(paid_at, '+8 hours') AS d FROM orders) " - "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", - ["原流程的 t.d 是 DATE(paid_at),盲算是 DATE(paid_at, '+8 hours')"]), - ("postgres", "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) " - "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", - "WITH t AS (SELECT CAST(paid_at AS DATE) AS d, amount FROM orders) " - "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", []), -], ids=["cast-between", "same-name-swapped", "time-zone", "date-or-cast"]) -def test_time_columns_inside_derived_tables(dialect, a, b, expected): - assert texts(a, b, dialect, labels=LABELS) == expected - - -# -------------------------------------------------------------------------- -# diff_facets:两边写法一样、意思不同时说清楚 -# -------------------------------------------------------------------------- - - -def test_same_named_derived_tables_with_swapped_definitions_are_told_apart(): - """两边的 CTE 同名、定义对调了(盲算的 t1 是原流程的 t2):按结构配对之后 t1.total 两边不是同一列,可只写 - 「t1.total > 100」两边一模一样,自相矛盾。带上各自的来历:原流程筛付款合计,盲算筛退款合计。""" - a = ("WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "t2 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " - "SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100") - b = ("WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " - "t2 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100") - assert texts(a, b, "postgres", labels=LABELS) == [ - "原流程按 t1.total > 100(t1.total 即 SUM(orders.amount))筛选," - "盲算按 t1.total > 100(t1.total 即 SUM(refunds.amount))筛选"] - a_sum = ("WITH a AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "b AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " - "SELECT SUM(a.total) FROM a JOIN b ON b.user_id = a.user_id") - b_sum = ("WITH a AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " - "b AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT SUM(a.total) FROM a JOIN b ON b.user_id = a.user_id") - assert texts(a_sum, b_sum, labels=LABELS) == [ - "原流程对 a.total(即 SUM(orders.amount))求和,盲算对 a.total(即 SUM(refunds.amount))求和"] - - -def test_same_named_derived_tables_that_differ_only_in_conditions_say_which_is_which(): - """两个 CTE 读同一张表、各列定义一样,只差里面的条件;盲算把两个名字的定义对调了、还都多了 is_deleted = 0。按条件 - 的重合配(不按名字:盲算的 paid 筛的是 status = 'open'),外面的「> 100」筛的就是未付合计;来历也一样(都是 - SUM(orders.amount)),说清两边同名的各对应哪一个。""" - a = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " - "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) " - "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > 100") - b = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' AND is_deleted = 0 " - "GROUP BY user_id), " - "open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' AND is_deleted = 0 " - "GROUP BY user_id) " - "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > 100") - assert texts(a, b, labels=LABELS) == [ - "原流程按 paid.total > 100 筛选," - "盲算按 paid.total > 100(盲算的 paid 对应原流程的 open、盲算的 open 对应原流程的 paid)筛选", - "盲算只保留 is_deleted = 0 的行,原流程没有这个条件"] - - -# -------------------------------------------------------------------------- -# diff_facets:同一个条件筛的先后 -# -------------------------------------------------------------------------- - - -@pytest.mark.parametrize("dialect,a,b,expected", [ - # 「第一单就是已付的人数」对「有已付订单的人数」 - ("postgres", - "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn " - "FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'", - "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn " - "FROM orders WHERE status = 'paid') t WHERE t.rn = 1", - ["原流程先算 t 的窗口函数、再按 status = 'paid' 筛选,盲算先按 status = 'paid' 筛选、再算 t 的窗口函数"]), - ("sqlite", "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t " - "WHERE status = 'paid'", - "WITH t AS (SELECT * FROM orders WHERE status = 'paid' ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t", - ["原流程先取 t 的前 10 行、再按 status = 'paid' 筛选,盲算先按 status = 'paid' 筛选、再取 t 的前 10 行"]), - ("sqlite", "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t " - "WHERE paid_at >= '2026-09-01'", - "WITH t AS (SELECT * FROM orders WHERE paid_at >= '2026-09-01' ORDER BY amount DESC LIMIT 10) " - "SELECT SUM(amount) FROM t", - ["原流程先取 t 的前 10 行、再按 paid_at >= '2026-09-01' 筛选," - "盲算先按 paid_at >= '2026-09-01' 筛选、再取 t 的前 10 行"]), -], ids=["window", "limit", "limit-time"]) -def test_filtering_before_or_after_a_window_or_limit_is_reported(dialect, a, b, expected): - """先取前 10 行再筛已付,和先筛已付再取前 10 行,取到的不是同一批:派生来源取前几行、算窗口函数时,外面的条件 - 不再透成里面那张表的条件,两边只差在筛的先后,照报。""" - assert texts(a, b, dialect, labels=LABELS) == expected - - -@pytest.mark.parametrize("dialect,a,b", [ - # 按窗口的分区键筛,窗口里外都一样 - ("postgres", - "SELECT COUNT(*) FROM (SELECT user_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn " - "FROM orders) t WHERE t.rn = 1 AND t.user_id > 100", - "SELECT COUNT(*) FROM (SELECT user_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn " - "FROM orders WHERE user_id > 100) t WHERE t.rn = 1"), - # 两边都是先取前 10 行再筛,只是起的名字不同 - ("sqlite", "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t " - "WHERE status = 'paid'", - "WITH top10 AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(t.amount) FROM top10 t " - "WHERE t.status = 'paid'"), -], ids=["partition-key", "same-order"]) -def test_filtering_at_an_equivalent_stage_is_no_difference(dialect, a, b): - assert texts(a, b, dialect) == [] - - -# -------------------------------------------------------------------------- -# diff_facets:没用到的 CTE -# -------------------------------------------------------------------------- - - -def test_a_cte_defined_but_not_used_does_not_count(): - """盲算定义了只筛已付的 CTE,却直接查 orders:少的那个条件要报出来;定义了没用的 CTE 里多一个条件、多一张表, - 不影响结果,不报。""" - paid = "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') " - assert texts(paid + "SELECT SUM(amount) FROM paid", paid + "SELECT SUM(amount) FROM orders", labels=LABELS) == \ - ["原流程只保留 status = 'paid' 的行,盲算没有这个条件"] - assert texts("SELECT SUM(amount) FROM orders WHERE status = 'paid'", - "WITH junk AS (SELECT * FROM users WHERE vip = 1) " - "SELECT SUM(amount) FROM orders WHERE status = 'paid'") == [] - assert texts(paid + "SELECT SUM(amount) FROM paid", - paid + "SELECT SUM(amount) FROM orders WHERE status = 'paid'") == [] - - -# -------------------------------------------------------------------------- -# diff_facets:自己关联自己 -# -------------------------------------------------------------------------- - -SELF_BASE = ("WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders " - "GROUP BY user_id, referrer_id) ") - - -@pytest.mark.parametrize("dialect,a,b,expected", [ - ("postgres", SELF_BASE + "SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id " - "WHERE b1.total > b2.total", - SELF_BASE + "SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id WHERE b2.total > b1.total", - ["原流程只保留 b1.total > b2.total 的行,盲算没有这个条件", "盲算只保留 b1.total < b2.total 的行,原流程没有这个条件"]), - ("sqlite", "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o1.amount > o2.amount", - "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o2.amount > o1.amount", - ["原流程只保留 o2.amount < o1.amount 的行,盲算没有这个条件", "盲算只保留 o2.amount > o1.amount 的行,原流程没有这个条件"]), - # 换了别名、换了关联先后:b1 还是出 referrer_id 的那一个 - ("postgres", SELF_BASE + "SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id " - "WHERE b1.total > b2.total", - SELF_BASE + "SELECT COUNT(*) FROM base y JOIN base x ON x.referrer_id = y.user_id WHERE y.total < x.total", []), - ("sqlite", "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o1.amount > o2.amount", - "SELECT COUNT(*) FROM orders c JOIN orders p ON c.parent_id = p.id WHERE c.amount > p.amount", []), -], ids=["cte-reversed", "table-reversed", "cte-renamed", "table-renamed"]) -def test_a_self_join_tells_its_two_aliases_apart(dialect, a, b, expected): - """同一个 CTE、同一张表自己关联自己:两个别名按各自在关联条件里出的列区分(出 referrer_id 的、出 user_id 的), - 比较写反了要报;只是换了别名、换了关联先后不报。""" - assert texts(a, b, dialect, labels=LABELS) == expected +def test_an_outer_join_turned_inner_by_where_says_what_was_written(): + """原文写的是 LEFT / RIGHT JOIN、WHERE 里的条件把没关联上的行筛掉了:按内连接比,说法照实写「写的是 LEFT JOIN, + 实际是 INNER JOIN」,不说成「用 INNER JOIN」。从 RIGHT JOIN 改写来的外连接说以哪一侧为主表,不说 LEFT JOIN。""" + a = "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'" + b = "SELECT SUM(o.amount) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE o.status = 'paid'" + assert texts(a, b, "postgres") == [ + "A 用 LEFT JOIN 关联 refunds 表,B 写的是 LEFT JOIN,但 WHERE 里的条件把关联不上 orders 的行筛掉了,实际是 INNER JOIN"] + a = "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id" + b = a + " WHERE o.status = 'paid'" + assert texts(a, b, "postgres") == [ + "A 以 refunds 为主表外连接 orders 表,B 写的是 RIGHT JOIN,但 WHERE 里的条件把关联不上 orders 的行筛掉了," + "实际是 INNER JOIN", "B 只保留 orders.status = 'paid' 的行,A 没有这个条件"] + # 本来就是内连接、和被 WHERE 变成内连接的 LEFT JOIN 是同一个意思 + assert explain_sql_diff("SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.amount > 10", + "SELECT COUNT(*) FROM refunds r JOIN orders o ON o.id = r.order_id WHERE r.amount > 10" + ).equivalent @pytest.mark.parametrize("dialect", DIALECTS) def test_each_dialect_gives_the_same_three_findings(dialect): - q = {"sqlite": '"', "postgres": '"', "mysql": "`"}[dialect] + q = "`" if dialect == "mysql" else '"' a = (f"SELECT COUNT(DISTINCT o.order_id) AS {q}订单数{q} FROM orders o " "WHERE o.paid_at >= '2026-09-01' AND o.is_deleted <> 1") b = f"SELECT COUNT(o.order_id) AS {q}订单数{q} FROM orders o WHERE o.created_at >= '2026-09-01'" @@ -1350,8 +716,128 @@ def test_each_dialect_gives_the_same_three_findings(dialect): def test_side_labels_in_chinese_have_no_stray_spaces(): a = "SELECT COUNT(DISTINCT order_id) FROM orders WHERE paid_at >= '2026-09-01'" b = "SELECT COUNT(order_id) FROM orders WHERE created_at >= '2026-09-01'" - assert texts(a, b, labels=("原流程", "盲算"), names={"paid_at": "支付时间"}) == [ - "原流程按支付时间 paid_at 筛时间,盲算按 created_at", "原流程对 order_id 去重计数,盲算没去重"] + assert texts(a, b, labels=LABELS) == ["原流程按 paid_at 筛时间,盲算按 created_at", "原流程对 order_id 去重计数,盲算没去重"] + + +def test_filters_on_real_columns_carry_the_columns_for_the_catalog(): + """筛选条件只涉及的实表列记在 detail 里:调用方据此对照数据目录里的有效行过滤。""" + [item] = explain_sql_diff("SELECT SUM(o.amount) FROM orders o WHERE o.is_deleted = 0", + "SELECT SUM(amount) FROM orders").items + assert item.aspect == "filter" and item.detail == {"columns": [{"table": "orders", "column": "is_deleted"}]} + + +def test_a_displayed_condition_means_what_the_sql_means(): + """给人看的条件按方言生成;方言生成的写法读回来不是同一个意思时(SQLite 把 CAST(x AS DATE) 生成 DATE(x))改用 + 通用写法,不拿 DATE('2026-09-01') 冒充原文。几个条件连起来时 OR 带上括号。""" + [item] = explain_sql_diff("SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= CAST('2026-09-01' AS DATE)").items + assert "CAST('2026-09-01' AS DATE)" in item.text and "DATE('2026-09-01')" not in item.text + a = "SELECT SUM(amount) FROM orders" + b = "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id AND (s.city = 'SH' OR s.city = 'BJ')" + [item] = explain_sql_diff(a, b).items + assert item.text == ("B 关联了 stores 表(关联的条件是 orders.store_id = stores.id AND " + "(stores.city = 'BJ' OR stores.city = 'SH')),A 没有") + + +# -------------------------------------------------------------------------- +# explain_sql_diff:兜底说明 +# -------------------------------------------------------------------------- + + +def test_nothing_specific_gives_one_fallback_item_and_the_lines_that_differ(): + a = "SELECT COUNT(*) FROM orders o WHERE o.id NOT IN (SELECT order_id FROM refunds)" + b = "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)" + diff = explain_sql_diff(a, b, "sqlite") + check_shape(diff) + assert diff.fallback and diff.items == [FacetDiff("other", None, None, FALLBACK)] + assert any(line.startswith("- ") for line in diff.text_diff) + assert any(line.startswith("+ ") for line in diff.text_diff) + assert diff_facets(facets(a), facets(b)) == diff.items + + +def test_specific_items_that_cover_only_part_end_with_the_rest_note(): + """一边多关联了 refunds(会把订单行按退款数复制),另一边只在 EXISTS 里用它:关联那一句说得准,EXISTS 那处说不准, + 补一句「另有未能归纳的差异」。""" + a = "SELECT SUM(o.amount) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)" + b = "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id" + diff = explain_sql_diff(a, b, labels=LABELS) + check_shape(diff) + assert diff.fallback and [d.text for d in diff.items] == [ + "盲算关联了 refunds 表(关联的条件是 orders.id = refunds.order_id),原流程没有关联它(只在子查询里用到了 refunds)", + FALLBACK_REST] + + +def test_columns_inside_a_cte_are_not_compared_because_canonical_forms_prune_them(): + """规范化会去掉 CTE 里没用到的列、重排剩下的:CTE 里选出来的列不同不说具体的(原文里两边的 CTE 都选了三列, + 说「A 在 t 里选出 2 列」就是错的),交给兜底说明。""" + a = ("SELECT COUNT(*) FROM (SELECT id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at, id) AS rn " + "FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'") + b = ("SELECT COUNT(*) FROM (SELECT id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at, id) AS rn " + "FROM orders WHERE status = 'paid') t WHERE t.rn = 1") + diff = explain_sql_diff(a, b) + check_shape(diff) + assert diff.fallback and not [d for d in diff.items if "选出" in d.text] + + +def test_text_diff_lists_at_most_forty_lines(): + cols_a = ", ".join(f"SUM(amount * {i}) AS c{i}" for i in range(60)) + cols_b = ", ".join(f"SUM(amount * {i + 100}) AS c{i}" for i in range(60)) + diff = explain_sql_diff(f"SELECT {cols_a} FROM orders", f"SELECT {cols_b} FROM orders") + assert not diff.equivalent and len(diff.text_diff) == 40 + + +@pytest.mark.parametrize("a,b", [ + ("SELECT FROM WHERE", BASE), + (BASE, "SELECT 1; SELECT 2"), + ("", BASE), + (None, BASE), + (BASE, "UPDATE orders SET amount = 0"), +]) +def test_an_unparsable_side_gives_one_parse_item_and_never_raises(a, b): + diff = explain_sql_diff(a, b, "sqlite") + assert not diff.equivalent and len(diff.items) == 1 and diff.items[0].aspect == "parse" + assert "没有逐项比较" in diff.items[0].text + json.dumps(diff.as_dict(), ensure_ascii=False) + + +# -------------------------------------------------------------------------- +# diff_facets:照旧收 facets,交给 explain_sql_diff +# -------------------------------------------------------------------------- + + +def test_diff_facets_returns_the_items_of_explain_sql_diff(): + pairs = [(BASE, "SELECT SUM(amount) FROM orders"), + ("SELECT COUNT(DISTINCT order_id) FROM orders", "SELECT COUNT(order_id) FROM orders"), + ("SELECT COUNT(*) FROM orders o WHERE o.id NOT IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)")] + for a, b in pairs: + for dialect in DIALECTS: + expected = explain_sql_diff(a, b, dialect, labels=LABELS, names={"paid_at": "支付时间"}).items + assert diff_facets(facets(a, dialect), facets(b, dialect), labels=LABELS, + names={"paid_at": "支付时间"}) == expected + + +def test_diff_facets_passes_the_schema_given_to_facets(): + schema = sql_pairs.schema_cache() + a = "SELECT SUM(amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE city = 'SH'" + b = "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE s.city = 'SH'" + assert diff_facets(facets(a, "sqlite", schema=schema), facets(b, "sqlite", schema=schema)) == [] + assert diff_facets(facets(a, "sqlite"), facets(b, "sqlite")) != [] + + +def test_an_unparsable_side_is_reported_once_without_item_by_item_comparison(): + [diff] = diff_facets(facets("SELECT FROM WHERE"), facets(BASE)) + assert diff.aspect == "parse" and diff.text.startswith("A:SQL 无法解析") and "没有逐项比较" in diff.text + assert diff.b is None + + +def test_facets_without_the_original_sql_cannot_be_called_equivalent(): + """不是 facets 算出来的 Facets(没记原文)比不了原文:facets 记的结构不全,两份一样也不说明 SQL 一样,只有同一个 + 对象才算没有不同,否则给兜底说明,不猜。""" + f = Facets(dialect="sqlite") + assert diff_facets(f, f) == [] + for other in (Facets(dialect="sqlite"), Facets(dialect="sqlite", limit="10")): + assert diff_facets(f, other) == [FacetDiff("other", None, None, FALLBACK)] def test_catalog_labels_come_from_visible_catalog_items(): @@ -1369,341 +855,5 @@ def test_catalog_labels_come_from_visible_catalog_items(): def test_a_facets_object_compares_with_itself_cleanly(): - f = facets(BASE) + f = facets("WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT order_id) FROM v") assert isinstance(f, Facets) and diff_facets(f, f) == [] - - -# -------------------------------------------------------------------------- -# diff_facets:读同一张表的几个派生来源(原样透出的 CTE、子查询) -# -------------------------------------------------------------------------- - -#: 两个原样透出的 CTE 读同一张表:一个是已付的订单,一个是未付的 -PASS_PO = ("WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), " - "open AS (SELECT * FROM orders WHERE status = 'open') ") -PO_JOIN = "FROM paid p JOIN open o ON o.user_id = p.user_id" -#: 同样两段写成 FROM 里的子查询 -PO_SUBQUERIES = ("FROM (SELECT * FROM orders WHERE status = 'paid') p " - "JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id") - - -@pytest.mark.parametrize("dialect", DIALECTS) -@pytest.mark.parametrize("form", ["cte", "subquery"]) -@pytest.mark.parametrize("a,b,expected", [ - ("SELECT SUM(p.amount) {j}", "SELECT SUM(o.amount) {j}", ["原流程对 paid.amount 求和,盲算对 open.amount 求和"]), - ("SELECT COUNT(*) {j} WHERE p.amount > 100", "SELECT COUNT(*) {j} WHERE o.amount > 100", - ["原流程按 paid.amount > 100 筛选,盲算按 open.amount > 100 筛选"]), - ("SELECT COUNT(*) {j} WHERE p.created_at >= '2026-09-01'", - "SELECT COUNT(*) {j} WHERE o.created_at >= '2026-09-01'", - ["原流程按 paid.created_at 筛时间,盲算按 open.created_at"]), - ("SELECT p.id {j} ORDER BY p.created_at DESC LIMIT 10", "SELECT p.id {j} ORDER BY o.created_at DESC LIMIT 10", - ["原流程取行前按 paid.created_at 降序排,盲算按 open.created_at 降序排"]), - ("SELECT COUNT(*) {j} GROUP BY p.store_id", "SELECT COUNT(*) {j} GROUP BY o.store_id", - ["原流程按 paid.store_id 分组,盲算按 open.store_id 分组"]), -], ids=["sum", "where", "time", "order-by", "group-by"]) -def test_which_pass_through_derived_table_a_column_comes_from_is_reported(dialect, form, a, b, expected): - """两个原样透出的 CTE(SELECT * FROM orders WHERE …)读同一张表:透过去都成了 orders 的列,外层用的是已付的那份 - 还是未付的那份就分不出来了(SQLite 实测求和 230 对 400)。同一段里有几个来源最后读的是同一张表时,这几个派生来源 - 不透过去,按配好对的编号认列。""" - if form == "cte": - head, j = PASS_PO, PO_JOIN - else: - head, j = "", PO_SUBQUERIES - expected = [t.replace("paid.", "p.").replace("open.", "o.") for t in expected] - assert texts(head + a.format(j=j), head + b.format(j=j), dialect, labels=LABELS) == expected - - -def test_a_time_column_of_a_pass_through_derived_table_names_the_real_table(): - """按哪个派生来源的同名时间列筛,说明里写派生来源的名字;给调用方对照数据目录的 detail 写透到底的实表和列。""" - [diff] = diff_facets(facets(PASS_PO + f"SELECT COUNT(*) {PO_JOIN} WHERE p.created_at::date >= '2026-09-01'", - "postgres"), - facets(PASS_PO + f"SELECT COUNT(*) {PO_JOIN} WHERE o.created_at::date >= '2026-09-01'", - "postgres"), labels=LABELS) - assert (diff.aspect, diff.text) == ("time", "原流程按 paid.created_at 筛时间,盲算按 open.created_at") - assert diff.detail == {"a": [{"table": "orders", "column": "created_at"}], - "b": [{"table": "orders", "column": "created_at"}]} - - -PO_COLUMNS = ("WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), " - "open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') ") -PO_CHAIN = ("WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), " - "paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') ") -#: 原样透出的 CTE 自己关联自己 -SELF_T = "WITH t AS (SELECT * FROM orders WHERE status = 'paid') " - - -@pytest.mark.parametrize("dialect", DIALECTS) -@pytest.mark.parametrize("a,b,expected", [ - # 直接选出各列的 CTE:关联条件透过去两边都是 orders.referrer_id = orders.user_id - (PO_COLUMNS + "SELECT SUM(p.amount) FROM paid p JOIN open o ON p.user_id = o.referrer_id", - PO_COLUMNS + "SELECT SUM(p.amount) FROM paid p JOIN open o ON p.referrer_id = o.user_id", - ["原流程关联 open 表的条件是 open.referrer_id = paid.user_id,盲算是 paid.referrer_id = open.user_id"]), - # CTE 套 CTE:paid、open 都从 base 取数 - (PO_CHAIN + "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", - PO_CHAIN + "SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100", - ["原流程按 paid.amount > 100 筛选,盲算按 open.amount > 100 筛选"]), - # 子查询里再套子查询 - ("SELECT COUNT(*) FROM (SELECT p.user_id " + PO_SUBQUERIES + " WHERE p.amount > 100) x", - "SELECT COUNT(*) FROM (SELECT p.user_id " + PO_SUBQUERIES + " WHERE o.amount > 100) x", - ["原流程按 p.amount > 100 筛选,盲算按 o.amount > 100 筛选"]), - # 派生列的定义里谁减谁 - (PASS_PO + "SELECT SUM(d) FROM (SELECT p.amount - o.amount AS d " + PO_JOIN + ") x", - PASS_PO + "SELECT SUM(d) FROM (SELECT o.amount - p.amount AS d " + PO_JOIN + ") x", - ["原流程的 x.d 是 paid.amount - open.amount,盲算是 open.amount - paid.amount"]), - # CTE 和它自己读的那张表关联:父单的金额还是子单的金额 - (SELF_T + "SELECT SUM(p.amount) FROM t p JOIN orders c ON c.parent_id = p.id", - SELF_T + "SELECT SUM(c.amount) FROM t p JOIN orders c ON c.parent_id = p.id", - ["原流程对 t.amount 求和,盲算对 orders.amount 求和"]), - # 原样透出的 CTE 自己关联自己:两个别名按各自在关联条件里出的列区分,透过去也不丢 - (SELF_T + "SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", - SELF_T + "SELECT SUM(b.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", - ["原流程对 a.amount 求和,盲算对 b.amount 求和"]), - # 自己关联自己、关联的方向反了:两边都写 a.amount,说清各自的 a 是哪一个 - (SELF_T + "SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", - SELF_T + "SELECT SUM(a.amount) FROM t a JOIN t b ON b.referrer_id = a.user_id", - ["原流程对 a.amount(a 是关联条件里出 referrer_id 的那个 t)求和," - "盲算对 a.amount(a 是关联条件里出 user_id 的那个 t)求和"]), - ("SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", - "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.referrer_id = a.user_id", - ["原流程对 a.amount(a 是关联条件里出 referrer_id 的那个 orders)求和," - "盲算对 a.amount(a 是关联条件里出 user_id 的那个 orders)求和"]), -], ids=["columns-join", "cte-chain", "nested-subquery", "derived-column", "with-own-table", "self-join", - "self-join-reversed", "table-self-join-reversed"]) -def test_derived_tables_reading_the_same_table_keep_apart(dialect, a, b, expected): - """读同一张表的几个派生来源(还有派生来源和它自己读的那张表)在同一段里,各自的列不透成那张表的列:关联条件、 - CTE 链、往里套的子查询、派生列的定义、自己关联自己都分得清是哪一份。""" - assert texts(a, b, dialect, labels=LABELS) == expected - - -@pytest.mark.parametrize("dialect,a,b", [ - # 关联先后对调、比较两边对调:意思一样 - ("sqlite", PASS_PO + f"SELECT COUNT(*) {PO_JOIN} WHERE p.amount > o.amount", - PASS_PO + "SELECT COUNT(*) FROM open o JOIN paid p ON p.user_id = o.user_id WHERE o.amount < p.amount"), - ("postgres", f"SELECT SUM(p.amount) {PO_SUBQUERIES}", - "SELECT SUM(x.amount) FROM (SELECT * FROM orders WHERE status = 'open') y " - "JOIN (SELECT * FROM orders WHERE status = 'paid') x ON x.user_id = y.user_id"), - # 两边给 CTE 起的名字不同 - ("mysql", PASS_PO + f"SELECT SUM(p.amount) {PO_JOIN}", - "WITH settled AS (SELECT * FROM orders WHERE status = 'paid'), " - "pending AS (SELECT * FROM orders WHERE status = 'open') " - "SELECT SUM(s.amount) FROM settled s JOIN pending q ON q.user_id = s.user_id"), - # 只有一个原样透出的 CTE:照旧透成那张表的列,写在 CTE 里还是外层的条件是同一个 - ("sqlite", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') " - "SELECT SUM(p.amount) FROM paid p JOIN refunds r ON r.order_id = p.id", - "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'"), - # 自己关联自己换了别名、换了关联先后 - ("sqlite", SELF_T + "SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", - SELF_T + "SELECT SUM(y.amount) FROM t x JOIN t y ON y.referrer_id = x.user_id"), -], ids=["swapped", "subquery-renamed-swapped", "cte-renamed", "single-pass-through", "self-join-renamed"]) -def test_derived_tables_reading_the_same_table_written_differently_are_no_difference(dialect, a, b): - assert texts(a, b, dialect) == [] - - -# -------------------------------------------------------------------------- -# diff_facets:相关子查询读的是外层那张表 -# -------------------------------------------------------------------------- - - -@pytest.mark.parametrize("dialect", DIALECTS) -@pytest.mark.parametrize("a,b,expected", [ - # 有子单的订单数对有父单的订单数:内外两个 orders 写出来一样,得分清哪一列是外层的 - ("SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", - "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)", - ["原流程按 EXISTS(SELECT 1 FROM orders AS c WHERE o.id = c.parent_id) 筛选," - "盲算按 EXISTS(SELECT 1 FROM orders AS c WHERE o.parent_id = c.id) 筛选"]), - # 之前有过订单对之后还有订单 - ("SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id " - "AND p.created_at < o.created_at)", - "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id " - "AND o.created_at < p.created_at)", - ["原流程按 EXISTS(SELECT 1 FROM orders AS p WHERE o.user_id = p.user_id AND o.created_at > p.created_at) 筛选," - "盲算按 EXISTS(SELECT 1 FROM orders AS p WHERE o.user_id = p.user_id AND o.created_at < p.created_at) 筛选"]), - # 外层是透过 CTE 的 orders - ("WITH p AS (SELECT * FROM orders WHERE status = 'paid') " - "SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = p.id)", - "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " - "SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = p.parent_id)", - ["原流程按 EXISTS(SELECT 1 FROM orders AS c WHERE p.id = c.parent_id) 筛选," - "盲算按 EXISTS(SELECT 1 FROM orders AS c WHERE p.parent_id = c.id) 筛选"]), - # SELECT 列表里的标量子查询:子单数对父单数 - ("SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o", - "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o", - ["原流程算的是 (SELECT COUNT(*) FROM orders AS c WHERE o.id = c.parent_id)," - "盲算算的是 (SELECT COUNT(*) FROM orders AS c WHERE o.parent_id = c.id)"]), -], ids=["exists-parent-child", "exists-before-after", "exists-through-cte", "scalar"]) -def test_a_correlated_subquery_on_the_outer_table_tells_inner_from_outer(dialect, a, b, expected): - """相关子查询读的表和外层是同一张(orders 里找子单、父单、同一用户更早的单):比较用的写法里外层的列另带标记, - c.parent_id = o.id 和 c.id = o.parent_id 才不会都成了 orders.id = orders.parent_id;给人看的写法照写别名(两边 - 表名一样时外层的列排在前面)。""" - assert texts(a, b, dialect, labels=LABELS) == expected - - -def test_a_correlated_lateral_subquery_on_the_outer_table_is_reported(): - a = ("SELECT o.id, x.s FROM orders o CROSS JOIN LATERAL " - "(SELECT SUM(c.amount) AS s FROM orders c WHERE c.parent_id = o.id) x") - b = a.replace("c.parent_id = o.id", "c.id = o.parent_id") - assert texts(a, b, "postgres", labels=LABELS) == ["原流程只保留 o.id = c.parent_id 的行,盲算没有这个条件", - "盲算只保留 o.parent_id = c.id 的行,原流程没有这个条件"] - - -@pytest.mark.parametrize("a,b", [ - ("SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", - "SELECT COUNT(*) FROM orders x WHERE EXISTS (SELECT 1 FROM orders y WHERE x.id = y.parent_id)"), - ("SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", - "SELECT COUNT(*) FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), -], ids=["renamed", "other-table"]) -def test_the_same_correlated_subquery_written_differently_is_no_difference(a, b): - assert texts(a, b) == [] - - -def test_a_difference_inside_a_condition_subquery_is_said_once(): - """条件里的子查询整个写进外层那个条件:里面多一个条件,外层那个条件已经报了,不再另说一句「只保留 state = - 'done' 的行」(它不筛外层的行,是 NOT EXISTS 里面的)。""" - a = "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)" - b = ("SELECT COUNT(*) FROM orders o WHERE NOT EXISTS " - "(SELECT 1 FROM refunds r WHERE r.order_id = o.id AND r.state = 'done')") - assert texts(a, b, labels=LABELS) == [ - "原流程按 NOT EXISTS(SELECT 1 FROM refunds WHERE orders.id = refunds.order_id) 筛选," - "盲算按 NOT EXISTS(SELECT 1 FROM refunds WHERE orders.id = refunds.order_id AND refunds.state = 'done') 筛选"] - - -# -------------------------------------------------------------------------- -# diff_facets:同一个条件、分组、连接出现在不同的层 -# -------------------------------------------------------------------------- - - -@pytest.mark.parametrize("dialect", DIALECTS) -@pytest.mark.parametrize("a,b,expected", [ - # 外层的条件和 IN 子查询里的同一个条件 - ("SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN " - "(SELECT user_id FROM orders WHERE status = 'paid')", - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", - ["原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件"]), - ("SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders x " - "WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", - "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders x " - "WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", - ["原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件"]), - # 关联进来的汇总 CTE 里有,外层没有 - ("WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) " - "SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id WHERE o.status = 'paid'", - "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) " - "SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id", - ["原流程在外层也只保留 status = 'paid' 的行,盲算在外层没有这个条件"]), - # UNION 的第二支少了条件 - ("SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' " - "UNION SELECT referrer_id FROM orders WHERE status = 'paid') t", - "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t", - ["原流程在 t 的 UNION 第 2 支里也只保留 status = 'paid' 的行,盲算在 t 的 UNION 第 2 支里没有这个条件"]), - # 外层的分组和 IN 子查询里的分组 - ("SELECT user_id, COUNT(*) FROM orders WHERE user_id IN " - "(SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1) GROUP BY user_id", - "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1)", - ["原流程在外层按 user_id 分组,盲算在外层没有分组"]), - # 外层的关联和 EXISTS 子查询里的关联 - ("SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE EXISTS " - "(SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", - "SELECT COUNT(*) FROM orders o WHERE EXISTS " - "(SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", - ["原流程在外层也关联了 refunds 表,盲算在外层没有"]), - # 外层算的聚合和标量子查询里的聚合 - ("SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", - "SELECT AVG(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", - ["原流程对 amount 求和,盲算求平均"]), - # 外层的时间条件和 IN 子查询里的同一个时间条件 - ("SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND user_id IN " - "(SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", - ["原流程在外层也按 paid_at >= '2026-09-01' 筛了时间,盲算在外层没有这个条件"]), - # 读同一张表的两个 CTE 各有一个 is_deleted = 0,盲算的 b 里少了:两个条件,不是一个 - ("WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), " - "b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'open') " - "SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", - "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), " - "b AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", - ["原流程在 b 里也只保留 is_deleted = 0 的行,盲算在 b 里没有这个条件"]), - # 最外层就是 UNION:第二支少了条件 - ("SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", - "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", - ["原流程在 UNION 第 2 支里也只保留 status = 'paid' 的行,盲算在 UNION 第 2 支里没有这个条件"]), -], ids=["in-subquery", "exists", "joined-cte", "union-branch", "group-by", "join", "aggregate", "time", - "two-ctes", "root-union"]) -def test_the_same_item_on_another_layer_is_counted_once_per_layer(dialect, a, b, expected): - """各段收集到的条件、分组、连接、聚合原来按整条语句去重成一个集合:外层的 status = 'paid' 和 IN 子查询(关联进来的 - 汇总 CTE、UNION 的另一支)里的同一个条件合成了一个,外层少了它报不出来(实测 350 对 700)。现在每一项记下在哪 - 一层,同一项在一边出现在几层就算几个:另一边层数少了,报多出来的那一层。""" - assert texts(a, b, dialect, labels=LABELS) == expected - - -@pytest.mark.parametrize("a,b", [ - # 有效记录条件写在原样透出的 CTE 里、外层又写了一遍:还是同一层的同一个条件 - ("WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT SUM(amount) FROM v WHERE is_deleted = 0", - "SELECT SUM(amount) FROM orders WHERE is_deleted = 0"), - # 两边外层都有、子查询里也都有 - ("SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN " - "(SELECT user_id FROM orders WHERE status = 'paid')", - "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT x.user_id FROM orders x WHERE x.status = 'paid') " - "AND o.status = 'paid'"), - # 条件从汇总的 CTE 里挪到外层(按分组键筛,先筛后筛一样) - ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 100 GROUP BY user_id) " - "SELECT SUM(total) FROM t", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT SUM(total) FROM t WHERE user_id > 100"), -], ids=["repeated-in-pass-through", "both-layers", "moved-group-key-filter"]) -def test_the_same_item_on_equivalent_layers_is_no_difference(a, b): - assert texts(a, b) == [] - - -# -------------------------------------------------------------------------- -# diff_facets:去重、UNION、最后出几行 -# -------------------------------------------------------------------------- - - -@pytest.mark.parametrize("dialect", DIALECTS) -@pytest.mark.parametrize("a,b,expected", [ - # 先在子查询里按用户分组、外层数用户数,对外层按用户分组、每人一行(实测 [(4,)] 对 [(3,),(2,),(1,),(1,)]) - ("SELECT COUNT(t.user_id) FROM (SELECT user_id FROM orders GROUP BY user_id) t", - "SELECT COUNT(user_id) FROM orders GROUP BY user_id", - ["原流程最后只出一行,盲算按 user_id 分组、每组一行"]), - # 同一批行 UNION ALL 了两遍(实测 8 对 4) - ("SELECT COUNT(*) FROM (SELECT id FROM orders WHERE status = 'paid' " - "UNION ALL SELECT id FROM orders WHERE status = 'paid') t", - "SELECT COUNT(*) FROM orders WHERE status = 'paid'", - ["原流程在 t 里把 2 支查询用 UNION ALL 合起来,盲算没有"]), - # 去重后的用户数对订单行数(实测 3 对 4) - ("SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", - "SELECT COUNT(*) FROM orders WHERE status = 'paid'", - ["原流程在 t 里对 user_id 去重,盲算没有去重"]), - ("SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", - "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid') t", - ["原流程在 t 里对 user_id 去重,盲算没有去重"]), - ("SELECT DISTINCT region FROM orders", "SELECT region FROM orders", ["原流程对 region 去重,盲算没有去重"]), - ("SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT referrer_id FROM orders) t", - "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT referrer_id FROM orders) t", - ["原流程在 t 里用 UNION 合并(去掉重复的行),盲算用 UNION ALL(重复的行都留着)"]), -], ids=["grain", "union-all-twice", "distinct-vs-rows", "distinct-vs-plain", "distinct-outer", "union-vs-all"]) -def test_dedup_union_and_how_many_rows_come_out_are_compared(dialect, a, b, expected): - """DISTINCT、UNION / UNION ALL 不进原来的任何一面,分组键挪了一层也只按整条语句比:同一批行合了两遍、去重后的 - 用户数对订单行数、外层出一行还是每个用户一行,都报不出来。""" - assert texts(a, b, dialect, labels=LABELS) == expected - - -@pytest.mark.parametrize("a,b", [ - # 分组写在 CTE 里、外面原样取出来,和直接分组出的行一样 - ("SELECT region, SUM(amount) FROM orders GROUP BY region", - "WITH t AS (SELECT region, SUM(amount) AS s FROM orders GROUP BY region) SELECT region, s FROM t"), - # 按某列分组不算别的,和对这一列去重一样 - ("SELECT user_id FROM orders GROUP BY user_id", "SELECT DISTINCT user_id FROM orders"), - # 两边一样的 UNION,派生来源起的名字不同 - ("SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION ALL " - "SELECT referrer_id FROM orders WHERE status = 'paid') t", - "WITH ids AS (SELECT user_id FROM orders WHERE status = 'paid' UNION ALL " - "SELECT referrer_id FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM ids"), - # 最外层直接 UNION,和包进子查询再原样取出来 - ("SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", - "SELECT * FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), - # 分组后的结果包一层原样取出来 - ("SELECT user_id, COUNT(*) FROM orders GROUP BY user_id", - "SELECT user_id, n FROM (SELECT user_id, COUNT(*) AS n FROM orders GROUP BY user_id) t"), -], ids=["group-in-cte", "group-or-distinct", "same-union", "union-wrapped", "grouped-wrapped"]) -def test_the_same_rows_written_differently_are_no_difference(a, b): - assert texts(a, b) == [] From f16860f704b7a2c1b46ccf67b87b254051354121 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 12:00:36 +0800 Subject: [PATCH 5/6] =?UTF-8?q?SQL=20=E5=B7=AE=E5=BC=82=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E9=87=8D=E5=81=9A=EF=BC=88=E7=AC=AC=202=20=E8=BD=AE=EF=BC=89?= =?UTF-8?q?=EF=BC=9A=E8=A7=84=E8=8C=83=E5=8C=96=E5=8F=AA=E7=95=99=E8=83=BD?= =?UTF-8?q?=E8=AE=B2=E6=B8=85=E4=B8=BA=E4=BB=80=E4=B9=88=E4=BF=9D=E6=8C=81?= =?UTF-8?q?=E8=AF=AD=E4=B9=89=E7=9A=84=E5=87=A0=E6=AD=A5=EF=BC=8C=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E7=9C=8B=E8=A1=8C=E5=87=BA=E6=9D=A5=E5=85=88=E5=90=8E?= =?UTF-8?q?=E7=9A=84=E4=B8=80=E6=AE=B5=E4=B8=8D=E5=8A=A8=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=EF=BC=9B=E5=85=B7=E4=BD=93=E8=AF=B4=E6=B3=95=E5=85=88=E7=A1=AE?= =?UTF-8?q?=E8=AE=A4=E4=B8=A4=E8=BE=B9=E7=9C=9F=E7=9A=84=E4=B8=8D=E5=90=8C?= =?UTF-8?q?=E3=80=81=E5=8F=A6=E4=B8=80=E8=BE=B9=E7=9C=9F=E7=9A=84=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=EF=BC=8C=E5=85=B6=E4=BD=99=E7=BB=99=E5=85=9C=E5=BA=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:第 2 轮复验抓到第 1 版规范化里一批不保持语义的改写(15 个 blocker),说明里一批假话、矛盾话(10 多个 major)。根子是借用了 sqlglot simplify 等现成改写、又把「相等」当成「一样」、没管结果看行出来先后的查询;说明这边 是先改名再比、只看这一段有没有那个条件。这一轮把改写收窄到每一步都讲得清为什么保持语义,说明这边每句话都先验证。 规范化(backend/app/data/sql_semantics.py): - 去掉 sqlglot 的 simplify(Decimal 折叠 0.1 + 0.2、-(-x) 化成 x、三项加乘拍平合常量、BETWEEN SYMMETRIC 丢了 SYMMETRIC)、pushdown_projections、HAVING 挪进 WHERE(MySQL 不分大小写时同组的值各不一样)、只有一列的 IN 列表 写成 =(SQLite 里 a IN (b) 是 a = +b)。只留自己写的几步:AND / OR、比较两边、一个节点的加乘两边排序,NOT 套 比较翻过来,BETWEEN 拆开,IN 常量列表去重排序,COUNT(1)、MIN/MAX(DISTINCT)、IIF / IF 写成 CASE、CASE ELSE NULL、 恒真的 1 = 1 去掉,IS NOT NULL / NOT LIKE 各方言统一成 NOT (…)。 - 新增 _Sensitivity:取行(含有 ORDER BY 并列)、DISTINCT ON、窗口函数、GROUP_CONCAT 这类聚合、聚合里的散列、当成标量 的子查询,以及去重、分组、MIN/MAX 留下的值「相等却不一样」(MySQL 字符串、PG numeric、算出来的表达式)的一段, 连同给它供行的子查询、CTE,都不调换来源、不排 GROUP BY 键、不改 DISTINCT、不并子查询。SQLite 的逗号连接和 CROSS JOIN 按原文分开(SQLite 不调换 CROSS JOIN 两边)。 - 列名表(CTE、子查询)写进里面那段的别名,实表别名带列名、递归 CTE(含 SQLite 不写 RECURSIVE、CTE 引用自己或后面的)、 BETWEEN SYMMETRIC、FETCH WITH TIES 不走规范化;FETCH FIRST 写成 LIMIT;USING 只在简单写法下写成 ON。 - EXISTS 改写前先查聚合(含 ORDER BY 里的)和 OFFSET;合并只调换最外层不取行的 UNION ALL(PG/MySQL 另加按位置排序、 各支列名一样的嵌套);后面有 RIGHT / FULL JOIN 时内连接不挪进 WHERE、开头的来源不调换。 - 子查询并进外层自己写(_merge_derived:只并内连接一侧、只筛行选列的),引用多次的简单 CTE 每处写回一份后整条语句 的别名重新改成互不相同的(压测撞到的:同一个 CTE 并进外层和相关子查询两处,里层别名遮住外层);外层 WHERE 只挪进 分组子查询的 HAVING(同一个值)。几个来源的 SELECT * 先写成各来源的 t.*(sqlglot 展开 * 时把实表排在子查询前面)。 - 解析丢了写法的不在语法树上判等价:0x10 和 x'10' 这类字面量、换了名字的类型名(SQLite 的 STRING 是数值亲和); SQLite 不带表名、带双引号、表结构里没有的列名保留大小写和引号(找不到这一列时被当成字符串)。 差异说明: - 换了表的一边照它自己的表说(以前把 B 改名成 A 的表,中文名、detail 归错表)。 - 新增 _Range:同一个对象只和常量比的条件按取值逐段算,留下的行一样的不说(IN 和 OR、冗余 IS NOT NULL、拆成两段的 时间、COUNT(*) > 1 和 >= 2);关联条件比相等关系的分组;聚合只在参数是列时说。 - 新增 _Usage:说「另一边没有这个条件 / 没有按时间筛选」之前,看另一边整条语句(含相关子查询、CTE、CASE、判空时连 聚合参数)是否用这一列筛选;条件子查询里不带表名的列没给表结构时不拿来说具体的。 - 子查询要外面写法一模一样(含 NOT、IN 左边的列)才配对,位置照实说「id NOT IN 的子查询」;EXCEPT 按先后逐支比; order 只在最外层、没取行、没 DISTINCT ON / 窗口函数时说;LEFT 对 INNER 只在最外层、可空一侧没进 WHERE、结果不只是 可空一侧的聚合时说;DISTINCT 对同列 GROUP BY、MySQL 表名只差大小写不说成不同;互相矛盾的一对不说。 测试:tests/fixtures/sql_pairs.py 加 nums 表和一组专挑写法差异的数据(0.1 + 0.2、像数字的文本、'SH' / 'sh'), 第 2 轮反例收进语料(_SAME_HARD 20 对、_SAME_R2 11 对、_DIFF_R2 36 对进执行比对;REGRESSIONS_R2 21 对、 DIALECT_SAME_R2 6 对、KNOWN_UNRECOGNIZED 4 对各方言只查结论);执行比对共 7 组数据、372 对,标 same 而没认出的 不许有具体说法;test_sql_semantics.py 补第 2 轮的说法用例和 normalize_sql「只改参数」的用例。这些新用例在上一版 上有 109 处失败。 验证:test_sql_semantics + test_verify_agent + test_sql_equivalence 973 passed;全量 -n 8:7853 passed、11 skipped。 等价改写 124 对认出 120 对(97%),不给表结构 117 对(94%)。复验者的脚本:vsprobe 变异压测 6 个种子约 4.3 万对无 误判等价;在私起的 PostgreSQL 18、MySQL 8.0 上重跑 vsadv 全部语料无 blocker;advfp 的 180 对等价对上具体说法 0 条; 自己的压测 3 个种子各约 1.5 万条可执行,规范形式相同的组执行结果全一致。 --- backend/app/data/sql_semantics.py | 4035 +++++++++++++++++-------- backend/tests/fixtures/sql_pairs.py | 349 ++- backend/tests/test_sql_equivalence.py | 37 +- backend/tests/test_sql_semantics.py | 258 +- 4 files changed, 3326 insertions(+), 1353 deletions(-) diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index 99f668b8..d1a45006 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -10,71 +10,97 @@ A / B,只差一个方面、又对得上数据目录里的事实(时间归属 - explain_sql_diff:两条 SQL 是不是等价(SqlDiff.equivalent),不是的话差在哪(items、text_diff)。 - diff_facets:照旧收两份 facets,交给 explain_sql_diff 比两份记下的原文,返回它的 items。 - facets:把一条查询拆成表、连接、WHERE 合取式、时间条件、分组、聚合、算出来的列、HAVING、取哪几行(给调用方取 - 结构用;CTE、子查询按位置编号,见 _Statement)。 + 结构用;CTE、子查询按位置编号,见 _Statement)。facets 和 normalize_sql 不受下面等价判定的改动影响。 **等价怎么判。** 过去几轮按方面归纳任意 SQL,每轮修好一批又冒出一批(还回退过),收不拢;现在先判等价、再说差在哪。 两边各自规范化,规范化的每一步都只做保持语义的变换,同一级的两个规范形式一字不差(比语法树逐个节点的类型和参数, -见 _fingerprint,不比生成的 SQL:sqlglot 按方言生成时会把不同的写法写成一样,SQLite 的 CAST(x AS DATE) 生成 -DATE(x))才算等价,否则一律不等价、条目一定不为空。规范形式分三级(见 _canonical_forms),哪一级对上都算: +见 _fingerprint,括号不算;不比生成的 SQL:sqlglot 按方言生成时会把不同的写法写成一样,SQLite 的 CAST(x AS DATE) +生成 DATE(x))才算等价,否则一律不等价、条目一定不为空。规范形式分三级(见 _canonical_forms),哪一级对上都算: -- optimizer:sqlglot 优化器里保持语义的几步——qualify(认列、按表结构展开 SELECT *、展开 GROUP BY 1 和别名), - eliminate_subqueries(FROM 里的子查询写成 CTE),merge_subqueries(只筛行、透列的 CTE、子查询并进外层;外连接 - 可空一侧选出算出来的列的不并),eliminate_ctes(去掉没用的 CTE),pushdown_projections(去掉没用到的列),simplify - (去掉和类型、排序规则、方言有关的三种化简,见 _simplify);之前把整条语句的别名改成互不相同的(merge_subqueries - 只在同一段里避重名,同一个 CTE 并进外层和相关子查询两处会互相遮住),没给表结构时把从一张表 SELECT * 的 CTE 换成 - 外面用到的那几列。不用的:unnest_subqueries(把 OR 里的标量子查询改成 CROSS JOIN,子查询没有行时整条都没了; - SQLite 的标量子查询取第一行,改成连接就成了每行都比),canonicalize(给日期比较加 CAST,SQLite 里 CAST(… AS DATE) - 取的是数字),normalize(合取范式改出来的条件人读不懂,也用不上)。认不全列(没给表结构、几张表又都没写表名)、 - WHERE / GROUP BY / HAVING 里有和列别名同名、又可能是表的列的名字(库里先认列,qualify 却当成别名)、SELECT 列表里 - 有和前面列别名同名、又不确定是表的列的名字(库里不认 SELECT 列表里的别名,qualify 没有表结构时却展开)、有递归 - CTE、LATERAL、表函数、不确定的函数(RANDOM)的不出这一级(见 _alias_or_column)。 -- light:轻量规范化,不认列:统一标识符大小写和引号、去注释、只引用一次的非递归 CTE 写回用到它的地方,再走下面不 - 需要认列的几步。 -- parsed:解析后重新生成(只统一大小写、空白、注释、引号)。 +- optimizer:先按方言统一名字(_normalize_names:SQLite 不带表名、带双引号、表结构里又没有的列名原样留着——找不到 + 这一列时 SQLite 把它当字符串),CTE、子查询的列名表写成里面那段的列别名,USING 写成 ON,FETCH FIRST 写成 LIMIT, + IS NOT NULL、NOT LIKE 这类带 negate 的写法统一成 NOT (…);然后 sqlglot 的 qualify 认列(给了数据源的 schema_cache + 就按它认、展开 SELECT *);再把整条语句的来源别名改成互不相同的,引用了好几次、只筛行选列的 CTE 每处写回一份,只 + 引用一次的 CTE 写回用到它的地方,只从几张表筛行选列的子查询并进外层(_merge_derived:只并内连接一侧的),外层 + WHERE 里只涉及分组子查询选出来的列的条件挪进它的 HAVING(同一个值),去掉没用的 CTE。认不全列(没给表结构、几张表 + 又都没写表名)、WHERE / GROUP BY / HAVING 里有和列别名同名又可能是表的列的名字、SELECT 列表里有和前面列别名同名的 + 名字、不确定的函数(RANDOM)的不出这一级。 +- light:不认列,只做上面不需要认列的几步。 +- parsed:解析后只统一名字的大小写、引号。 -两级共用的本模块自己的几步(见 _finish):不影响结合的括号、ORDER BY 的 ASC、OFFSET 0 去掉;COUNT(1) 写成 COUNT(*); -IN 列表去重排序、只有一项写成 =;EXISTS 子查询写成 SELECT 1;IN 子查询里的 DISTINCT 去掉;SELECT DISTINCT 写成按 -选出来的列 GROUP BY,已经按选出来的列分组的 DISTINCT 去掉;原样选出一个子查询全部列的一段换成那个子查询(SQLite 里 -UNION 不换:从子查询读合并的结果会按第一支的类型亲和转换别的支的值);外层 WHERE 里只涉及一个分组 CTE 的条件挪进它的 -HAVING、HAVING 里只涉及分组键的挪进 WHERE;只有两个来源的 RIGHT JOIN 写成 LEFT JOIN;WHERE 把外连接可空一侧没关联上的 -行筛掉了的写成内连接;内连接的条件统一挪进 WHERE、来源写成逗号连接;最外层和条件里子查询的列名去掉;最后起规范的名字 -(CTE 按定义排序叫 cte1、cte2…,CTE 的列按定义排序叫 c1、c2…,来源的别名叫表名、内层带 @层数、同名第二个起带 #序号, -同名来源谁先谁后逐个排列着试、取写出来最小的;取到哪几行、哪一行的值看扫描先后的一段——没有 ORDER BY 的 LIMIT、 -聚合里既不分组也不聚合的列——来源不调换先后,见 _unstable),交换律成立的地方(AND、OR、比较两边、两项的加乘、IN -列表、GROUP BY、PARTITION BY)排好序。合并的各支只有 UNION ALL、而且调换先后不改变结果时才排(见 _Namer._reorderable):去重的 UNION -和 INTERSECT 留先出现的那个值(SQLite 里 2 和 2.0、MySQL 不分大小写时 'SH' 和 'sh' 算重复),带 LIMIT、按名字 ORDER BY -的要看哪支在前,SQLite 合并结果的排序规则取最左一支的(只排整条语句最外层、不排序的)。 +两级共用的几步(见 _finish):原样透出子查询全部列的一段换成那个子查询;两个来源的 RIGHT JOIN 写成 LEFT JOIN;WHERE +把外连接可空一侧没关联上的行筛掉了的写成内连接;开头那组内连接的条件挪进 WHERE(后面有 RIGHT / FULL JOIN 的不挪); +IN 子查询里的 DISTINCT 和不带聚合的 ORDER BY 去掉、SELECT DISTINCT 写成按选出来的列 GROUP BY、已经按选出来的列分组的 +DISTINCT 去掉;COUNT(1) 写成 COUNT(*)、MIN / MAX 里的 DISTINCT 去掉;SQLite 的 IIF、MySQL 的 IF 写成 CASE,CASE 的 +ELSE NULL 去掉;WHERE / HAVING / ON 里的 TRUE、1 = 1 去掉;EXISTS 子查询写成 SELECT 1(只取前 n 行的也去掉取行); +IN 列表去重排序、只有一个常量的写成 =;BETWEEN 写成两个比较(SYMMETRIC 不走这两级);NOT 套比较写成反过来的比较、 +NOT NOT 条件去掉;ORDER BY 的 ASC、OFFSET 0 去掉;最外层 ORDER BY 引用的别名换成表达式,最外层和条件里的子查询选出来 +的列名去掉;最后起规范的名字(CTE 叫 cte1…、CTE 和子查询选出来的列叫 c1…、来源叫表名、内层带 @层数、同名第二个起带 +#序号,同名来源谁先谁后排列着试、取写出来最小的),交换律成立的地方(AND、OR、比较两边、一个节点的加乘两边、IN 列表、 +GROUP BY)排好序;合并只调换整条语句最外层不取行的 UNION ALL 各支(PostgreSQL、MySQL 另加:按位置排序的,和套在 +里面、各支列名一样、外面不看先后的)。 -等价的结论另有执行比对守门(tests/test_sql_equivalence.py):虚构库上五组随机数据加一组极端数据,判等价的两条每组 -结果都得一样,结果不一样的绝不能判等价。给了表结构时 SELECT * 按它展开,表结构得和库对得上。原文里有 sqlglot 解析时 -会丢掉、语义却不同的写法(一元加号)的,不在语法树上判等价。假设列没有声明非默认的排序规则:SQLite 比较两列时取左边 -那列声明的排序规则,表结构里看不到,比较两边对调当成保持语义(两边都写了 COLLATE 的不对调)。 +**故意不做的改写**(第 2 轮复验在这些地方抓到了不保持语义的例子):sqlglot 的 simplify(数字常量按 Decimal 折叠, +SQLite 按浮点数算 0.1 + 0.2 ≠ 0.3;把 -(-x) 化成 x,丢了取负时的数值转换;三项以上的加乘拍平再合常量;BETWEEN +SYMMETRIC 改写时丢了 SYMMETRIC)、unnest_subqueries(OR 里的标量子查询改成 CROSS JOIN,子查询没有行时整条都没了)、 +canonicalize(给日期比较加 CAST)、pushdown_projections;HAVING 里的条件挪进 WHERE(MySQL 默认不分大小写的排序规则下 +'SH'、'sh' 在同一组,HAVING BINARY city = 'SH' 看的是这一组留下的那个值);只有一项、又不是常量的 IN 列表写成 = +(SQLite 里 a IN (b) 是 a = +b)。 -**具体说法的门槛(见 _Explainer)。** 在两边的规范形式上逐段比,只把有把握的不同写成具体的中文句子:两段读的来源 -对得上(一模一样,或者只差一处说得清的:换了一张表、一边多关联了一张表、同一张表的关联类型或条件不同、外连接的 -主表对调)之后,才逐项比时间条件(实表上的时间列不同、同一列的范围不同、只有一边按时间筛)、筛选条件(只涉及 -这一段实表列的条件一边有一边没有、同一列上两边的条件不同)、选出来的列(聚合函数、去重计数、算法、列数和先后)、 -分组键、HAVING、取行和排序、UNION 还是 UNION ALL。条件里套着的子查询两边各一个、外面写法一样的,往里比(第 5 轮 -漏报的「只在 IN / EXISTS / 标量子查询里用到的 CTE」并进子查询之后就在这里说出来);CTE 两边定义一样的对齐,各剩 -一个对不上的配成一对往里比。说的都是「这一段的这一项两边各写的是什么」,不推断对结果的影响;原文写的是 LEFT / -RIGHT JOIN、规范化成了内连接或者调了方向的,照原文说(「写的是 LEFT JOIN,实际是 INNER JOIN」「以 X 为主表外连接」)。 -牵涉 CTE 算出来的列、外层的列、自己关联自己的表、说不准的子查询,以及 CTE 里选出来的列(规范化去掉、重排过,和原文 -对不上)的,宁可不说具体的。两边给人看的写法一样的话不说(读起来自相矛盾)。 +**结果看行出来的先后的一段(见 _Sensitivity)不调换来源的先后、不把 CROSS JOIN 写成逗号连接、不排 GROUP BY 的键、 +不改 DISTINCT、不往里并子查询**:取行(LIMIT / OFFSET / FETCH,ORDER BY 有并列时取哪几行也看先后)、DISTINCT ON、 +窗口函数、拼串拼数组的聚合(GROUP_CONCAT)、聚合里既不分组也不聚合的列、当成标量用的子查询(SQLite 取第一行),以及 +去重、分组、MIN / MAX、去重的 UNION 留下的值「相等却不一样」的时候(MySQL 默认不分大小写的字符串、PostgreSQL numeric +的 2.0 和 2.00、SQLite 没有类型亲和的列、算出来的表达式:留下的是先扫到的那个);给这样一段供行的子查询、CTE 也算。 +SQLite 里逗号连接和 CROSS JOIN 解析成同一个节点,按原文分开(_mark_comma_joins:SQLite 不调换 CROSS JOIN 两边)。 + +**假设(做不到的)**:SQLite 没给表结构时按列有声明的类型、默认的 BINARY 排序规则算——声明了 COLLATE NOCASE 的列、 +没有类型亲和的列(能同时存 1 和 1.0)表结构里看不出来,这样的列上去重、分组再调换关联先后可能留下不同的代表值; +比较两边都是列时对调两边(SQLite 比较两列用左边那列声明的排序规则,两边都写了 COLLATE 的不对调)。PostgreSQL、 +MySQL 没给表结构时一律不当成「相等就是一样」,去重、分组、MIN / MAX 的一段不调换来源。MySQL 的表名按 Linux 上分 +大小写算(BILLS 和 bills 不判等价)。 + +等价的结论另有执行比对守门(tests/test_sql_equivalence.py):虚构库上五组随机数据、一组极端数据、一组专挑写法差异的 +数据,判等价的两条每组结果都得一样,结果不一样的绝不能判等价。原文里有 sqlglot 解析时会丢掉、语义却不同的写法 +(一元加号、0x10 和 x'10' 这类字面量、换了名字的类型名——SQLite 的 CAST(x AS STRING) 是数值亲和,解析成和 TEXT +一样的节点,见 _lost_in_parsing)的,不在语法树上判等价。 + +**具体说法的门槛(见 _Explainer)。** 在两边的规范形式上逐段比,只把有把握、确实不同的地方写成具体的中文句子: + +- 来源对得上(一模一样,或者只差一处说得清的:换了一张表、一边多关联了一张表、同一张表的关联类型或条件不同、 + 外连接的主表对调)才往下比;换了表的那一边,列照它自己的表说(中文名、detail 都归到它自己的表)。FROM 里的子查询 + 只有一处不同时往里比(「在 t 里」)。 +- 说「X 不一样」之前先确认真的不一样:同一个对象只和常量比的几个条件(=、<>、大小比较、IN 常量、IS NULL、BETWEEN, + AND / OR / NOT 连起来)按取值逐段算一遍(_Range),留下的行一样的不说(IN 和 OR、多写一个 IS NOT NULL、一段时间拆成 + 两段、多一个被覆盖的下界、分组后 COUNT(*) > 1 和 >= 2);关联条件只是两列相等的,比相等关系分出的组;聚合只在参数 + 是同一列、或者各是一列时说(IIF 和 CASE、COALESCE 和 CASE 这类写法不同的算法不说);两列被关联条件定成相等的, + 选哪一列不说。 +- 说「另一边没有这个条件 / 没有按时间筛选」之前,先确认另一边整条语句(含子查询、CTE、相关子查询里引用外层的列、 + CASE 里、判空时连聚合的参数)都没用这些列筛选(_Usage;合并里别的支不算),没有像时间的列、像日期的常量。 +- 条件里的子查询:两边除了子查询之外写法一模一样(含 NOT、IN 左边的列)的才配成一对往里比,位置照实说(「id NOT IN + 的子查询」「NOT EXISTS 子查询」);子查询里不带表名的列没给表结构时分不清是里层还是外层的,不拿它说具体的。 +- 只是排序不同(order)只在整条语句最外层、没有取行、DISTINCT ON、窗口函数、不是只出一行时说;取行前的排序算 + limit;子查询里的排序不说(它可能决定 GROUP_CONCAT 的先后)。取行在标量子查询、EXISTS 里、只出一行时不说。 +- LEFT JOIN 对 INNER JOIN 只在整条语句最外层、WHERE 里没用到可空一侧的列、结果不只是可空一侧的列的聚合(那样两种 + 写法结果一样)时说;EXCEPT 按先后逐支比,两边对调了就对不上;一边 DISTINCT、一边按同样的列 GROUP BY 不说成一边 + 去重一边没有;MySQL 表名只差大小写不说成换了表。 +- 说的都是「这一段的这一项两边各写的是什么」;两边给人看的写法一样、和已经说过的一句正好反过来的不说。 **兜底。** 规范形式不一致、一条具体说法都没有时,给一条 aspect="other" 的「两条 SQL 的写法不同,未能归纳成具体差异, 请对照原文」;具体说法只覆盖了一部分时追加「另有未能归纳的差异,请对照原文」;fallback 为真。text_diff 给出规范化后 不同的行(difflib 比两边 sqlglot pretty 的输出,最多 40 行)。 仍做不到的(只给兜底说明,或者认不出等价): -- 认不出的等价改写:IN 子查询和 EXISTS、NOT IN 和 NOT EXISTS 互换(空值时本来就不等价)、IN 子查询和关联同一张表 - (关联的键不唯一时会多出行,不知道主键就不能当成一回事)、窗口函数前后按分区键筛、三项以上的加乘换了结合方式、 - 括号展开、和方言有关的同义写法(PostgreSQL 的 DATE(x) 和 CAST(x AS DATE))、SQLite 里套一层子查询的 UNION、 - 去重的 UNION 和 INTERSECT 调换各支、SQLite 里套在子查询里或带 ORDER BY 的 UNION ALL 调换各支、LATERAL、递归 CTE; - 没给表结构时 WHERE、GROUP BY、HAVING 里用列别名的(GROUP BY r 的 r 可能是表的列)、SELECT 列表里和前面的列别名 - 同名的;没有 ORDER BY 的 LIMIT、聚合里既不分组也不聚合的列的一段调换关联先后。 -- 说不出具体差异、只给兜底的:自己关联自己的方向或角色不同、两个 CTE 算出同名的列用了哪一个、相关子查询内外层分不清 - 的、读的来源对不上(一边多一个分组 CTE 再关联、子查询里换了表又改了条件)、CTE 里选出来的列不同、NOT IN 对 NOT - EXISTS、几处不同交错在一起的。 +- 认不出的等价改写:IN 子查询和 EXISTS、NOT IN 和 NOT EXISTS 互换(空值时本来就不等价)、IN 子查询和关联同一张表、 + 条件挪进相关子查询、IN 和 OR、冗余的条件、COALESCE 和 CASE 这类写法不同的等价表达式、窗口函数的 PARTITION BY 换 + 先后、HAVING 里分组键上的条件和 WHERE(只认外层 WHERE 挪进 HAVING 那个方向)、去重的 UNION / INTERSECT 调换各支、 + SQLite 里套在子查询里的 UNION、三项以上的加乘换结合、和方言有关的同义写法(DATE(x) 和 CAST(x AS DATE))、LATERAL、 + 递归 CTE、实表别名带列名、USING 用在第一个连接以外的地方;没给表结构时 WHERE、GROUP BY、HAVING 里用列别名的、 + SELECT 列表里和前面的列别名同名的;结果看行出来的先后的一段调换关联先后、DISTINCT 改 GROUP BY;PostgreSQL、MySQL + 没给表结构时去重、分组的一段调换关联先后。 +- 说不出具体差异、只给兜底的:自己关联自己的方向或角色不同、几个派生来源互相交错、相关子查询内外层的条件、 + 另一边别处(别的子查询、CTE)也用到同一列筛选的、读的来源对不上、CTE 和子查询里选出来的列不同、条件或算法写法 + 不同又判断不了是不是真不一样的、EXCEPT 两边对调、几处不同交错在一起的。 **和 sqlcheck 一样宁可漏报。** 解析不了(语法错、方言特性、多条语句、不是查询)就只给一条 aspect="parse";从不抛异常, 规范化或说明本身出了意外就当作这一级没有、说不清,记日志。只读语法树,不执行 SQL。 @@ -84,6 +110,7 @@ from __future__ import annotations import hashlib import itertools import logging +import math import re from collections import deque from collections.abc import Iterable, Iterator, Mapping @@ -2241,19 +2268,49 @@ _MAX_ORDERINGS = 64 _ORDERING_BUDGET = 2000 #: 原来的名字(CTE 名、FROM 里子查询的别名)记在里面那段查询的 meta 里:起了规范的名字之后,差异说明还要照原样说 _ORIGINAL = "sql_semantics_name" -#: GROUP BY 是从 SELECT DISTINCT 改写来的:差异说明里说「去重」 -_FROM_DISTINCT = "sql_semantics_distinct" #: 写的是 LEFT / RIGHT JOIN、WHERE 把没关联上的行筛掉了、按内连接比的来源(记在那张表上,值是原来写的连接):差异 #: 说明里照实说「写的是 LEFT JOIN,实际是 INNER JOIN」 _REJECTED = "sql_semantics_rejected" #: 从 RIGHT JOIN 改写成的 LEFT JOIN(记在 JOIN 上):差异说明里不说「用 LEFT JOIN」,说以哪一侧为主表 _FLIPPED = "sql_semantics_flipped" -#: 不确定的函数:每次求值结果都可能不同,合并子查询会把一次求值复制成几次,不走优化器那一级 +#: GROUP BY 是从 SELECT DISTINCT 改写来的:差异说明里说「去重」 +_FROM_DISTINCT = "sql_semantics_distinct" +#: 认不准是哪一列的列(记在列名的标识符上):SQLite 里不带表名、带双引号、表结构里又没有的名字(找不到这一列时被当成 +#: 字符串);条件、标量子查询里不带表名的列(没给表结构时 qualify 归给子查询里的表,可它也可能是外层的列)。差异说明 +#: 不拿它说具体的 +_UNSURE = "sql_semantics_unsure" +#: 不确定的函数:每次求值结果都可能不同,并子查询、合并定义一样的 CTE 会改变求值的次数,不走优化器那一级 _VOLATILE = frozenset({"random", "rand", "randn", "newid", "uuid", "gen_random_uuid", "uuid_generate_v4", "random_uuid", - "randomblob", "setseed", "nextval", "sys_guid"}) -#: 拿空值比较、算术都得空值(外连接没关联上的那一侧整行是空值)的写法,WHERE 里用到它就把没关联上的行筛掉了 -_STRICT_UNARY = ("Paren", "Neg", "Cast", "Lower", "Upper", "Length", "Abs") -_STRICT_BINARY = ("Add", "Sub", "Mul", "Div", "Mod", "DPipe") + "randomblob", "setseed", "nextval", "sys_guid", "changes", "last_insert_rowid", "total_changes"}) +#: 结果看行出来的先后的聚合:拼成串、拼成数组、取任意一个、取第一个 / 最后一个(类名和函数名都按小写去下划线比) +_ORDERED_AGGS = frozenset({"groupconcat", "stringagg", "arrayagg", "arrayconcatagg", "arrayuniqueagg", "jsonarrayagg", + "jsonobjectagg", "jsonagg", "jsonbagg", "jsongrouparray", "jsongroupobject", "listagg", + "xmlagg", "anyvalue", "first", "last", "firstvalue", "lastvalue", "nthvalue", + "jsonbobjectagg", "jsonobjectaggstrict", "arrayaggdistinct", "bitstringagg"}) +#: 拿空值做比较、算术都得空值(外连接没关联上的那一侧整行是空值) +_STRICT_UNARY = ("Neg", "Cast", "Lower", "Upper", "Length", "Abs") +_STRICT_BINARY = ("Add", "Sub", "Mul", "Div", "Mod") +#: 解析时只留下值、丢了写法的字面量:SQLite 里 0x10 是整数 16、x'10' 是 BLOB,解析出来是同一个节点;N'…'、E'…'、 +#: $$…$$ 这些也一样,带了就不在语法树上判等价 +_LOSSY_LITERALS = frozenset({"HEX_STRING", "BIT_STRING", "BYTE_STRING", "NATIONAL_STRING", "UNICODE_STRING", + "HEREDOC_STRING", "RAW_STRING"}) +#: 类型名的同义写法:解析后重新生成换了个名字、在这个方言里却是同一个类型(SQLite 按名字里的字样定类型亲和:INT 和 +#: INTEGER 都是整数亲和,VARCHAR 和 TEXT 都是文本亲和)。不在这里的换了名字就算解析丢了写法(SQLite 的 STRING 是数值 +#: 亲和、TEXT 是文本亲和,sqlglot 解析成同一个类型) +_TYPE_SYNONYMS = { + "sqlite": {"INT": "INTEGER", "VARCHAR": "TEXT", "CHAR": "TEXT", "NVARCHAR": "TEXT", "CLOB": "TEXT", + "DOUBLE": "REAL", "FLOAT": "REAL"}, + "postgres": {"INTEGER": "INT", "INT4": "INT", "INT8": "BIGINT", "INT2": "SMALLINT", "FLOAT8": "DOUBLE", + "FLOAT4": "REAL", "BOOL": "BOOLEAN", "NUMERIC": "DECIMAL", "CHARACTER": "CHAR"}, + "mysql": {"INTEGER": "INT", "DEC": "DECIMAL", "NUMERIC": "DECIMAL", "BOOL": "BOOLEAN"}, +} +#: FROM 列表到这些词就结束了(之后的逗号不再是连接) +_CLAUSE_ENDS = frozenset({"WHERE", "GROUP_BY", "HAVING", "ORDER_BY", "LIMIT", "OFFSET", "FETCH", "UNION", "INTERSECT", + "EXCEPT", "WINDOW", "QUALIFY", "SEMICOLON", "RETURNING", "FOR", "CONNECT_BY", "START_WITH", + "CLUSTER_BY", "DISTRIBUTE_BY", "SORT_BY", "PREWHERE", "LOCK"}) +_ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") +_FLIP = {"GT": "LT", "LT": "GT", "GTE": "LTE", "LTE": "GTE", "EQ": "EQ", "NEQ": "NEQ"} +_NEGATE = {"EQ": "NEQ", "NEQ": "EQ", "GT": "LTE", "LTE": "GT", "GTE": "LT", "LT": "GTE"} class _Unsupported(Exception): @@ -2267,32 +2324,92 @@ _OPERAND_KEYWORDS = frozenset({"SELECT", "WHERE", "AND", "OR", "NOT", "ON", "WHE "EXISTS", "ANY", "SOME", "FROM", "JOIN", "WITH", "ORDER", "GROUP"}) -def _lost_in_parsing(text: str, read: str | None) -> bool: - """原文里有 sqlglot 解析时丢掉、语义却不一样的写法:一元加号(SQLite 里 +a 去掉了列的类型亲和,+a = '10' 和 - a = '10' 结果不同,解析出来却是同一棵树)。有就不在语法树上判等价。""" +def _type_words(tokens: list[Any], read: str | None) -> list[str]: + from sqlglot.parser import Parser + + synonyms = _TYPE_SYNONYMS.get(read or "", {}) + words = [t.text.upper() for t in tokens if t.token_type in Parser.TYPE_TOKENS] + return sorted(synonyms.get(w, w) for w in words) + + +def _lost_in_parsing(text: str, read: str | None, tree: Any) -> bool: + """原文里有 sqlglot 解析时丢掉、语义却不一样的写法,有就不在语法树上判等价: + + - 一元加号:SQLite 里 +a 去掉了列的类型亲和,+a = '10' 和 a = '10' 结果不同,解析出来却是同一棵树; + - 只留下值、丢了写法的字面量(0x10 和 x'10',见 _LOSSY_LITERALS); + - 类型名:解析后重新生成换了名字、又不是这个方言里的同义写法(SQLite 的 CAST(x AS STRING) 是数值亲和,解析成 + 和 CAST(x AS TEXT) 一样的节点)。""" try: tokens = _tokens(text, read) except Exception: # noqa: BLE001 - 切词都不行的,解析也过不了,那边会说 return False prev = None for token in tokens: - if token.token_type.name == "PLUS": + kind = token.token_type.name + if kind in _LOSSY_LITERALS: + return True + if kind == "PLUS": if prev is None: return True - kind, word = prev.token_type.name, prev.text - operand = kind in ("VAR", "IDENTIFIER", "NUMBER", "R_PAREN", "R_BRACKET", "NULL", "TRUE", "FALSE", "END", - "STAR") or kind.endswith("STRING") or ( + pk, word = prev.token_type.name, prev.text + operand = pk in ("VAR", "IDENTIFIER", "NUMBER", "R_PAREN", "R_BRACKET", "NULL", "TRUE", "FALSE", "END", + "STAR") or pk.endswith("STRING") or ( _PLAIN_NAME.match(word) is not None and word.upper() not in _OPERAND_KEYWORDS) if not operand: return True prev = token - return False + try: + again = _tokens(tree.sql(dialect=read, comments=False), read) + except Exception: # noqa: BLE001 - 重新生成不了:当成丢了写法 + return True + return _type_words(tokens, read) != _type_words(again, read) + + +def _mark_comma_joins(tree: Any, text: str, read: str | None) -> None: + """SQLite 里逗号连接和 CROSS JOIN 解析成同一个节点(都是 kind=CROSS),可 SQLite 不调换 CROSS JOIN 两边的先后、 + 逗号连接会调换:结果看行出来的先后时(GROUP_CONCAT、有并列的 LIMIT)两种写法不一样。按原文认出哪些是逗号连接, + 去掉它们的 CROSS(和别的方言解析出来的一样)。认不出时整条语句的 CROSS 连接标成认不出(见 _fingerprint)。""" + from sqlglot import exp + + joins = [j for j in tree.find_all(exp.Join) if str(j.args.get("kind") or "").upper() == "CROSS"] + if not joins: + return + try: + tokens = _tokens(text, read) + except Exception: # noqa: BLE001 + tokens = [] + marks: list[tuple[int, bool]] = [] + depth = 0 + in_from: set[int] = set() + for tok in tokens: + name = tok.token_type.name + if name == "L_PAREN": + depth += 1 + elif name == "R_PAREN": + in_from.discard(depth) + depth -= 1 + elif name == "FROM": + in_from.add(depth) + elif name in _CLAUSE_ENDS: + in_from.discard(depth) + elif name == "COMMA" and depth in in_from: + marks.append((tok.start, True)) + elif name in ("JOIN", "STRAIGHT_JOIN"): + marks.append((tok.start, False)) + for join in joins: + starts = [i.meta["start"] for i in join.this.find_all(exp.Identifier) if isinstance(i.meta.get("start"), int)] + before = [m for m in marks if starts and m[0] < min(starts)] + if not before: + join.meta["sql_semantics_join"] = f"unknown:{id(tree)}" + continue + comma = max(before)[1] + if comma: + join.set("kind", None) @dataclass class _Form: - """一条语句在某一级的规范形式。key 一样才算等价,text 是生成的 SQL(给人看);pre 是起规范名字之前的树(差异 - 说明按另一边的 CTE 名重新起名时用,见 _Explainer.statement)。""" + """一条语句在某一级的规范形式。key 一样才算等价,text 是生成的 SQL(给人看);pre 是起规范名字之前的树。""" level: str text: str @@ -2308,57 +2425,72 @@ class _Form: def _fingerprint(node: Any) -> str: - """语法树的确定写法:每个节点写类名和全部参数(按参数名排序,字符串原样),不同的树一定写得不一样。类型标注、 - 注释、meta 不算。""" + """语法树的确定写法:每个节点写类名和全部参数(按参数名排序,字符串原样),不同的树一定写得不一样。括号不算 + (语法树的结构已经定了怎么结合:SQLite 里 (a) 和 a 的类型亲和也一样);类型标注、注释、meta 不算,只有认不出 + 是逗号还是 CROSS JOIN 的连接带上标记(见 _mark_comma_joins)。""" from sqlglot import exp + if isinstance(node, exp.Paren): + return _fingerprint(node.this) if isinstance(node, exp.Expr): parts = [f"{k}={_fingerprint(v)}" for k, v in sorted(node.args.items()) if v is not None and v is not False and v != []] + mark = node.meta.get("sql_semantics_join") if isinstance(node, exp.Join) else None + if mark: + parts.append(f"@join={mark}") return f"{type(node).__name__}({','.join(parts)})" if isinstance(node, list): return "[" + ",".join(_fingerprint(v) for v in node) + "]" return repr(node) -def _key(node: Any, read: str | None) -> tuple[str, str]: - """排序、去重用的键:先按生成的 SQL 排(好读),同样写法的再按 _fingerprint 分开。""" - return node.sql(dialect=read, comments=False), _fingerprint(node) +def _key(node: Any, read: str | None = None) -> tuple[str, str]: + """排序、去重用的键:只看语法树(见 _fingerprint),括号写没写不影响先后。""" + return _fingerprint(node), "" -def _schema_mapping(schema: Mapping[str, Any] | None) -> dict[str, dict[str, str]] | None: - """schema_cache → sqlglot 的表结构 {表: {列: 类型}}。类型一律记 UNKNOWN:只要列名(认列属于哪张表、展开 - SELECT *);按类型做的改写和方言有关(canonicalize 给日期比较加 CAST,SQLite 里 CAST(… AS DATE) 取的是数字), - 不做。""" +@dataclass +class _Schema: + """数据源的表结构:mapping 给 qualify(只要列名),types 记各列声明的类型(判断「相等的值是不是写法也一样」,见 + _Sensitivity),columns 是所有表的列名(小写)。""" + + mapping: dict[str, dict[str, str]] | None = None + types: dict[str, dict[str, str]] = field(default_factory=dict) + columns: set[str] = field(default_factory=set) + + +def _schema_info(schema: Mapping[str, Any] | None) -> _Schema: + """schema_cache → _Schema。qualify 用的类型一律记 UNKNOWN:只要列名(认列属于哪张表、展开 SELECT *);按类型做 + 的改写和方言有关,不做。""" tables = schema.get("tables") if isinstance(schema, Mapping) else None if not isinstance(tables, Mapping): - return None - out: dict[str, dict[str, str]] = {} + return _Schema() + mapping: dict[str, dict[str, str]] = {} + types: dict[str, dict[str, str]] = {} for name, meta in tables.items(): columns = meta.get("columns") if isinstance(meta, Mapping) else None - names = [c["name"] for c in columns or [] if isinstance(c, Mapping) and isinstance(c.get("name"), str) - and c["name"]] - if names: - out[str(name)] = {n: "UNKNOWN" for n in names} - return out or None + cols = [c for c in columns or [] if isinstance(c, Mapping) and isinstance(c.get("name"), str) and c["name"]] + if cols: + mapping[str(name)] = {c["name"]: "UNKNOWN" for c in cols} + types[str(name).translate(_ASCII_LOWER)] = { + c["name"].translate(_ASCII_LOWER): str(c.get("type") or "") for c in cols} + return _Schema(mapping or None, types, {c for cols in types.values() for c in cols}) -def _canonical_forms(root: Any, read: str | None, schema: dict[str, dict[str, str]] | None) -> dict[str, _Form]: +def _canonical_forms(root: Any, read: str | None, info: _Schema) -> dict[str, _Form]: """一条语句的几级规范形式: - - optimizer:sqlglot 优化器的几步(qualify 认列、展开 SELECT *、别名展开;eliminate_subqueries 把 FROM 里的子查询 - 写成 CTE;merge_subqueries 把只用一次、只是筛行透列的 CTE、子查询并进外层;eliminate_ctes 去掉没用的 CTE; - pushdown_projections 去掉没用到的列;simplify 化简条件),加上本模块自己的几步(见 _finish)。认不全列(没给表 - 结构、几张表又都没写表名)、有递归 CTE、LATERAL 之类的不出这一级。 - - light:轻量规范化,不认列:统一标识符大小写、去注释、只引用一次的非递归 CTE 写回用到它的地方,再走 _finish 里 - 不需要认列的几步。 - - parsed:解析后重新生成的写法(只统一了大小写、空白、注释),总是有。 + - optimizer:认列(sqlglot 的 qualify:认清每一列属于哪个来源,给了表结构就展开 SELECT *)之后的全套改写(见 + _optimized、_finish)。认不全列、有和列别名同名又可能是表的列的名字、不确定的函数、递归 CTE、LATERAL 之类的 + 不出这一级。 + - light:不认列,只做不需要认列的改写(见 _light)。 + - parsed:解析后只统一名字的大小写、引号(见 _normalize_names),总是有。 每一级的每一步都只做保持语义的变换,某一级出不来就没有这一级,从不抛异常。""" out: dict[str, _Form] = {} for level, build in ((_OPTIMIZER, _optimized), (_LIGHT, _light)): try: - out[level] = build(root, read, schema) + out[level] = build(root, read, info) except _Unsupported as e: logger.debug("SQL 规范化(%s)跳过:%s", level, e) except Exception as e: # noqa: BLE001 - sqlglot 认不了列、写法不支持:这一级不出,别的级照样比 @@ -2369,32 +2501,35 @@ def _canonical_forms(root: Any, read: str | None, schema: dict[str, dict[str, st else: logger.warning("SQL 规范化(%s)失败", level, exc_info=True) try: - from sqlglot.optimizer.normalize_identifiers import normalize_identifiers - - tree = normalize_identifiers(root.copy(), dialect=read) - _unquote(tree, read) + tree = root.copy() + _normalize_names(tree, read, info) out[_PARSED] = _Form(_PARSED, tree.sql(dialect=read, comments=False), tree) except Exception: # noqa: BLE001 logger.warning("SQL 重新生成失败", exc_info=True) return out -def _optimized(root: Any, read: str | None, schema: dict[str, dict[str, str]] | None) -> _Form: +def _optimized(root: Any, read: str | None, info: _Schema) -> _Form: + """认列之后的规范形式:统一名字 → 列名表写进定义 → USING 写成 ON → FETCH 写成 LIMIT → qualify 认列(给了表结构 + 先按表结构,表结构里没有的列再不给表结构认一遍)→ 来源别名改成整条语句里互不相同的 → 只引用一次的 CTE 写回用到 + 它的地方 → 只筛行、透列的子查询并进外层 → 去掉没用的 CTE → 两级共用的几步(_finish)。""" from sqlglot.errors import OptimizeError - from sqlglot.optimizer.eliminate_ctes import eliminate_ctes - from sqlglot.optimizer.eliminate_subqueries import eliminate_subqueries - from sqlglot.optimizer.merge_subqueries import merge_subqueries - from sqlglot.optimizer.pushdown_projections import pushdown_projections from sqlglot.optimizer.qualify import qualify tree = root.copy() - why = _unsupported(tree) or ("不确定的函数" if _volatile(tree) else None) + why = _unsupported(tree, read) or ("不确定的函数" if _volatile(tree) else None) if why: raise _Unsupported(why) + _normalize_names(tree, read, info) _remember_names(tree) - # 给了表结构先按表结构认列;表结构里没有的列(表结构旧了)再按不给表结构认一遍:两遍都认不全就不出这一级 + _mark_unsure(tree) + _push_column_lists(tree) + _expand_using(tree) + _qualify_stars(tree) + _fetch_to_limit(tree) + _normalize_negation(tree) qualified = None - for mapping in ([schema, None] if schema else [None]): + for mapping in ([info.mapping, None] if info.mapping else [None]): if _alias_or_column(tree, read, mapping): continue try: @@ -2406,87 +2541,86 @@ def _optimized(root: Any, read: str | None, schema: dict[str, dict[str, str]] | if qualified is None: raise _Unsupported("认不全列") tree = qualified - if _has_star(tree): - _narrow_stars(tree, read) - if not _has_star(tree) and not _correlated_derived(tree): - tree = eliminate_subqueries(tree) - _split_simple_ctes(tree) - # merge_subqueries 只在同一段里避免重名:同一个 CTE 并进外层和相关子查询两处,两处都叫 orders,子查询里本来 - # 指外层的 orders.id 就被里层遮住了。先把整条语句的别名都改成互不相同的 - _unique_aliases(tree, read) - # merge_subqueries 会把外连接可空一侧的子查询里算出来的列(常量、COALESCE)原样并进外层:没关联上的行本该 - # 是空值,并进来就成了常量。这种写法不并 - if _merge_safe(tree): - tree = merge_subqueries(tree) - tree = eliminate_ctes(tree) - _push_into_groups(tree) - tree = pushdown_projections(tree) - else: - tree = eliminate_ctes(tree) - return _finish(tree, read, level=_OPTIMIZER, qualified=True) + _split_simple_ctes(tree) + _inline_ctes(tree) + # 写回的 CTE 定义各是一份副本,里面的别名一样:同一个 CTE 写回外层和相关子查询两处,并进去之后子查询里本来指外层 + # 的列就被里层同名的别名遮住了。写回之后再把整条语句的别名改成互不相同的 + _unique_aliases(tree) + _merge_derived(tree, read, info) + _push_into_groups(tree) + _drop_unused_ctes(tree) + return _finish(tree, read, info, level=_OPTIMIZER, qualified=True) -def _light(root: Any, read: str | None, schema: Any = None) -> _Form: - from sqlglot.optimizer.normalize_identifiers import normalize_identifiers - from sqlglot.optimizer.qualify_tables import qualify_tables - - tree = normalize_identifiers(root.copy(), dialect=read) - why = _unsupported(tree) +def _light(root: Any, read: str | None, info: _Schema) -> _Form: + """不认列的规范形式:统一名字、列名表写进定义、USING 写成 ON、FETCH 写成 LIMIT、只引用一次的 CTE 写回用到它的 + 地方,再走 _finish 里不需要认列的几步。""" + tree = root.copy() + why = _unsupported(tree, read) if why: raise _Unsupported(why) + _normalize_names(tree, read, info) _remember_names(tree) + _mark_unsure(tree) + _push_column_lists(tree) + _expand_using(tree) + _qualify_stars(tree) + _fetch_to_limit(tree) + _normalize_negation(tree) _inline_ctes(tree) - tree = qualify_tables(tree, dialect=read) - return _finish(tree, read, level=_LIGHT, qualified=False) + _drop_unused_ctes(tree) + return _finish(tree, read, info, level=_LIGHT, qualified=False) -def _finish(tree: Any, read: str | None, *, level: str, qualified: bool) -> _Form: - """两级共用的本模块自己的几步,都只做保持语义的变换(为什么保持语义见各函数的说明): +def _finish(tree: Any, read: str | None, info: _Schema, *, level: str, qualified: bool) -> _Form: + """两级共用的几步,都只做保持语义的变换(为什么保持语义见各函数的说明): - 1. 和名字无关的改写:不需要的引号、不影响结合的括号、ORDER BY 的 ASC、OFFSET 0 去掉;COUNT(1) 写成 COUNT(*); - IN 列表去重排序、只有一项写成 =;EXISTS 子查询写成 SELECT 1;IN 子查询里的 DISTINCT、不取行时的 ORDER BY - 去掉;SELECT DISTINCT 写成按选出来的列 GROUP BY,已经按选出来的列分组的 DISTINCT 去掉;原样选出一个子查询 - 全部列的一段换成那个子查询;两张表的 RIGHT JOIN 写成 LEFT JOIN;WHERE 把外连接可空一侧没关联上的行筛掉了的 - 写成内连接;内连接的条件挪进 WHERE、来源写成逗号连接;最外层 ORDER BY 引用的别名换成表达式,最外层和条件里 - 的子查询选出来的列名去掉(列名不影响取值)。 - 2. 化简条件(_simplify:sqlglot 的 simplify 去掉和类型、排序规则、方言有关的几种化简)。 - 3. 起规范的名字、排好交换律成立的地方(见 _named、_Namer、_sort_own)。 + 1. 动来源的:两个来源的 RIGHT JOIN 写成 LEFT JOIN;WHERE 把外连接可空一侧没关联上的行筛掉了的写成内连接;开头那组 + 内连接的条件挪进 WHERE、写成逗号连接。结果看行出来的先后的一段(见 _Sensitivity)都不做。 + 0. 原样选出 FROM 里一个子查询全部列、什么别的都不做的一段换成那个子查询(见 _unwrap_passthrough)。 + 1.5 去重的:SELECT DISTINCT 写成按选出来的列 GROUP BY、已经按选出来的列分组的 DISTINCT 去掉、IN 子查询里的 + DISTINCT 和 ORDER BY 去掉(见各函数:只在「相等」就是「一样」、不看行出来的先后时做)。 + 2. 和来源无关的:COUNT(1) 写成 COUNT(*)、MIN / MAX 里的 DISTINCT 去掉;EXISTS 子查询写成 SELECT 1;IN 列表去重 + 排序、只有一个常量的写成 =;BETWEEN 写成两个比较;NOT 套比较写成反过来的比较、NOT NOT 去掉;不影响结合的括号、 + ORDER BY 的 ASC、OFFSET 0 去掉;最外层 ORDER BY 引用的别名换成表达式,最外层和条件里的子查询选出来的列名去掉。 + 3. 起规范的名字、排好交换律成立的地方(见 _named)。 """ - _unquote(tree, read) + tree = _unwrap_passthrough(tree, read) + sens = _Sensitivity(tree, read, info) + _right_to_left(tree, sens) + sens = _Sensitivity(tree, read, info) + _reject_nulls(tree, sens, read) + _flatten_prefix(tree, sens) + _in_subquery_distinct(tree, sens) + _distinct_to_group(tree, sens) + _count_star(tree) + _minmax_distinct(tree) + _case_forms(tree, read) + _drop_true_conjuncts(tree) + _exists_subqueries(tree) + _in_lists(tree) + _expand_between(tree) + _fold_not(tree) _strip_parens(tree) _strip_asc(tree) - _count_star(tree) - _in_lists(tree, read) - _exists_projection(tree) - _in_subquery_distinct(tree) - _distinct_to_group(tree, read) - _redundant_distinct(tree, read) _zero_offset(tree) - tree = _unwrap_passthrough(tree, read) - _right_to_left(tree) - _reject_nulls(tree) - _flatten_inner(tree) _order_by_aliases(tree) _drop_output_aliases(tree) - tree = _simplify(tree, read) pre = tree.copy() - tree = _named(tree, read, qualified) + sens = _Sensitivity(tree, read, info) + tree = _named(tree, read, qualified, sens) return _Form(level, tree.sql(dialect=read, comments=False), tree, pre, qualified) -def _named(tree: Any, read: str | None, qualified: bool, hint: "_Hint | None" = None) -> Any: +def _named(tree: Any, read: str | None, qualified: bool, sens: "_Sensitivity", hint: "_Hint | None" = None) -> Any: """起规范的名字、排好交换律成立的地方,最后统一引号(只给需要的标识符加)。""" - from sqlglot import exp from sqlglot.optimizer.qualify_columns import quote_identifiers - _Namer(tree, read, qualified, hint).canon(tree, 0, {}) - tree = _simplify(tree, read) - for select in list(tree.find_all(exp.Select)): - _sort_own(select, read) + _Namer(tree, read, qualified, sens, hint).canon(tree, 0, {}) return quote_identifiers(tree, dialect=read, identify=False) -# ---- 守门:哪些结构不走某一级 +# ---- 守门:哪些结构不走规范化的两级 def _volatile(tree: Any) -> bool: from sqlglot import exp @@ -2499,149 +2633,115 @@ def _volatile(tree: Any) -> bool: return False -def _unsupported(tree: Any) -> str | None: - """递归 CTE、LATERAL、UNNEST、PIVOT、表函数、VALUES、SEMI / ANTI / ASOF 这些连接:起名、合并都不处理。""" +def _unsupported(tree: Any, read: str | None) -> str | None: + """规范化的两级不处理的结构:递归 CTE(写了 RECURSIVE,或者 CTE 引用自己、引用后面的 CTE——SQLite 里不写 + RECURSIVE 也是递归)、LATERAL、UNNEST、PIVOT、表函数、VALUES、SEMI / ANTI / ASOF / NATURAL 这些连接、实表别名 + 带列名(PostgreSQL 的 orders AS o(a, b) 给列改了名)、BETWEEN SYMMETRIC、FETCH … WITH TIES / PERCENT、ROLLUP / + CUBE / GROUPING SETS、SELECT INTO、加锁。""" from sqlglot import exp for node in tree.walk(): - if isinstance(node, exp.With) and node.args.get("recursive"): - return "递归 CTE" - if isinstance(node, (exp.Lateral, exp.Unnest, exp.Pivot, exp.TableSample, exp.Values)): + if isinstance(node, exp.With): + if node.args.get("recursive"): + return "递归 CTE" + names = [c.alias_or_name for c in node.expressions] + for i, cte in enumerate(node.expressions): + later = set(names[i:]) + if any(t.name in later and not t.args.get("db") for t in cte.this.find_all(exp.Table)): + return "CTE 引用自己或后面的 CTE" + if isinstance(node, (exp.Lateral, exp.Unnest, exp.Pivot, exp.TableSample, exp.Values, exp.Into, exp.Lock)): return type(node).__name__ - if isinstance(node, exp.Table) and not isinstance(node.this, exp.Identifier): - return "表函数" + if isinstance(node, exp.Table): + if not isinstance(node.this, exp.Identifier): + return "表函数" + alias = node.args.get("alias") + if isinstance(alias, exp.TableAlias) and alias.columns: + return "实表别名带列名" if isinstance(node, exp.Join) and (node.args.get("method") or str(node.kind or "").upper() in - ("SEMI", "ANTI", "ASOF", "POSITIONAL", "STRAIGHT_JOIN")): + ("SEMI", "ANTI", "ASOF", "POSITIONAL", "STRAIGHT_JOIN", "ARRAY")): return "连接方式" + if isinstance(node, exp.Between) and node.args.get("symmetric"): + return "BETWEEN SYMMETRIC" + if isinstance(node, exp.Fetch): + options = node.args.get("limit_options") + if options is not None and (options.args.get("percent") or options.args.get("with_ties")): + return "FETCH WITH TIES / PERCENT" + if isinstance(node, exp.Group) and any(node.args.get(k) for k in ("rollup", "cube", "grouping_sets", "totals")): + return "ROLLUP / CUBE / GROUPING SETS" return None -def _starry(select: Any) -> bool: +# ---- 名字 + +def _identifier_role(ident: Any) -> str: + """一个标识符在语法树里是什么:column(列名)/ column_alias(列的别名、CTE 和子查询的列名表)/ table(表名、 + 列前面的表名或别名)/ table_alias(表、子查询、CTE 的别名)/ other。""" from sqlglot import exp - return any(isinstance(p, exp.Star) or (isinstance(p, exp.Column) and p.is_star) for p in select.expressions) + parent, key = ident.parent, ident.arg_key + if isinstance(parent, exp.Column): + return "column" if key == "this" else "table" + if isinstance(parent, exp.Table): + return "table" + if isinstance(parent, exp.TableAlias): + return "column_alias" if key == "columns" else "table_alias" + if isinstance(parent, exp.Alias): + return "column_alias" + return "other" -def _takes_any(select: Any) -> bool: - """没有 ORDER BY 的 LIMIT / OFFSET:取哪几行 SQL 没规定,库里取先出来的几行。出行的先后会变的改写(调换来源、 - DISTINCT 写成 GROUP BY、RIGHT JOIN 写成 LEFT JOIN)这一段都不做,免得取到的行不同。""" - return (select.args.get("limit") is not None or select.args.get("offset") is not None) and \ - select.args.get("order") is None +def _normalize_names(tree: Any, read: str | None, info: _Schema) -> None: + """按方言的规则统一名字的大小写和引号,只做库里认成同一个名字的改写;处理过的标识符标成 case_sensitive,后面 + qualify 不再按 sqlglot 的规则改(sqlglot 认为 SQLite 带引号的名字也不分大小写,可 SQLite 找不到这一列时把双引号 + 里的当成字符串,'Paid' 和 'paid' 就不同了)。只认 ASCII 字母的大小写(库里也只折 ASCII)。 - -def _unstable(select: Any) -> bool: - """这一段取到哪几行、取到哪一行的值 SQL 本来就没规定、库里看扫描的先后:没有 ORDER BY 的 LIMIT / OFFSET;聚合的 - 一段(有 GROUP BY 或聚合函数)里既不在聚合函数里、也不是分组键(的一部分)的列(SQLite 取组里任意一行的值)。 - 这样的一段来源不调换先后:调换之后扫描的先后跟着变,实际取到的行就不同了。""" + - SQLite:不带引号的名字、带引号的表名和别名不分大小写,统一成小写;条件、SELECT 列表里不带表名、带引号的列名 + ("Paid")原样留着,除非表结构里确实有这一列(有这一列就不会被当成字符串)。 + - PostgreSQL(和认不出的方言):不带引号的名字折成小写;带引号的里面没有大写字母就和不带引号的是同一个名字。 + - MySQL:列名、列的别名不分大小写,统一成小写;表名、表的别名在 Linux 上分大小写,原样。 + """ from sqlglot import exp - if _takes_any(select): - return True - group = select.args.get("group") - keys = {_fingerprint(k) for k in (group.expressions if group is not None else [])} - - def aggregates(node: Any) -> bool: - return any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) and not _nested(n, node) - for n in node.walk()) - - clauses = [c for c in (*select.expressions, select.args.get("having"), select.args.get("order")) if c is not None] - if group is None and not any(aggregates(p) for p in select.expressions): - return False - - def loose(node: Any) -> bool: - if isinstance(node, (exp.Query, exp.Subquery)) or _fingerprint(node) in keys: - return False - if isinstance(node, exp.AggFunc) and not isinstance(node.parent, exp.Window): - return False - if isinstance(node, exp.Column): - return not node.is_star - return any(loose(c) for c in node.iter_expressions()) - - return any(loose(c) for c in clauses) - - -def _has_star(tree: Any) -> bool: - from sqlglot import exp - - return any(_starry(s) for s in tree.find_all(exp.Select)) - - -def _correlated_derived(tree: Any) -> bool: - """FROM 里的子查询、CTE 引用了外层的列(相关派生表):写成顶层的 CTE 就断了关联,不改写。""" - from sqlglot.optimizer.scope import traverse_scope - - return any((s.is_derived_table or s.is_cte) and s.external_columns for s in traverse_scope(tree)) - - -def _merge_safe(tree: Any) -> bool: - """外连接可空一侧(LEFT JOIN 关联进来的、RIGHT JOIN 之前的、FULL JOIN 两侧)的子查询、CTE,只有「直接读实表、 - 原样选列」的才让 merge_subqueries 并:选出来的是常量、COALESCE、CASE 这类表达式时,没关联上的行本该整行是空值, - 并进外层后成了算出来的值;里面再套着子查询的,里层先并上来也会变成这样。""" - from sqlglot import exp - - ctes = {c.alias_or_name: c.this for c in tree.find_all(exp.CTE)} - for select in tree.find_all(exp.Select): - joins = select.args.get("joins") or [] - items = _items_of(select) - nullable: list[Any] = [] - for i, join in enumerate(joins): - side = str(join.side or "").upper() - if side in ("LEFT", "FULL"): - nullable.append(join.this) - if side in ("RIGHT", "FULL"): - nullable += items[:i + 1] - for item in nullable: - body = item.this if isinstance(item, exp.Subquery) else ( - ctes.get(item.name) if isinstance(item, exp.Table) and not item.args.get("db") else None) - if body is None: - continue - if not isinstance(body, exp.Select): - return False - if not all(isinstance(p.unalias(), exp.Column) for p in body.expressions): - return False - if any(not isinstance(s, exp.Table) or s.name in ctes for s in _items_of(body)): - return False - return True - - -def _alias_or_column(tree: Any, read: str | None, schema: dict[str, dict[str, str]] | None) -> bool: - """有不带表名、又和这一段的列别名同名的名字,qualify 会把它当成别名展开,库里认的却可能是表的列,这一遍不认列: - - - WHERE、GROUP BY、HAVING 里:库里认的是列还是别名各方言不一样(SQLite 的 GROUP BY、HAVING 都先认列:HAVING - amount > 10 里的 amount 是组里任意一行的 amount,不是 SUM(amount) AS amount),qualify 却一律当成别名(GROUP BY - created_at 成了按 DATE(created_at) 分组)。表结构里有这一段的每张表、这个名字又不是其中哪张表的列时,它只能是 - 别名,照常认。 - - SELECT 列表里引用前面列的别名:SQLite、PostgreSQL、MySQL 都不认,名字指的是表的列(SELECT user_id AS amount, - amount * 2 里的 amount 是表的 amount 列),不是表的列就报错;qualify 没有表结构时却当成别名展开成 user_id * 2, - 有表结构时不是表的列的也展开。表结构里有这一段的每张表、这个名字又是其中的列时,qualify 先认列,照常认。""" - from sqlglot import exp - from sqlglot.optimizer.normalize_identifiers import normalize_identifiers - - columns = {name.lower(): {c.lower() for c in cols} for name, cols in (schema or {}).items()} - copy = normalize_identifiers(tree.copy(), dialect=read) - for select in copy.find_all(exp.Select): - aliases = {p.alias.lower() for p in select.expressions if isinstance(p, exp.Alias) and p.alias} - if not aliases: + for ident in list(tree.find_all(exp.Identifier)): + name = ident.this + if not isinstance(name, str): continue - items = _items_of(select) - known = all(isinstance(i, exp.Table) and i.name.lower() in columns for i in items) - real = set().union(*(columns[i.name.lower()] for i in items)) if known and items else set() - for key in ("where", "group", "having"): - node = select.args.get(key) - if node is None: - continue - names = {c.name.lower() for c in node.find_all(exp.Column) if not c.table and not _nested(c, node)} - if (names & aliases) - (set() if not known else (aliases - real)): - return True - earlier: set[str] = set() - for projection in select.expressions: - names = {c.name.lower() for c in projection.find_all(exp.Column) - if not c.table and (c is projection or not _nested(c, projection))} - if (names & earlier) - real: - return True - if isinstance(projection, exp.Alias) and projection.alias: - earlier.add(projection.alias.lower()) - return False + role = _identifier_role(ident) + quoted = bool(ident.args.get("quoted")) + lower = name.translate(_ASCII_LOWER) + if read == "sqlite": + keep = quoted and role == "column" and not ident.parent.args.get("table") and lower not in info.columns + if keep: + ident.meta[_UNSURE] = True + else: + name, quoted = lower, False + elif read == "mysql": + if role in ("column", "column_alias"): + name = lower + quoted = False + elif not quoted or lower == name: + name, quoted = lower, False + if quoted or not _PLAIN_NAME.match(name): + quoted = True + ident.set("this", name) + ident.set("quoted", quoted) + ident.meta["case_sensitive"] = True + + +def _mark_unsure(tree: Any) -> None: + """条件里的子查询(IN、EXISTS、标量)里不带表名的列标成认不准(见 _UNSURE):它可能是子查询里的表的列,也可能是 + 外层的列,表结构里没有就分不清。只影响差异说明说不说具体的,不影响规范形式。""" + from sqlglot import exp + + for col in tree.find_all(exp.Column): + if col.table or not isinstance(col.this, exp.Identifier): + continue + node = col.parent + while node is not None: + if isinstance(node, exp.Select) and (_scalar_context(node) or _condition_query(node)): + col.this.meta[_UNSURE] = True + break + node = node.parent def _remember_names(tree: Any) -> None: @@ -2655,31 +2755,214 @@ def _remember_names(tree: Any) -> None: sub.this.meta.setdefault(_ORIGINAL, sub.alias) +def _push_column_lists(tree: Any) -> None: + """CTE、FROM 里的子查询自己写了列名表的(WITH p(uid, total) AS (…)、(…) AS t(x, y)),列名写成里面那段(UNION 是 + 第一支)选出来的列的别名,列名表去掉:外面按这些名字引用,两种写法是一回事。里面那段选的是 *(不知道有几列)、 + 列数对不上、本段 ORDER BY / GROUP BY / HAVING 还按原来的别名引用的,没法改写,不走规范化的两级。""" + from sqlglot import exp + + for holder in list(tree.find_all(exp.CTE, exp.Subquery)): + alias = holder.args.get("alias") + if not isinstance(alias, exp.TableAlias) or not alias.columns: + continue + body = holder.this + target = _leaves(body)[0] if isinstance(body, exp.SetOperation) else body + names = [c.name for c in alias.columns] + if not isinstance(target, exp.Select) or _starry(target) or len(target.expressions) != len(names): + raise _Unsupported("列名表对不上选出来的列") + own = {p.alias for p in target.expressions if isinstance(p, exp.Alias) and p.alias} + clauses = [target.args.get(k) for k in ("order", "group", "having")] + if isinstance(body, exp.SetOperation): + if body.args.get("order") is not None: + raise _Unsupported("合并的结果按名字排序") + for clause in clauses: + if clause is not None and any(not c.table and c.name in own for c in clause.find_all(exp.Column)): + raise _Unsupported("列名表改名之后本段的引用对不上") + target.set("expressions", [exp.alias_(p.unalias().copy() if isinstance(p, exp.Alias) else p.copy(), n, + copy=False) for p, n in zip(target.expressions, names)]) + alias.set("columns", None) + + +def _expand_using(tree: Any) -> None: + """JOIN … USING (c) 写成 ON 左表.c = 右表.c,这一段里不带表名的 c 写成左表.c(内连接、LEFT JOIN 时合并出来的 c + 就是左表的 c)。只改得了「FROM 只有一个来源、第一个连接就是 USING、SELECT 里没有 *」的写法:别的(左边好几个 + 来源时 c 是哪一张的、SELECT * 列的先后、RIGHT / FULL JOIN 合并出来的值、子查询里引用 c)没法照原样改写,不走 + 规范化的两级。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + if not any(j.args.get("using") for j in joins): + continue + if len(joins) < 1 or not joins[0].args.get("using") or any(j.args.get("using") for j in joins[1:]): + raise _Unsupported("USING") + join = joins[0] + if str(join.side or "").upper() not in ("", "LEFT") or join.args.get("on") is not None or \ + any(_is_star(p) for p in select.expressions): + raise _Unsupported("USING") + left, right = select.args["from_"].this, join.this + la, ra = left.alias_or_name, right.alias_or_name + if not la or not ra: + raise _Unsupported("USING") + names = [u.name for u in join.args["using"]] + for sub in _nested_queries(select): + if any(not c.table and c.name in names for c in sub.find_all(exp.Column)): + raise _Unsupported("子查询里引用 USING 的列") + for node in list(_block_nodes(select)): + if isinstance(node, exp.Column) and not node.table and node.name in names: + node.set("table", exp.to_identifier(la)) + conds = [exp.EQ(this=exp.column(n, table=la), expression=exp.column(n, table=ra)) for n in names] + join.set("using", None) + join.set("on", _and_all(conds)) + + +def _qualify_stars(tree: Any) -> None: + """几个来源的 SELECT * 写成按 FROM 的先后逐个来源的 t.*:两种写法是一回事(没有 USING / NATURAL 时 * 就是各来源的 + 全部列依次排开)。sqlglot 的 qualify 展开 * 时把实表的列排在 FROM 里的子查询前面,和库里的先后不一样,先写成 t.* + 它就按写的先后展开。有来源没有名字的不动(这时也不走规范化的两级)。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + if not any(isinstance(p, exp.Star) for p in select.expressions): + continue + items = _items_of(select) + if len(items) < 2 or any(not i.alias_or_name for i in items): + continue + stars = [exp.Column(this=exp.Star(), table=exp.to_identifier(i.alias_or_name)) for i in items] + new: list[Any] = [] + for p in select.expressions: + new += [c.copy() for c in stars] if isinstance(p, exp.Star) else [p] + select.set("expressions", new) + + +def _fetch_to_limit(tree: Any) -> None: + """FETCH FIRST n ROWS ONLY(不带 WITH TIES / PERCENT)就是 LIMIT n(sqlglot 把 FETCH 放在 limit 这个参数上)。""" + from sqlglot import exp + + for query in list(tree.find_all(exp.Select, exp.SetOperation)): + for key in ("limit", "fetch"): + fetch = query.args.get(key) + if not isinstance(fetch, exp.Fetch): + continue + count = fetch.args.get("count") or exp.Literal.number(1) + query.set(key, None) + query.set("limit", exp.Limit(expression=count.copy())) + + +# ---- 认列属于哪个来源 + +class _Resolver: + """按别名认清每一列属于哪个来源(相关子查询里的列到外层找):refs[id(来源)] 是引用它的列,cte_of[id(表)] 是它 + 引用的 CTE,owner[id(来源)] 是来源所在的那段 SELECT。不带表名的列不认(轻量一级)。""" + + def __init__(self, tree: Any) -> None: + self.refs: dict[int, list[Any]] = {} + self.cte_of: dict[int, Any] = {} + self.owner: dict[int, Any] = {} + self.source_of: dict[int, Any] = {} + self._walk(tree, [], {}) + + def _walk(self, query: Any, frames: list[dict[str, Any]], ctes: dict[str, Any]) -> None: + from sqlglot import exp + + if isinstance(query, exp.Subquery): + self._walk(query.this, frames, ctes) + return + with_ = query.args.get("with_") + if with_ is not None: + ctes = dict(ctes) + for cte in with_.expressions: + self._walk(cte.this, frames, ctes) + ctes[cte.alias_or_name] = cte + if isinstance(query, exp.SetOperation): + self._walk(query.left, frames, ctes) + self._walk(query.right, frames, ctes) + return + if not isinstance(query, exp.Select): + raise _Unsupported(type(query).__name__) + frame: dict[str, Any] = {} + for item in _items_of(query): + if isinstance(item, exp.Subquery): + self._walk(item.this, frames, ctes) + elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier): + if not item.args.get("db") and item.name in ctes: + self.cte_of[id(item)] = ctes[item.name] + else: + raise _Unsupported(type(item).__name__) + name = item.alias_or_name + if not name or name in frame: + raise _Unsupported("来源没有名字或重名") + frame[name] = item + self.owner[id(item)] = query + self.refs.setdefault(id(item), []) + stack = frames + [frame] + for node in _block_nodes(query): + if isinstance(node, exp.Column) and node.table: + for f in reversed(stack): + if node.table in f: + self.refs[id(f[node.table])].append(node) + self.source_of[id(node)] = f[node.table] + break + for sub in _nested_queries(query): + self._walk(sub, stack, ctes) + + def body(self, item: Any) -> Any: + """派生来源里面那段查询(FROM 里的子查询、引用的 CTE);实表是 None。""" + from sqlglot import exp + + if isinstance(item, exp.Subquery): + return item.this + cte = self.cte_of.get(id(item)) + return cte.this if cte is not None else None + + +def _unique_aliases(tree: Any) -> None: + """(认列之后)整条语句里每个来源的别名改成互不相同的(_s1、_s2…),引用它的列跟着改:按原来的别名认清每一列 + 属于哪个来源之后再改,相关子查询里引用外层的列不会认错;之后并子查询、写回 CTE 时名字不会撞上。""" + from sqlglot import exp + + resolver = _Resolver(tree) + taken = {t.name for t in tree.find_all(exp.Table)} | {c.alias_or_name for c in tree.find_all(exp.CTE)} + n = 0 + for item in [i for s in tree.find_all(exp.Select) for i in _items_of(s)]: + n += 1 + name = f"_s{n}" + while name in taken: + name += "_" + alias = item.args.get("alias") + if isinstance(alias, exp.TableAlias): + alias.set("this", exp.to_identifier(name)) + else: + item.set("alias", exp.TableAlias(this=exp.to_identifier(name))) + for col in resolver.refs.get(id(item), []): + col.set("table", exp.to_identifier(name)) + + +def _nested_with(tree: Any) -> bool: + from sqlglot import exp + + top = tree.args.get("with_") + return any(isinstance(n, exp.With) and n is not top for n in tree.walk()) + + def _inline_ctes(tree: Any) -> None: - """(轻量一级)只引用一次、在 FROM / JOIN 里引用的非递归顶层 CTE 写回引用它的地方,成为 FROM 里的子查询:写成 - CTE 还是子查询是同一个意思。CTE 自己写了列名的、语句里还有别的 WITH 的(名字可能被遮住)不动。""" + """只引用一次的非递归顶层 CTE 写回引用它的地方,成为 FROM 里的子查询(别名照旧):写成 CTE 还是子查询是同一个 + 意思。语句里还有别的 WITH 的不动(名字可能被里层的 CTE 遮住)。""" from sqlglot import exp with_ = tree.args.get("with_") - if with_ is None or with_.args.get("recursive"): - return - if any(isinstance(n, exp.With) for n in tree.walk() if n is not with_): + if with_ is None or with_.args.get("recursive") or _nested_with(tree): return changed = True - while changed: + while changed and with_.expressions: changed = False + resolver = _Resolver(tree) for cte in list(with_.expressions): - name = cte.alias_or_name - if cte.args["alias"].columns: - continue - refs = [t for t in tree.find_all(exp.Table) if t.name == name and not t.args.get("db") - and isinstance(t.this, exp.Identifier)] - if len(refs) != 1 or refs[0].find_ancestor(exp.CTE) is cte or \ - not isinstance(refs[0].parent, (exp.From, exp.Join)): + refs = [t for t in tree.find_all(exp.Table) if resolver.cte_of.get(id(t)) is cte] + if len(refs) != 1: continue ref = refs[0] - body = cte.this.copy() - sub = exp.Subquery(this=body, alias=exp.TableAlias(this=exp.to_identifier(ref.alias or name))) + sub = exp.Subquery(this=cte.this.copy(), alias=exp.TableAlias(this=exp.to_identifier(ref.alias_or_name))) ref.replace(sub) cte.pop() changed = True @@ -2688,18 +2971,912 @@ def _inline_ctes(tree: Any) -> None: tree.set("with_", None) -def _unquote(tree: Any, read: str | None) -> None: - """不需要的引号去掉:按方言的规则,不加引号时认成的还是同一个名字(PostgreSQL 的 "orders" 就是 orders,"Orders" - 不是)。最后统一引号时需要的再加回来(见 _named)。""" +def _drop_unused_ctes(tree: Any) -> None: + """没有被引用的非递归 CTE 去掉:查询语句里没用到的 CTE 不影响结果。""" from sqlglot import exp - from sqlglot.dialects.dialect import Dialect - dialect = Dialect.get_or_raise(read) - for ident in tree.find_all(exp.Identifier): - if ident.quoted and isinstance(ident.this, str): - bare = exp.Identifier(this=ident.this, quoted=False) - if dialect.normalize_identifier(bare).this == ident.this: - ident.set("quoted", False) + changed = True + while changed: + changed = False + resolver = _Resolver(tree) + used = {id(c) for c in resolver.cte_of.values()} + for with_ in list(tree.find_all(exp.With)): + if with_.args.get("recursive"): + continue + for cte in list(with_.expressions): + if id(cte) not in used: + cte.pop() + changed = True + if not with_.expressions and isinstance(with_.parent, exp.Expr): + with_.parent.set("with_", None) + + +def _mergeable(body: Any) -> bool: + """FROM 里的子查询能不能并进外层:只从几张实表(内连接)里筛行、选列(没有分组、聚合、去重、排序、取行、窗口函数、 + UNION、WITH,选出来的列里没有子查询),并进去之后每一行、每个值都一样。""" + from sqlglot import exp + + if not isinstance(body, exp.Select): + return False + if {k for k, v in body.args.items() if v} - {"expressions", "from_", "where", "joins"}: + return False + sources = _items_of(body) + if not sources or any(not isinstance(i, exp.Table) for i in sources): + return False + if any(not _inner(j) for j in body.args.get("joins") or []): + return False + stars = [p for p in body.expressions if _is_star(p)] + if stars and (len(body.expressions) != 1 or not isinstance(body.expressions[0], exp.Star) or len(sources) != 1): + return False + for p in body.expressions: + if any(isinstance(n, (exp.AggFunc, exp.Window, exp.Query, exp.Subquery)) for n in p.walk()): + return False + names = [p.alias_or_name for p in body.expressions] + if not stars and (not all(names) or len(set(names)) != len(names)): + return False + where = body.args.get("where") + return where is None or not any(isinstance(n, (exp.AggFunc, exp.Window)) for n in where.walk()) + + +def _merge_derived(tree: Any, read: str | None, info: _Schema) -> None: + """(认列之后)只筛行、选列的子查询(见 _mergeable)并进外层:外层引用它的列换成里面的定义,里面的 WHERE 和外层的 + WHERE(和它里面内连接的条件)用 AND 连起来,来源换成里面那几张表。只并内连接一侧(FROM 或内连接进来的)的:外连接 + 可空一侧没关联上的行该是空值,并进来就成了算出来的值。结果看行出来的先后的外层(见 _Sensitivity)不并。 + 外层有 SELECT * 的:子查询选的就是 *(换成那张表,列一样)或者它是唯一的来源(* 换成它选出来的列)才并。""" + from sqlglot import exp + + def substitute(col: Any, expr: Any) -> None: + new = expr.copy() + if not isinstance(new, (exp.Column, exp.Literal, exp.Null, exp.Boolean)): + new = exp.Paren(this=new) + col.replace(new) + + for _ in range(64): + resolver = _Resolver(tree) + sens = _Sensitivity(tree, read, info, resolver) + done = False + for select in reversed(list(tree.find_all(exp.Select))): + if sens.of(select): + continue + joins = select.args.get("joins") or [] + if any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + items = _items_of(select) + for index, item in enumerate(items): + if not isinstance(item, exp.Subquery) or not _mergeable(item.this): + continue + if index and not _inner(joins[index - 1]): + continue + body = item.this + sources = _items_of(body) + src = sources[0] + star_body = isinstance(body.expressions[0], exp.Star) + defs = {} if star_body else {p.alias_or_name: p.unalias() for p in body.expressions} + refs = resolver.refs.get(id(item), []) + if not star_body and any(c.name not in defs for c in refs): + continue + stars = [p for p in select.expressions if _is_star(p)] + bare = [p for p in stars if isinstance(p, exp.Star)] + if bare and not star_body and len(items) != 1: + continue + alias = item.alias_or_name + for col in refs: + if star_body: + col.set("table", exp.to_identifier(src.alias_or_name)) + else: + substitute(col, defs[col.name]) + if not star_body: + expanded = [exp.alias_(e.copy(), n, copy=False) for n, e in defs.items()] + new_list = [] + for p in select.expressions: + if isinstance(p, exp.Star) or (isinstance(p, exp.Column) and p.is_star and p.table == alias): + new_list += [e.copy() for e in expanded] + else: + new_list.append(p) + select.set("expressions", new_list) + else: + for p in stars: + if isinstance(p, exp.Column) and p.table == alias: + p.set("table", exp.to_identifier(src.alias_or_name)) + conds = [] + inner_where = body.args.get("where") + if inner_where is not None: + conds += _conjuncts(inner_where.this) + for join in body.args.get("joins") or []: + if join.args.get("on") is not None: + conds += _conjuncts(join.args["on"]) + if index: + own = joins[index - 1] + if own.args.get("on") is not None: + conds += _conjuncts(own.args["on"]) + position = index - 1 + new_joins = joins[:position] + [exp.Join(this=s.copy()) for s in sources] + joins[position + 1:] + select.set("joins", new_joins) + else: + select.args["from_"].set("this", src.copy()) + select.set("joins", [exp.Join(this=s.copy()) for s in sources[1:]] + list(joins) or None) + outer = select.args.get("where") + conds += _conjuncts(outer.this) if outer is not None else [] + if conds: + select.set("where", exp.Where(this=_and_all([c.copy() for c in conds]))) + done = True + break + if done: + break + if not done: + return + + +def _split_simple_ctes(tree: Any) -> None: + """引用了好几次、只从几张表里筛行选列的 CTE(见 _mergeable),每处引用各写回一份 FROM 里的子查询(别名照旧): + 这样的 CTE 是确定的(有不确定函数的语句不走这一级),算几遍结果一样,写回之后每处都能并进外层。语句里还有别的 + WITH 的不动(名字可能被里层的 CTE 遮住)。""" + from sqlglot import exp + + with_ = tree.args.get("with_") + if with_ is None or with_.args.get("recursive") or _nested_with(tree): + return + resolver = _Resolver(tree) + for cte in list(with_.expressions): + if not _mergeable(cte.this): + continue + refs = [t for t in tree.find_all(exp.Table) if resolver.cte_of.get(id(t)) is cte] + if len(refs) < 2 or any(r.find_ancestor(exp.CTE) is not None for r in refs): + continue + for ref in refs: + ref.replace(exp.Subquery(this=cte.this.copy(), alias=exp.TableAlias(this=exp.to_identifier(ref.alias_or_name)))) + cte.pop() + if not with_.expressions: + tree.set("with_", None) + + +def _grouped(expression: Any, keys: set[str]) -> bool: + """分组的子查询选出来的一列在每组里只有一个值:整个是分组键,或者每一列都在聚合函数里、或者本身是分组键。""" + from sqlglot import exp + + if _fingerprint(expression) in keys: + return True + if any(isinstance(n, (exp.Window, exp.Query, exp.Subquery)) for n in expression.walk()): + return False + for col in expression.find_all(exp.Column): + if _fingerprint(col) not in keys and col.find_ancestor(exp.AggFunc) is None: + return False + return True + + +def _push_into_groups(tree: Any) -> None: + """(认列之后)外层 WHERE 里只涉及一个分组子查询(FROM 或内连接进来的)选出来的列的条件,换成子查询里对应的表达式、 + 挪进它的 HAVING:子查询的每一行就是一组,外层按这一行的值筛和 HAVING 按这一组的值筛,用的是同一个值(分组键取的 + 是同一个代表值,聚合是同一个结果)。子查询有窗口函数、取行、去重、ROLLUP 这些的不挪;HAVING 里的条件不往 WHERE + 挪(相等不等于一样时,组里的行各是各的值)。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + where = select.args.get("where") + joins = select.args.get("joins") or [] + if where is None or any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + for index, item in enumerate(_items_of(select)): + if not isinstance(item, exp.Subquery) or (index and not _inner(joins[index - 1])): + continue + body = item.this + if not isinstance(body, exp.Select) or body.args.get("group") is None or \ + any(body.args.get(k) for k in ("limit", "offset", "distinct", "qualify", "windows", "with_")) or \ + any(n.find(exp.Window) for n in body.expressions): + continue + outputs = {p.alias_or_name: p.unalias() for p in body.expressions if not _is_star(p)} + keys = {_fingerprint(k) for k in body.args["group"].expressions} + alias = item.alias_or_name + moved, kept = [], [] + for cond in _conjuncts(where.this): + cols = list(cond.find_all(exp.Column)) + if not cols or cond.find(exp.Query, exp.Subquery) is not None or \ + any(c.table != alias or c.name not in outputs or not _grouped(outputs[c.name], keys) + for c in cols): + kept.append(cond) + continue + new = cond.copy() + for col in list(new.find_all(exp.Column)): + col.replace(exp.Paren(this=outputs[col.name].copy())) + moved.append(new) + if moved: + having = body.args.get("having") + body.set("having", exp.Having(this=_and_all(([having.this] if having else []) + moved))) + select.set("where", exp.Where(this=_and_all(kept)) if kept else None) + where = select.args.get("where") + if where is None: + break + + +def _distinct_to_group(tree: Any, sens: "_Sensitivity") -> None: + """SELECT DISTINCT a, b(没有分组、聚合、窗口函数)和 GROUP BY a, b 出的行一样(空值也算一组)——只在选出来的都是 + 「相等就是一样」的列(见 _Sensitivity._identity_safe)、这一段不看行出来的先后(DISTINCT 按扫到的先后出行,GROUP BY + 按分组键排好了出)时改:不然两种写法留下的代表值、取到的行可能不同。ORDER BY 用到没选出来的列时不改(两种写法对 + 这种排序的规矩不一样)。已经按选出来的列分组的 DISTINCT 去掉(每组一行、各行的分组键都不同)。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + distinct = select.args.get("distinct") + if distinct is None or distinct.args.get("on") or sens.of(select) or _starry(select): + continue + group = select.args.get("group") + if group is not None: + selected = {_fingerprint(p.unalias()) for p in select.expressions} + if group.expressions and all(_fingerprint(k) in selected for k in group.expressions) and \ + not any(group.args.get(k) for k in ("rollup", "cube", "grouping_sets")) and \ + not any(p.find(exp.Window) for p in select.expressions): + select.set("distinct", None) + continue + if select.args.get("having") is not None or _aggregating(select) or \ + any(p.find(exp.Window, exp.Query, exp.Subquery) for p in select.expressions): + continue + keys = [p.unalias() for p in select.expressions] + if not all(isinstance(k, exp.Column) and sens._identity_safe(k, select) for k in keys): + continue + order = select.args.get("order") + if order is not None: + allowed = {_fingerprint(k) for k in keys} + aliases = {p.alias for p in select.expressions if p.alias} + if any(_fingerprint(o.this) not in allowed and not ( + isinstance(o.this, exp.Column) and not o.this.table and o.this.name in aliases) + for o in order.expressions): + continue + new = exp.Group(expressions=[k.copy() for k in keys]) + new.meta[_FROM_DISTINCT] = True + select.set("distinct", None) + select.set("group", new) + + +def _in_subquery_distinct(tree: Any, sens: "_Sensitivity") -> None: + """IN (子查询) 只看取值的集合:里面不取行时,没有聚合的 ORDER BY 不影响(有聚合的 ORDER BY 让它成了聚合查询,只出 + 一行);选出来的那一列「相等就是一样」时 DISTINCT 也不影响(MySQL 默认不分大小写,DISTINCT 留下 'SH' 之后 + BINARY 'sh' 就找不到了)。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.In)): + query = node.args.get("query") + sub = query.this if isinstance(query, exp.Subquery) else query + if not isinstance(sub, exp.Select) or any(sub.args.get(k) for k in ("limit", "offset", "fetch")): + continue + if any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) for n in _block_nodes(sub)) or \ + sub.args.get("group") is not None or sub.args.get("having") is not None: + continue + if not any(n.find(exp.Window) for n in sub.expressions): + sub.set("order", None) + distinct = sub.args.get("distinct") + if distinct is not None and not distinct.args.get("on") and len(sub.expressions) == 1 and \ + isinstance(sub.expressions[0].unalias(), exp.Column) and \ + sens._identity_safe(sub.expressions[0].unalias(), sub): + sub.set("distinct", None) + + +# ---- 结果看不看行出来的先后 + +class _Sensitivity: + """哪几段 SELECT 的结果看行出来的先后:这样的一段不调换来源的先后、不把 CROSS JOIN 写成逗号连接、不排 GROUP BY + 的键、不把 RIGHT JOIN 写成 LEFT JOIN,不往里并子查询——这些改写都会改变库里扫描的先后,实际取到的行、值就不同了。 + + 本段自己看先后的:取行(LIMIT / OFFSET / FETCH,有 ORDER BY 也可能有并列);DISTINCT ON;窗口函数;拼串、拼数组、 + 取任意一个这类聚合(GROUP_CONCAT);聚合里既不分组也不聚合的列(SQLite、MySQL 取组里任意一行的值);当成标量 + 用的子查询(SQLite 取第一行),除非是不分组的聚合(只出一行);去重、分组、MIN / MAX、去重的 UNION 留下的是先 + 出现的那个值,而「相等」不等于「写法一样」的时候(MySQL 默认不分大小写的排序规则下的 'SH' 和 'sh'、PostgreSQL + numeric 的 2.0 和 2.00、SQLite 没有类型亲和的列里的 1 和 1.0、算出来的表达式)看先后,见 _identity_safe。 + 给看先后的一段供行的(它 FROM 里的子查询、引用的 CTE,层层往里)也看先后。""" + + def __init__(self, tree: Any, read: str | None, info: _Schema, resolver: _Resolver | None = None) -> None: + from sqlglot import exp + + self.read = read + self.info = info + try: + self.resolver = resolver or _Resolver(tree) + except _Unsupported: + self.resolver = None + selects = list(tree.find_all(exp.Select)) + marked = {id(s) for s in selects if self._own(s)} + for op in tree.find_all(exp.SetOperation): + if op.args.get("distinct") or isinstance(op, (exp.Intersect, exp.Except)): + for leaf in _leaves(op): + if isinstance(leaf, exp.Select) and \ + not all(self._identity_safe(p.unalias(), leaf) for p in leaf.expressions): + marked.add(id(leaf)) + by_id = {id(s): s for s in selects} + stack = [by_id[i] for i in marked if i in by_id] + while stack: + select = stack.pop() + for item in _items_of(select): + body = self.resolver.body(item) if self.resolver is not None else ( + item.this if isinstance(item, exp.Subquery) else None) + for leaf in _leaves(body) if body is not None else []: + if isinstance(leaf, exp.Select) and id(leaf) not in marked: + marked.add(id(leaf)) + stack.append(leaf) + self.marked = marked + + def of(self, select: Any) -> bool: + return id(select) in self.marked + + def _own(self, select: Any) -> bool: + from sqlglot import exp + + if any(select.args.get(k) for k in ("limit", "offset", "fetch")): + return True + distinct = select.args.get("distinct") + if distinct is not None and distinct.args.get("on"): + return True + own = list(_block_nodes(select)) + if any(isinstance(n, exp.Window) or _order_dependent(n) for n in own): + return True + if _scalar_context(select) and not (_aggregating(select) and select.args.get("group") is None): + return True + if _loose_columns(select): + return True + if distinct is not None and not all(self._identity_safe(p.unalias(), select) for p in select.expressions): + return True + group = select.args.get("group") + if group is not None and not all(self._identity_safe(k, select) for k in group.expressions): + return True + for n in own: + if isinstance(n, (exp.Min, exp.Max)) and not n.expressions and \ + not self._identity_safe(n.this.expressions[0] if isinstance(n.this, exp.Distinct) and + n.this.expressions else n.this, select): + return True + return False + + def _identity_safe(self, node: Any, select: Any, depth: int = 0) -> bool: + """node 的取值里「相等」是不是就是「一样」:COUNT 是整数;光一列的话看它是哪张表的哪一列、声明的什么类型。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, exp.Count): + return True + if not isinstance(node, exp.Column) or node.is_star or depth > 16: + return False + item = self.resolver.source_of.get(id(node)) if self.resolver is not None else None + if item is None: + items = _items_of(select) + item = items[0] if len(items) == 1 and not node.table else None + if item is None: + return self._safe_type(None) + body = self.resolver.body(item) if self.resolver is not None else ( + item.this if isinstance(item, exp.Subquery) else None) + if body is None: + if not isinstance(item, exp.Table): + return False + columns = self.info.types.get(item.name.translate(_ASCII_LOWER)) + declared = columns.get(node.name.translate(_ASCII_LOWER)) if columns is not None else None + return self._safe_type(declared) + if not isinstance(body, exp.Select): + return False + for p in body.expressions: + if not _is_star(p) and p.alias_or_name == node.name: + return self._identity_safe(p.unalias(), body, depth + 1) + sources = _items_of(body) + if any(isinstance(p, exp.Star) for p in body.expressions) and len(sources) == 1: + return self._through_star(sources[0], node.name, depth + 1) + return False + + def _through_star(self, item: Any, name: str, depth: int) -> bool: + """SELECT * 原样透出来的一列:到它唯一的来源里找同名的列。""" + from sqlglot import exp + + body = self.resolver.body(item) if self.resolver is not None else None + if body is None: + if not isinstance(item, exp.Table): + return False + columns = self.info.types.get(item.name.translate(_ASCII_LOWER)) + return self._safe_type(columns.get(name.translate(_ASCII_LOWER)) if columns is not None else None) + if not isinstance(body, exp.Select) or depth > 16: + return False + for p in body.expressions: + if not _is_star(p) and p.alias_or_name == name: + return self._identity_safe(p.unalias(), body, depth + 1) + sources = _items_of(body) + if any(isinstance(p, exp.Star) for p in body.expressions) and len(sources) == 1: + return self._through_star(sources[0], name, depth + 1) + return False + + def _safe_type(self, declared: str | None) -> bool: + """声明的类型下「相等」是不是就是「一样」。SQLite 没给表结构时按列有声明的类型、默认的排序规则算(假设); + 有类型的话 BLOB、没写类型(没有类型亲和)的列可能混着 1 和 1.0。PostgreSQL、MySQL 不知道类型就不算:numeric + 的 2.0 和 2.00、MySQL 默认不分大小写的字符串都是相等写法却不一样。""" + if self.read == "sqlite": + if declared is None: + return True + upper = declared.upper() + if "INT" in upper or any(w in upper for w in ("CHAR", "CLOB", "TEXT", "REAL", "FLOA", "DOUB")): + return True + return bool(upper) and "BLOB" not in upper + if declared is None: + return False + lower = declared.lower().strip() + if self.read == "mysql": + return any(lower.startswith(w) for w in ("int", "integer", "bigint", "smallint", "tinyint", "mediumint", + "date", "datetime", "timestamp", "time", "year", "decimal", + "numeric", "bit", "bool")) + return any(lower.startswith(w) for w in ("int", "integer", "bigint", "smallint", "serial", "bigserial", + "text", "varchar", "character", "char", "bpchar", "date", + "timestamp", "time", "bool", "uuid")) + + +def _order_dependent(node: Any) -> bool: + from sqlglot import exp + + name = type(node).__name__.lower() + if isinstance(node, exp.Anonymous): + name = str(node.name).lower().replace("_", "") + return name in _ORDERED_AGGS + + +def _aggregating(select: Any) -> bool: + """这一段是聚合查询:有 GROUP BY,或者自己的 SELECT 列表、HAVING、ORDER BY 里有聚合函数(不算窗口函数)。""" + from sqlglot import exp + + if select.args.get("group") is not None or select.args.get("having") is not None: + return True + return any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) for n in _block_nodes(select) + if not isinstance(n.parent, exp.Window)) + + +def _scalar_context(query: Any) -> bool: + """这段查询当成一个值用(标量子查询):不在 FROM / JOIN、IN、EXISTS、ANY / ALL、CTE 里,也不是整条语句。""" + from sqlglot import exp + + node = query + while True: + parent = node.parent + if parent is None: + return False + if isinstance(parent, exp.Subquery): + node = parent + continue + if isinstance(parent, exp.SetOperation): + node = parent + continue + if isinstance(parent, (exp.From, exp.Join, exp.Exists, exp.CTE, exp.Any, exp.All)): + return False + if isinstance(parent, exp.In) and node.arg_key == "query": + return False + return True + + +def _loose_columns(select: Any) -> bool: + """聚合的一段里既不在聚合函数里、也不是分组键(的一部分)的列:SQLite、MySQL 取组里任意一行的值。""" + from sqlglot import exp + + if not _aggregating(select): + return False + group = select.args.get("group") + keys = {_fingerprint(k) for k in (group.expressions if group is not None else [])} + + def loose(node: Any) -> bool: + if isinstance(node, (exp.Query, exp.Subquery)) or _fingerprint(node) in keys: + return False + if isinstance(node, exp.AggFunc) and not isinstance(node.parent, exp.Window): + return False + if isinstance(node, exp.Window): + return False + if isinstance(node, exp.Column): + return not node.is_star + return any(loose(c) for c in node.iter_expressions()) + + clauses = [c for c in (*select.expressions, select.args.get("having"), select.args.get("order")) if c is not None] + return any(loose(c) for c in clauses) + + +# ---- 动来源的改写 + +def _unwrap_passthrough(tree: Any, read: str | None) -> Any: + """原样选出 FROM 里唯一一个子查询的全部列、什么别的都不做的一段(SELECT * FROM (…) t、SELECT t.a, t.b FROM (…) t), + 换成那个子查询:出的行一样,列的先后也一样。最外层和条件里的子查询列名不影响取值,列名可以不同;别的地方(FROM 里 + 的子查询、CTE)外面按名字引用,名字也得一样。不换的:子查询自己带 ORDER BY / LIMIT / OFFSET(套一层之后顺序、取行 + 不一定照旧);子查询是 UNION 这类合并、库是 SQLite(和不知道是哪种库时)——SQLite 从子查询里读合并的结果时,按第一支 + 那一列的类型亲和转换别的支的值(REAL 列在前,另一支的整数 2 读出来成了 2.0),套不套这一层结果不一样。""" + from sqlglot import exp + + def outputs(query: Any) -> list[str] | None: + if any(query.args.get(k) is not None for k in ("order", "limit", "offset", "fetch")): + return None + if isinstance(query, exp.SetOperation): + if read in (None, "sqlite"): + return None + leaves = _leaves(query) + return outputs(leaves[0]) if leaves and isinstance(leaves[0], exp.Select) else None + if not isinstance(query, exp.Select) or _starry(query) or query.args.get("with_") is not None: + return None + names = [p.alias_or_name for p in query.expressions] + return names if all(names) and len(set(names)) == len(names) else None + + changed = True + while changed: + changed = False + for select in list(tree.find_all(exp.Select)): + if select.args.get("joins") or select.args.get("from_") is None or \ + {k for k, v in select.args.items() if v} - {"expressions", "from_", "with_"} or \ + (select.args.get("with_") is not None and select is not tree): + continue + item = select.args["from_"].this + if not isinstance(item, exp.Subquery): + continue + body = item.this + names = outputs(body) + if names is None: + continue + alias = item.alias_or_name + star = len(select.expressions) == 1 and isinstance(select.expressions[0], exp.Star) + cols = [p.unalias() for p in select.expressions] + if not star and (len(cols) != len(names) or any( + not isinstance(c, exp.Column) or c.is_star or c.name != n or (c.table and c.table != alias) + for c, n in zip(cols, names))): + continue + renamed = not star and any(p.alias_or_name != n for p, n in zip(select.expressions, names)) + parent = select.parent + loose = select is tree or isinstance(parent, (exp.In, exp.Exists)) or ( + isinstance(parent, exp.Subquery) and not isinstance(parent.parent, (exp.From, exp.Join, exp.CTE))) + if renamed and not loose: + continue + new = body.copy() + if select is tree: + if select.args.get("with_") is not None: + if new.args.get("with_") is not None: + continue + new.set("with_", select.args["with_"]) + tree = new + else: + select.replace(new) + changed = True + break + return tree + + +def _bare_star(select: Any) -> bool: + from sqlglot import exp + + return any(isinstance(p, exp.Star) for p in select.expressions) + + +def _right_to_left(tree: Any, sens: _Sensitivity) -> None: + """只有两个来源的 a RIGHT JOIN b ON p 就是 b LEFT JOIN a ON p。选的是 * 时列的先后跟着变、结果看行出来的先后的 + 一段扫描的先后跟着变,不改。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + if len(joins) != 1 or str(joins[0].side or "").upper() != "RIGHT" or not _plain_join(joins[0]) or \ + joins[0].args.get("on") is None or _bare_star(select) or sens.of(select): + continue + first = select.args["from_"] + left, right = first.this, joins[0].this + first.set("this", right.copy()) + joins[0].set("this", left.copy()) + joins[0].set("side", "LEFT") + joins[0].set("kind", None) + joins[0].meta[_FLIPPED] = True + for join in tree.find_all(exp.Join): + if join.side and str(join.kind or "").upper() == "OUTER": + join.set("kind", None) + + +def _null_propagating(node: Any, alias: str, read: str | None) -> bool: + """alias 那一侧整行是空值时,node 一定是空值:那一侧的列,或者对它做比较、算术、取大小写、转类型。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, exp.Column): + return bool(node.table) and node.table == alias + if type(node).__name__ in _STRICT_UNARY: + return _null_propagating(node.this, alias, read) + if type(node).__name__ in _STRICT_BINARY or (isinstance(node, exp.DPipe) and read != "mysql"): + return _null_propagating(node.left, alias, read) or _null_propagating(node.right, alias, read) + return False + + +def _rejects(cond: Any, alias: str, read: str | None) -> bool: + """alias 那一侧整行是空值时 cond 不成立(是空值或假):比较、LIKE、IN、BETWEEN 的一边一定是空值,IS NOT NULL, + NOT 套着这样的比较,AND 里有一项不成立、OR 里每一项都不成立。IS NULL、COALESCE、CASE、<=> 之类不算。""" + from sqlglot import exp + + cond = _unparen(cond) + comparisons = (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.Like, exp.ILike) + if isinstance(cond, comparisons): + return _null_propagating(cond.left, alias, read) or _null_propagating(cond.right, alias, read) + if isinstance(cond, (exp.In, exp.Between)): + return _null_propagating(cond.this, alias, read) + if isinstance(cond, exp.Not): + inner = _unparen(cond.this) + if isinstance(inner, exp.Is) and isinstance(inner.expression, exp.Null): + return _null_propagating(inner.this, alias, read) + if isinstance(inner, comparisons): + return _null_propagating(inner.left, alias, read) or _null_propagating(inner.right, alias, read) + if isinstance(inner, (exp.In, exp.Between)): + return _null_propagating(inner.this, alias, read) + return False + if isinstance(cond, exp.And): + return _rejects(cond.left, alias, read) or _rejects(cond.right, alias, read) + if isinstance(cond, exp.Or): + return _rejects(cond.left, alias, read) and _rejects(cond.right, alias, read) + return False + + +def _reject_nulls(tree: Any, sens: _Sensitivity, read: str | None) -> None: + """LEFT JOIN 关联进来的那一侧,同一段 WHERE 里有把它整行为空的行筛掉的条件(见 _rejects):没关联上的行反正留 + 不下来,这次外连接就是内连接。有 RIGHT / FULL JOIN 的段、结果看行出来的先后的段不动。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + where = select.args.get("where") + if where is None or sens.of(select) or any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + conds = _conjuncts(where.this) + for join in joins: + if str(join.side or "").upper() != "LEFT" or not _plain_join(join): + continue + alias = join.this.alias_or_name + if alias and any(_rejects(c, alias, read) for c in conds): + join.set("side", None) + join.set("kind", None) + join.this.meta[_REJECTED] = "RIGHT JOIN" if join.meta.get(_FLIPPED) else "LEFT JOIN" + + +def _flatten_prefix(tree: Any, sens: _Sensitivity) -> None: + """开头那组内连接(FROM 和紧跟着的内连接、CROSS JOIN、逗号连接,到第一个外连接为止)的 ON 条件挪进 WHERE、写成 + 逗号连接:内连接的条件写在 ON 还是 WHERE 一样;后面的 LEFT JOIN 对左边每一行各自关联,左边先筛后筛一样。后面有 + RIGHT / FULL JOIN 的不动:没关联上的那一侧整行空值,挪到 WHERE 的条件会把它们筛掉(PostgreSQL、MySQL 里逗号还比 + JOIN 结合得松,a, b RIGHT JOIN c 是 a 乘上 b RIGHT JOIN c)。结果看行出来的先后的一段也不动(SQLite 不调换 CROSS + JOIN 两边、会调换逗号连接)。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + if not joins or select.args.get("from_") is None or sens.of(select) or \ + any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + moved: list[Any] = [] + for join in joins: + if not _inner(join): + break + on = join.args.get("on") + if on is not None: + moved += _conjuncts(on) + join.set("on", None) + join.set("kind", None) + join.meta.pop("sql_semantics_join", None) + if moved: + where = select.args.get("where") + select.set("where", exp.Where(this=_and_all((_conjuncts(where.this) if where else []) + moved))) + + +# ---- 和来源无关的改写 + +def _items_of(select: Any) -> list[Any]: + first = select.args.get("from_") + return ([first.this] if first is not None else []) + [j.this for j in select.args.get("joins") or []] + + +def _and_all(items: list[Any]) -> Any: + from sqlglot import exp + + if not items: + return None + return exp.and_(*items, copy=False) if len(items) > 1 else items[0] + + +def _plain_join(join: Any) -> bool: + return not (join.args.get("using") or join.args.get("method") or join.args.get("natural")) + + +def _comma(join: Any) -> bool: + """开头那组内连接里的一项:写成逗号连接、条件都在 WHERE 里(见 _flatten_prefix)。""" + return _plain_join(join) and not join.side and not join.args.get("kind") and join.args.get("on") is None + + +def _inner(join: Any) -> bool: + return _plain_join(join) and not join.side and str(join.kind or "").upper() in ("", "INNER", "CROSS") + + +def _constant(node: Any) -> bool: + from sqlglot import exp + + return isinstance(node, (exp.Literal, exp.Null, exp.Boolean)) or ( + isinstance(node, exp.Neg) and isinstance(node.this, exp.Literal)) + + +def _strip_asc(tree: Any) -> None: + from sqlglot import exp + + for o in tree.find_all(exp.Ordered): + if o.args.get("desc") is False: + o.set("desc", None) + + +def _normalize_negation(tree: Any) -> None: + """x IS NOT NULL、x NOT LIKE y 这类带 negate 的写法统一写成 NOT (x IS NULL)、NOT (x LIKE y):各方言解析出来的形状 + 不一样(PostgreSQL 的 IS NOT NULL 是 Is(negate=True),SQLite 的是 Not(Is)),意思一样。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.Predicate)): + if node.args.get("negate") is True: + node.set("negate", None) + copy = node.copy() + node.replace(exp.Not(this=exp.Paren(this=copy))) + + +def _case_forms(tree: Any, read: str | None) -> None: + """CASE … ELSE NULL END 的 ELSE NULL 去掉(不写 ELSE 就是空值);SQLite 的 IIF(c, a, b)、MySQL 的 IF(c, a, b) 写成 + CASE WHEN c THEN a ELSE b END(SQLite 文档:两者生成同样的字节码;MySQL 的 IF 在 c 为真时取 a,否则取 b)。""" + from sqlglot import exp + + if read in ("sqlite", "mysql"): + for node in list(tree.find_all(exp.If)): + if isinstance(node.parent, exp.Case) or node.args.get("true") is None: + continue + case = exp.Case(ifs=[exp.If(this=node.this.copy(), true=node.args["true"].copy())]) + if node.args.get("false") is not None: + case.set("default", node.args["false"].copy()) + node.replace(case) + for case in tree.find_all(exp.Case): + if isinstance(case.args.get("default"), exp.Null): + case.set("default", None) + + +def _drop_true_conjuncts(tree: Any) -> None: + """WHERE、HAVING、ON 里恒为真的项(TRUE、1 = 1 这种同一个常量和自己相等)去掉:x AND TRUE 就是 x。""" + from sqlglot import exp + + def true(node: Any) -> bool: + node = _unparen(node) + if isinstance(node, exp.Boolean): + return bool(node.this) + if isinstance(node, (exp.EQ, exp.GTE, exp.LTE)): + left, right = _unparen(node.this), _unparen(node.expression) + return isinstance(left, exp.Literal) and isinstance(right, exp.Literal) and \ + _fingerprint(left) == _fingerprint(right) + return False + + holders = [(w, "this") for w in tree.find_all(exp.Where, exp.Having)] + \ + [(j, "on") for j in tree.find_all(exp.Join) if j.args.get("on") is not None] + for holder, key in holders: + cond = holder.args.get(key) + parts = _conjuncts(cond) + kept = [p for p in parts if not true(p)] + if len(kept) == len(parts): + continue + if kept: + holder.set(key, _and_all([p.copy() for p in kept])) + elif isinstance(holder, exp.Join): + holder.set(key, exp.true()) + else: + holder.parent.set(holder.arg_key, None) + + +def _count_star(tree: Any) -> None: + """COUNT(1)、COUNT('x'):数的是不为空的常量,就是行数。COUNT(DISTINCT 1)(有行就是 1)、COUNT(NULL) 不是。""" + from sqlglot import exp + + for c in list(tree.find_all(exp.Count)): + if isinstance(c.this, exp.Literal) and not c.args.get("distinct"): + c.set("this", exp.Star()) + + +def _minmax_distinct(tree: Any) -> None: + """MIN(DISTINCT x)、MAX(DISTINCT x) 就是 MIN(x)、MAX(x):去不去重最小最大的都是那个值。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.Min, exp.Max)): + arg = node.this + if not node.expressions and isinstance(arg, exp.Distinct) and len(arg.expressions) == 1 and \ + not arg.args.get("on"): + node.set("this", arg.expressions[0]) + + +def _simple_value(node: Any) -> bool: + """只是列、常量、* 和它们的加减乘除(没有函数调用:PostgreSQL 里 SELECT 列表的集合返回函数会改变行数)。""" + from sqlglot import exp + + return all(isinstance(n, (exp.Column, exp.Literal, exp.Null, exp.Boolean, exp.Star, exp.Identifier, exp.Paren, + exp.Neg, exp.Add, exp.Sub, exp.Mul, exp.Div, exp.Alias)) for n in node.walk()) + + +def _exists_subqueries(tree: Any) -> None: + """EXISTS 只看有没有行:选什么、去不去重、怎么排都不影响,统一写成 SELECT 1;只取前 n 行(n ≥ 1、不跳过行)也 + 不影响有没有行。动之前先确认这一段不是聚合查询(SELECT、HAVING、ORDER BY 里有聚合函数时总出一行,ORDER BY + COUNT(*) 去掉就成了普通查询)、不跳过行(DISTINCT 在 OFFSET 之前算,去掉 DISTINCT 行数就变了)、选的只是列和 + 常量(PostgreSQL 的集合返回函数会改变行数)。""" + from sqlglot import exp + + for ex in list(tree.find_all(exp.Exists)): + sub = ex.this.this if isinstance(ex.this, exp.Subquery) else ex.this + if not isinstance(sub, exp.Select) or _aggregating(sub) and sub.args.get("group") is None: + continue + if sub.args.get("having") is not None or sub.args.get("offset") is not None or \ + sub.args.get("fetch") is not None: + continue + if any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) for n in _block_nodes(sub)): + continue + distinct = sub.args.get("distinct") + if distinct is not None and distinct.args.get("on"): + continue + limit = sub.args.get("limit") + if limit is not None: + value = limit.expression + if not (isinstance(value, exp.Literal) and value.is_int and int(value.this) >= 1): + continue + if not all(_simple_value(p) for p in sub.expressions): + continue + sub.set("limit", None) + sub.set("distinct", None) + sub.set("order", None) + sub.set("expressions", [exp.Literal.number(1)]) + + +def _in_lists(tree: Any) -> None: + """IN 列表是个集合:一模一样的项去掉、排序;只有一个常量的写成 =(NOT IN 跟着成了 <>,空值的结果一样)。只有 + 一项、又不是常量的不改:SQLite 里 a IN (b) 是 a = +b,右边那一列不带类型亲和,和 a = b 结果不同。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.In)): + if not node.expressions or node.args.get("query") or node.args.get("unnest") or node.args.get("field"): + continue + uniq = {_fingerprint(i): i for i in node.expressions} + ordered = [uniq[k] for k in sorted(uniq)] + if len(ordered) == 1 and _constant(ordered[0]): + node.replace(exp.EQ(this=node.this, expression=ordered[0])) + else: + node.set("expressions", ordered) + + +def _expand_between(tree: Any) -> None: + """x BETWEEN a AND b 就是 x >= a AND x <= b(SQLite 文档原话,PostgreSQL、MySQL 也是)。x 里有子查询、不确定的 + 函数的不改(BETWEEN 只算一次 x)。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.Between)): + x = node.this + if node.args.get("symmetric") or x is None or x.find(exp.Query, exp.Subquery) is not None or _volatile(x): + continue + pair = exp.And(this=exp.GTE(this=x.copy(), expression=node.args["low"].copy()), + expression=exp.LTE(this=x.copy(), expression=node.args["high"].copy())) + node.replace(exp.Paren(this=pair)) + + +def _predicate(node: Any) -> bool: + """取值只会是真、假、空值的条件(比较、IS、IN、LIKE、EXISTS、BETWEEN,和它们的 AND / OR / NOT)。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.Predicate, exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.Is, exp.In, + exp.Like, exp.ILike, exp.Exists, exp.Between)): + return True + if isinstance(node, (exp.And, exp.Or)): + return _predicate(node.left) and _predicate(node.right) + if isinstance(node, exp.Not): + return _predicate(node.this) + return False + + +def _fold_not(tree: Any) -> None: + """NOT 套比较写成反过来的比较(NOT a = b 就是 a <> b,NOT a > b 就是 a <= b:两边有空值时都是空值);NOT NOT p 写成 p + (p 是条件时;NOT NOT 5 是 1,不是 5)。""" + from sqlglot import exp + + changed = True + while changed: + changed = False + for node in list(tree.find_all(exp.Not)): + if node.parent is None and node is not tree: + continue + inner = _unparen(node.this) + name = type(inner).__name__ + if name in _NEGATE and not isinstance(inner.this, (exp.Any, exp.All)) and \ + not isinstance(inner.expression, (exp.Any, exp.All)): + node.replace(getattr(exp, _NEGATE[name])(this=inner.this, expression=inner.expression)) + changed = True + elif isinstance(inner, exp.Not) and _predicate(inner.this): + node.replace(inner.this) + changed = True + if changed: + break #: 括号里是这些(运算符的一部分)时,外面是比较运算的话括号可以去掉:算术比比较结合得紧 @@ -2708,8 +3885,9 @@ _COMPARISON = ("EQ", "NEQ", "GT", "GTE", "LT", "LTE") def _strip_parens(tree: Any) -> None: - """不影响结合的括号去掉:括号外面不是运算符(函数参数、别名、SELECT 列表、ORDER BY、GROUP BY、WHERE 整个条件、 - IN 列表的一项……)、括号里是一列一个常量一次函数调用,或者括号里是算术、外面是比较。""" + """不影响结合的括号去掉(比较时本来就不看括号,见 _fingerprint;这里只为生成的写法好读):括号外面不是运算符 + (函数参数、别名、SELECT 列表、ORDER BY、GROUP BY、WHERE 整个条件、IN 列表的一项……)、括号里是一列一个常量一次 + 函数调用,或者括号里是算术、外面是比较。""" from sqlglot import exp for paren in list(tree.find_all(exp.Paren)): @@ -2737,547 +3915,6 @@ def _zero_offset(tree: Any) -> None: offset.pop() -def _redundant_distinct(tree: Any, read: str | None) -> None: - """已经按选出来的列分组(每个分组键都原样选出来了)的 SELECT DISTINCT:每组一行、各行的分组键都不同,去重 - 去不掉什么。""" - from sqlglot import exp - - for select in list(tree.find_all(exp.Select)): - distinct, group = select.args.get("distinct"), select.args.get("group") - if distinct is None or distinct.args.get("on") or group is None or not group.expressions or \ - any(group.args.get(k) for k in ("rollup", "cube", "grouping_sets")): - continue - if any(p.find(exp.Window) for p in select.expressions): - continue - selected = {_fingerprint(p.unalias()) for p in select.expressions} - if all(_fingerprint(k) in selected for k in group.expressions): - select.set("distinct", None) - - -def _narrow_stars(tree: Any, read: str | None) -> None: - """没给表结构时 SELECT * 展不开:从一张实表 SELECT * 的 CTE、FROM 里的子查询(不去重、不分组、不取行、没有窗口 - 函数),外面只按名字引用其中几列的,* 换成用到的那几列(外面有 SELECT * 或 t.* 读它的不换)。不去重时一段出几行 - 和选几列无关,外面用不到的列不影响结果;用到的列那张表里没有的话,原文本来就出错。""" - from sqlglot import exp - - namer = _Namer(tree, read, qualified=True) - ctes = {c.alias_or_name: c for c in tree.find_all(exp.CTE)} - starred: set[int] = set() - for select in tree.find_all(exp.Select): - for p in select.expressions: - if isinstance(p, exp.Star): - starred |= {id(i) for i in _items_of(select)} - elif isinstance(p, exp.Column) and p.is_star: - starred |= {id(i) for i in _items_of(select) if i.alias_or_name == p.table} - bodies: list[tuple[Any, list[Any]]] = [] - for cte in ctes.values(): - refs = [t for s in tree.find_all(exp.Select) for t in _items_of(s) - if isinstance(t, exp.Table) and namer.cte_of.get(id(t)) is cte] - bodies.append((cte.this, refs)) - for sub in tree.find_all(exp.Subquery): - if isinstance(sub.parent, (exp.From, exp.Join)): - bodies.append((sub.this, [sub])) - for body, refs in bodies: - if not isinstance(body, exp.Select) or len(body.expressions) != 1 or \ - not isinstance(body.expressions[0], exp.Star) or not _simple_body(body): - continue - items = _items_of(body) - if len(items) != 1 or not isinstance(items[0], exp.Table) or id(items[0]) in namer.cte_of or \ - any(id(r) in starred for r in refs): - continue - names = sorted({col.name for r in refs for col in namer.refs.get(id(r), [])}) - alias = items[0].alias_or_name - body.set("expressions", [exp.alias_(exp.column(n, table=alias), n, copy=False) for n in names] or - [exp.alias_(exp.Literal.number(1), "_one", copy=False)]) - - -def _unique_aliases(tree: Any, read: str | None) -> None: - """整条语句里每个来源(实表、CTE 的引用)的别名改成互不相同的(表名__序号),引用它的列跟着改:按原来的别名 - 认清每一列属于哪个来源之后再改(见 _Namer),相关子查询里引用外层的列不会认错。""" - from sqlglot import exp - - namer = _Namer(tree, read, qualified=True) - taken = {t.name for t in tree.find_all(exp.Table)} | {c.alias_or_name for c in tree.find_all(exp.CTE)} - n = 0 - for item in [i for s in tree.find_all(exp.Select) for i in _items_of(s)]: - n += 1 - name = f"{item.name if isinstance(item, exp.Table) else 'sub'}__{n}" - while name in taken: - name += "_" - alias = item.args.get("alias") - if isinstance(alias, exp.TableAlias): - alias.set("this", exp.to_identifier(name)) - else: - item.set("alias", exp.TableAlias(this=exp.to_identifier(name))) - for col in namer.refs.get(id(item), []): - col.set("table", exp.to_identifier(name)) - - -def _cte_refs(tree: Any, name: str) -> list[Any]: - """引用顶层 CTE name 的表(没有被里层同名的 CTE 遮住的)。""" - from sqlglot import exp - - out = [] - for table in tree.find_all(exp.Table): - if table.name != name or table.args.get("db") or not isinstance(table.this, exp.Identifier): - continue - shadowed = False - node = table.parent - while node is not None and node is not tree: - with_ = node.args.get("with_") if isinstance(node, exp.Query) else None - if with_ is not None and any(c.alias_or_name == name for c in with_.expressions): - shadowed = True - break - node = node.parent - if not shadowed: - out.append(table) - return out - - -def _simple_body(body: Any) -> bool: - """只是从来源里筛行、算列的一段(没有分组、聚合、去重、取行、窗口函数、UNION):在哪儿算、算几遍结果都一样。""" - from sqlglot import exp - - if not isinstance(body, exp.Select) or body.args.get("with_") is not None: - return False - if any(body.args.get(k) for k in ("group", "having", "distinct", "limit", "offset", "qualify", "windows")): - return False - return not any(isinstance(n, (exp.AggFunc, exp.Window)) for p in body.expressions for n in p.walk()) - - -def _split_simple_ctes(tree: Any) -> None: - """引用了好几次的「简单」CTE(见 _simple_body)每处引用各复制一份:merge_subqueries 只并只引用一次的 CTE, - 复制之后每处都能并进去。CTE 是确定的(有不确定函数的语句不走这一级),算几遍结果一样。并不进去的几份定义 - 一模一样,起名时又合回一个(见 _Namer._name_ctes)。""" - from sqlglot import exp - - with_ = tree.args.get("with_") - if with_ is None: - return - taken = {c.alias_or_name for c in with_.expressions} | {t.name for t in tree.find_all(exp.Table)} - for cte in list(with_.expressions): - name = cte.alias_or_name - refs = _cte_refs(tree, name) - if len(refs) < 2 or cte.args["alias"].columns or not _simple_body(cte.this): - continue - position = with_.expressions.index(cte) - for i, ref in enumerate(refs[1:], 2): - copy_name = f"{name}_{i}" - while copy_name in taken: - copy_name += "_" - taken.add(copy_name) - twin = exp.CTE(this=cte.this.copy(), alias=exp.TableAlias(this=exp.to_identifier(copy_name))) - with_.expressions.insert(position + i - 1, twin) - twin.parent = with_ - alias = ref.alias_or_name - ref.set("this", exp.to_identifier(copy_name)) - if not ref.alias: - ref.set("alias", exp.TableAlias(this=exp.to_identifier(alias))) - with_.set("expressions", list(with_.expressions)) - - -def _push_into_groups(tree: Any) -> None: - """外层 WHERE 里只涉及一个分组 CTE(只引用一次、在开头那组内连接里)的条件,换成 CTE 里对应的表达式,挪进它的 - HAVING:每一行就是一组,筛行和筛组是一回事。CTE 有窗口函数、取行、去重、没写成分组键也没聚合的列时不挪。 - 然后(各段都做)HAVING 里只涉及分组键、不含聚合的条件挪进 WHERE:同一组的分组键都一样,先筛后筛一样。两步 - 之后,这类条件写在里层 WHERE、HAVING 还是外层 WHERE 都落到同一处。""" - from sqlglot import exp - - with_ = tree.args.get("with_") - ctes = {c.alias_or_name: c for c in (with_.expressions if with_ is not None else [])} - for select in list(tree.find_all(exp.Select)): - where = select.args.get("where") - joins = select.args.get("joins") or [] - if where is None or select.args.get("from_") is None or any(not _inner(j) for j in joins): - continue - for item in _items_of(select): - cte = ctes.get(item.name) if isinstance(item, exp.Table) and not item.args.get("db") else None - if cte is None or len(_cte_refs(tree, item.name)) != 1: - continue - body = cte.this - if not isinstance(body, exp.Select) or body.args.get("group") is None or \ - any(body.args.get(k) for k in ("limit", "offset", "distinct", "qualify", "windows")) or \ - any(n.find(exp.Window) for n in body.expressions): - continue - outputs = {p.alias_or_name: p.unalias() for p in body.expressions} - keys = {_fingerprint(k) for k in body.args["group"].expressions} - alias = item.alias_or_name - moved, kept = [], [] - for cond in _conjuncts(where.this): - cols = list(cond.find_all(exp.Column)) - if not cols or cond.find(exp.Select, exp.Subquery) is not None or \ - any(c.table != alias or c.name not in outputs for c in cols) or \ - not all(_grouped(outputs[c.name], keys) for c in cols): - kept.append(cond) - continue - new = cond.copy() - for col in list(new.find_all(exp.Column)): - col.replace(exp.paren(outputs[col.name].copy(), copy=False)) - moved.append(new) - if moved: - having = body.args.get("having") - body.set("having", exp.Having(this=_and_all(([having.this] if having else []) + moved))) - select.set("where", exp.Where(this=_and_all(kept)) if kept else None) - where = select.args.get("where") - if where is None: - break - for select in list(tree.find_all(exp.Select)): - having, group = select.args.get("having"), select.args.get("group") - if having is None or group is None or any(group.args.get(k) for k in ("rollup", "cube", "grouping_sets")): - continue - keys = {_fingerprint(k) for k in group.expressions if isinstance(k, exp.Column)} - moved, kept = [], [] - for cond in _conjuncts(having.this): - cols = list(cond.find_all(exp.Column)) - if cols and all(_fingerprint(c) in keys for c in cols) and not any( - isinstance(n, (exp.AggFunc, exp.Window, exp.Select, exp.Subquery)) for n in cond.walk()): - moved.append(cond) - else: - kept.append(cond) - if moved: - where = select.args.get("where") - select.set("where", exp.Where(this=_and_all(([where.this] if where else []) + moved))) - select.set("having", exp.Having(this=_and_all(kept)) if kept else None) - - -def _grouped(expression: Any, keys: set[str]) -> bool: - """分组 CTE 选出来的一列在每组里只有一个值:整个是分组键,或者每一列都在聚合函数里、或者本身是分组键。""" - from sqlglot import exp - - if _fingerprint(expression) in keys: - return True - for col in expression.find_all(exp.Column): - if _fingerprint(col) not in keys and col.find_ancestor(exp.AggFunc) is None: - return False - return not any(isinstance(n, exp.Window) for n in expression.walk()) - - -def _unwrap_passthrough(tree: Any, read: str | None) -> Any: - """原样选出一个子查询(FROM 里的子查询,或只引用一次的 CTE)全部列、什么别的都不做的一段(SELECT t.a, t.b - FROM (…) t),换成那个子查询:出的行一样,列的先后、名字也一样。最外层和条件里的子查询列名不影响取值,列名 - 可以不同;别的地方(FROM 里的子查询、CTE)外面按名字引用,名字也得一样。 - - 子查询是 UNION 这类合并的,SQLite(和不知道是哪种库时)不换:SQLite 从子查询里读合并的结果时,按第一支那一列的 - 类型亲和转换别的支的值(REAL 列在前,另一支的整数 2 读出来成了 2.0,像数字的文本也会被转成数),套不套这一层 - 结果不一样。""" - from sqlglot import exp - - def outputs(query: Any) -> list[str] | None: - if isinstance(query, exp.SetOperation) and read not in (None, "sqlite"): - leaves = _leaves(query) - return outputs(leaves[0]) if leaves else None - if not isinstance(query, exp.Select) or _starry(query): - return None - names = [p.alias_or_name for p in query.expressions] - return names if all(names) and len(set(names)) == len(names) else None - - changed = True - while changed: - changed = False - for select in list(tree.find_all(exp.Select)): - if select.args.get("joins") or select.args.get("from_") is None or any( - select.args.get(k) for k in ("where", "group", "having", "order", "limit", "offset", "distinct", - "qualify", "windows", "hint")): - continue - item = select.args["from_"].this - with_ = tree.args.get("with_") - cte = None - if isinstance(item, exp.Subquery): - body = item.this - elif isinstance(item, exp.Table) and with_ is not None and not item.args.get("db"): - cte = next((c for c in with_.expressions if c.alias_or_name == item.name), None) - if cte is None or len(_cte_refs(tree, item.name)) != 1 or select.find_ancestor(exp.CTE) is cte: - continue - body = cte.this - else: - continue - names = outputs(body) - alias = item.alias_or_name - cols = [p.unalias() for p in select.expressions] - if names is None or len(cols) != len(names) or body.args.get("with_") is not None or any( - not isinstance(c, exp.Column) or c.name != n or (c.table and c.table != alias) - for c, n in zip(cols, names)): - continue - renamed = any(p.alias_or_name != n for p, n in zip(select.expressions, names)) - parent = select.parent - loose = select is tree or isinstance(parent, (exp.In, exp.Exists)) or ( - isinstance(parent, exp.Subquery) and not isinstance(parent.parent, (exp.From, exp.Join, exp.CTE))) - if renamed and not loose: - continue - new = body.copy() - if select is tree: - if select.args.get("with_") is not None: - new.set("with_", select.args["with_"]) - tree = new - else: - select.replace(new) - if cte is not None: - cte.pop() - with_ = tree.args.get("with_") - if with_ is not None and not with_.expressions: - tree.set("with_", None) - changed = True - break - return tree - - -# ---- 和名字无关的改写 - -def _items_of(select: Any) -> list[Any]: - first = select.args.get("from_") - return ([first.this] if first is not None else []) + [j.this for j in select.args.get("joins") or []] - - -def _and_all(items: list[Any]) -> Any: - from sqlglot import exp - - if not items: - return None - return exp.and_(*items, copy=False) if len(items) > 1 else items[0] - - -def _plain_join(join: Any) -> bool: - return not (join.args.get("using") or join.args.get("method") or join.args.get("natural")) - - -def _comma(join: Any) -> bool: - """开头那组内连接里的一项:逗号连接(或者没有条件的 CROSS JOIN,sqlglot 把逗号连接也读成 CROSS),条件都在 WHERE - 里(见 _flatten_inner)。规范形式里写成逗号连接:SQLite 不会调换 CROSS JOIN 的先后,按它执行规范形式会很慢。""" - return _plain_join(join) and not join.side and str(join.kind or "").upper() in ("", "CROSS") and \ - join.args.get("on") is None - - -def _inner(join: Any) -> bool: - return _plain_join(join) and not join.side and str(join.kind or "").upper() in ("", "INNER", "CROSS") - - -def _constant(node: Any) -> bool: - from sqlglot import exp - - return isinstance(node, (exp.Literal, exp.Null, exp.Boolean)) or ( - isinstance(node, exp.Neg) and isinstance(node.this, exp.Literal)) - - -def _strip_asc(tree: Any) -> None: - from sqlglot import exp - - for o in tree.find_all(exp.Ordered): - if o.args.get("desc") is False: - o.set("desc", None) - - -def _count_star(tree: Any) -> None: - """COUNT(1)、COUNT('x'):数的是不为空的常量,就是行数。COUNT(DISTINCT 1) 不是(有行就是 1)。""" - from sqlglot import exp - - for c in list(tree.find_all(exp.Count)): - if isinstance(c.this, exp.Literal) and not c.args.get("distinct"): - c.set("this", exp.Star()) - - -def _in_lists(tree: Any, read: str | None) -> None: - """IN 列表是个集合:去重、按写法排序;只有一项的写成 =(NOT IN 跟着成了 <>,空值的结果一样)。""" - from sqlglot import exp - - for node in list(tree.find_all(exp.In)): - if not node.expressions or node.args.get("query") or node.args.get("unnest") or node.args.get("field"): - continue - uniq = {_key(i, read): i for i in node.expressions} - ordered = [uniq[k] for k in sorted(uniq)] - if len(ordered) == 1: - node.replace(exp.EQ(this=node.this, expression=ordered[0])) - else: - node.set("expressions", ordered) - - -def _aggregate_free(select: Any) -> bool: - from sqlglot import exp - - return not any(isinstance(n, exp.AggFunc) and not n.find_ancestor(exp.Window) - for p in select.expressions for n in p.walk()) - - -def _exists_projection(tree: Any) -> None: - """EXISTS 只看有没有行:选什么、去不去重、怎么排(取行也只看行数)都不影响,统一写成 SELECT 1。没有 GROUP BY - 却有聚合(总出一行)的、有 HAVING 的(HAVING、GROUP BY 1 可能引用选出来的列)不动。""" - from sqlglot import exp - - for ex in list(tree.find_all(exp.Exists)): - sub = ex.this.this if isinstance(ex.this, exp.Subquery) else ex.this - if not isinstance(sub, exp.Select) or sub.args.get("having") or \ - (not sub.args.get("group") and not _aggregate_free(sub)): - continue - distinct = sub.args.get("distinct") - if distinct is not None and not distinct.args.get("on"): - sub.set("distinct", None) - sub.set("expressions", [exp.Literal.number(1)]) - sub.set("order", None) - - -def _in_subquery_distinct(tree: Any) -> None: - """IN (子查询) 只看取值的集合:里面的 DISTINCT、不取行时的 ORDER BY 不影响(NOT IN 也一样,空值还在)。""" - from sqlglot import exp - - for node in list(tree.find_all(exp.In)): - query = node.args.get("query") - sub = query.this if isinstance(query, exp.Subquery) else query - if not isinstance(sub, exp.Select) or any(sub.args.get(k) for k in ("limit", "offset", "fetch")): - continue - sub.set("order", None) - distinct = sub.args.get("distinct") - if distinct is not None and not distinct.args.get("on") and not sub.args.get("group") and _aggregate_free(sub): - sub.set("distinct", None) - - -def _distinct_to_group(tree: Any, read: str | None) -> None: - """SELECT DISTINCT a, b(没有分组、聚合、窗口函数)和 GROUP BY a, b 出的行一样(空值也算一组)。ORDER BY 用到 - 没选出来的列时不改(两种写法对这种排序的规矩不一样);没有 ORDER BY 的 LIMIT 不改(DISTINCT 按扫到的先后出行, - GROUP BY 按分组键排好了出,取到的行不同)。""" - from sqlglot import exp - - for select in list(tree.find_all(exp.Select)): - distinct = select.args.get("distinct") - if distinct is None or distinct.args.get("on") or select.args.get("group") or select.args.get("having"): - continue - if _starry(select) or _takes_any(select) or not _aggregate_free(select) or \ - any(p.find(exp.Window, exp.Select, exp.Subquery) for p in select.expressions): - continue - keys = [p.unalias().copy() for p in select.expressions] - if any(_constant(k) for k in keys): - continue - order = select.args.get("order") - if order is not None: - allowed = {_fingerprint(k) for k in keys} - aliases = {p.alias for p in select.expressions if p.alias} - if any(_fingerprint(o.this) not in allowed and not ( - isinstance(o.this, exp.Column) and not o.this.table and o.this.name in aliases) - for o in order.expressions): - continue - group = exp.Group(expressions=keys) - group.meta[_FROM_DISTINCT] = True - select.set("distinct", None) - select.set("group", group) - - -def _right_to_left(tree: Any) -> None: - """只有两个来源的 a RIGHT JOIN b ON p 就是 b LEFT JOIN a ON p。选的是 * 时列的先后跟着变、没有 ORDER BY 的 LIMIT - 取先扫到的几行,不改。""" - from sqlglot import exp - - for select in list(tree.find_all(exp.Select)): - joins = select.args.get("joins") or [] - if len(joins) != 1 or str(joins[0].side or "").upper() != "RIGHT" or not _plain_join(joins[0]) or \ - _starry(select) or _takes_any(select): - continue - first = select.args["from_"] - left, right = first.this, joins[0].this - first.set("this", right.copy()) - joins[0].set("this", left.copy()) - joins[0].set("side", "LEFT") - joins[0].set("kind", None) - joins[0].meta[_FLIPPED] = True - for join in tree.find_all(exp.Join): - if join.side and str(join.kind or "").upper() == "OUTER": - join.set("kind", None) - - -def _null_propagating(node: Any, alias: str) -> bool: - """alias 那一侧整行是空值时,node 一定是空值:那一侧的列,或者对它做比较、算术、取大小写、转类型。""" - from sqlglot import exp - - if isinstance(node, exp.Column): - return bool(node.table) and node.table == alias - if type(node).__name__ in _STRICT_UNARY: - return _null_propagating(node.this, alias) - if type(node).__name__ in _STRICT_BINARY: - return _null_propagating(node.left, alias) or _null_propagating(node.right, alias) - return False - - -def _rejects(cond: Any, alias: str) -> bool: - """alias 那一侧整行是空值时 cond 不成立(是空值或假):比较、LIKE、IN、BETWEEN 的一边一定是空值,IS NOT NULL, - 或者 AND 里有一项不成立、OR 里每一项都不成立。IS NULL、COALESCE、CASE 不算。""" - from sqlglot import exp - - if isinstance(cond, exp.Paren): - return _rejects(cond.this, alias) - if isinstance(cond, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.Like, exp.ILike)): - return _null_propagating(cond.left, alias) or _null_propagating(cond.right, alias) - if isinstance(cond, (exp.In, exp.Between)): - return _null_propagating(cond.this, alias) - if isinstance(cond, exp.Not): - inner = cond.this.unnest() if isinstance(cond.this, exp.Paren) else cond.this - if isinstance(inner, exp.Is) and isinstance(inner.expression, exp.Null): - return _null_propagating(inner.this, alias) - if isinstance(inner, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.Like, exp.ILike)): - return _null_propagating(inner.left, alias) or _null_propagating(inner.right, alias) - return False - if isinstance(cond, exp.And): - return _rejects(cond.left, alias) or _rejects(cond.right, alias) - if isinstance(cond, exp.Or): - return _rejects(cond.left, alias) and _rejects(cond.right, alias) - return False - - -def _reject_nulls(tree: Any) -> None: - """LEFT JOIN 关联进来的那一侧,同一段 WHERE 里有把它整行为空的行筛掉的条件(见 _rejects):没关联上的行反正留 - 不下来,这次外连接就是内连接。有 RIGHT / FULL JOIN 的段不动。""" - from sqlglot import exp - - for select in list(tree.find_all(exp.Select)): - joins = select.args.get("joins") or [] - where = select.args.get("where") - if where is None or any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): - continue - conds = _conjuncts(where.this) - for join in joins: - if str(join.side or "").upper() != "LEFT" or not _plain_join(join): - continue - alias = join.this.alias_or_name - if alias and any(_rejects(c, alias) for c in conds): - join.set("side", None) - join.set("kind", None) - join.this.meta[_REJECTED] = "RIGHT JOIN" if join.meta.get(_FLIPPED) else "LEFT JOIN" - - -def _flatten_inner(tree: Any) -> None: - """内连接的 ON 条件挪进 WHERE:开头连续的内连接(和后面只引用它们的内连接)是一组来源,写成逗号连接,条件写在 - ON 还是 WHERE 都一样,来源的先后也不影响结果(后面的 LEFT JOIN 只保留左边的行,左边先筛后筛一样)。有 RIGHT / - FULL JOIN、USING、NATURAL 的段不动;引用了 LEFT JOIN 进来的表的内连接留在原处。""" - from sqlglot import exp - - for select in list(tree.find_all(exp.Select)): - joins = select.args.get("joins") or [] - if not joins or select.args.get("from_") is None: - continue - if any(not _plain_join(j) or str(j.side or "").upper() in ("RIGHT", "FULL") or - str(j.kind or "").upper() in ("SEMI", "ANTI", "ASOF") for j in joins): - continue - prefix = [select.args["from_"].this] - lefts: list[Any] = [] - tail: list[Any] = [] - moved: list[Any] = [] - for join in joins: - if tail: - tail.append(join) - continue - if _inner(join): - on = join.args.get("on") - refs = {c.table for c in (on.find_all(exp.Column) if on is not None else []) if c.table} - if refs & {j.this.alias_or_name for j in lefts}: - tail.append(join) - continue - prefix.append(join.this) - moved += _conjuncts(on) if on is not None else [] - else: - lefts.append(join) - select.args["from_"].set("this", prefix[0]) - select.set("joins", [exp.Join(this=s) for s in prefix[1:]] + lefts + tail) - if moved: - where = select.args.get("where") - select.set("where", exp.Where(this=_and_all((_conjuncts(where.this) if where else []) + moved))) - - def _order_by_aliases(tree: Any) -> None: """最外层 ORDER BY 引用 SELECT 的别名的,换成那个表达式:别名接下来要去掉。不换的:别名指的是数字常量(ORDER BY 1 是按第一列排);别名指的是一个不带表名、又和某个别名同名的列(ORDER BY 里单独一个名字先认别名:SELECT user_id AS @@ -3309,7 +3946,8 @@ def _drop_aliases(query: Any) -> None: def _drop_output_aliases(tree: Any) -> None: - """最外层(不是 UNION 时)和条件里的子查询选出来的列名不影响任何取值,去掉;本段里还按名字引用的留着。""" + """最外层(UNION 不按名字排序时各支都是)和条件里的子查询选出来的列名不影响任何取值,去掉;本段里还按名字引用 + 的留着。""" from sqlglot import exp if isinstance(tree, exp.SetOperation): @@ -3378,53 +4016,56 @@ def _nested_queries(select: Any) -> list[Any]: return out -# ---- 化简:sqlglot 的 simplify 去掉和类型、排序规则、方言有关的几种 - -def _simplify(tree: Any, read: str | None) -> Any: - """sqlglot 的 simplify(AND / OR 去重排序、吸收律、德摩根、BETWEEN 拆开、常量折叠、比较的方向……),去掉三种: - - - 同一列上的几个比较合成一个(x > 5 AND x > 3 → x > 5)、字符串常量之间的比较:结果取决于列的类型和排序规则 - (SQLite 里 TEXT 列和数字比是按文本比,PostgreSQL 的语言排序规则下 'a' < 'B'); - - 等式两边移项(x + 1 = 3 → x = 2):TEXT 列在 SQLite 里不成立; - - DATE_TRUNC 比较改成区间:取决于方言的日期语义。 - - 另外两边都写了 COLLATE 的比较不对调两边(SQLite 用左边那个排序规则)。 - """ +def _starry(select: Any) -> bool: from sqlglot import exp - from sqlglot.optimizer.simplify import Simplifier - class _Safe(Simplifier): - def _simplify_comparison(self, expression, left, right, or_=False): # noqa: ANN001 - return None + return any(isinstance(p, exp.Star) or (isinstance(p, exp.Column) and p.is_star) for p in select.expressions) - def sort_comparison(self, expression): # noqa: ANN001 - if isinstance(expression, exp.Binary) and expression.this.find(exp.Collate) is not None and \ - expression.expression.find(exp.Collate) is not None: - return expression - return super().sort_comparison(expression) - def simplify_equality(self, expression): # noqa: ANN001 - return expression +def _alias_or_column(tree: Any, read: str | None, schema: dict[str, dict[str, str]] | None) -> bool: + """有不带表名、又和这一段的列别名同名的名字,qualify 会把它当成别名展开,库里认的却可能是表的列,这一遍不认列: - def simplify_datetrunc(self, expression): # noqa: ANN001 - return expression + - WHERE、GROUP BY、HAVING 里:库里认的是列还是别名各方言不一样(SQLite 的 GROUP BY、HAVING 都先认列:HAVING + amount > 10 里的 amount 是组里任意一行的 amount,不是 SUM(amount) AS amount),qualify 却一律当成别名(GROUP BY + created_at 成了按 DATE(created_at) 分组)。表结构里有这一段的每张表、这个名字又不是其中哪张表的列时,它只能是 + 别名,照常认。 + - SELECT 列表里引用前面列的别名:SQLite、PostgreSQL、MySQL 都不认,名字指的是表的列(SELECT user_id AS amount, + amount * 2 里的 amount 是表的 amount 列),不是表的列就报错;qualify 没有表结构时却当成别名展开成 user_id * 2, + 有表结构时不是表的列的也展开。表结构里有这一段的每张表、这个名字又是其中的列时,qualify 先认列,照常认。""" + from sqlglot import exp - def _simplify_binary(self, expression, a, b): # noqa: ANN001 - if isinstance(expression, self.COMPARISONS) and not isinstance(expression, exp.Is) and \ - not (a.is_number and b.is_number): - return None - return super()._simplify_binary(expression, a, b) - - return _Safe(dialect=read).simplify(tree) + columns = {name.lower(): {c.lower() for c in cols} for name, cols in (schema or {}).items()} + for select in tree.find_all(exp.Select): + aliases = {p.alias.lower() for p in select.expressions if isinstance(p, exp.Alias) and p.alias} + if not aliases: + continue + items = _items_of(select) + known = all(isinstance(i, exp.Table) and i.name.lower() in columns for i in items) + real = set().union(*(columns[i.name.lower()] for i in items)) if known and items else set() + for key in ("where", "group", "having"): + node = select.args.get(key) + if node is None: + continue + names = {c.name.lower() for c in node.find_all(exp.Column) if not c.table and not _nested(c, node)} + if (names & aliases) - (set() if not known else (aliases - real)): + return True + earlier: set[str] = set() + for projection in select.expressions: + names = {c.name.lower() for c in projection.find_all(exp.Column) + if not c.table and (c is projection or not _nested(c, projection))} + if (names & earlier) - real: + return True + if isinstance(projection, exp.Alias) and projection.alias: + earlier.add(projection.alias.lower()) + return False # ---- 起规范的名字 @dataclass class _Hint: - """按另一边起名(差异说明用,见 _Explainer.statement):定义一样的 CTE 叫另一边的名字;names 是这一边原来的 - CTE 名 → 另一边的名字(两边各剩一个配不上的 CTE 时配成一对);taken 是另一边用过的名字,配不上的另起名字时 - 避开。""" + """按另一边起名(差异说明用):定义一样的 CTE 叫另一边的名字;names 是这一边原来的 CTE 名 → 另一边的名字;taken + 是另一边用过的名字,配不上的另起名字时避开。""" bodies: dict[str, str] = field(default_factory=dict) names: dict[str, str] = field(default_factory=dict) @@ -3434,67 +4075,42 @@ class _Hint: class _Namer: """给一条语句起规范的名字(见 _finish 第 3 步)。 - 先按原来的别名认清每一列属于哪个来源(相关子查询里的列到外层找),记成「来源 → 引用它的列」;之后改来源的别名 - 时把这些列一起改掉,不再按名字找,换名不会把列认错。别名带层数:内外层的别名不会同名,相关子查询里引用外层的 - 列不会被里层遮住;同一层的兄弟子查询各自看不见对方,可以重名。""" + 先按原来的别名认清每一列属于哪个来源(见 _Resolver);之后改来源的别名时把这些列一起改掉,不再按名字找,换名不会 + 把列认错。别名带层数:内外层的别名不会同名,相关子查询里引用外层的列不会被里层遮住;同一层的兄弟子查询各自看不见 + 对方,可以重名。""" - def __init__(self, tree: Any, read: str | None, qualified: bool, hint: _Hint | None = None) -> None: + def __init__(self, tree: Any, read: str | None, qualified: bool, sens: _Sensitivity, + hint: _Hint | None = None) -> None: from sqlglot import exp self.tree = tree self.read = read self.qualified = qualified + self.sens = sens self.hint = hint - self.refs: dict[int, list[Any]] = {} - self.cte_of: dict[int, Any] = {} + resolver = _Resolver(tree) + self.refs = resolver.refs + self.cte_of = resolver.cte_of self.base_names = {t.name.lower() for t in tree.find_all(exp.Table) if t.name} self.budget = _ORDERING_BUDGET - self._resolve(tree, [], {}) + self.starred = self._starred_sources(tree) + + def _starred_sources(self, tree: Any) -> set[int]: + """被 SELECT * 读着的来源(* 没展开):它选出来的列名、先后会原样传到外面,不能改名、重排。""" + from sqlglot import exp + + out: set[int] = set() + for select in tree.find_all(exp.Select): + for p in select.expressions: + if isinstance(p, exp.Star): + out |= {id(i) for i in _items_of(select)} + elif isinstance(p, exp.Column) and p.is_star: + out |= {id(i) for i in _items_of(select) if i.alias_or_name == p.table} + return out def _text(self, node: Any) -> tuple[str, str]: return _key(node, self.read) - def _resolve(self, query: Any, frames: list[dict[str, Any]], ctes: dict[str, Any]) -> None: - from sqlglot import exp - - if isinstance(query, exp.Subquery): - self._resolve(query.this, frames, ctes) - return - with_ = query.args.get("with_") - if with_ is not None: - ctes = dict(ctes) - for cte in with_.expressions: - self._resolve(cte.this, frames, ctes) - ctes[cte.alias_or_name] = cte - if isinstance(query, exp.SetOperation): - self._resolve(query.left, frames, ctes) - self._resolve(query.right, frames, ctes) - return - if not isinstance(query, exp.Select): - raise _Unsupported(type(query).__name__) - frame: dict[str, Any] = {} - for item in _items_of(query): - if isinstance(item, exp.Subquery): - self._resolve(item.this, frames, ctes) - elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier): - if not item.args.get("db") and item.name in ctes: - self.cte_of[id(item)] = ctes[item.name] - else: - raise _Unsupported(type(item).__name__) - name = item.alias_or_name - if not name or name in frame: - raise _Unsupported("来源没有名字或重名") - frame[name] = item - stack = frames + [frame] - for node in _block_nodes(query): - if isinstance(node, exp.Column) and node.table: - for f in reversed(stack): - if node.table in f: - self.refs.setdefault(id(f[node.table]), []).append(node) - break - for sub in _nested_queries(query): - self._resolve(sub, stack, ctes) - def _alias(self, base: str, depth: int, k: int) -> str: name = base if depth == 0 else f"{base}@{depth}" if k > 1: @@ -3537,7 +4153,8 @@ class _Namer: for cte in ready: original = cte.alias_or_name body = self.canon(cte.this, depth + 1 if depth else 0, cte_names) - if self.qualified and self._rename_columns(cte): + refs = [t for t in self.tree.find_all(exp.Table) if self.cte_of.get(id(t)) is cte] + if self._rename_outputs(cte.this, refs): body = self._text(cte.this) texts.append((body, original, cte)) texts.sort(key=lambda x: x[0]) @@ -3546,7 +4163,7 @@ class _Namer: if body in by_text: cte_names[id(cte)] = by_text[body] continue - name = (hint.names.get(original) or hint.bodies.get(body[1])) if hint else None + name = (hint.names.get(original) or hint.bodies.get(body[0])) if hint else None while name is None or name.lower() in self.base_names or name in by_text.values(): n += 1 name = f"{prefix}{n}" @@ -3558,36 +4175,34 @@ class _Namer: kept.append(cte) with_.set("expressions", kept) - def _rename_columns(self, cte: Any) -> bool: - """CTE 选出来的列按定义排序后叫 c1、c2…,引用它们的列跟着改:外面按名字引用,名字是各自起的(total 还是 - sum_amt),先后也不影响。CTE 自己写了列名、是 UNION、列名重复、本段里还按名字引用自己的列的不改。""" + def _rename_outputs(self, body: Any, holders: list[Any]) -> bool: + """(认列之后)CTE、FROM 里的子查询选出来的列改叫 c1、c2…,引用它们的列跟着改:外面按名字引用,名字是各自起的 + (total 还是 sum_amt)。没有被 SELECT * 原样读着(* 没展开时列名、先后会传到外面)的,先按定义排序:外面按名字 + 引用,先后不影响。UNION、选了 *、列名重复、本段里还按名字引用自己的列、GROUP BY / ORDER BY 按位置引用的不改。""" from sqlglot import exp - body = cte.this - if not isinstance(body, exp.Select) or cte.args["alias"].columns: + if not self.qualified or not isinstance(body, exp.Select) or any(id(h) in self.starred for h in holders): return False projections = body.expressions names = [p.alias_or_name for p in projections] - if not projections or any(not n or isinstance(p, exp.Star) for n, p in zip(names, projections)) or \ + if not projections or any(not n or _is_star(p) for n, p in zip(names, projections)) or \ len(set(names)) != len(names): return False inside = {id(n) for p in projections for n in p.walk()} if {n.name for n in _block_nodes(body) if isinstance(n, exp.Column) and not n.table and id(n) not in inside} & \ set(names): return False - # GROUP BY 2、ORDER BY 1 按位置引用选出来的列:重排之后就指到别的列上了 for clause in ("group", "order"): node = body.args.get(clause) keys = [k.this if isinstance(k, exp.Ordered) else k for k in (node.expressions if node is not None else [])] if any(k.is_number for k in keys): return False - refs = [t for t in self.tree.find_all(exp.Table) if self.cte_of.get(id(t)) is cte] ordered = sorted(projections, key=lambda p: self._text(p.unalias())) mapping = {p.alias_or_name: f"c{i}" for i, p in enumerate(ordered, 1)} body.set("expressions", [exp.alias_(p.unalias().copy(), mapping[p.alias_or_name], copy=False) for p in ordered]) - for table in refs: - for col in self.refs.get(id(table), []): + for holder in holders: + for col in self.refs.get(id(holder), []): if col.name in mapping: col.set("this", exp.to_identifier(mapping[col.name])) return True @@ -3620,35 +4235,40 @@ class _Namer: query.set("expression", ordered[-1]) def _reorderable(self, query: Any) -> bool: - """这一处合并的各支调换先后,结果(行、值、排序)一样不一样。只有 UNION ALL 可能一样: + """这一处合并的各支调换先后,结果(行、值)一样不一样。只认整条语句最外层、不排序不取行的 UNION ALL: - - UNION(去重)、INTERSECT 留先出现的那个值,相等不等于一样:SQLite 里 REAL 列的 2.0 和 INTEGER 列的 2、MySQL - 默认不分大小写的排序规则下的 'SH' 和 'sh'、PostgreSQL numeric 的 2 和 2.00;EXCEPT 本来就分左右。 - - 这一层自己带 LIMIT / OFFSET 的,取的是先出来的几行(没有 ORDER BY 时就看哪支在前)。 - - ORDER BY 里有按名字排的:合并结果的列名取第一支的,调换之后 ORDER BY id 排的可能是另一列。 - - SQLite(没说方言的也按它)合并结果的列的排序规则取最左一支那列声明的(排序、从子查询读出来再比较都用它), - 表结构里看不到列声明的排序规则:只排整条语句最外层、没有 ORDER BY 的。""" + - UNION(去重)、INTERSECT 留先出现的那个值,相等不等于一样(SQLite 里 REAL 列的 2.0 和 INTEGER 列的 2、MySQL + 默认不分大小写的 'SH' 和 'sh');EXCEPT 本来就分左右。 + - 带 LIMIT / OFFSET 的取的是先出来的几行;ORDER BY 按名字排的看第一支的列名;SQLite 给合并的结果排序时用最左 + 一支那列声明的排序规则。PostgreSQL、MySQL 按位置排序(ORDER BY 1)的可以排。 + - 套在 CTE、子查询里的:SQLite 从子查询读合并的结果时按第一支的类型亲和、排序规则转换别的支,不排;别的方言 + 只在各支选出来的列名、先后都一样(外面按列名引用,列名取第一支的)、外面不看行出来的先后(不去重、不取行, + 见 _Sensitivity)时排。""" from sqlglot import exp if not isinstance(query, exp.Union) or query.args.get("distinct"): return False - if query.args.get("limit") is not None or query.args.get("offset") is not None: + if any(query.args.get(k) is not None for k in ("limit", "offset", "fetch")): return False + sqlite = self.read in (None, "sqlite") order = query.args.get("order") - keys = [o.this if isinstance(o, exp.Ordered) else o for o in (order.expressions if order is not None else [])] - if any(not k.is_number for k in keys): + if order is not None and (sqlite or any(not (isinstance(o.this, exp.Literal) and o.this.is_int) + for o in order.expressions)): return False - if self.read in (None, "sqlite"): - if keys: - return False - top = query - while isinstance(top.parent, exp.Subquery): - top = top.parent - if top is not self.tree: - return False - return True + top = query + while isinstance(top.parent, exp.Subquery): + top = top.parent + if top is self.tree: + return True + if sqlite or order is not None: + return False + leaves = _leaves(query) + if any(not isinstance(leaf, exp.Select) or self.sens.of(leaf) or _starry(leaf) for leaf in leaves): + return False + names = [[p.alias_or_name for p in leaf.expressions] for leaf in leaves] + return all(n == names[0] for n in names) and all(names[0]) - def _signature(self, item: Any, cte_names: dict[int, str], subs: dict[int, str]) -> tuple[str, str]: + def _signature(self, item: Any, cte_names: dict[int, str], subs: dict[int, tuple[str, str]]) -> tuple[str, str]: """来源的 (排序用的标识, 起别名用的名字):实表是表名,CTE 是它的规范名,FROM 里的子查询是规范化后的写法。""" from sqlglot import exp @@ -3658,7 +4278,7 @@ class _Namer: name = cte_names.get(id(cte), cte.alias_or_name) return f"cte:{name}", name return "table:" + ".".join(p for p in (item.catalog, item.db, item.name) if p), item.name - return f"sub:{subs[id(item)][1]}", "sub" + return f"sub:{subs[id(item)][0]}", "sub" def _canon_select(self, select: Any, depth: int, cte_names: dict[int, str]) -> tuple[str, str]: from sqlglot import exp @@ -3668,6 +4288,8 @@ class _Namer: for item in items: if isinstance(item, exp.Subquery): subs[id(item)] = self.canon(item.this, depth + 1, cte_names) + if self._rename_outputs(item.this, [item]): + subs[id(item)] = self._text(item.this) elif id(item) in self.cte_of: cte = self.cte_of[id(item)] item.set("this", exp.to_identifier(cte_names.get(id(cte), cte.alias_or_name))) @@ -3682,7 +4304,10 @@ class _Namer: else: break prefix, rest = items[:n_prefix], joins[max(n_prefix - 1, 0):] if items else joins - movable = not _starry(select) and not _unstable(select) + # 后面有 RIGHT / FULL JOIN 时开头那组来源不调换:PostgreSQL、MySQL 里 a, b RIGHT JOIN c 是 a 乘上 (b RIGHT JOIN c), + # 换成 b, a 就成了 b 乘上 (a RIGHT JOIN c) + movable = not _bare_star(select) and not self.sens.of(select) and \ + not any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins) free_lefts = movable and bool(rest) and self._independent_lefts(prefix, rest) def groups(xs: list[Any], key: Any) -> list[list[Any]]: @@ -3691,14 +4316,18 @@ class _Namer: prefix_groups = groups(prefix, lambda s: keys[id(s)][0]) if movable else [[s] for s in prefix] rest_groups = groups(rest, lambda j: keys[id(j.this)][0]) if free_lefts else [[j] for j in rest] - options = [[list(p) for p in itertools.permutations(g)] if len(g) > 1 else [g] - for g in prefix_groups + rest_groups] total = 1 - for o in options: - total *= len(o) + for g in prefix_groups + rest_groups: + total *= math.factorial(len(g)) + if total > _MAX_ORDERINGS: + break if total > _MAX_ORDERINGS or total > self.budget: - options = [[o[0]] for o in options] + # 排列太多(十几个同名来源):先算个数再展开,超了就只按排好的先后 + options = [[g] for g in prefix_groups + rest_groups] total = 1 + else: + options = [[list(p) for p in itertools.permutations(g)] if len(g) > 1 else [g] + for g in prefix_groups + rest_groups] self.budget -= total combos = list(itertools.product(*options)) best: tuple[tuple[str, str], int] | None = None @@ -3748,13 +4377,10 @@ class _Namer: col.set("table", exp.to_identifier(name)) for sub in _nested_queries(select): self.canon(sub, depth + 1, cte_names) - _sort_own(select, self.read) + _sort_own(select, self.read, self.sens.of(select)) return self._text(select) -_FLIP = {"GT": "LT", "LT": "GT", "GTE": "LTE", "LTE": "GTE", "EQ": "EQ", "NEQ": "NEQ"} - - def _fixed(node: Any) -> bool: """不涉及任何列、子查询的表达式(常量、常量算出来的日期):排比较、加乘的两边时放在右边,好读。""" from sqlglot import exp @@ -3762,10 +4388,11 @@ def _fixed(node: Any) -> bool: return node.find(exp.Column, exp.Select, exp.Subquery) is None -def _sort_own(select: Any, read: str | None) -> None: - """这一段自己的表达式里交换律成立的地方按写出来的文本排(里层先排):AND、OR(去重)、比较(不涉及列的一边在 - 右,两边都涉及列时按文本排、大于小于跟着换方向)、两项的加和乘(不涉及列的一项在右,三项以上的不跨项挪动: - 浮点数加法不满足结合律)、IN 列表、GROUP BY、PARTITION BY。""" +def _sort_own(select: Any, read: str | None, sensitive: bool = False) -> None: + """这一段自己的表达式里交换律成立的地方排好(里层先排):AND、OR(各项都是条件时去掉一模一样的项:x AND x 在 + SELECT 列表里不是 x)、比较(不涉及列的一边在右,两边都涉及列时排好、大于小于跟着换方向;两边都写了 COLLATE 的 + 不换,SQLite 用左边那个)、两项的加和乘(只换这一个节点的两边,不跨项挪动:浮点数加法不满足结合律)、IN 列表、 + GROUP BY(结果看行出来的先后时不排:SQLite 按分组键排好了出行)。""" from sqlglot import exp def key(node: Any) -> tuple[str, str]: @@ -3782,12 +4409,16 @@ def _sort_own(select: Any, read: str | None) -> None: if node is not select and node.parent is None: continue if isinstance(node, (exp.And, exp.Or)): - if isinstance(node.parent, type(node)): + if isinstance(_unparen_parent(node), type(node)): continue - uniq: dict[str, Any] = {} - for part in node.flatten(): - uniq.setdefault(key(part), part) - ordered = [uniq[k] for k in sorted(uniq)] + parts = _flat(node, type(node)) + if all(_predicate(p) for p in parts): + uniq: dict[tuple[str, str], Any] = {} + for part in parts: + uniq.setdefault(key(part), part) + ordered = [uniq[k] for k in sorted(uniq)] + else: + ordered = sorted(parts, key=key) new = ordered[0] if len(ordered) == 1 else ( exp.and_(*ordered, copy=False) if isinstance(node, exp.And) else exp.or_(*ordered, copy=False)) node.replace(new) @@ -3795,7 +4426,6 @@ def _sort_own(select: Any, read: str | None) -> None: left, right = node.this, node.expression if isinstance(left, (exp.Any, exp.All)) or isinstance(right, (exp.Any, exp.All)): continue - # 两边都写了 COLLATE 时 SQLite 用左边那个:不对调 if left.find(exp.Collate) is not None and right.find(exp.Collate) is not None: continue if (_fixed(left) and not _fixed(right)) or (_fixed(left) == _fixed(right) and key(left) > key(right)): @@ -3806,16 +4436,36 @@ def _sort_own(select: Any, read: str | None) -> None: node.set("this", right) node.set("expression", left) elif isinstance(node, exp.Group): - if node.expressions and not any(node.args.get(k) for k in ("rollup", "cube", "grouping_sets")): + if node.expressions and not sensitive: node.set("expressions", sorted(node.expressions, key=key)) - elif isinstance(node, exp.Window): - if node.args.get("partition_by"): - node.set("partition_by", sorted(node.args["partition_by"], key=key)) elif isinstance(node, exp.In) and node.expressions: uniq = {key(i): i for i in node.expressions} node.set("expressions", [uniq[k] for k in sorted(uniq)]) +def _flat(node: Any, cls: type) -> list[Any]: + """AND / OR 连着的各项(穿过括号:BETWEEN 改写出来的 (x >= a AND x <= b) 也拆开)。""" + out: list[Any] = [] + stack = [node] + while stack: + n = stack.pop() + inner = _unparen(n) + if isinstance(inner, cls): + stack += [inner.expression, inner.this] + else: + out.append(n) + return out + + +def _unparen_parent(node: Any) -> Any: + from sqlglot import exp + + parent = node.parent + while isinstance(parent, exp.Paren): + parent = parent.parent + return parent + + # ========================================================================== # 差异说明 # ========================================================================== @@ -3824,11 +4474,12 @@ def _sort_own(select: Any, read: str | None) -> None: @dataclass(frozen=True) class FacetDiff: """一处不同。aspect:time(时间列)/ time_range(同一时间列、范围不同)/ filter(筛选条件)/ aggregate(聚合函数、 - 去重计数)/ expression(选出来的列怎么算)/ join(关联的表、关联类型、关联条件、外连接的主表)/ tables(查的是 - 哪张表)/ set_op(几支查询怎么合并)/ group_by(分组键、去重)/ having / limit(取哪几行)/ order(只是排序不同)/ - parse(解析不了)/ other(兜底:未能归纳成具体差异,见 explain_sql_diff)。a、b 是两边各自的写法(给人看,没有 - 就是 None),text 是一句中文说明。detail 只在 time 上有:两边的时间列 [{table, column}],调用方据此对照数据目录 - (业务日期);在 filter 上有 {"columns": [{table, column}]}:条件只涉及的实表列,对照有效行过滤用。""" + 去重计数)/ expression(选出来的列)/ join(关联的表、关联类型、关联条件、外连接的主表)/ tables(查的是哪张表)/ + set_op(几支查询怎么合并)/ group_by(分组键、去重)/ having / limit(取哪几行,含取行前的排序)/ order(只是最终 + 结果的排序不同,不影响取到哪些行)/ parse(解析不了)/ other(兜底:未能归纳成具体差异,见 explain_sql_diff)。a、b + 是两边各自的写法(给人看,没有就是 None),text 是一句中文说明。detail 只在 time 上有:两边的时间列 + [{table, column}](各是各的表),调用方据此对照数据目录(业务日期);在 filter 上有 {"columns": [{table, column}]}: + 条件只涉及的实表列,对照有效行过滤用。""" aspect: str a: Any @@ -3873,6 +4524,9 @@ _MAX_DIFF_LINES = 40 _CJK_PUNCT = frozenset(",。;:、!?「」『』()《》【】…—") #: 聚合函数的说法 _VERBS = {"sum": "求和", "count": "计数", "avg": "求平均", "min": "取最小值", "max": "取最大值"} +#: 只取日期的字符串(2026-09-01):PostgreSQL、MySQL 里拿它和时间列比,按字符串比的先后和按日期比的一样 +_DATE_ONLY = re.compile(r"^\d{4}-\d{2}-\d{2}$") +_COMPARE_OPS = ("EQ", "NEQ", "GT", "GTE", "LT", "LTE") def _wide(ch: str) -> bool: @@ -3926,22 +4580,406 @@ def _labeler(names: Mapping[str, str] | None) -> Any: return label +# ---- 同一个对象上的条件:留下的取值一样不一样 + +def _literal_value(node: Any) -> tuple[str, Any] | None: + """常量的值:("n", Decimal) 或 ("s", 字符串);不是数字、字符串常量返回 None。""" + from decimal import Decimal, InvalidOperation + + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, exp.Neg): + inner = _literal_value(node.this) + return ("n", -inner[1]) if inner is not None and inner[0] == "n" else None + if isinstance(node, exp.Literal): + if node.is_string: + return "s", str(node.this) + try: + return "n", Decimal(str(node.this)) + except (InvalidOperation, ValueError): + return None + return None + + +class _Range: + """一个对象(一列、一个聚合、一个函数)上只和常量比的条件(=、<>、<、<=、>、>=、IN 常量列表、IS [NOT] NULL、 + BETWEEN,用 AND / OR / NOT 连起来),在对象的各段取值上成立不成立:用到的常量把取值切成一段段(小于最小的、 + 每个常量本身、相邻两个之间、大于最大的),每段里条件的真假都一样,各取一个代表算一遍,再算对象是空值时。两组 + 条件在每一段、空值上都同真同不真,留下的行就一样。 + + 字符串的大小先后:SQLite 按默认的 BINARY 排序规则;PostgreSQL(语言排序规则下 'a' < 'B')、MySQL(默认不分 + 大小写)只认等不等,除非常量都是只取日期的字符串(2026-09-01,按字符串比和按日期比先后一样)。MySQL 的字符串 + 等不等不分大小写。COUNT 的取值是整数(分组时至少是 1)。""" + + def __init__(self, read: str | None) -> None: + self.read = read + + def subject(self, cond: Any) -> tuple[str, Any, list[tuple[str, Any]], bool] | None: + """cond 只是同一个对象和常量比时,返回 (对象的指纹, 对象, 常量, 有没有比大小);不然 None。""" + from sqlglot import exp + + subjects: dict[str, Any] = {} + literals: list[tuple[str, Any]] = [] + ranged = [False] + + def side(subj: Any, lits: list[Any]) -> bool: + if subj is None or _literal_value(subj) is not None or isinstance(_unparen(subj), exp.Null) or \ + subj.find(exp.Query, exp.Subquery) is not None or _volatile(subj): + return False + subjects[_fingerprint(subj)] = subj + literals.extend(lits) + return True + + def walk(n: Any) -> bool: + n = _unparen(n) + if isinstance(n, (exp.And, exp.Or)): + return walk(n.left) and walk(n.right) + if isinstance(n, exp.Not): + return walk(n.this) + name = type(n).__name__ + if name in _COMPARE_OPS: + lv, rv = _literal_value(n.expression), _literal_value(n.this) + if name not in ("EQ", "NEQ"): + ranged[0] = True + if lv is not None and rv is None: + return side(n.this, [lv]) + if rv is not None and lv is None: + return side(n.expression, [rv]) + return False + if isinstance(n, exp.In): + if n.args.get("query") or n.args.get("unnest") or n.args.get("field") or not n.expressions: + return False + lits = [] + for item in n.expressions: + if isinstance(_unparen(item), exp.Null): + continue + value = _literal_value(item) + if value is None: + return False + lits.append(value) + return side(n.this, lits) + if isinstance(n, exp.Is): + return isinstance(_unparen(n.expression), exp.Null) and side(n.this, []) + if n.args.get("negate"): + return False + if isinstance(n, exp.Between): + low, high = _literal_value(n.args.get("low")), _literal_value(n.args.get("high")) + ranged[0] = True + return not n.args.get("symmetric") and low is not None and high is not None and \ + side(n.this, [low, high]) + return False + + if not walk(cond) or len(subjects) != 1: + return None + kinds = {k for k, _ in literals} + if len(kinds) > 1: + return None + if ranged[0] and kinds == {"s"} and self.read != "sqlite" and \ + not all(_DATE_ONLY.match(v) for _, v in literals): + return None + fp, subj = next(iter(subjects.items())) + return fp, subj, literals, ranged[0] + + def _norm(self, value: tuple[str, Any]) -> Any: + kind, v = value + if kind == "s" and self.read == "mysql": + return v.casefold() + return v + + def _pieces(self, subject: Any, literals: list[tuple[str, Any]], ranged: bool, grouped: bool) -> list[Any]: + """代表各段取值的值(不含空值)。""" + from decimal import Decimal + + from sqlglot import exp + + values = sorted({self._norm(v) for v in literals}) + kind = literals[0][0] if literals else "n" + if not values: + return [Decimal(0)] if kind == "n" else ["x"] + if not ranged: + other = values[-1] + 1 if kind == "n" else values[-1] + "\x00~" + return values + [other] + out: list[Any] = [] + if kind == "n": + integral = isinstance(_unparen(subject), exp.Count) + low = Decimal(1) if integral and grouped and isinstance(_unparen(subject).this, exp.Star) else ( + Decimal(0) if integral else None) + candidates = [values[0] - 1] + [x for i, v in enumerate(values) + for x in ([v] + ([(v + values[i + 1]) / 2] if i + 1 < len(values) else []))] + candidates.append(values[-1] + 1) + if integral: + ints: set[Decimal] = set() + for i, v in enumerate(values): + ints |= {v.to_integral_value(rounding="ROUND_FLOOR") + d for d in (-1, 0, 1)} + ints |= {v.to_integral_value(rounding="ROUND_CEILING") + d for d in (-1, 0, 1)} + candidates = sorted(ints) + for c in candidates: + if low is not None and c < low: + continue + out.append(c) + return out or [low or Decimal(0)] + if values[0] != "": + out.append("") + for i, v in enumerate(values): + out.append(v) + between = v + "\x00" + if i + 1 >= len(values) or between < values[i + 1]: + out.append(between) + return out + + def _truth(self, n: Any, v: Any) -> str: + """n 在对象取 v(None 是空值)时是真(T)、假(F)还是空值(N)。""" + from sqlglot import exp + + n = _unparen(n) + if isinstance(n, exp.And): + a, b = self._truth(n.left, v), self._truth(n.right, v) + return "F" if "F" in (a, b) else ("N" if "N" in (a, b) else "T") + if isinstance(n, exp.Or): + a, b = self._truth(n.left, v), self._truth(n.right, v) + return "T" if "T" in (a, b) else ("N" if "N" in (a, b) else "F") + if isinstance(n, exp.Not): + t = self._truth(n.this, v) + return {"T": "F", "F": "T"}.get(t, "N") + if isinstance(n, exp.Is): + hit = v is None + return "T" if hit != bool(n.args.get("negate")) else "F" + if v is None: + return "N" + name = type(n).__name__ + if name in _COMPARE_OPS: + right = _literal_value(n.expression) + if right is not None: + lit, op = self._norm(right), name + else: + lit, op = self._norm(_literal_value(n.this)), _FLIP[name] + if type(v) is not type(lit): + return "N" + result = {"EQ": v == lit, "NEQ": v != lit, "GT": v > lit, "GTE": v >= lit, "LT": v < lit, + "LTE": v <= lit}[op] + return "T" if result else "F" + if isinstance(n, exp.In): + hit = any(not isinstance(_unparen(i), exp.Null) and self._norm(_literal_value(i)) == v + for i in n.expressions) + if hit: + return "T" + return "N" if any(isinstance(_unparen(i), exp.Null) for i in n.expressions) else "F" + if isinstance(n, exp.Between): + low, high = self._norm(_literal_value(n.args["low"])), self._norm(_literal_value(n.args["high"])) + return "T" if low <= v <= high else "F" + return "N" + + def compare(self, conds_a: list[Any], conds_b: list[Any], grouped: bool = False) -> bool | None: + """两组条件(各自 AND 连,可以一边为空 = 不筛)留下的取值是不是一样:一样 True,不一样 False,认不出 None。""" + parsed = [self.subject(c) for c in conds_a + conds_b] + if any(p is None for p in parsed) or len({p[0] for p in parsed}) != 1: + return None + subject = parsed[0][1] + literals = [lit for p in parsed for lit in p[2]] + if len({k for k, _ in literals}) > 1: + return None + ranged = any(p[3] for p in parsed) + from sqlglot import exp + + values = self._pieces(subject, literals, ranged, grouped) + if not isinstance(_unparen(subject), exp.Count): + values.append(None) + for v in values: + ta = all(self._truth(c, v) == "T" for c in conds_a) + tb = all(self._truth(c, v) == "T" for c in conds_b) + if ta != tb: + return False + return True + + +# ---- 一边用到了哪些列(判断「另一边没有」能不能说) + +class _Usage: + """一条语句里每一列(按定义找到实表列,认不出来源的只记列名)用在了哪儿: + + - cond:WHERE、ON、HAVING 里,条件里的子查询(IN、EXISTS、标量)选出来的列; + - case:CASE / IF / FILTER 里(SUM(CASE WHEN status = 'paid' THEN amount END) 等于先按 status 筛再求和); + - agg:聚合函数、窗口函数的参数(COUNT(x)、SUM(x) 跳过空值,等于先筛掉 x 为空的行)。 + + 光选出来、分组、排序的列不记。说「另一边没有这个条件」「另一边没有按时间筛选」之前,先看另一边在这一段能影响到的 + 地方(整条语句,除了合并里别的支)用没用到这些列。""" + + def __init__(self, tree: Any) -> None: + from sqlglot import exp + + self.nodes: dict[tuple[str | None, str], dict[int, str]] = {} + self.temporal = False + try: + self.resolver: _Resolver | None = _Resolver(tree) + except _Unsupported: + self.resolver = None + for col in tree.find_all(exp.Column): + if col.is_star or not col.name: + continue + kind = self._kind(col) + if kind is None: + continue + bases = self._bases(col, 0) + if isinstance(col.this, exp.Identifier) and col.this.meta.get(_UNSURE): + bases.append((None, col.name)) + for table, column in bases: + self.nodes.setdefault((table, column), {})[id(col)] = kind + if kind != "agg" and _temporal_name(column): + self.temporal = True + for node in tree.find_all(exp.Where, exp.Having, exp.Join): + target = node.args.get("on") if isinstance(node, exp.Join) else node.this + if target is not None and _datey(target): + self.temporal = True + + @staticmethod + def _owner(node: Any) -> tuple[Any, str | None, bool, bool]: + """node 属于哪一段 SELECT、在它的哪个子句里、路上有没有 CASE / IF / FILTER、有没有聚合 / 窗口函数。""" + from sqlglot import exp + + case = agg = False + child, parent = node, node.parent + while parent is not None: + if isinstance(parent, (exp.Case, exp.If, exp.Filter)): + case = True + if isinstance(parent, (exp.AggFunc, exp.Window)): + agg = True + if isinstance(parent, exp.Select): + return parent, child.arg_key, case, agg + child, parent = parent, parent.parent + return None, None, case, agg + + def _kind(self, col: Any) -> str | None: + select, key, case, agg = self._owner(col) + if select is None: + return None + if key == "expressions": + if _scalar_context(select) or _condition_query(select): + return "cond" + return "case" if case else ("agg" if agg else None) + if key in ("group", "order", "distinct"): + return "case" if case else None + return "cond" + + def _bases(self, col: Any, depth: int) -> list[tuple[str | None, str]]: + from sqlglot import exp + + if depth > 16 or self.resolver is None: + return [(None, col.name)] + item = self.resolver.source_of.get(id(col)) + if item is None: + select = self._owner(col)[0] + items = _items_of(select) if select is not None else [] + if col.table or len(items) != 1: + return [(None, col.name)] + item = items[0] + return self._from_item(item, col.name, depth) + + def _from_item(self, item: Any, name: str, depth: int) -> list[tuple[str | None, str]]: + from sqlglot import exp + + body = self.resolver.body(item) if self.resolver is not None else None + if body is None: + return [(item.name, name)] if isinstance(item, exp.Table) else [(None, name)] + leaves = _leaves(body) + if not leaves or not isinstance(leaves[0], exp.Select) or depth > 16: + return [(None, name)] + names = [p.alias_or_name for p in leaves[0].expressions] + if name not in names: + sources = _items_of(leaves[0]) + if len(leaves) == 1 and any(isinstance(p, exp.Star) for p in leaves[0].expressions) and len(sources) == 1: + return self._from_item(sources[0], name, depth + 1) + return [(None, name)] + index = names.index(name) + out: list[tuple[str | None, str]] = [] + for leaf in leaves: + if not isinstance(leaf, exp.Select) or index >= len(leaf.expressions): + return [(None, name)] + expr = leaf.expressions[index].unalias() + for c in ([expr] if isinstance(expr, exp.Column) else list(expr.find_all(exp.Column))): + if not c.is_star: + out += self._bases(c, depth + 1) + return out or [(None, name)] + + def touches(self, refs: Iterable[tuple[str, str]], skip: Iterable[Any] = (), + kinds: tuple[str, ...] = ("cond", "case"), scope: Any = None) -> bool: + """这些实表列用在了 kinds 里(认不出来源的同名列也算);skip 里的条件中的那几处不算,scope 所在合并的别的支 + (不碰 scope 的行)不算。""" + from sqlglot import exp + + skipped = {id(c) for cond in skip for c in cond.find_all(exp.Column)} + node = scope + while node is not None and node.parent is not None: + parent = node.parent + if isinstance(parent, exp.SetOperation): + for branch in (parent.this, parent.expression): + if branch is not node: + skipped |= {id(c) for c in branch.find_all(exp.Column)} + node = parent + for table, column in refs: + for key in ((table, column), (None, column)): + if any(i not in skipped and k in kinds for i, k in self.nodes.get(key, {}).items()): + return True + return False + + +def _kinds_for(conds: list[Any]) -> tuple[str, ...]: + """说某个条件只在一边有之前,要看另一边哪些地方用到了这些列:条件里有判空(IS NULL / IS NOT NULL)的连聚合的 + 参数也看(COUNT(x) 本来就跳过 x 为空的行)。""" + from sqlglot import exp + + if any(isinstance(n, exp.Is) for c in conds for n in c.walk()): + return "cond", "case", "agg" + return "cond", "case" + + +def _condition_query(select: Any) -> bool: + """这段查询是条件里的子查询(IN、EXISTS、ANY / ALL):选出来的列参与比较。""" + from sqlglot import exp + + node = select + while isinstance(node.parent, (exp.Subquery, exp.SetOperation)): + node = node.parent + parent = node.parent + return isinstance(parent, (exp.Exists, exp.Any, exp.All)) or (isinstance(parent, exp.In) and + node.arg_key == "query") + + +# ---- 一段 SELECT + @dataclass class _Rel: - """一段里的一个来源:alias 是规范别名,name 是实表名或 CTE 的规范名,table 表示是不是实表,shown 是给人看的 - 名字(实表是表名,CTE 是原来起的名字),rejected 表示写的是 LEFT JOIN、被 WHERE 变成了内连接。""" + """一段里的一个来源:alias 是规范别名,name 是实表名(CTE 是规范名,子查询是 sub),kind 是 table / cte / derived, + shown 是给人看的名字(实表是表名,CTE、子查询是原来起的名字),rejected 表示写的是 LEFT JOIN、被 WHERE 变成了 + 内连接。""" alias: str name: str - table: bool + kind: str shown: str + node: Any rejected: str | None = None + @property + def table(self) -> bool: + return self.kind == "table" + + +def _rel_of(item: Any, ctes: Mapping[str, str]) -> _Rel: + from sqlglot import exp + + alias = item.alias_or_name + if isinstance(item, exp.Subquery): + return _Rel(alias, "sub", "derived", item.this.meta.get(_ORIGINAL) or alias, item, item.meta.get(_REJECTED)) + if item.name in ctes: + return _Rel(alias, item.name, "cte", ctes[item.name], item, item.meta.get(_REJECTED)) + return _Rel(alias, item.name, "table", item.name, item, item.meta.get(_REJECTED)) + @dataclass class _Block: - """一段 SELECT 拆开来比:开头那组内连接的来源(rels)、后面的连接(joins:LEFT JOIN 等,按先后)、WHERE、 - GROUP BY、HAVING 的各项,选出来的列,排序和取行。""" + """一段 SELECT 拆开来比:开头那组逗号连接的来源(rels)、后面的连接(joins:LEFT JOIN、没写成逗号连接的内连接, + 按先后)、WHERE、GROUP BY、HAVING 的各项,选出来的列,排序和取行。""" select: Any rels: list[_Rel] @@ -3954,17 +4992,19 @@ class _Block: limit: str | None offset: str | None distinct: bool - deduped: bool - ctes: dict[str, str] = field(default_factory=dict) + distinct_on: bool + windowed: bool + ctes: dict[str, str] + deduped: bool = False @classmethod - def of(cls, select: Any, read: str | None, ctes: Mapping[str, str]) -> "_Block | None": + def of(cls, select: Any, read: str | None, ctes: Mapping[str, str], root: bool = False) -> "_Block | None": from sqlglot import exp - if not isinstance(select, exp.Select) or select.args.get("with_") is not None: + if not isinstance(select, exp.Select) or (select.args.get("with_") is not None and not root): return None items = _items_of(select) - if any(not isinstance(i, exp.Table) for i in items): + if not items or any(not isinstance(i, (exp.Table, exp.Subquery)) for i in items): return None joins = select.args.get("joins") or [] n = 1 @@ -3973,54 +5013,67 @@ class _Block: n += 1 else: break - ctes = dict(ctes) - rels = [_Rel(i.alias_or_name, i.name, i.name not in ctes, ctes.get(i.name, i.name), - i.meta.get(_REJECTED)) for i in items[:n]] where, group, having = select.args.get("where"), select.args.get("group"), select.args.get("having") order, limit, offset = select.args.get("order"), select.args.get("limit"), select.args.get("offset") + distinct = select.args.get("distinct") def number(node: Any) -> str | None: return node.expression.sql(dialect=read) if node is not None and node.expression is not None else None - return cls(select, rels, joins[n - 1:] if items else joins, + return cls(select, [_rel_of(i, ctes) for i in items[:n]], joins[n - 1:], _conjuncts(where.this) if where is not None else [], list(group.expressions) if group is not None else [], _conjuncts(having.this) if having is not None else [], list(select.expressions), list(order.expressions) if order is not None else [], - number(limit), number(offset), select.args.get("distinct") is not None, - bool(group is not None and group.meta.get(_FROM_DISTINCT)), ctes) + number(limit), number(offset), distinct is not None, + bool(distinct is not None and distinct.args.get("on")), + any(isinstance(n, exp.Window) for n in _block_nodes(select)), dict(ctes), + bool(group is not None and group.meta.get(_FROM_DISTINCT))) + + def all_rels(self) -> list[_Rel]: + return self.rels + [_rel_of(j.this, self.ctes) for j in self.joins] def rel(self, alias: str) -> _Rel | None: - for r in self.rels: + for r in self.all_rels(): if r.alias == alias: return r - for join in self.joins: - if join.this.alias_or_name == alias: - name = join.this.name - return _Rel(alias, name, name not in self.ctes, self.ctes.get(name, name)) return None def names(self) -> list[str]: - return [r.name for r in self.rels] + [j.this.name for j in self.joins] + return [r.name for r in self.all_rels()] class _Explainer: """在两边的规范形式上逐段比,只把有把握的不同写成具体的中文句子,同时记下是不是所有不同都说到了。 - 有把握的门槛:两段读的来源对得上(一模一样,或者只差一处说得清的:换了一张表、一边多关联了一张表、同一张表的 - 关联类型或关联条件不同、外连接的主表对调)之后,才逐项比 WHERE、选出来的列、分组、HAVING、取行;每一项只在它 - 只涉及这一段的实表列(不涉及 CTE 算出来的列、外层的列、子查询,同名的表在这一段里只出现一次)时才说具体的, - 说的都是「这一段的这一项两边各是什么」,不推断对结果的影响。来源对不上、两段结构不同、条件里套着子查询又配不成 - 一对的,宁可不说具体的,交给兜底说明。CTE 两边定义一样的按另一边的名字起名后对齐;各剩一个对不上的配成一对, - 往里比。CTE 里选出来的列不比:规范化时去掉了没用到的列、按定义重排过,和原文的列对不上。""" + 有把握的门槛: + - 两段读的来源对得上(一模一样,或者只差一处说得清的:换了一张表、一边多关联了一张表、同一张表的关联类型或 + 关联条件不同、外连接的主表对调)才往下比;换了表的那一边,列照它自己的表说(不借另一边的表名)。 + - 说「X 不一样」之前先确认真的不一样:同一个对象只和常量比的几个条件按取值逐段算一遍(_Range),留下的行一样的 + 不说(IN 和 OR、多写一个 IS NOT NULL、把一段时间拆成两段);关联条件只是列相等的,比相等关系分出的组。认不出 + 的写法不说。 + - 说「另一边没有」之前先确认另一边整条语句(含子查询、CTE、相关子查询里引用外层的列)的筛选里都没用到这些列 + (_Usage);说「没有按时间筛选」之前确认另一边哪儿都没有像时间的列、像日期的常量。 + - 条件里的子查询两边写法除子查询外一模一样(含 NOT、IN 左边的列)才配成一对往里比,说清是哪个子查询。 + - 只是排序不同的说法(order)只在整条语句最外层、没有取行、DISTINCT ON、窗口函数时说;取行前的排序算 limit。 + - LEFT JOIN 对 INNER JOIN 只在整条语句最外层、WHERE 里没用到可空一侧的列、结果不只是可空一侧的列的聚合(那样 + 两种写法结果一样)时说。 + 说的都是「这一段的这一项两边各写的是什么」。两边给人看的写法一样的话不说(读起来自相矛盾),说过的一对反过来再说 + 一遍的也不说。""" - def __init__(self, read: str | None, labels: tuple[str, str], names: Mapping[str, str] | None) -> None: + def __init__(self, read: str | None, labels: tuple[str, str], names: Mapping[str, str] | None, + info: _Schema) -> None: self.read = read self.la, self.lb = labels self.label = _labeler(names) + self.info = info self.items: list[FacetDiff] = [] self.ctes: dict[str, str] = {} - #: 有一句话没说出口(两边给人看的写法一样,说出来自相矛盾):这一处不同就没说到 + self.range = _Range(read) + self.usage_a: _Usage | None = None + self.usage_b: _Usage | None = None + self.roots: set[int] = set() + #: 有一句话没说出口(两边给人看的写法一样,或者和前面的话矛盾):这一处不同就没说到 self.withheld = False def sql(self, node: Any) -> str: @@ -4038,7 +5091,6 @@ class _Explainer: @staticmethod def fp(node: Any) -> str: - """比「是不是同一个」用的写法(见 _fingerprint):生成的 SQL 可能把不同的写法写成一样的。""" return _fingerprint(node) # ---- 入口:整条语句 @@ -4051,21 +5103,20 @@ class _Explainer: names_a = {c.alias_or_name: c for c in (wa.expressions if wa is not None else [])} if form_b.pre is not None and names_a: tb = self._renamed(form_a, form_b, names_a) + self.usage_a, self.usage_b = _Usage(ta), _Usage(tb) wb = tb.args.get("with_") names_b = {c.alias_or_name: c for c in (wb.expressions if wb is not None else [])} shown_a = {n: c.this.meta.get(_ORIGINAL) or n for n, c in names_a.items()} shown_b = {n: c.this.meta.get(_ORIGINAL) or n for n, c in names_b.items()} covered = set(names_a) == set(names_b) - main_a, main_b = ta.copy(), tb.copy() - main_a.set("with_", None) - main_b.set("with_", None) self.ctes = {**shown_b, **shown_a} - covered &= self.query(main_a, main_b, ("", ""), outputs=True) + self.roots = {id(ta), id(tb)} + covered &= self.query(ta, tb, ("", ""), "top") for name in sorted(set(names_a) & set(names_b)): body_a, body_b = names_a[name].this, names_b[name].this if self.fp(body_a) != self.fp(body_b): where = (_say("在", shown_a[name], "里"), _say("在", shown_b[name], "里")) - covered &= self.query(body_a, body_b, where, outputs=False) + covered &= self.query(body_a, body_b, where, "derived") return covered and not self.withheld def _renamed(self, form_a: _Form, form_b: _Form, names_a: dict[str, Any]) -> Any: @@ -4073,43 +5124,54 @@ class _Explainer: from sqlglot import exp hint = _Hint(bodies={_fingerprint(c.this): name for name, c in names_a.items()}, taken=set(names_a)) - first = _named(form_b.pre.copy(), self.read, form_b.qualified, hint) + + def name(pre: Any) -> Any: + return _named(pre, self.read, form_b.qualified, _Sensitivity(pre, self.read, self.info), hint) + + first = name(form_b.pre.copy()) wb = first.args.get("with_") left_b = [c for c in (wb.expressions if wb is not None else []) if c.alias_or_name not in names_a] used = {c.alias_or_name for c in (wb.expressions if wb is not None else [])} - left_a = [name for name in names_a if name not in used] + left_a = [n for n in names_a if n not in used] if len(left_a) == 1 and len(left_b) == 1: pre = form_b.pre.copy() original = left_b[0].this.meta.get(_ORIGINAL) matches = [c.alias_or_name for c in pre.find_all(exp.CTE) if c.this.meta.get(_ORIGINAL) == original] if original is not None and len(matches) == 1: hint.names[matches[0]] = left_a[0] - return _named(pre, self.read, form_b.qualified, hint) + return name(pre) return first - def query(self, qa: Any, qb: Any, where: tuple[str, str], *, outputs: bool) -> bool: + def query(self, qa: Any, qb: Any, where: tuple[str, str], ctx: str) -> bool: from sqlglot import exp if self.fp(qa) == self.fp(qb): return True if isinstance(qa, exp.Select) and isinstance(qb, exp.Select): - return self.select(qa, qb, where, outputs) + return self.select(qa, qb, where, ctx) if isinstance(qa, exp.SetOperation) and isinstance(qb, exp.SetOperation): - return self.setop(qa, qb, where, outputs) + return self.setop(qa, qb, where, ctx) return False # ---- 说法的拼法 - def sides(self, where: tuple[str, str], first: int = 0) -> tuple[str, str]: - """两边的叫法,带上在哪一段(「A 在 paid 里」),按 A、B 的先后返回。两边都说在哪:只说一边,「另一边没有」 - 会读成整条语句都没有。""" + def sides(self, where: tuple[str, str]) -> tuple[str, str]: + """两边的叫法,带上在哪一段(「A 在 paid 里」)。两边都说在哪:只说一边,「另一边没有」会读成整条语句都没有。""" return _say(self.la, where[0]), _say(self.lb, where[1]) def add(self, aspect: str, a: Any, b: Any, text: str, detail: dict[str, Any] | None = None) -> None: - """记一句话。两边给人看的写法一样(「A 算的是 1,B 算的是 1」)就不说:读起来自相矛盾,这一处算没说到。""" + """记一句话。两边给人看的写法一样(「A 算的是 1,B 算的是 1」)、和已经说过的一句正好反过来(A 说 X、B 说 Y + 对上 A 说 Y、B 说 X)的不说:读起来自相矛盾,这一处算没说到。""" if isinstance(a, str) and isinstance(b, str) and a == b: self.withheld = True return + for old in self.items: + if old.aspect == aspect and old.a == b and old.b == a and (a is not None or b is not None): + self.items.remove(old) + self.withheld = True + return + if old.text == text: + return self.items.append(FacetDiff(aspect, a, b, text, detail)) def base_columns(self, node: Any, block: _Block) -> list[tuple[_Rel, Any]] | None: @@ -4119,21 +5181,32 @@ class _Explainer: if node.find(exp.Select, exp.Subquery) is not None: return None names = block.names() + aliases = {e.alias for e in block.exprs if isinstance(e, exp.Alias) and e.alias} out = [] for col in node.find_all(exp.Column): - rel = block.rel(col.table) if col.table else None + if col.is_star or (not col.table and col.name in aliases): + return None + rel = block.rel(col.table) if col.table else (block.rels[0] if len(block.all_rels()) == 1 else None) if rel is None or not rel.table or names.count(rel.name) != 1: return None + if isinstance(col.this, exp.Identifier) and col.this.meta.get(_UNSURE): + columns = self.info.types.get(rel.name.translate(_ASCII_LOWER)) + if columns is None or col.name.translate(_ASCII_LOWER) not in columns: + return None out.append((rel, col)) return out + def refs(self, node: Any, block: _Block) -> set[tuple[str, str]] | None: + cols = self.base_columns(node, block) + return None if cols is None else {(r.name, c.name) for r, c in cols} + def show(self, node: Any, block: _Block, *, full: bool = False) -> str: """给人看的写法:只有一个来源时列不写表名,否则写实表名(规范别名换回表名,CTE 换回原来的名字);选出来的 列去掉规范化起的别名。full 时一律写表名。""" from sqlglot import exp copy = node.unalias().copy() if isinstance(node, exp.Alias) else node.copy() - single = len(block.rels) + len(block.joins) == 1 and not full + single = len(block.all_rels()) == 1 and not full for col in copy.find_all(exp.Column): rel = block.rel(col.table) if col.table else None if rel is not None: @@ -4148,107 +5221,134 @@ class _Explainer: return self.show(conds[0], block) return self.show(exp.and_(*[c.copy() for c in conds], copy=False), block) - def named(self, rel: _Rel, column: str) -> str: + def named(self, table: str, column: str) -> str: """列带中文名(有的话):「支付时间 paid_at」。""" - label = self.label((rel.name, column)) + label = self.label((table, column)) return _say(label, column) if label else column def what(self, rel: _Rel) -> str: return _say(rel.shown, "表") if rel.table else _say("子查询", rel.shown) + @staticmethod + def mapped(node: Any, mapping: Mapping[str, str]) -> Any: + """B 的一个节点按来源的对应关系(B 的别名 → A 的别名)改名后的副本:只用来和 A 比是不是同一个写法。""" + from sqlglot import exp + + if not mapping: + return node + copy = node.copy() + for col in copy.find_all(exp.Column): + if col.table in mapping: + col.set("table", exp.to_identifier(mapping[col.table])) + return copy + + def mfp(self, node: Any, mapping: Mapping[str, str]) -> str: + return self.fp(self.mapped(node, mapping)) + # ---- 一段 SELECT - def select(self, a: Any, b: Any, where: tuple[str, str], outputs: bool) -> bool: - A, B = _Block.of(a, self.read, self.ctes), _Block.of(b, self.read, self.ctes) + def select(self, a: Any, b: Any, where: tuple[str, str], ctx: str) -> bool: + A = _Block.of(a, self.read, self.ctes, id(a) in self.roots) + B = _Block.of(b, self.read, self.ctes, id(b) in self.roots) if A is None or B is None: return False - matched = self.relations(A, B, where) + matched = self.relations(A, B, where, ctx) if matched is None: return False - covered, skip_a, skip_b, B = matched - covered &= self.conditions(A, B, where, skip_a, skip_b) - if outputs: - covered &= self.outputs(A, B, where) - else: - # CTE 里选出来的列规范化时去掉过没用到的、重排过,和原文的列对不上:不同就不说具体的 - covered &= self.groups(A, B, where) - covered &= [self.fp(e) for e in A.exprs] == [self.fp(e) for e in B.exprs] - covered &= self.having(A, B, where) - covered &= self.rows(A, B, where) + covered, mapping, skip_a, skip_b = matched + covered &= self.derived(A, B, mapping, where) + covered &= self.conditions(A, B, mapping, where, skip_a, skip_b) + covered &= self.outputs(A, B, mapping, where, ctx) + covered &= self.having(A, B, mapping, where) + covered &= self.rows(A, B, mapping, where, ctx) return covered - def relations(self, A: _Block, B: _Block, where: tuple[str, str] - ) -> tuple[bool, set[str], set[str], _Block] | None: - """两段读的来源对不对得上。对得上返回 (说到了没有, A 已经说过的 WHERE 项, B 已经说过的 WHERE 项, B);B 可能换了 - 一张表(换成 A 的表名之后再比别的)。对不上返回 None。""" + @staticmethod + def _join_sig(join: Any) -> tuple[str, str]: + return str(join.side or "").upper() or str(join.kind or "").upper(), join.this.alias_or_name + + def relations(self, A: _Block, B: _Block, where: tuple[str, str], ctx: str + ) -> tuple[bool, dict[str, str], set[str], set[str]] | None: + """两段读的来源对不对得上。对得上返回 (说到了没有, B 的别名 → A 的别名, A 已经说过的 WHERE 项, B 已经说过的 + WHERE 项);对不上返回 None。""" la, lb = self.sides(where) ra, rb = [r.alias for r in A.rels], [r.alias for r in B.rels] - ja, jb = [self.fp(j) for j in A.joins], [self.fp(j) for j in B.joins] - if ra == rb and ja == jb: - return True, set(), set(), B - # 换了一张表:开头那组来源只差一张实表,其余一样 - if len(ra) == len(rb) and ja == jb and not A.joins: + ja = [self._join_sig(j) + (self.fp(j.args.get("on")) if j.args.get("on") is not None else "",) + for j in A.joins] + jb = [self._join_sig(j) + (self.fp(j.args.get("on")) if j.args.get("on") is not None else "",) + for j in B.joins] + kinds_a = {r.alias: r.kind for r in A.all_rels()} + kinds_b = {r.alias: r.kind for r in B.all_rels()} + if ra == rb and ja == jb and all(kinds_a[k] == kinds_b.get(k) for k in kinds_a): + return True, {}, set(), set() + # 换了一张表:开头那组来源只差一张实表(各自只出现一次),后面的连接一样 + if len(ra) == len(rb) and not A.joins and not B.joins: only_a = [r for r in A.rels if r.alias not in rb] only_b = [r for r in B.rels if r.alias not in ra] if len(only_a) == 1 and len(only_b) == 1 and only_a[0].table and only_b[0].table and \ - only_a[0].name == only_a[0].alias and only_b[0].name == only_b[0].alias: + A.names().count(only_a[0].name) == 1 and B.names().count(only_b[0].name) == 1: x, y = only_a[0], only_b[0] + if x.name.translate(_ASCII_LOWER) == y.name.translate(_ASCII_LOWER): + return None self.add("tables", x.name, y.name, _say(la, "查的是", x.name, "表,", lb, "查的是", y.name, "表")) - return True, set(), set(), self._swap(B, y, x) + return True, {y.alias: x.alias}, set(), set() # 一边多关联了一张表(内连接) - if ja == jb and not A.joins: + if not A.joins and not B.joins: if set(rb) < set(ra) and len(ra) == len(rb) + 1: return self._extra(A, B, la, lb, 0) if set(ra) < set(rb) and len(rb) == len(ra) + 1: - first_b = self.sides(where, 1) - return self._extra(B, A, first_b[1], first_b[0], 1) - # 同一张表:LEFT JOIN 对 INNER JOIN、关联条件不同、只有一边关联了(LEFT JOIN)、外连接的主表对调 + return self._extra(B, A, lb, la, 1) + if ctx != "top": + return None return self._outer(A, B, where) - def _swap(self, B: _Block, old: _Rel, new: _Rel) -> _Block | None: - from sqlglot import exp - - select = B.select.copy() - for col in select.find_all(exp.Column): - if col.table == old.alias: - col.set("table", exp.to_identifier(new.alias)) - for table in select.find_all(exp.Table): - if table.alias_or_name == old.alias: - table.set("this", exp.to_identifier(new.name)) - table.set("alias", exp.TableAlias(this=exp.to_identifier(new.alias))) - for s in list(select.find_all(exp.Select)): - _sort_own(s, self.read) - return _Block.of(select, self.read, self.ctes) - - def _uses(self, block: _Block, name: str) -> bool: - """这一段在子查询里(不是开头的来源、连接)用到了 name 这张表。""" - from sqlglot import exp - - own = {id(i) for i in _items_of(block.select)} - return any(t.name == name and id(t) not in own for t in block.select.find_all(exp.Table)) - def _extra(self, big: _Block, small: _Block, lbig: str, lsmall: str, side: int - ) -> tuple[bool, set[str], set[str], _Block] | None: + ) -> tuple[bool, dict[str, str], set[str], set[str]] | None: from sqlglot import exp [rel] = [r for r in big.rels if r.alias not in {s.alias for s in small.rels}] - if big.names().count(rel.name) != 1: + if not rel.table or big.names().count(rel.name) != 1: return None conds = [c for c in big.where if rel.alias in {col.table for col in c.find_all(exp.Column)}] what = self.what(rel) shown = _say("(关联的条件是", self.show_all(conds, big), ")") if conds and \ all(self.base_columns(c, big) is not None for c in conds) else "" - tail = ("没有关联它(只在子查询里用到了", rel.shown, ")") if self._uses(small, rel.name) else ("没有",) + used = any(t.name == rel.name and id(t) not in {id(i) for i in _items_of(small.select)} + for t in small.select.find_all(exp.Table)) + tail = ("没有关联它(只在子查询里用到了", rel.shown, ")") if used else ("没有",) text = _say(lbig, "关联了", what, shown, ",", lsmall, *tail) self.add("join", what if side == 0 else None, what if side == 1 else None, text) skip = {self.fp(c) for c in conds} - return (True, skip, set(), small) if side == 0 else (True, set(), skip, big) + return (True, {}, skip, set()) if side == 0 else (True, {}, set(), skip) - def _outer(self, A: _Block, B: _Block, where: tuple[str, str]) -> tuple[bool, set[str], set[str], _Block] | None: + def _nullable_used(self, block: _Block, alias: str) -> bool: + """这一段在 ON 之外(WHERE、HAVING)用到了外连接可空一侧的列:可能把没关联上的行筛掉了(COALESCE(x, 0) > 1 + 这类规范化认不出的写法),LEFT JOIN 就不一定是外连接。""" from sqlglot import exp + clauses = block.where + block.having + return any(c.table == alias for cond in clauses for c in cond.find_all(exp.Column)) + + def _null_blind(self, block: _Block, alias: str) -> bool: + """这一段不分组、选出来的都是只涉及可空一侧的列、会跳过空值的聚合(COUNT(x)、SUM、AVG、MIN、MAX):LEFT JOIN + 多出来的没关联上的行在这些聚合里都是空值、被跳过,和 INNER JOIN 结果一样。""" + from sqlglot import exp + + if block.group: + return False + for e in block.exprs: + agg = _aggregate(e.unalias()) + if agg is None or agg[1] is None: + return False + if any(c.table != alias for c in e.find_all(exp.Column)): + return False + return True + + def _outer(self, A: _Block, B: _Block, where: tuple[str, str] + ) -> tuple[bool, dict[str, str], set[str], set[str]] | None: + """(整条语句最外层)同一张表:LEFT JOIN 对 INNER JOIN、关联条件不同、只有一边关联了(LEFT JOIN)、外连接的 + 主表对调。""" la, lb = self.sides(where) - first_b = self.sides(where, 1) def kind(join: Any) -> str: return str(join.side or "").lower() or "inner" @@ -4256,17 +5356,18 @@ class _Explainer: ra, rb = [r.alias for r in A.rels], [r.alias for r in B.rels] ja = {j.this.alias_or_name: j for j in A.joins} jb = {j.this.alias_or_name: j for j in B.joins} - if len(A.joins) != len(ja) or len(B.joins) != len(jb): + if len(A.joins) != len(ja) or len(B.joins) != len(jb) or any(not r.table for r in A.all_rels() + B.all_rels()): return None # 外连接的主表对调:两个来源,一边 x LEFT JOIN y,一边 y LEFT JOIN x,条件一样 if len(ra) == len(rb) == 1 and len(ja) == len(jb) == 1 and ra != rb and set(ra) == set(jb) and \ set(rb) == set(ja): x, y = A.joins[0], B.joins[0] - if kind(x) == kind(y) == "left" and self.fp(x.args["on"]) == self.fp(y.args["on"]): + if kind(x) == kind(y) == "left" and self.fp(x.args.get("on")) == self.fp(y.args.get("on")) and \ + not self._nullable_used(A, x.this.alias_or_name) and not self._nullable_used(B, y.this.alias_or_name): wa = _say("以", A.rels[0].shown, "为主表外连接", B.rels[0].shown) wb = _say("以", B.rels[0].shown, "为主表外连接", A.rels[0].shown) self.add("join", wa, wb, _say(la, wa, ",", lb, wb)) - return True, set(), set(), B + return True, {}, set(), set() return None if [r for r in ra if r not in rb and r not in jb] or [r for r in rb if r not in ra and r not in ja]: return None @@ -4279,14 +5380,18 @@ class _Explainer: for alias in sorted(set(ja) | set(jb)): x, y = ja.get(alias), jb.get(alias) if x is not None and y is not None: - if kind(x) != kind(y): + if kind(x) != kind(y) or not _plain_join(x) or not _plain_join(y): return None if self.fp(x) != self.fp(y): - on_x, on_y = self._on(x, A), self._on(y, B) - if on_x is None or on_y is None: + on_x, on_y = x.args.get("on"), y.args.get("on") + if on_x is None or on_y is None or self._same_condition(_conjuncts(on_x), _conjuncts(on_y), + A, B) is not False: + return None + if kind(x) == "left" and (self._nullable_used(A, alias) or self._nullable_used(B, alias)): return None rel = A.rel(alias) - self.add("join", on_x, on_y, _say(la, "关联", self.what(rel), "的条件是", on_x, ",", lb, "是", on_y)) + wa, wb = self.show(on_x, A, full=True), self.show(on_y, B, full=True) + self.add("join", wa, wb, _say(la, "关联", self.what(rel), "的条件是", wa, ",", lb, "是", wb)) continue # 一边 LEFT JOIN,另一边内连接(在开头那组来源里,关联条件在 WHERE 里) if x is not None and alias in rb and kind(x) == "left": @@ -4303,32 +5408,24 @@ class _Explainer: continue # 只有一边关联了 join = x if x is not None else y - if join is None or kind(join) != "left": + if join is None or kind(join) != "left" or not _plain_join(join): + return None + block, me, other = (A, la, lb) if x is not None else (B, lb, la) + if self._nullable_used(block, alias): return None - block, me, other = (A, la, lb) if x is not None else (B, first_b[1], first_b[0]) rel = block.rel(alias) - conds = [c for c in block.where if alias in {col.table for col in c.find_all(exp.Column)}] text = _say(me, *self._outer_words(join, block), self.what(rel), ",", other, "没有") - if conds: - if not all(self.base_columns(c, block) is not None for c in conds): - return None - text += _say("(WHERE 里还有", self.show_all(conds, block), ")") self.add("join", rel.shown if x is not None else None, rel.shown if y is not None else None, text) - (skip_a if x is not None else skip_b).update(self.fp(c) for c in conds) - return True, skip_a, skip_b, B - - def _on(self, join: Any, block: _Block) -> str | None: - on = join.args.get("on") - if on is None or self.base_columns(on, block) is None: - return None - return self.show(on, block) + return True, {}, skip_a, skip_b def _left_inner(self, join: Any, left: _Block, inner: _Block, alias: str, la: str, lb: str, side: int) -> set[str] | None: """left 这边 LEFT JOIN alias,inner 那边同一张表是内连接:inner 的 WHERE 里有 left 的 ON 的每一项才算关联条件 - 一样,只差连接类型。返回 inner 那边已经说过的 WHERE 项。照原文说:left 那边原来写的是 RIGHT JOIN 的说以哪一侧 - 为主表;inner 那边写的是 LEFT / RIGHT JOIN、被 WHERE 里的条件变成了内连接的,说写的是什么、实际是什么。side 是 - left 在哪一边(0 是 A);la、lb 是 A、B 的叫法。""" + 一样,只差连接类型。left 那边 WHERE 里用到可空一侧的列(可能已经把没关联上的行筛掉了)、结果只是可空一侧 + 的列的聚合(两种写法结果一样)时不说。照原文说:left 那边原来写的是 RIGHT JOIN 的说以哪一侧为主表;inner + 那边写的是 LEFT / RIGHT JOIN、被 WHERE 里的条件变成了内连接的,说写的是什么、实际是什么。""" + if not _plain_join(join) or self._nullable_used(left, alias) or self._null_blind(left, alias): + return None on = join.args.get("on") conds = [self.fp(c) for c in _conjuncts(on)] if on is not None else [] if not conds or not set(conds) <= {self.fp(c) for c in inner.where}: @@ -4358,10 +5455,29 @@ class _Explainer: return ("以", "、".join(r.shown for r in block.rels), "为主表外连接") return ("用 LEFT JOIN 关联",) + def derived(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str]) -> bool: + """FROM 里的子查询:两边别名对得上、定义不同的,只有一处时往里比(说「在 t 里」,照原来起的名字)。""" + pairs = [] + for ra in A.all_rels(): + if ra.kind != "derived": + continue + rb = next((r for r in B.all_rels() if mapping.get(r.alias, r.alias) == ra.alias), None) + if rb is None or rb.kind != "derived": + return False + if self.fp(ra.node.this) != self.fp(rb.node.this): + pairs.append((ra, rb)) + if not pairs: + return True + if len(pairs) > 1: + return False + ra, rb = pairs[0] + inner = (_deeper(where[0], ra.shown, plain=True), _deeper(where[1], rb.shown, plain=True)) + return self.query(ra.node.this, rb.node.this, inner, "derived") + # ---- WHERE - def timed(self, cond: Any, block: _Block) -> tuple[_Rel, Any] | None: - """时间条件:一列(可以套着日期函数、CAST)和常量比较,列名像时间或者常量像日期。返回 (来源, 列)。""" + def timed(self, cond: Any, block: _Block) -> tuple[str, str] | None: + """时间条件:一列(可以套着日期函数、CAST)和常量比较,列名像时间或者常量像日期。返回 (实表, 列)。""" from sqlglot import exp if not isinstance(cond, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.In)): @@ -4381,119 +5497,191 @@ class _Explainer: return None rel, col = cols[0] if _temporal_name(col.name) or any(_datey(o) for o in others) or _datey(column_side): - return rel, col + return rel.name, col.name return None - def conditions(self, A: _Block, B: _Block, where: tuple[str, str], skip_a: set[str], skip_b: set[str]) -> bool: + def conditions(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str], + skip_a: set[str], skip_b: set[str]) -> bool: la, lb = self.sides(where) - only_a, only_b = self._unmatched([c for c in A.where if self.fp(c) not in skip_a], - [c for c in B.where if self.fp(c) not in skip_b]) + xa = [c for c in A.where if self.fp(c) not in skip_a] + xb = [c for c in B.where if self.fp(c) not in skip_b] + only_a, only_b = self._unmatched(xa, xb, mapping) if not only_a and not only_b: return True covered = True - only_a, only_b = self._times(A, B, la, lb, only_a, only_b) - # 同一列上的两个条件配成一对 - for c in list(only_a): - col_a = self._single_column(c, A) - if col_a is None: - continue - mine = [x for x in only_a if self._single_column(x, A) == col_a] - theirs = [y for y in only_b if self._single_column(y, B) == col_a] - if len(mine) == 1 and len(theirs) == 1: - d = theirs[0] - wa, wb = self.show(c, A), self.show(d, B) - if wa == wb: - wa, wb = self.show(c, A, full=True), self.show(d, B, full=True) - self.add("filter", wa, wb, _say(la, "按", wa, "筛选,", lb, "按", wb, "筛选"), - {"columns": self._columns(c, A)}) - only_a.remove(c) - only_b.remove(d) - # 两边各剩一个、涉及的列一样的条件,也配成一对说 - if len(only_a) == 1 and len(only_b) == 1: - ca, cb = self.base_columns(only_a[0], A), self.base_columns(only_b[0], B) - if ca and cb and {(r.name, c.name) for r, c in ca} == {(r.name, c.name) for r, c in cb}: - wa, wb = self.show(only_a[0], A), self.show(only_b[0], B) - self.add("filter", wa, wb, _say(la, "按", wa, "筛选,", lb, "按", wb, "筛选"), - {"columns": self._columns(only_a[0], A)}) - only_a, only_b = [], [] - # 只在一边的条件:「另一边没有」带上在哪一段,免得读成整条语句都没有 + only_a, only_b, ok = self._times(A, B, mapping, la, lb, only_a, only_b) + covered &= ok + only_a, only_b, ok = self._same_subject(A, B, mapping, la, lb, only_a, only_b) + covered &= ok + only_a, only_b, ok = self._join_equalities(A, B, mapping, la, lb, only_a, only_b) + covered &= ok fa, fb = _say(self.la, where[0]), _say(self.lb, where[1]) - for side, rest, block, me, other in ((0, only_a, A, fa, fb), (1, only_b, B, fb, fa)): + for side, rest, block, other_block, other_usage, me, other in ( + (0, only_a, A, B, self.usage_b, fa, fb), (1, only_b, B, A, self.usage_a, fb, fa)): for c in list(rest): - if self.base_columns(c, block) is None: + refs = self.refs(c, block) + if not refs or other_usage is None or \ + other_usage.touches(refs, kinds=_kinds_for([c]), scope=other_block.select): + continue + if self.range.subject(c) is not None and self.range.compare([c], []) is not False: continue shown = self.show(c, block) excluded = self._excludes(c, block) text = _say(me, "排除了", excluded, "的行,", other, "没有") if excluded else \ _say(me, "只保留", shown, "的行,", other, "没有这个条件") self.add("filter", shown if side == 0 else None, shown if side == 1 else None, text, - {"columns": self._columns(c, block)}) + {"columns": [{"table": t, "column": col} for t, col in sorted(refs)]}) rest.remove(c) - # 条件里套着的子查询:两边各一个、套子查询之外写法一样的配成一对,往里比 - for c in list(only_a): - template = self._template(c) - if template is None: - continue - mine = [x for x in only_a if self._template(x) == template] - theirs = [d for d in only_b if self._template(d) == template] - if len(mine) == 1 and len(theirs) == 1: - d = theirs[0] - place = self._place(c) - inner = (_deeper(where[0], place), _deeper(where[1], place)) - covered &= self.query(self._subquery(c), self._subquery(d), inner, outputs=True) - only_a.remove(c) - only_b.remove(d) + only_a, only_b, ok = self._subqueries(A, B, mapping, where, only_a, only_b) + covered &= ok return covered and not only_a and not only_b - def _times(self, A: _Block, B: _Block, la: str, lb: str, only_a: list[Any], only_b: list[Any] - ) -> tuple[list[Any], list[Any]]: - """时间条件:两边差在按哪一列筛时间、同一列上的范围,或者只有一边按时间筛。说到了的从两边拿掉。""" - ta = [(c, t) for c in only_a if (t := self.timed(c, A)) is not None] - tb = [(c, t) for c in only_b if (t := self.timed(c, B)) is not None] - cols_a = {(t[0].name, t[1].name) for _, t in ta} - cols_b = {(t[0].name, t[1].name) for _, t in tb} - if not (ta or tb) or len(cols_a) > 1 or len(cols_b) > 1: - return only_a, only_b - all_a = [(c, (t[0].name, t[1].name)) for c in A.where if (t := self.timed(c, A)) is not None] - all_b = [(c, (t[0].name, t[1].name)) for c in B.where if (t := self.timed(c, B)) is not None] - detail = {"a": [{"table": t, "column": c} for t, c in sorted(cols_a)], - "b": [{"table": t, "column": c} for t, c in sorted(cols_b)]} - if cols_a and cols_b and cols_a != cols_b: - (_, ca), (_, cb) = next(iter(cols_a)), next(iter(cols_b)) - text = _say(la, "按", self.named(ta[0][1][0], ca), "筛时间,", lb, "按", self.named(tb[0][1][0], cb)) - if self._shape([c for c, _ in ta]) != self._shape([c for c, _ in tb]): - text += _say("(", la, "是", self.show_all(_by_bound([c for c, _ in ta]), A), ",", lb, "是", - self.show_all(_by_bound([c for c, _ in tb]), B), ")") - self.add("time", ca, cb, text, detail) - else: - [col] = list(cols_a | cols_b) - ra = _by_bound([c for c, k in all_a if k == col]) - rb = _by_bound([c for c, k in all_b if k == col]) - rel = (ta or tb)[0][1][0] - if ra and rb: - wa, wb = self.show_all(ra, A), self.show_all(rb, B) - self.add("time_range", wa, wb, _say(la, "的时间范围是", wa, ",", lb, "是", wb)) - elif ra and not all_b: - self.add("time", col[1], None, _say(la, "按", self.named(rel, col[1]), "筛时间,", lb, "没有按时间筛选"), - detail) - elif rb and not all_a: - self.add("time", None, col[1], _say(la, "没有按时间筛选,", lb, "按", self.named(rel, col[1]), "筛时间"), - detail) - else: - return only_a, only_b - return [c for c in only_a if all(c is not x for x, _ in ta)], [c for c in only_b if all(c is not x for x, _ in tb)] - - def _unmatched(self, xs: list[Any], ys: list[Any]) -> tuple[list[Any], list[Any]]: + def _unmatched(self, xs: list[Any], ys: list[Any], mapping: Mapping[str, str]) -> tuple[list[Any], list[Any]]: rest = list(ys) only_x = [] for x in xs: - found = next((y for y in rest if self.fp(y) == self.fp(x)), None) + found = next((y for y in rest if self.mfp(y, mapping) == self.fp(x)), None) if found is None: only_x.append(x) else: rest.remove(found) return only_x, rest + def _on_column(self, conds: list[Any], block: _Block, key: tuple[str, str]) -> list[Any]: + return [c for c in conds if self.refs(c, block) == {key}] + + def _times(self, A: _Block, B: _Block, mapping: Mapping[str, str], la: str, lb: str, only_a: list[Any], + only_b: list[Any]) -> tuple[list[Any], list[Any], bool]: + """时间条件:两边差在按哪一列筛时间、同一列上的范围,或者只有一边按时间筛。说到了的从两边拿掉。""" + ta = [(c, t) for c in only_a if (t := self.timed(c, A)) is not None] + tb = [(c, t) for c in only_b if (t := self.timed(c, B)) is not None] + cols_a, cols_b = {t for _, t in ta}, {t for _, t in tb} + if not (ta or tb) or len(cols_a) > 1 or len(cols_b) > 1 or self.usage_a is None or self.usage_b is None: + return only_a, only_b, True + detail = {"a": [{"table": t, "column": c} for t, c in sorted(cols_a)], + "b": [{"table": t, "column": c} for t, c in sorted(cols_b)]} + all_a = {t for c in A.where if (t := self.timed(c, A)) is not None} + all_b = {t for c in B.where if (t := self.timed(c, B)) is not None} + if cols_a and cols_b and cols_a != cols_b: + (ka,), (kb,) = cols_a, cols_b + if all_a != cols_a or all_b != cols_b or self.usage_b.touches([ka], scope=B.select) or \ + self.usage_a.touches([kb], scope=A.select): + return only_a, only_b, False + text = _say(la, "按", self.named(*ka), "筛时间,", lb, "按", self.named(*kb)) + if self._shape([c for c, _ in ta]) != self._shape([c for c, _ in tb]): + text += _say("(", la, "是", self.show_all(_by_bound([c for c, _ in ta]), A), ",", lb, "是", + self.show_all(_by_bound([c for c, _ in tb]), B), ")") + self.add("time", ka[1], kb[1], text, detail) + elif cols_a == cols_b: + (key,) = cols_a + ra = _by_bound(self._on_column(A.where, A, key)) + rb = _by_bound(self._on_column(B.where, B, key)) + same = self.range.compare(ra, [self.mapped(c, mapping) for c in rb]) + if same is not False or self.usage_a.touches([key], ra, scope=A.select) or \ + self.usage_b.touches([key], rb, scope=B.select): + return only_a, only_b, False + wa, wb = self.show_all(ra, A), self.show_all(rb, B) + self.add("time_range", wa, wb, _say(la, "的时间范围是", wa, ",", lb, "是", wb)) + else: + (key,) = cols_a or cols_b + if cols_a: + if self.usage_b.temporal or self.usage_b.touches([key], scope=B.select): + return only_a, only_b, False + self.add("time", key[1], None, _say(la, "按", self.named(*key), "筛时间,", lb, "没有按时间筛选"), detail) + else: + if self.usage_a.temporal or self.usage_a.touches([key], scope=A.select): + return only_a, only_b, False + self.add("time", None, key[1], _say(la, "没有按时间筛选,", lb, "按", self.named(*key), "筛时间"), detail) + return [c for c in only_a if all(c is not x for x, _ in ta)], \ + [c for c in only_b if all(c is not x for x, _ in tb)], True + + def _same_subject(self, A: _Block, B: _Block, mapping: Mapping[str, str], la: str, lb: str, + only_a: list[Any], only_b: list[Any]) -> tuple[list[Any], list[Any], bool]: + """同一个对象只和常量比的条件:两边这个对象上的条件合起来按取值逐段算(_Range),留下的行一样的不说;不一样 + 的,两边都有没配上的条件时说「A 按 X 筛选,B 按 Y 筛选」,只有一边有时说「A 只保留 X 的行,B 没有这个条件」 + (另一边在这个对象上的条件这一边都有,多出来的确实多筛掉了行)。这一列在两边别的地方(别的条件、子查询、 + CTE)还用来筛选的不说:留下哪些行就不只是这几个条件定的。""" + covered = True + done_a: list[Any] = [] + done_b: list[Any] = [] + + def subject(c: Any, block: _Block, m: Mapping[str, str]) -> str | None: + s = self.range.subject(self.mapped(c, m)) + return s[0] if s is not None and self.refs(c, block) is not None else None + + keys_a = {id(c): subject(c, A, {}) for c in A.where} + keys_b = {id(c): subject(c, B, mapping) for c in B.where} + wanted = [k for k in dict.fromkeys([keys_a.get(id(c)) for c in only_a] + [keys_b.get(id(c)) for c in only_b]) + if k is not None] + for key in wanted: + mine = [c for c in only_a if keys_a.get(id(c)) == key] + theirs = [c for c in only_b if keys_b.get(id(c)) == key] + all_a = [c for c in A.where if keys_a.get(id(c)) == key] + all_b = [c for c in B.where if keys_b.get(id(c)) == key] + refs_a = set().union(*(self.refs(c, A) for c in all_a)) if all_a else set() + refs_b = set().union(*(self.refs(c, B) for c in all_b)) if all_b else set() + done_a += mine + done_b += theirs + same = self.range.compare(all_a, [self.mapped(c, mapping) for c in all_b]) + usage_a, usage_b = self.usage_a, self.usage_b + kinds = _kinds_for(all_a + all_b) + if same is not False or usage_a is None or usage_b is None or \ + usage_a.touches(refs_a | refs_b, all_a, kinds, A.select) or \ + usage_b.touches(refs_a | refs_b, all_b, kinds, B.select): + covered = False + continue + detail_a = {"columns": [{"table": t, "column": col} for t, col in sorted(refs_a)]} + detail_b = {"columns": [{"table": t, "column": col} for t, col in sorted(refs_b)]} + if mine and theirs: + wa, wb = self.show_all(mine, A), self.show_all(theirs, B) + if wa == wb: + wa, wb = self.show(exp_and(mine), A, full=True), self.show(exp_and(theirs), B, full=True) + self.add("filter", wa, wb, _say(la, "按", wa, "筛选,", lb, "按", wb, "筛选"), detail_a) + else: + side, conds, block, me, other, detail = (0, mine, A, la, lb, detail_a) if mine else \ + (1, theirs, B, lb, la, detail_b) + shown = self.show_all(conds, block) + excluded = self._excludes(conds[0], block) if len(conds) == 1 else None + text = _say(me, "排除了", excluded, "的行,", other, "没有") if excluded else \ + _say(me, "只保留", shown, "的行,", other, "没有这个条件") + self.add("filter", shown if side == 0 else None, shown if side == 1 else None, text, detail) + return [c for c in only_a if all(c is not d for d in done_a)], \ + [c for c in only_b if all(c is not d for d in done_b)], covered + + def _join_equalities(self, A: _Block, B: _Block, mapping: Mapping[str, str], la: str, lb: str, + only_a: list[Any], only_b: list[Any]) -> tuple[list[Any], list[Any], bool]: + """两边没配上的条件都是「两列相等」的关联条件时,比相等关系把列分成的组:分法不同就是关联的条件不同(列各是各的 + 表);分法一样(a = b AND b = c 和 a = b AND a = c)不说。""" + ea = [c for c in only_a if _column_equality(c) and self.base_columns(c, A) is not None] + eb = [c for c in only_b if _column_equality(c) and self.base_columns(c, B) is not None] + if not ea or not eb or len(ea) != len(only_a) or len(eb) != len(only_b): + return only_a, only_b, True + all_a = [c for c in A.where if _column_equality(c)] + all_b = [self.mapped(c, mapping) for c in B.where if _column_equality(c)] + if _partition(all_a) == _partition(all_b): + return [], [], False + wa, wb = self.show_all(ea, A), self.show_all(eb, B) + if wa == wb: + wa, wb = self.show(exp_and(ea), A, full=True), self.show(exp_and(eb), B, full=True) + self.add("join", wa, wb, _say(la, "关联的条件是", wa, ",", lb, "是", wb)) + return [], [], True + + def _same_condition(self, xs: list[Any], ys: list[Any], A: _Block, B: _Block) -> bool | None: + """两组条件(各自 AND 连)是不是一样:只差在两列相等的分法上、或者同一个对象只和常量比时能判断;不然 None。""" + only_x, only_y = self._unmatched(xs, ys, {}) + if not only_x and not only_y: + return True + if all(_column_equality(c) for c in only_x + only_y): + return _partition([c for c in xs if _column_equality(c)]) == \ + _partition([c for c in ys if _column_equality(c)]) + subjects = {self.range.subject(c)[0] if self.range.subject(c) else None for c in only_x + only_y} + if None in subjects or len(subjects) != 1: + return None + (fp,) = subjects + all_x = [c for c in xs if (s := self.range.subject(c)) is not None and s[0] == fp] + all_y = [c for c in ys if (s := self.range.subject(c)) is not None and s[0] == fp] + return self.range.compare(all_x, all_y) + def _shape(self, conds: list[Any]) -> list[str]: """时间条件把那一列换成占位符之后的写法:两边只差时间列时一样。""" from sqlglot import exp @@ -4506,16 +5694,6 @@ class _Explainer: out.append(self.fp(copy)) return sorted(out) - def _single_column(self, cond: Any, block: _Block) -> tuple[str, str] | None: - cols = self.base_columns(cond, block) - if not cols or len({(r.name, c.name) for r, c in cols}) != 1: - return None - return cols[0][0].name, cols[0][1].name - - def _columns(self, cond: Any, block: _Block) -> list[dict[str, str]]: - cols = self.base_columns(cond, block) or [] - return [{"table": t, "column": c} for t, c in sorted({(r.name, col.name) for r, col in cols})] - def _excludes(self, cond: Any, block: _Block) -> str | None: """「排除」的写法去掉否定之后排除的是什么:<> 写成 =,NOT x 写成 x,IS NOT NULL 写成「列 为空」。""" from sqlglot import exp @@ -4529,7 +5707,30 @@ class _Explainer: return self.show(inner, block) return None - def _template(self, cond: Any) -> str | None: + def _subqueries(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str], + only_a: list[Any], only_b: list[Any]) -> tuple[list[Any], list[Any], bool]: + """条件里套着的子查询:两边各一个、除了子查询之外写法一模一样(含 NOT、IN 左边的列)的配成一对往里比。""" + covered = True + done_a, done_b = [], [] + for c in only_a: + template = self._template(c, {}) + if template is None: + continue + mine = [x for x in only_a if self._template(x, {}) == template] + theirs = [d for d in only_b if self._template(d, mapping) == template] + if len(mine) != 1 or len(theirs) != 1: + continue + d = theirs[0] + place_a, place_b = self._place(c, A), self._place(d, B) + inner = (_deeper(where[0], place_a), _deeper(where[1], place_b)) + kind = _subquery_kind(c) + covered &= self.query(self._subquery(c), self._subquery(d), inner, kind) + done_a.append(c) + done_b.append(d) + return [c for c in only_a if all(c is not x for x in done_a)], \ + [c for c in only_b if all(c is not x for x in done_b)], covered + + def _template(self, cond: Any, mapping: Mapping[str, str]) -> str | None: """条件里只套着一个子查询时,子查询换成占位符之后的写法;不然 None。""" from sqlglot import exp @@ -4537,36 +5738,53 @@ class _Explainer: not any(isinstance(p, (exp.Select, exp.SetOperation)) for p in _ancestors(n, cond))] if len(tops) != 1: return None - copy = cond.copy() + copy = self.mapped(cond, mapping).copy() if mapping else cond.copy() for node in copy.walk(): if isinstance(node, (exp.Select, exp.SetOperation)): node.replace(exp.Literal.string("")) break return self.fp(copy) - def _subquery(self, cond: Any) -> Any: + @staticmethod + def _subquery(cond: Any) -> Any: from sqlglot import exp return next(n for n in cond.walk() if isinstance(n, (exp.Select, exp.SetOperation))) - def _place(self, cond: Any) -> str: + def _place(self, cond: Any, block: _Block) -> str: + """子查询在条件里的位置:「users.id NOT IN 的子查询」「NOT EXISTS 子查询」「条件的子查询」。""" from sqlglot import exp - if cond.find(exp.Exists) is not None: - return "EXISTS 子查询" - if cond.find(exp.In) is not None: - return "IN 子查询" + sub = self._subquery(cond) + node = sub + while isinstance(node.parent, (exp.Subquery, exp.Paren)): + node = node.parent + parent = node.parent + negated = isinstance(_unparen_parent(parent), exp.Not) if parent is not None else False + if isinstance(parent, exp.Exists): + return "NOT EXISTS 子查询" if negated else "EXISTS 子查询" + if isinstance(parent, exp.In): + return _say(self.show(parent.this, block), "NOT IN" if negated else "IN", "的子查询") return "条件的子查询" # ---- 选出来的列、分组 - def groups(self, A: _Block, B: _Block, where: tuple[str, str]) -> bool: - """分组键:两边不同、又都只涉及实表列时说各按什么分组(从 SELECT DISTINCT 改写来的说「去重」)。""" + def groups(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str]) -> bool: + """分组键:两边不同、又都只涉及实表列、多出来的键不是另一边的键算出来的时,说各按什么分组。""" la, lb = self.sides(where) - if sorted(self.fp(g) for g in A.group) == sorted(self.fp(g) for g in B.group): + ka = sorted(self.fp(g) for g in A.group) + kb = sorted(self.mfp(g, mapping) for g in B.group) + if ka == kb: return True - if not all(self.base_columns(g, A) is not None for g in A.group) or \ - not all(self.base_columns(g, B) is not None for g in B.group): + ra = [self.refs(g, A) for g in A.group] + rb = [self.refs(g, B) for g in B.group] + if any(r is None for r in ra + rb): + return False + cols_a = set().union(*ra) if ra else set() + cols_b = set().union(*rb) if rb else set() + extra_a = [g for g, r in zip(A.group, ra) if self.fp(g) not in kb] + extra_b = [g for g, r in zip(B.group, rb) if self.mfp(g, mapping) not in ka] + if any(self.refs(g, A) <= cols_b for g in extra_a) or any(self.refs(g, B) <= cols_a for g in extra_b): return False def say(block: _Block, other: _Block) -> tuple[str, ...]: @@ -4580,26 +5798,48 @@ class _Explainer: self.add("group_by", wa, wb, _say(la, *say(A, B), ",", lb, *say(B, A))) return True - def outputs(self, A: _Block, B: _Block, where: tuple[str, str]) -> bool: - """分组键、选出来的列。分组键不同时,选出来的列只差在分组键那几列上(都排在前面、各是各的分组键)的,算在 - 分组键的不同里,不另说;其余逐列比,列数不同、只是先后不同各说一句。""" + def _dedup_swap(self, A: _Block, B: _Block, mapping: Mapping[str, str]) -> bool: + """一边 SELECT DISTINCT(不分组),另一边按选出来的那几列分组、不去重、不聚合:两种写法都是去掉重复的行, + 不能说成一边去重一边没去重。""" + def plain(block: _Block) -> bool: + return block.distinct and not block.distinct_on and not block.group and not _aggregating(block.select) + + def grouped(block: _Block, m: Mapping[str, str]) -> set[str] | None: + if block.distinct or not block.group or block.having or \ + any(_aggregate(e.unalias()) is not None for e in block.exprs): + return None + keys = {self.mfp(g, m) for g in block.group} + outs = {self.mfp(e.unalias(), m) for e in block.exprs} + return keys if keys == outs else None + + outs_a = {self.fp(e.unalias()) for e in A.exprs} + outs_b = {self.mfp(e.unalias(), mapping) for e in B.exprs} + return (plain(A) and grouped(B, mapping) == outs_a) or (plain(B) and grouped(A, {}) == outs_b) + + def outputs(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str], ctx: str) -> bool: + """分组键、选出来的列。选出来的列只在整条语句最外层、合并的一支、标量子查询里比(CTE、FROM 里的子查询选出来的 + 列规范化时改过名、排过序,和原文对不上);分组键不同时,选出来的列只差在分组键那几列上的算在分组键的不同里。 + 其余逐列比:都是一个聚合、都只是一列时才说,两列被关联条件定成相等的不说。""" la, lb = self.sides(where) - ga, gb = [self.fp(g) for g in A.group], [self.fp(g) for g in B.group] + ga = [self.fp(g) for g in A.group] + gb = [self.mfp(g, mapping) for g in B.group] regrouped = sorted(ga) != sorted(gb) - covered = self.groups(A, B, where) + if self._dedup_swap(A, B, mapping): + return False + covered = self.groups(A, B, mapping, where) ea, eb = A.exprs, B.exprs - ta, tb = [self.fp(e) for e in ea], [self.fp(e) for e in eb] + ta, tb = [self.fp(e.unalias()) for e in ea], [self.mfp(e.unalias(), mapping) for e in eb] if ta == tb: return covered + if ctx not in ("top", "branch", "scalar") or any(_is_star(e) for e in ea + eb): + return False if regrouped and covered: - ka = next((i for i, t in enumerate(ta) if t not in ga), len(ta)) - kb = next((i for i, t in enumerate(tb) if t not in gb), len(tb)) - if set(ta[:ka]) == set(ga) and set(tb[:kb]) == set(gb) and ta[ka:] == tb[kb:]: + na = next((i for i, t in enumerate(ta) if t not in ga), len(ta)) + nb = next((i for i, t in enumerate(tb) if t not in gb), len(tb)) + if set(ta[:na]) == set(ga) and set(tb[:nb]) == set(gb) and ta[na:] == tb[nb:]: return covered - confident = all(self.base_columns(e, A) is not None for e in ea) and \ - all(self.base_columns(e, B) is not None for e in eb) if len(ea) != len(eb) or sorted(ta) == sorted(tb): - if not confident: + if any(self.base_columns(e, A) is None for e in ea) or any(self.base_columns(e, B) is None for e in eb): return False wa = "、".join(self.show(e, A) for e in ea) wb = "、".join(self.show(e, B) for e in eb) @@ -4610,25 +5850,47 @@ class _Explainer: self.add("expression", wa, wb, text) return covered for i, (x, y) in enumerate(zip(ea, eb), 1): - if self.fp(x) == self.fp(y): + if self.fp(x.unalias()) == self.mfp(y.unalias(), mapping): continue - # 只差在列名上(最外层还按名字排序时别名留着):取值一样,没什么可说的,也说不清排序那边,交给别的项 - if self.fp(x.unalias()) == self.fp(y.unalias()) or not self.output(x, y, A, B, la, lb, i, len(ea)): + if not self.output(x, y, A, B, mapping, la, lb, i, len(ea)): covered = False return covered - def output(self, x: Any, y: Any, A: _Block, B: _Block, la: str, lb: str, i: int, n: int) -> bool: + def output(self, x: Any, y: Any, A: _Block, B: _Block, mapping: Mapping[str, str], la: str, lb: str, i: int, + n: int) -> bool: from sqlglot import exp if self.base_columns(x, A) is None or self.base_columns(y, B) is None: return False - ga, gb = _aggregate(x.unalias()), _aggregate(y.unalias()) + xu, yu = x.unalias(), y.unalias() + equal = _partition([c for c in A.where if _column_equality(c)] + + [self.mapped(c, mapping) for c in B.where if _column_equality(c)]) + ga, gb = _aggregate(xu), _aggregate(yu) if ga is not None and gb is not None: (fa, arg_a, da), (fb, arg_b, db) = ga, gb + same_arg = (arg_a is None and arg_b is None) or ( + arg_a is not None and arg_b is not None and (self.fp(arg_a) == self.mfp(arg_b, mapping) or ( + isinstance(_unparen(arg_a), exp.Column) and isinstance(_unparen(arg_b), exp.Column) and + self.refs(arg_a, A) == self.refs(arg_b, B)))) + if not same_arg and any(arg is not None and not isinstance(_unparen(arg), exp.Column) + for arg in (arg_a, arg_b)): + return False + if same_arg and fa == fb and da == db: + return False + if not same_arg and arg_a is not None and arg_b is not None and \ + _same_class(equal, self.fp(arg_a), self.mfp(arg_b, mapping)): + return False + if fa == fb and same_arg and fa in ("min", "max"): + return False + if {fa, fb} == {"count"} and (arg_a is None) != (arg_b is None): + arg = arg_a if arg_a is not None else arg_b + block = A if arg_a is not None else B + refs = self.refs(arg, block) or set() + if self.usage_a is None or self.usage_b is None or \ + self.usage_a.touches(refs, [xu], scope=A.select) or self.usage_b.touches(refs, [yu], scope=B.select): + return False wa = self.show(arg_a, A) if arg_a is not None else "*" wb = self.show(arg_b, B) if arg_b is not None else "*" - same_arg = (arg_a is None and arg_b is None) or ( - arg_a is not None and arg_b is not None and self.fp(arg_a) == self.fp(arg_b)) if wa == wb and not same_arg: wa, wb = self.show(arg_a, A, full=True), self.show(arg_b, B, full=True) if same_arg and fa == fb: @@ -4642,58 +5904,99 @@ class _Explainer: text = _say(la, *what_a, ",", lb, *what_b) self.add("aggregate", self.show(x, A), self.show(y, B), text) return True - wa, wb = self.show(x, A), self.show(y, B) - if wa == wb: - wa, wb = self.show(x, A, full=True), self.show(y, B, full=True) - verb = "选的是" if isinstance(x.unalias(), exp.Column) and isinstance(y.unalias(), exp.Column) else "算的是" - where_col = _say("第", str(i), "列") if n > 1 else "" - self.add("expression", wa, wb, _say(la, where_col, verb, wa, ",", lb, verb, wb)) - return True + if isinstance(xu, exp.Column) and isinstance(yu, exp.Column): + if _same_class(equal, self.fp(xu), self.mfp(yu, mapping)) or self.refs(xu, A) == self.refs(yu, B): + return False + wa, wb = self.show(x, A), self.show(y, B) + if wa == wb: + wa, wb = self.show(x, A, full=True), self.show(y, B, full=True) + where_col = _say("第", str(i), "列") if n > 1 else "" + self.add("expression", wa, wb, _say(la, where_col, "选的是", wa, ",", lb, "选的是", wb)) + return True + return False # ---- HAVING、取行 - def having(self, A: _Block, B: _Block, where: tuple[str, str]) -> bool: + def having(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str]) -> bool: + """HAVING:同一个对象(聚合)只和常量比的,两边合起来逐段算,留下的组一样的不说(COUNT(*) > 1 和 >= 2、分组 + 之后 COUNT(*) > 0);只有一边有、另一边 HAVING 里没有别的条件的,说「分组后只保留」。""" la, lb = self.sides(where) - only_a, only_b = self._unmatched(A.having, B.having) + only_a, only_b = self._unmatched(A.having, B.having, mapping) if not only_a and not only_b: return True - if len(only_a) == 1 and len(only_b) == 1 and self.base_columns(only_a[0], A) is not None and \ - self.base_columns(only_b[0], B) is not None: - wa, wb = self.show(only_a[0], A), self.show(only_b[0], B) + grouped = bool(A.group) and bool(B.group) + if any(self.base_columns(c, A) is None for c in only_a) or any(self.base_columns(c, B) is None for c in only_b): + return False + subjects = {s[0] for c in only_a + [self.mapped(c, mapping) for c in only_b] + if (s := self.range.subject(c)) is not None} + if len(subjects) == 1: + (fp,) = subjects + all_a = [c for c in A.having if (s := self.range.subject(c)) is not None and s[0] == fp] + all_b = [self.mapped(c, mapping) for c in B.having + if (s := self.range.subject(self.mapped(c, mapping))) is not None and s[0] == fp] + if len(all_a) != len(A.having) or len(all_b) != len(B.having) or \ + self.range.compare(all_a, all_b, grouped=grouped) is not False: + return False + elif subjects or only_a and only_b: + return False + if only_a and only_b: + wa, wb = self.show_all(only_a, A), self.show_all(only_b, B) self.add("having", wa, wb, _say(la, "分组后只保留", wa, "的组,", lb, "只保留", wb, "的组")) return True - covered = True - fa, fb = _say(self.la, where[0]), _say(self.lb, where[1]) - for side, rest, block, me, other in ((0, only_a, A, fa, fb), (1, only_b, B, fb, fa)): - for c in rest: - if self.base_columns(c, block) is None: - covered = False - continue - shown = self.show(c, block) - self.add("having", shown if side == 0 else None, shown if side == 1 else None, - _say(me, "分组后只保留", shown, "的组,", other, "没有这个条件")) - return covered + side, rest, block, me, other = (0, only_a, A, la, lb) if only_a else (1, only_b, B, lb, la) + usage = self.usage_b if side == 0 else self.usage_a + refs = set().union(*(self.refs(c, block) or set() for c in rest)) + if (B.having if side == 0 else A.having) or usage is None or \ + usage.touches(refs, scope=(B if side == 0 else A).select): + return False + shown = self.show_all(rest, block) + self.add("having", shown if side == 0 else None, shown if side == 1 else None, + _say(me, "分组后只保留", shown, "的组,", other, "没有这个条件")) + return True - def rows(self, A: _Block, B: _Block, where: tuple[str, str]) -> bool: + def rows(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str], ctx: str) -> bool: + """取行、排序、去重。取行在标量子查询、EXISTS 子查询里不说(SQLite 的标量子查询本来就取第一行),不分组的聚合 + (只出一行:取行、排序、去重都不起作用)不说;排序只在整条语句最外层、没有取行、DISTINCT ON、窗口函数时说成 + order,有取行时算 limit,别的地方不说(子查询里的排序可能决定 GROUP_CONCAT 的先后),按常量排的不说。""" la, lb = self.sides(where) - if A.limit != B.limit: - self.add("limit", A.limit, B.limit, _say(la, _rows(A.limit), ",", lb, _rows(B.limit))) - if A.offset != B.offset: - self.add("limit", A.offset, B.offset, _say(la, _skip(A.offset), ",", lb, _skip(B.offset))) - if [self.fp(o) for o in A.order] != [self.fp(o) for o in B.order]: - if not all(self.base_columns(o, A) is not None for o in A.order) or \ - not all(self.base_columns(o, B) is not None for o in B.order): - return False - wa, wb = self._sorting(A.order, A), self._sorting(B.order, B) - if A.limit is not None or B.limit is not None: - self.add("limit", wa, wb, _say(la, "取行前", wa, ",", lb, wb)) + covered = True + + def single(block: _Block) -> bool: + return _aggregating(block.select) and not block.group + + if A.limit != B.limit or A.offset != B.offset: + if ctx in ("scalar", "exists") or single(A) or single(B): + covered = False else: - self.add("order", wa, wb, _say(la, wa, ",", lb, wb)) + if A.limit != B.limit: + self.add("limit", A.limit, B.limit, _say(la, _rows(A.limit), ",", lb, _rows(B.limit))) + if A.offset != B.offset: + self.add("limit", A.offset, B.offset, _say(la, _skip(A.offset), ",", lb, _skip(B.offset))) + if [self.fp(o) for o in A.order] != [self.mfp(o, mapping) for o in B.order]: + limited = A.limit is not None or B.limit is not None or A.offset is not None or B.offset is not None + if A.distinct_on or B.distinct_on or A.windowed or B.windowed or single(A) or single(B) or \ + any(not self.base_columns(o.this, A) for o in A.order) or \ + any(not self.base_columns(o.this, B) for o in B.order) or \ + (not limited and ctx != "top") or (limited and ctx in ("scalar", "exists")): + covered = False + else: + wa, wb = self._sorting(A.order, A), self._sorting(B.order, B) + if limited: + self.add("limit", wa, wb, _say(la, "取行前", wa, ",", lb, wb)) + else: + self.add("order", wa, wb, _say(la, wa, ",", lb, wb)) if A.distinct != B.distinct: + if A.distinct_on or B.distinct_on or self._dedup_swap(A, B, mapping) or ctx in ("in", "exists") or \ + single(A) or single(B): + return False + block = A if A.distinct else B + keys = {self.fp(g) for g in block.group} + if keys and keys <= {self.fp(e.unalias()) for e in block.exprs}: + return False me, other = (la, lb) if A.distinct else (lb, la) self.add("group_by", "DISTINCT" if A.distinct else None, "DISTINCT" if B.distinct else None, _say(me, "用 SELECT DISTINCT 去掉了重复的行,", other, "没有")) - return True + return covered def _sorting(self, order: list[Any], block: _Block) -> str: if not order: @@ -4703,28 +6006,44 @@ class _Explainer: # ---- UNION - def setop(self, a: Any, b: Any, where: tuple[str, str], outputs: bool) -> bool: + def setop(self, a: Any, b: Any, where: tuple[str, str], ctx: str) -> bool: + """合并:种类不同(UNION 对 INTERSECT)不说具体的;UNION 对 UNION ALL 说一句。最外层的 UNION ALL 各支排过序, + 按写法配对;别的(去重的 UNION、EXCEPT、INTERSECT、套在里面的)按先后逐支比(EXCEPT 两边对调了就对不上)。 + 只有一支不同的往里比。""" from sqlglot import exp la, lb = self.sides(where) if type(a) is not type(b): return False xa, xb = _set_branches(a), _set_branches(b) + covered = True + if bool(a.args.get("distinct")) != bool(b.args.get("distinct")) and isinstance(a, exp.Union): + wa, wb = _merging(a), _merging(b) + self.add("set_op", wa, wb, _say(la, "用", wa, ",", lb, "用", wb)) if len(xa) != len(xb): if isinstance(a, exp.Union) and bool(a.args.get("distinct")) == bool(b.args.get("distinct")): self.add("set_op", str(len(xa)), str(len(xb)), _say(la, "合并了", str(len(xa)), "支查询,", lb, "合并了", str(len(xb)), "支")) return False - covered = True - if bool(a.args.get("distinct")) != bool(b.args.get("distinct")) and isinstance(a, exp.Union): - wa, wb = _merging(a), _merging(b) - self.add("set_op", wa, wb, _say(la, "用", wa, ",", lb, "用", wb)) - only_a, only_b = self._unmatched(xa, xb) - if len(only_a) == 1 and len(only_b) == 1: - branch = (_deeper(where[0], "合并的一支"), _deeper(where[1], "合并的一支")) - covered &= self.query(only_a[0], only_b[0], branch, outputs=outputs) - elif only_a or only_b: - covered = False + free = isinstance(a, exp.Union) and not a.args.get("distinct") and not b.args.get("distinct") and \ + ctx == "top" and not any(a.args.get(k) or b.args.get(k) for k in ("order", "limit", "offset")) + if free: + only_a, only_b = self._unmatched(xa, xb, {}) + pairs = list(zip(only_a, only_b)) if len(only_a) == len(only_b) == 1 else None + if only_a and pairs is None: + return False + else: + pairs = [(x, y) for x, y in zip(xa, xb) if self.fp(x) != self.fp(y)] + if len(pairs) > 1: + return False + for x, y in pairs or []: + if isinstance(a, exp.Except): + index = next(i for i, v in enumerate(xa) if v is x) + place = "EXCEPT 前面那支" if index == 0 else "EXCEPT 后面那支" + else: + place = "合并的一支" + branch = (_deeper(where[0], place), _deeper(where[1], place)) + covered &= self.query(x, y, branch, "branch" if ctx == "top" else ctx) for key in ("order", "limit", "offset"): va = self.fp(a.args[key]) if a.args.get(key) else "" vb = self.fp(b.args[key]) if b.args.get(key) else "" @@ -4733,7 +6052,61 @@ class _Explainer: return covered -def _deeper(where: str, place: str) -> str: +def exp_and(conds: list[Any]) -> Any: + from sqlglot import exp + + return conds[0] if len(conds) == 1 else exp.and_(*[c.copy() for c in conds], copy=False) + + +def _column_equality(cond: Any) -> bool: + from sqlglot import exp + + cond = _unparen(cond) + return isinstance(cond, exp.EQ) and isinstance(_unparen(cond.this), exp.Column) and \ + isinstance(_unparen(cond.expression), exp.Column) + + +def _partition(conds: list[Any]) -> frozenset[frozenset[str]]: + """「两列相等」的几个条件把列分成的组(传递:a = b、b = c 时 a、b、c 一组)。""" + parent: dict[str, str] = {} + + def find(x: str) -> str: + parent.setdefault(x, x) + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + for c in conds: + c = _unparen(c) + a, b = _fingerprint(_unparen(c.this)), _fingerprint(_unparen(c.expression)) + parent[find(a)] = find(b) + groups: dict[str, set[str]] = {} + for x in list(parent): + groups.setdefault(find(x), set()).add(x) + return frozenset(frozenset(g) for g in groups.values() if len(g) > 1) + + +def _same_class(partition: frozenset[frozenset[str]], x: str, y: str) -> bool: + return any(x in g and y in g for g in partition) + + +def _subquery_kind(cond: Any) -> str: + from sqlglot import exp + + sub = next(n for n in cond.walk() if isinstance(n, (exp.Select, exp.SetOperation))) + node = sub + while isinstance(node.parent, (exp.Subquery, exp.Paren)): + node = node.parent + parent = node.parent + if isinstance(parent, exp.Exists): + return "exists" + if isinstance(parent, exp.In): + return "in" + return "scalar" + + +def _deeper(where: str, place: str, plain: bool = False) -> str: """在哪一段里再往里一层:「在 t 里」+「IN 子查询」→「在 t 的 IN 子查询里」。""" if not where: return _say("在", place, "里") @@ -4781,12 +6154,13 @@ def _merging(query: Any) -> str: def _aggregate(node: Any) -> tuple[str, Any, bool] | None: - """光一个聚合(SUM / AVG / COUNT / MIN / MAX):(函数, 参数(COUNT(*) 是 None), 是否去重)。""" + """光一个聚合(SUM / AVG / COUNT / MIN / MAX,MIN / MAX 只有一个参数:两个参数的 max(a, b) 是取较大的那个,不是 + 聚合):(函数, 参数(COUNT(*) 是 None), 是否去重)。""" from sqlglot import exp kinds = {exp.Sum: "sum", exp.Avg: "avg", exp.Count: "count", exp.Min: "min", exp.Max: "max"} func = kinds.get(type(node)) - if func is None: + if func is None or node.args.get("expressions"): return None arg = node.this distinct = isinstance(arg, exp.Distinct) @@ -4835,16 +6209,14 @@ def explain_sql_diff(a: str, b: str, dialect: str | None = None, *, schema: Mapp names: Mapping[str, str] | None = None, labels: tuple[str, str] = ("A", "B")) -> SqlDiff: """两条 SQL 是不是等价、不是的话差在哪。从不抛异常。 - 等价怎么判:两边各自规范化(见 _canonical_forms:sqlglot 优化器那一级、轻量那一级、只重新生成的那一级),每一步都 + 等价怎么判:两边各自规范化(见 _canonical_forms:认列之后的那一级、不认列的轻量一级、只统一名字的一级),每一步都 只做保持语义的变换;同一级的两个规范形式完全一样(比语法树,见 _Form.key)才算等价,否则一律不等价,items 一定 - 不为空。schema 是数据源的 schema_cache,给了就按它认列、展开 SELECT *(表结构要和库对得上)。 + 不为空。schema 是数据源的 schema_cache,给了就按它认列、展开 SELECT *(表结构要和库对得上)。原文里有解析时丢了 + 写法的(一元加号、0x10、换了名字的类型名,见 _lost_in_parsing)不在语法树上判等价。 - 不等价时怎么说:在两边的规范形式上逐段比(见 _Explainer),只把有把握的不同写成具体的中文句子——实表上的时间列 - 不同、某个只涉及实表列的筛选条件一边有一边没有(或者同一列上两边的条件不同)、聚合函数或去重计数不同、选出来的 - 某一列算法不同、分组键不同、关联的表或关联类型、关联条件、外连接的主表不同、换了一张表、取行和排序不同、UNION - 还是 UNION ALL;两段读的来源对不上、条件里套着说不准的子查询、牵涉 CTE 算出来的列的,不说具体的。具体说法一条都 - 没有时给一条 aspect="other" 的兜底说明(「两条 SQL 的写法不同,未能归纳成具体差异,请对照原文」),只覆盖了一部分 - 时追加一条「另有未能归纳的差异」,fallback 为真;text_diff 给出规范化之后不同的行。 + 不等价时怎么说:在两边的规范形式上逐段比(见 _Explainer),只把有把握、确实不同的地方写成具体的中文句子;具体说法 + 一条都没有时给一条 aspect="other" 的兜底说明(「两条 SQL 的写法不同,未能归纳成具体差异,请对照原文」),只覆盖了 + 一部分时追加一条「另有未能归纳的差异」,fallback 为真;text_diff 给出规范化之后不同的行。 labels 是两边的叫法(缺省 A / B,双盲复算传「原流程」「盲算」);names 是列的中文名({「表.列」或「列」: 中文名}, 可以用 catalog_labels 从数据目录取),时间列写成「支付时间 paid_at」。任一边解析不了:只给一条 aspect="parse"。 @@ -4860,9 +6232,18 @@ def explain_sql_diff(a: str, b: str, dialect: str | None = None, *, schema: Mapp side, error = (la, ea) if ea else (lb, eb) text = _say(side, ":", error or "", ",没有逐项比较") return SqlDiff(False, [FacetDiff("parse", ea, eb, text)]) - mapping = _schema_mapping(schema) - forms_a = {} if _lost_in_parsing(a, read) else _canonical_forms(ra, read, mapping) - forms_b = {} if _lost_in_parsing(b, read) else _canonical_forms(rb, read, mapping) + info = _schema_info(schema) + forms_a: dict[str, _Form] = {} + forms_b: dict[str, _Form] = {} + try: + if read == "sqlite": + _mark_comma_joins(ra, a.strip(), read) + _mark_comma_joins(rb, b.strip(), read) + forms_a = {} if _lost_in_parsing(a.strip(), read, ra) else _canonical_forms(ra, read, info) + forms_b = {} if _lost_in_parsing(b.strip(), read, rb) else _canonical_forms(rb, read, info) + except Exception: # noqa: BLE001 - 规范化本身的缺陷不能连累调用方:当作说不清,记下来修 + logger.warning("SQL 规范化失败", exc_info=True) + forms_a, forms_b = {}, {} for level in (_OPTIMIZER, _LIGHT, _PARSED): fa, fb = forms_a.get(level), forms_b.get(level) if fa is not None and fb is not None and fa.key == fb.key: @@ -4871,7 +6252,7 @@ def explain_sql_diff(a: str, b: str, dialect: str | None = None, *, schema: Mapp items: list[FacetDiff] = [] covered = False if level is not None: - explainer = _Explainer(read, (la, lb), names) + explainer = _Explainer(read, (la, lb), names, info) try: covered = explainer.statement(forms_a[level], forms_b[level]) items = explainer.items diff --git a/backend/tests/fixtures/sql_pairs.py b/backend/tests/fixtures/sql_pairs.py index 46fb6fd6..452071fb 100644 --- a/backend/tests/fixtures/sql_pairs.py +++ b/backend/tests/fixtures/sql_pairs.py @@ -10,8 +10,10 @@ OR 里的标量子查询改成 CROSS JOIN,都是复验时真踩到的),所 2 的负幂次的倍数,不同的求和先后得到的浮点数一样;比较结果时浮点数另按 6 位小数取整(乘 0.9 这类每行都有舍入)。 语料(PAIRS,SQLite,拿去执行)每一项 (编号, A, B, 预期):same 是已知等价的改写(别名、CTE 改名、关联先后、条件 -左右、子查询和 CTE 互换……),diff 是不许判等价的(换时间列、少一个条件、去重与否、外连接与内连接,以及只在某个方言、 -某种排序规则下才等价的)。LIMIT 都带能定出唯一顺序的排序,没有 ORDER BY 的 LIMIT 不进语料。过去复验报过的反例都 +左右、子查询和 CTE 互换……),编号以 hard: 开头的是要按语义推理才看得出等价的(条件挪进相关子查询、冗余条件),规范化 +不保证认得出、认不出时只许给兜底说明;diff 是不许判等价的(换时间列、少一个条件、去重与否、外连接与内连接,以及只在 +某个方言、某种排序规则下才等价的)。第 2 轮复验的反例在 _SAME_HARD、_SAME_R2、_DIFF_R2(执行)和 REGRESSIONS_R2、 +DIALECT_SAME_R2、KNOWN_UNRECOGNIZED(各方言,只查结论)里。LIMIT 都带能定出唯一顺序的排序,没有 ORDER BY 的 LIMIT 不进语料。过去复验报过的反例都 收进来了:第 5 轮回退的「只在 IN 子查询里用到的 CTE」在 PAIRS 里;复验者留在 /private/tmp 的脚本里、在这几组数据上 结果不同的对是 VALIDATOR_DIFF(也进 PAIRS 执行);旧测试(五轮复核修正按复验报告补的用例,各方言,用到的列这个库里 不全有)里报过不同的对是 REGRESSIONS、意思一样的对是 DIALECT_SAME,这两份不执行,只查判没判等价。 @@ -36,6 +38,9 @@ DDL = { "refunds": "id INTEGER PRIMARY KEY, order_id INTEGER, user_id INTEGER, amount REAL, state TEXT, refunded_at TEXT", "payments": "id INTEGER PRIMARY KEY, order_id INTEGER, user_id INTEGER, amount REAL, method TEXT, paid_at TEXT", "shipments": "id INTEGER PRIMARY KEY, order_id INTEGER, carrier TEXT, shipped_at TEXT", + # 一张叫 nums 的实表:SQLite 里不写 RECURSIVE 的递归 CTE(WITH nums(n) AS (… FROM nums …))引用的是它自己,换个名字 + # 引用的就是这张表了 + "nums": "n INTEGER", } @@ -95,8 +100,9 @@ def _rows(seed: int) -> dict[str, list[tuple]]: # 关联不上订单的退款、付款 refunds.append((len(refunds) + 1, pick([None, 999]), pick(user_ids), pick(AMOUNTS), "done", pick(DATES))) payments.append((len(payments) + 1, None, pick(user_ids), pick(AMOUNTS), "cash", pick(DATES))) + nums = [(n,) for n in rng.sample([1, 2, 3, 4, 10, 20], rng.randint(1, 4))] return {"users": users, "stores": stores, "orders": orders, "orders_archive": archive, "order_items": items, - "refunds": refunds, "payments": payments, "shipments": shipments} + "refunds": refunds, "payments": payments, "shipments": shipments, "nums": nums} def _edge_rows() -> dict[str, list[tuple]]: @@ -107,17 +113,41 @@ def _edge_rows() -> dict[str, list[tuple]]: (2, 1, None, 1, 2, None, 0, 0, "paid", "east", None, None, 0), (3, None, 1, None, None, 100, 0, 1, "paid", None, "2026-09-30 23:59:59", "2026-09-01", 1)] return {"users": users, "stores": stores, "orders": orders, "orders_archive": [], "order_items": [], - "refunds": [], "payments": [], "shipments": []} + "refunds": [], "payments": [], "shipments": [], "nums": []} -#: 生成数据用的种子(每个种子一组),外加一组极端数据 +def _tricky_rows() -> dict[str, list[tuple]]: + """一组专挑写法差异的数据(第 2 轮复验的反例在这上面现形):金额 0.1、0.2、0.3(浮点数 0.1 + 0.2 ≠ 0.3)、 + 100 * 0.07;状态里有像数字的文本('3.0'、'12')和大小写不同的 'PAID';用户名是像数字的文本('0100' 取两次负是 + 100);城市 'SH'、'sh';id 16 的订单(0x10);nums 表有几行。""" + users = [(1, "0100", "SH", 1, 3, None, "2026-09-01", 0), (2, "5", "sh", 0, 1, 1, "2026-09-02", 0), + (3, "alice", "Sh", None, None, 2, None, 1), (4, "Bob", None, 1, 5, None, "2026-08-31 23:59:59", None)] + stores = [(1, "s1", "SH", "east", 1), (2, "s2", "sh", None, None), (3, "3.0", "12", "west", 2)] + amounts = [0.1, 0.2, 0.3, 0.3, 0.30000000000000004, 7.0, 100 * 0.07, 12.5, None, 100.0, 0, 0.5] + statuses = ["paid", "PAID", "3.0", "12", None, "open", "void", "paid"] + orders = [] + for i in range(1, 21): + orders.append((i, (i % 4) + 1 if i % 7 else None, (i % 3) + 1, i - 1 if i % 5 == 0 else None, (i % 2) + 1, + amounts[i % len(amounts)], [0, 0.5, None][i % 3], [0, 1, None][i % 3], + statuses[i % len(statuses)], ["east", "west", None][i % 3], DATES[i % len(DATES)], + DATES[(i * 3) % len(DATES)], [0, 1, None][i % 3])) + refunds = [(1, 1, 2, 0.1, "done", "2026-09-05"), (2, 16, 1, 0.2, None, None), (3, 3, None, 0.3, "done", "2026-09-30"), + (4, 99, 3, 7.0, "pending", "2026-10-01")] + payments = [(1, 1, 2, 0.3, "card", "2026-09-02"), (2, 16, 1, 0.1, "cash", None)] + items = [(1, 1, 1, 2, 0.1), (2, 16, 2, None, 0.2), (3, 3, 1, 1, None)] + shipments = [(1, 1, "sf", "2026-09-03"), (2, 16, None, None)] + return {"users": users, "stores": stores, "orders": orders, "orders_archive": orders[:5], "order_items": items, + "refunds": refunds, "payments": payments, "shipments": shipments, "nums": [(1,), (2,), (10,)]} + + +#: 生成数据用的种子(每个种子一组),外加一组极端数据、一组专挑写法差异的数据 SEEDS = (7, 11, 23, 42, 2026) def build_databases() -> list[sqlite3.Connection]: """每组数据一个内存库。""" out = [] - for rows in [_rows(seed) for seed in SEEDS] + [_edge_rows()]: + for rows in [_rows(seed) for seed in SEEDS] + [_edge_rows(), _tricky_rows()]: conn = sqlite3.connect(":memory:") for name, ddl in DDL.items(): conn.execute(f"CREATE TABLE {name} ({ddl})") @@ -847,9 +877,6 @@ DIALECT_SAME: list[tuple[str, str, str, str]] = [ ("swapped_join_order_of_two_derived_tables_is_no_difference", "sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM t JOIN u ON u.user_id = t.user_id WHERE t.total > u.cnt", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM u JOIN t ON t.user_id = u.user_id WHERE t.total > u.cnt"), - ("the_same_item_on_equivalent_layers_is_no_difference", "sqlite", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 100 GROUP BY user_id) SELECT SUM(total) FROM t", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(total) FROM t WHERE user_id > 100"), ("derived_columns_renamed_on_one_side_are_paired", "sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100 AND n > 1", "WITH t AS (SELECT user_id, SUM(amount) AS s, COUNT(*) AS c FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 100 AND c > 1"), @@ -1860,6 +1887,308 @@ VALIDATOR_DIFF: list[tuple[str, str, str]] = [ ] +# ---------------------------------------------------------------------------- 第 2 轮复验的反例 +# 第 2 轮复验(重做之后的第一版)报的 blocker / major 和几位复验者留在 /private/tmp 的脚本里的对,改写到这个库的 +# 表上(列名换成这里有的),SQLite 能执行的进执行比对;只在 PostgreSQL / MySQL 才有的写法进 REGRESSIONS_R2 只查结论。 + +#: 意思一样、但要按语义推理才看得出的(条件挪进相关子查询、冗余条件、写法不同的等价条件……):规范化不保证认得出, +#: 认不出时只许给兜底说明,不许给具体说法(复验专挑这类对查「说出不存在的差异」) +_SAME_HARD = [ + # 条件挪进相关子查询(EXISTS 里引用外层的列):B 是有这个条件的,不能说「B 没有」 + ("r2-exists-moved-time", "SELECT COUNT(*) FROM orders o WHERE o.paid_at >= '2026-09-01' AND EXISTS " + "(SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id " + "AND o.paid_at >= '2026-09-01')"), + ("r2-exists-moved-filter", "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS " + "(SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND o.status = 'paid')"), + ("r2-scalar-moved-filter", "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND o.amount > " + "(SELECT AVG(r.amount) FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE o.amount > (SELECT AVG(r.amount) FROM refunds r WHERE r.order_id = o.id " + "AND o.status = 'paid')"), + # refunds 没有 status 列:子查询里的 status 是外层 orders 的 + ("r2-in-moved-outer-column", "SELECT COUNT(*) FROM orders WHERE status = 'paid' AND id IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE id IN (SELECT order_id FROM refunds WHERE status = 'paid')"), + # 一段时间拆成两段、IN 和 OR、多写一个下界:留下的行一样 + ("r2-time-split-range", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT COUNT(*) FROM orders WHERE (paid_at >= '2026-09-01' AND paid_at < '2026-09-16') OR " + "(paid_at >= '2026-09-16' AND paid_at < '2026-10-01')"), + ("r2-time-in-vs-or", "SELECT COUNT(*) FROM orders WHERE paid_at IN ('2026-09-01', '2026-09-30')", + "SELECT COUNT(*) FROM orders WHERE paid_at = '2026-09-01' OR paid_at = '2026-09-30'"), + ("r2-time-redundant-lower", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01' AND paid_at > '2026-08-01'"), + ("r2-in-vs-or", "SELECT COUNT(*) FROM orders WHERE status IN ('paid', 'done')", + "SELECT COUNT(*) FROM orders WHERE status = 'paid' OR status = 'done'"), + ("r2-redundant-not-null", "SELECT SUM(tax) FROM orders WHERE tax > 1 AND tax IS NOT NULL", + "SELECT SUM(tax) FROM orders WHERE tax > 1"), + ("r2-redundant-not-null-join", "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id " + "WHERE o.user_id IS NOT NULL", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id"), + ("r2-redundant-not-null-in", "SELECT COUNT(*) FROM orders WHERE store_id IN (SELECT id FROM stores " + "WHERE region = 'east') AND store_id IS NOT NULL", + "SELECT COUNT(*) FROM orders WHERE store_id IN (SELECT id FROM stores WHERE region = 'east')"), + ("r2-count-col-vs-not-null-filter", "SELECT COUNT(tax) FROM orders", "SELECT COUNT(*) FROM orders WHERE tax IS NOT NULL"), + ("r2-count-distinct-not-null", "SELECT COUNT(DISTINCT user_id) FROM orders WHERE user_id IS NOT NULL", + "SELECT COUNT(DISTINCT user_id) FROM orders"), + ("r2-coalesce-gt", "SELECT COUNT(*) FROM orders WHERE COALESCE(amount, 0) > 10", + "SELECT COUNT(*) FROM orders WHERE amount > 10"), + ("r2-left-coalesce-reject", "SELECT o.id FROM orders o LEFT JOIN users u ON u.id = o.user_id " + "WHERE COALESCE(u.grade, 0) > 1", + "SELECT o.id FROM orders o JOIN users u ON u.id = o.user_id WHERE COALESCE(u.grade, 0) > 1"), + ("r2-left-count-nullable-side", "SELECT COUNT(r.id) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(r.id) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("r2-having-count-gt-1-vs-ge-2", "SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1", + "SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) >= 2"), + ("r2-having-noop", "SELECT store_id, SUM(amount) FROM orders GROUP BY store_id HAVING COUNT(*) > 0", + "SELECT store_id, SUM(amount) FROM orders GROUP BY store_id"), + ("r2-scalar-max-commute", "SELECT max(amount, tax) FROM orders", "SELECT max(tax, amount) FROM orders"), + ("r2-case-vs-coalesce", "SELECT SUM(CASE WHEN discount IS NULL THEN 0 ELSE discount END) FROM orders", + "SELECT SUM(COALESCE(discount, 0)) FROM orders"), +] + +#: 意思一样、规范化认得出的改写(第 2 轮新增的几步) +_SAME_R2 = [ + ("r2-one-eq-one", "SELECT COUNT(*) FROM orders WHERE 1 = 1 AND status = 'paid'", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("r2-not-is-not-null", "SELECT COUNT(*) FROM orders WHERE NOT (paid_at IS NOT NULL)", + "SELECT COUNT(*) FROM orders WHERE paid_at IS NULL"), + ("r2-not-like", "SELECT COUNT(*) FROM orders WHERE status NOT LIKE 'p%'", + "SELECT COUNT(*) FROM orders WHERE NOT status LIKE 'p%'"), + ("r2-iif-vs-case", "SELECT SUM(IIF(status = 'paid', amount, 0)) FROM orders", + "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE 0 END) FROM orders"), + ("r2-case-else-null", "SELECT SUM(CASE WHEN status = 'paid' THEN amount END) FROM orders", + "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE NULL END) FROM orders"), + ("r2-min-distinct", "SELECT MIN(DISTINCT amount), MAX(DISTINCT tax) FROM orders", + "SELECT MIN(amount), MAX(tax) FROM orders"), + ("r2-exists-limit-1", "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id LIMIT 1)", + "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"), + ("r2-in-subquery-distinct", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT DISTINCT user_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM refunds)"), + ("r2-using-vs-on-select", "SELECT user_id FROM orders o LEFT JOIN refunds r USING (user_id)", + "SELECT o.user_id FROM orders o LEFT JOIN refunds r ON o.user_id = r.user_id"), + ("r2-multi-use-cte-correlated", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p " + "WHERE EXISTS (SELECT 1 FROM p AS p2 WHERE p2.parent_id = p.id)", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders c " + "WHERE c.parent_id = o.id AND c.status = 'paid')"), + ("r2-outer-where-into-having", "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id " + "HAVING SUM(amount) > 100) SELECT COUNT(*) FROM t", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id) t WHERE t.s > 100"), +] + +#: 不许判等价的(第 2 轮复验报的 blocker 改写到这个库上;在这几组数据上大多结果不同) +_DIFF_R2 = [ + # 数字常量不折叠:SQLite 按浮点数算,0.1 + 0.2 ≠ 0.3;三项以上的加乘不换结合 + ("r2-float-fold", "SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2", "SELECT COUNT(*) FROM orders WHERE amount = 0.3"), + ("r2-float-fold-chain", "SELECT COUNT(*) FROM orders WHERE amount + 0.1 + 0.2 = 0.3", + "SELECT COUNT(*) FROM orders WHERE amount + 0.3 = 0.3"), + ("r2-float-fold-mul", "SELECT id, amount * 0.1 * 3 FROM orders", "SELECT id, amount * 0.3 FROM orders"), + ("r2-float-fold-compare", "SELECT COUNT(*) FROM orders WHERE amount < 3 * 0.1", + "SELECT COUNT(*) FROM orders WHERE amount < 0.3"), + ("r2-float-fold-007", "SELECT COUNT(*) FROM orders WHERE amount >= 100 * 0.07", + "SELECT COUNT(*) FROM orders WHERE amount >= 7.00"), + # 取两次负会把文本转成数 + ("r2-double-negation", "SELECT id, -(-name) FROM users", "SELECT id, name FROM users"), + ("r2-double-negation-compare", "SELECT COUNT(*) FROM users WHERE -(-name) = 100", + "SELECT COUNT(*) FROM users WHERE name = 100"), + # 列名表:CTE 自己写的列名决定了外面的名字指哪一列 + ("r2-cte-column-list-swap", "WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM p WHERE total > 100", + "WITH p(total, uid) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM p WHERE total > 100"), + ("r2-cte-column-list-swap-2", "WITH t(user_id, amount) AS (SELECT amount, user_id FROM orders) SELECT SUM(amount) FROM t", + "WITH t(amount, user_id) AS (SELECT amount, user_id FROM orders) SELECT SUM(amount) FROM t"), + # SQLite 不写 RECURSIVE 也是递归:A 引用的是自己,B 引用的是实表 nums + ("r2-recursive-without-keyword", "WITH nums(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM nums WHERE n < 5) " + "SELECT COUNT(*) FROM nums", + "WITH seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM nums WHERE n < 5) SELECT COUNT(*) FROM seq"), + # ORDER BY 有并列再取行、拼串、窗口函数:结果看行出来的先后,调换来源、DISTINCT 改 GROUP BY 都会变 + ("r2-distinct-group-limit-ties", "SELECT DISTINCT user_id, status FROM orders ORDER BY user_id LIMIT 4", + "SELECT user_id, status FROM orders GROUP BY user_id, status ORDER BY user_id LIMIT 4"), + ("r2-group-key-order-limit-ties", "SELECT user_id, store_id, COUNT(*) FROM orders GROUP BY user_id, store_id " + "ORDER BY COUNT(*) DESC LIMIT 2", + "SELECT user_id, store_id, COUNT(*) FROM orders GROUP BY store_id, user_id ORDER BY COUNT(*) DESC LIMIT 2"), + ("r2-cross-join-swap-limit-ties", "SELECT o.id, u.id FROM orders o CROSS JOIN users u WHERE u.id = o.user_id " + "ORDER BY o.amount LIMIT 3", + "SELECT o.id, u.id FROM users u CROSS JOIN orders o WHERE u.id = o.user_id ORDER BY o.amount LIMIT 3"), + ("r2-group-concat-join-order", "SELECT u.city, GROUP_CONCAT(o.id) FROM users u CROSS JOIN orders o " + "WHERE o.user_id = u.id GROUP BY u.city", + "SELECT u.city, GROUP_CONCAT(o.id) FROM orders o CROSS JOIN users u WHERE o.user_id = u.id GROUP BY u.city"), + ("r2-group-concat-cross-vs-comma", "SELECT u.city, GROUP_CONCAT(o.id) FROM users u CROSS JOIN orders o " + "WHERE o.user_id = u.id GROUP BY u.city", + "SELECT u.city, GROUP_CONCAT(o.id) FROM users u, orders o WHERE o.user_id = u.id GROUP BY u.city"), + ("r2-row-number-join-order", "SELECT o.id, ROW_NUMBER() OVER () AS rn FROM users u CROSS JOIN orders o " + "WHERE o.user_id = u.id", + "SELECT o.id, ROW_NUMBER() OVER () AS rn FROM orders o CROSS JOIN users u WHERE o.user_id = u.id"), + ("r2-row-number-ties-join-order", "SELECT o.id, ROW_NUMBER() OVER (ORDER BY o.status) AS rn FROM users u " + "JOIN orders o ON o.user_id = u.id", + "SELECT o.id, ROW_NUMBER() OVER (ORDER BY o.status) AS rn FROM orders o JOIN users u ON o.user_id = u.id"), + ("r2-group-concat-subquery-order", "SELECT GROUP_CONCAT(id) FROM (SELECT id FROM orders ORDER BY amount DESC, id) t", + "SELECT GROUP_CONCAT(id) FROM (SELECT id FROM orders) t"), + # SQLite 里 a IN (b) 是 a = +b:右边那一列不带类型亲和 + ("r2-in-single-column", "SELECT COUNT(*) FROM orders WHERE status IN (store_id)", + "SELECT COUNT(*) FROM orders WHERE status = store_id"), + ("r2-not-in-single-column", "SELECT COUNT(*) FROM orders WHERE status NOT IN (store_id)", + "SELECT COUNT(*) FROM orders WHERE status <> store_id"), + # 解析时丢了写法的:类型名(STRING 是数值亲和)、0x10 和 x'10' + ("r2-cast-string", "SELECT store_id, CAST(store_id AS STRING) FROM orders", + "SELECT store_id, CAST(store_id AS TEXT) FROM orders"), + ("r2-cast-bytea", "SELECT id, CAST(status AS BLOB) FROM orders", "SELECT id, CAST(status AS BYTEA) FROM orders"), + ("r2-hex-literal", "SELECT COUNT(*) FROM orders WHERE id = 0x10", "SELECT COUNT(*) FROM orders WHERE id = x'10'"), + # SELECT * 的列先后 + ("r2-star-cte-column-order", "WITH c AS (SELECT user_id, amount FROM orders) SELECT * FROM c JOIN users " + "ON users.id = c.user_id", + "WITH c AS (SELECT amount, user_id FROM orders) SELECT * FROM c JOIN users ON users.id = c.user_id"), + ("r2-star-join-reorder", "SELECT * FROM orders o LEFT JOIN refunds r ON r.order_id = o.id JOIN users u ON u.id = o.user_id", + "SELECT * FROM orders o JOIN users u ON u.id = o.user_id LEFT JOIN refunds r ON r.order_id = o.id"), + # EXISTS 里:ORDER BY 有聚合就是聚合查询(总出一行);DISTINCT 在 OFFSET 之前算(SQLite 恰好不管这个 DISTINCT) + ("r2-exists-order-by-aggregate", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r " + "WHERE r.order_id = o.id ORDER BY COUNT(*))", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)"), + ("r2-exists-distinct-offset", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT DISTINCT r.state FROM refunds r " + "WHERE r.amount > 0 LIMIT 1 OFFSET 1)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT r.state FROM refunds r WHERE r.amount > 0 LIMIT 1 OFFSET 1)"), + # HAVING 不往 WHERE 挪:不分大小写的排序规则下同一组里的值各不一样 + ("r2-having-key-vs-where", "SELECT city, COUNT(*) FROM users GROUP BY city HAVING city = 'SH'", + "SELECT city, COUNT(*) FROM users WHERE city = 'SH' GROUP BY city"), + # 两边对调、子查询配错位置(复验说成自相矛盾的话的几对) + ("r2-except-sides-swapped", "SELECT user_id FROM orders WHERE paid_at >= '2026-09-01' EXCEPT SELECT user_id FROM refunds", + "SELECT user_id FROM refunds EXCEPT SELECT user_id FROM orders WHERE created_at >= '2026-09-01'"), + ("r2-in-not-in-swapped", "WITH p AS (SELECT user_id FROM orders WHERE status = 'paid' AND user_id IS NOT NULL), " + "v AS (SELECT user_id FROM orders WHERE status = 'void' AND user_id IS NOT NULL) SELECT COUNT(*) FROM users " + "WHERE id IN (SELECT user_id FROM p) AND id NOT IN (SELECT user_id FROM v)", + "WITH p AS (SELECT user_id FROM orders WHERE status = 'paid' AND user_id IS NOT NULL), " + "v AS (SELECT user_id FROM orders WHERE status = 'void' AND user_id IS NOT NULL) SELECT COUNT(*) FROM users " + "WHERE id IN (SELECT user_id FROM v) AND id NOT IN (SELECT user_id FROM p)"), + ("r2-not-in-filter", "WITH p AS (SELECT user_id FROM orders WHERE status = 'paid'), v AS (SELECT user_id FROM orders " + "WHERE status = 'void' AND user_id IS NOT NULL) SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM p) " + "AND id NOT IN (SELECT user_id FROM v)", + "WITH p AS (SELECT user_id FROM orders WHERE status = 'paid'), v AS (SELECT user_id FROM orders " + "WHERE status = 'open' AND user_id IS NOT NULL) SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM p) " + "AND id NOT IN (SELECT user_id FROM v)"), + ("r2-exists-not-exists-swapped", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r " + "WHERE r.order_id = o.id AND r.state = 'done') AND NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id " + "AND r.state = 'pending')", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND r.state = 'pending') " + "AND NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND r.state = 'done')"), + ("r2-in-left-column-swapped", "SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM orders WHERE status = 'paid') " + "AND referrer_id IN (SELECT user_id FROM orders WHERE status = 'void')", + "SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM orders WHERE status = 'void') " + "AND referrer_id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + # 换了表:各自的列是各自表的(复验报的 detail 归错了表) + ("r2-table-swap-time", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM users WHERE created_at >= '2026-09-01'"), + ("r2-table-swap-join", "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id WHERE u.city = 'SH'"), + ("r2-table-swap-time-2", "SELECT SUM(amount) FROM refunds WHERE refunded_at >= '2026-09-01'", + "SELECT SUM(amount) FROM payments WHERE paid_at >= '2026-09-01'"), +] + +#: 只在 PostgreSQL / MySQL 才有的写法(不执行,只查不判等价):第 2 轮复验在真实的库上跑出结果不同的 +REGRESSIONS_R2: list[tuple[str, str, str, str]] = [ + ("having_binary_is_not_where", "mysql", + "SELECT city, COUNT(*) FROM users GROUP BY city HAVING BINARY city = 'SH'", + "SELECT city, COUNT(*) FROM users WHERE BINARY city = 'SH' GROUP BY city"), + ("having_like_binary_is_not_where", "mysql", + "SELECT city, COUNT(*) FROM users GROUP BY city HAVING city LIKE BINARY 'S%'", + "SELECT city, COUNT(*) FROM users WHERE city LIKE BINARY 'S%' GROUP BY city"), + ("having_collate_is_not_where", "mysql", + "SELECT city, COUNT(*) FROM users GROUP BY city HAVING city COLLATE utf8mb4_bin = 'SH'", + "SELECT city, COUNT(*) FROM users WHERE city COLLATE utf8mb4_bin = 'SH' GROUP BY city"), + ("distinct_city_join_order_ci", "mysql", + "SELECT DISTINCT u.city FROM users u JOIN orders o ON o.user_id = u.id", + "SELECT DISTINCT u.city FROM orders o JOIN users u ON o.user_id = u.id"), + ("group_city_join_order_ci", "mysql", + "SELECT u.city, COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id GROUP BY u.city", + "SELECT u.city, COUNT(*) FROM orders o JOIN users u ON o.user_id = u.id GROUP BY u.city"), + ("order_by_aggregate_in_exists", "postgres", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id ORDER BY COUNT(*))", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)"), + ("order_by_aggregate_in_in", "postgres", + "SELECT COUNT(*) FROM users WHERE vip IN (SELECT 1 FROM refunds WHERE amount > 1000 ORDER BY COUNT(*))", + "SELECT COUNT(*) FROM users WHERE vip IN (SELECT 1 FROM refunds WHERE amount > 1000)"), + ("cte_column_list_swap", "postgres", + "WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) SELECT COUNT(*) FROM p WHERE total > 100", + "WITH p(total, uid) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) SELECT COUNT(*) FROM p WHERE total > 100"), + ("cte_column_list_swap", "mysql", + "WITH t(user_id, amount) AS (SELECT amount, user_id FROM orders) SELECT SUM(amount) FROM t", + "WITH t(amount, user_id) AS (SELECT amount, user_id FROM orders) SELECT SUM(amount) FROM t"), + ("derived_column_list_swap", "postgres", + "SELECT SUM(t.x) FROM (SELECT amount, tax FROM orders) AS t(x, y)", + "SELECT SUM(t.x) FROM (SELECT amount, tax FROM orders) AS t(y, x)"), + ("derived_column_list_swap", "mysql", + "SELECT t.b FROM (SELECT user_id, amount FROM orders) AS t(a, b) WHERE t.a > 3", + "SELECT t.b FROM (SELECT user_id, amount FROM orders) AS t(b, a) WHERE t.a > 3"), + ("derived_column_list_renames", "postgres", + "SELECT amount, user_id FROM (SELECT amount, user_id FROM orders) AS t(user_id, amount)", + "SELECT amount, user_id FROM orders"), + ("table_alias_column_list", "postgres", + "SELECT COUNT(*) FROM orders AS o(user_id, id) WHERE o.user_id > 5", + "SELECT COUNT(*) FROM orders AS o WHERE o.user_id > 5"), + ("union_all_in_cte_alias_order", "postgres", + "WITH u AS (SELECT user_id AS a, amount AS b FROM orders UNION ALL SELECT amount AS b, user_id AS a FROM refunds) " + "SELECT SUM(b), SUM(a) FROM u", + "WITH u AS (SELECT amount AS b, user_id AS a FROM refunds UNION ALL SELECT user_id AS a, amount AS b FROM orders) " + "SELECT SUM(b), SUM(a) FROM u"), + ("union_all_in_cte_alias_order", "mysql", + "WITH u AS (SELECT user_id AS a, amount AS b FROM orders UNION ALL SELECT amount AS b, user_id AS a FROM refunds) " + "SELECT SUM(b), SUM(a) FROM u", + "WITH u AS (SELECT amount AS b, user_id AS a FROM refunds UNION ALL SELECT user_id AS a, amount AS b FROM orders) " + "SELECT SUM(b), SUM(a) FROM u"), + ("between_symmetric", "postgres", + "SELECT COUNT(*) FROM orders WHERE amount BETWEEN SYMMETRIC 100 AND 10", + "SELECT COUNT(*) FROM orders WHERE amount BETWEEN 100 AND 10"), + ("group_key_order_with_limit", "postgres", + "SELECT user_id, store_id, COUNT(*) FROM orders GROUP BY user_id, store_id ORDER BY COUNT(*) DESC LIMIT 2", + "SELECT user_id, store_id, COUNT(*) FROM orders GROUP BY store_id, user_id ORDER BY COUNT(*) DESC LIMIT 2"), + ("exists_distinct_offset", "postgres", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT DISTINCT r.state FROM refunds r WHERE r.amount > 0 LIMIT 1 OFFSET 1)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT r.state FROM refunds r WHERE r.amount > 0 LIMIT 1 OFFSET 1)"), + ("distinct_on_keeps_a_different_row", "postgres", + "SELECT DISTINCT ON (user_id) user_id, status FROM orders ORDER BY user_id, created_at DESC NULLS LAST, id", + "SELECT DISTINCT ON (user_id) user_id, status FROM orders ORDER BY user_id, paid_at DESC NULLS LAST, id"), + ("float_folding", "postgres", + "SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2", "SELECT COUNT(*) FROM orders WHERE amount = 0.3"), + ("double_negation", "mysql", "SELECT id, -(-name) FROM users", "SELECT id, name FROM users"), +] + +#: 意思一样、规范化认得出的(PostgreSQL / MySQL 的写法) +DIALECT_SAME_R2: list[tuple[str, str, str, str]] = [ + ("fetch_first_is_limit", "postgres", "SELECT id FROM orders ORDER BY id FETCH FIRST 4 ROWS ONLY", + "SELECT id FROM orders ORDER BY id LIMIT 4"), + ("offset_fetch_is_limit_offset", "postgres", "SELECT id FROM orders ORDER BY id OFFSET 2 ROWS FETCH NEXT 4 ROWS ONLY", + "SELECT id FROM orders ORDER BY id LIMIT 4 OFFSET 2"), + ("mysql_column_names_ignore_case", "mysql", "SELECT SUM(amount) FROM orders WHERE is_deleted = 0", + "SELECT SUM(AMOUNT) FROM orders WHERE IS_DELETED = 0"), + ("is_not_null_forms", "postgres", "SELECT COUNT(*) FROM orders WHERE NOT (paid_at IS NOT NULL)", + "SELECT COUNT(*) FROM orders WHERE paid_at IS NULL"), + ("not_like_forms", "postgres", "SELECT COUNT(*) FROM orders WHERE status NOT LIKE 'p%'", + "SELECT COUNT(*) FROM orders WHERE NOT status LIKE 'p%'"), + ("if_vs_case", "mysql", "SELECT SUM(IF(status = 'paid', amount, 0)) FROM orders", + "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE 0 END) FROM orders"), +] + +#: 意思一样、规范化故意不认的(认了就要假设「相等就是一样」,见模块说明):只给兜底说明,不给具体说法 +KNOWN_UNRECOGNIZED: list[tuple[str, str, str, str]] = [ + ("having_on_group_key_vs_where", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 100 GROUP BY user_id) SELECT SUM(total) FROM t", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(total) FROM t WHERE user_id > 100"), + ("union_dedup_branch_order", "sqlite", + "SELECT id, amount FROM orders UNION SELECT id, amount FROM orders_archive", + "SELECT id, amount FROM orders_archive UNION SELECT id, amount FROM orders"), + ("window_partition_key_order", "sqlite", + "SELECT id, SUM(amount) OVER (PARTITION BY user_id, store_id) AS s FROM orders", + "SELECT id, SUM(amount) OVER (PARTITION BY store_id, user_id) AS s FROM orders"), + ("distinct_vs_group_mysql_strings", "mysql", + "SELECT DISTINCT city FROM users", "SELECT city FROM users GROUP BY city"), +] + + PAIRS: list[Pair] = [Pair(f"same:{i}", a, b, "same") for i, a, b in _SAME] + \ [Pair(f"diff:{i}", a, b, "diff") for i, a, b in _DIFF] + \ - [Pair(f"validator:{n}:{k}", a, b, "diff") for k, (n, a, b) in enumerate(VALIDATOR_DIFF)] + [Pair(f"validator:{n}:{k}", a, b, "diff") for k, (n, a, b) in enumerate(VALIDATOR_DIFF)] + \ + [Pair(f"same:{i}", a, b, "same") for i, a, b in _SAME_R2] + \ + [Pair(f"hard:{i}", a, b, "same") for i, a, b in _SAME_HARD] + \ + [Pair(f"diff:{i}", a, b, "diff") for i, a, b in _DIFF_R2] + +#: 按编号找语料里的一对(测试里引用第 2 轮的反例用) +PAIRS_BY_ID: dict[str, Pair] = {p.id: p for p in PAIRS} diff --git a/backend/tests/test_sql_equivalence.py b/backend/tests/test_sql_equivalence.py index b2f8e817..dd6e38a7 100644 --- a/backend/tests/test_sql_equivalence.py +++ b/backend/tests/test_sql_equivalence.py @@ -6,11 +6,13 @@ explain_sql_diff 判两条 SQL 等价,当且仅当两边经过只保持语义 - 判了等价的两条,在每组数据上结果都必须一样(结果集按行排序比较); - 在某组数据上结果不一样的两条,绝不能判等价; - 语料里标 diff 的(语义不同,或者只在某个方言、某种排序规则下才等价),一律不能判等价; -- 标 same 的(已知等价的改写),记下认出来的比例:目标是大多数都认得出,但不强求全部。 +- 标 same 的(已知等价),没认出来时只许给兜底说明,不许给具体说法(具体说法说的是两边确实不同的地方); +- 等价改写(编号不以 hard: 开头的 same)记下认出来的比例:目标是大多数都认得出,但不强求全部。 -库和语料见 tests/fixtures/sql_pairs.py:5 组按固定种子生成的数据加一组极端数据,带空值、重复、边界值、已删除的 -行、退款;语料里有手写的等价改写和语义不同的对,也有复验者以前造的、在这几组数据上结果不同的对。每一对都在不给表 -结构和给表结构(数据源的 schema_cache)两种情况下各判一次,两种都要守住上面的规矩。 +库和语料见 tests/fixtures/sql_pairs.py:5 组按固定种子生成的数据加一组极端数据、一组专挑写法差异的数据(0.1 + 0.2、 +像数字的文本、大小写不同的值),带空值、重复、边界值、已删除的行、退款;语料里有手写的等价改写和语义不同的对,也有 +复验者以前造的、在这几组数据上结果不同的对,以及第 2 轮复验报的反例。每一对都在不给表结构和给表结构(数据源的 +schema_cache)两种情况下各判一次,两种都要守住上面的规矩。 """ from __future__ import annotations @@ -59,18 +61,25 @@ def test_equivalence_is_guarded_by_execution(databases, pair): assert not diff.equivalent, f"{pair.id}:第 {differs} 组数据上结果不同,却判了等价(schema={bool(schema)})" if pair.expect == "diff": assert not diff.equivalent, f"{pair.id} 不该判等价(schema={bool(schema)})" + if pair.expect == "same" and not diff.equivalent: + specific = [d.text for d in diff.items if d.aspect != "other"] + assert not specific, f"{pair.id} 意思一样,却说了具体的不同(schema={bool(schema)}):{specific}" # 换个先后问,结论一样 assert explain_sql_diff(pair.b, pair.a, "sqlite", schema=schema).equivalent == diff.equivalent, pair.id def test_most_known_equivalent_rewrites_are_recognized(): - """已知等价的改写里认出来的比例(给表结构时)。不强求全部认出来:认不出的给的是兜底说明,不是错误的结论。""" - same = [p for p in sql_pairs.PAIRS if p.expect == "same"] + """已知等价的改写(别名、CTE 改名、关联先后、条件左右、子查询和 CTE 互换……)里认出来的比例。不强求全部认出来: + 认不出的给的是兜底说明,不是错误的结论。要按语义推理才看得出的(hard:)另记,不算进比例。""" + same = [p for p in sql_pairs.PAIRS if p.expect == "same" and not p.id.startswith("hard:")] hits = [p.id for p in same if explain_sql_diff(p.a, p.b, "sqlite", schema=SCHEMA).equivalent] bare = [p.id for p in same if explain_sql_diff(p.a, p.b, "sqlite").equivalent] ratio, bare_ratio = len(hits) / len(same), len(bare) / len(same) - print(f"\n已知等价的 {len(same)} 对里认出 {len(hits)} 对({ratio:.0%}),不给表结构时 {len(bare)} 对({bare_ratio:.0%})") + print(f"\n已知等价的改写 {len(same)} 对里认出 {len(hits)} 对({ratio:.0%}),不给表结构时 {len(bare)} 对({bare_ratio:.0%})") print("没认出来的:", sorted(set(p.id for p in same) - set(hits))) + hard = [p for p in sql_pairs.PAIRS if p.id.startswith("hard:")] + hard_hits = [p.id for p in hard if explain_sql_diff(p.a, p.b, "sqlite", schema=SCHEMA).equivalent] + print(f"要按语义推理的 {len(hard)} 对里认出 {len(hard_hits)} 对(其余只给兜底说明)") assert ratio >= 0.9 and bare_ratio >= 0.9 @@ -78,5 +87,17 @@ def test_the_corpus_is_large_and_varied_enough(): assert len(sql_pairs.PAIRS) >= 150 and len(sql_pairs.SEEDS) >= 5 text = " ".join(p.a + " " + p.b for p in sql_pairs.PAIRS).upper() for construct in ("WITH ", "EXISTS", " IN (SELECT", "UNION", "OVER (", "DISTINCT", "LEFT JOIN", "HAVING", - "LIMIT", "ORDER BY", "RIGHT JOIN", "FULL JOIN", "NOT IN", "CROSS JOIN"): + "LIMIT", "ORDER BY", "RIGHT JOIN", "FULL JOIN", "NOT IN", "CROSS JOIN", "EXCEPT", "GROUP_CONCAT", + "USING (", "IIF("): assert construct in text, construct + + +def test_the_tricky_dataset_exposes_the_round_two_counterexamples(databases): + """第 2 轮复验的反例在专挑写法差异的那组数据上确实结果不同(不然执行比对守不住它们)。""" + tricky = databases[-1] + pairs = {p.id: p for p in sql_pairs.PAIRS} + for pid in ("diff:r2-float-fold", "diff:r2-float-fold-chain", "diff:r2-double-negation", "diff:r2-in-single-column", + "diff:r2-cast-string", "diff:r2-hex-literal", "diff:r2-recursive-without-keyword", + "diff:r2-cte-column-list-swap-2"): + pair = pairs[pid] + assert sql_pairs.run(tricky, pair.a) != sql_pairs.run(tricky, pair.b), pid diff --git a/backend/tests/test_sql_semantics.py b/backend/tests/test_sql_semantics.py index 95199efa..61c9e967 100644 --- a/backend/tests/test_sql_semantics.py +++ b/backend/tests/test_sql_semantics.py @@ -604,8 +604,6 @@ def test_time_column_differs_with_catalog_names(): "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", [("aggregate", "A 对 orders.amount 求和,B 对 refunds.amount 求和")]), ("SELECT DISTINCT region FROM orders", "SELECT region FROM orders", [("group_by", "A 对 region 去重,B 没有去重")]), - ("SELECT SUM(amount) FROM orders", "SELECT ROUND(SUM(amount), 2) FROM orders", - [("expression", "A 算的是 SUM(amount),B 算的是 ROUND(SUM(amount), 2)")]), ("SELECT user_id, SUM(amount) FROM orders GROUP BY user_id", "SELECT SUM(amount), user_id FROM orders GROUP BY user_id", [("expression", "两边选出来的列一样,先后不同:A 是 user_id、SUM(amount),B 是 SUM(amount)、user_id")]), # 分组、HAVING、取行 @@ -661,9 +659,9 @@ def test_specific_differences_are_said_in_one_sentence_each(a, b, expected): def test_a_difference_inside_a_cte_used_only_in_a_condition_subquery_is_reported(): """第 5 轮回退的例子:只在 IN 子查询里用到的 CTE 里的条件不同,以前整个漏报(返回空列表)。现在 CTE 并进了 IN - 子查询,往里比,说得出是哪个条件。EXISTS、标量子查询一样。""" + 子查询,往里比,说得出是哪个条件,位置带上 IN 左边的列(两个 IN 子查询时才分得清)。EXISTS、标量子查询一样。""" head = "WITH p AS (SELECT * FROM orders WHERE status = '{}') SELECT COUNT(*) FROM users u WHERE " - for tail, place in (("u.id IN (SELECT user_id FROM p)", "在 IN 子查询里"), + for tail, place in (("u.id IN (SELECT user_id FROM p)", "在 id IN 的子查询里"), ("EXISTS (SELECT 1 FROM p WHERE p.user_id = u.id)", "在 EXISTS 子查询里"), ("(SELECT COUNT(*) FROM p WHERE p.user_id = u.id) > 0", "在条件的子查询里")): a, b = head.format("paid") + tail, head.format("open") + tail @@ -728,10 +726,10 @@ def test_filters_on_real_columns_carry_the_columns_for_the_catalog(): def test_a_displayed_condition_means_what_the_sql_means(): """给人看的条件按方言生成;方言生成的写法读回来不是同一个意思时(SQLite 把 CAST(x AS DATE) 生成 DATE(x))改用 - 通用写法,不拿 DATE('2026-09-01') 冒充原文。几个条件连起来时 OR 带上括号。""" - [item] = explain_sql_diff("SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", - "SELECT COUNT(*) FROM orders WHERE paid_at >= CAST('2026-09-01' AS DATE)").items - assert "CAST('2026-09-01' AS DATE)" in item.text and "DATE('2026-09-01')" not in item.text + 通用写法,不拿 DATE(paid_at) 冒充原文。几个条件连起来时 OR 带上括号。""" + [item] = explain_sql_diff("SELECT COUNT(*) FROM orders WHERE CAST(paid_at AS DATE) >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE CAST(paid_at AS DATE) >= '2026-10-01'", "sqlite").items + assert item.aspect == "time_range" and "CAST(paid_at AS DATE)" in item.text and "DATE(paid_at)" not in item.text a = "SELECT SUM(amount) FROM orders" b = "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id AND (s.city = 'SH' OR s.city = 'BJ')" [item] = explain_sql_diff(a, b).items @@ -779,6 +777,19 @@ def test_columns_inside_a_cte_are_not_compared_because_canonical_forms_prune_the assert diff.fallback and not [d for d in diff.items if "选出" in d.text] +def test_differences_that_cannot_be_checked_get_only_the_fallback(): + """两边算法不同、又判断不了是不是真的不同的(ROUND 包了一层、CASE 和 COALESCE、两种写法的时间常量),不说具体的: + 「A 算的是 X,B 算的是 Y」读起来就是两边不同,说错了比不说更糟。""" + for a, b in (("SELECT SUM(amount) FROM orders", "SELECT ROUND(SUM(amount), 2) FROM orders"), + ("SELECT SUM(COALESCE(discount, 0)) FROM orders", + "SELECT SUM(CASE WHEN discount IS NULL THEN 0 ELSE discount END) FROM orders"), + ("SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= DATE('2026-09-01')")): + diff = explain_sql_diff(a, b, "sqlite") + check_shape(diff) + assert diff.fallback and diff.items == [FacetDiff("other", None, None, FALLBACK)], (a, b, diff.items) + + def test_text_diff_lists_at_most_forty_lines(): cols_a = ", ".join(f"SUM(amount * {i}) AS c{i}" for i in range(60)) cols_b = ", ".join(f"SUM(amount * {i + 100}) AS c{i}" for i in range(60)) @@ -857,3 +868,234 @@ def test_catalog_labels_come_from_visible_catalog_items(): def test_a_facets_object_compares_with_itself_cleanly(): f = facets("WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT order_id) FROM v") assert isinstance(f, Facets) and diff_facets(f, f) == [] + + +# -------------------------------------------------------------------------- +# 第 2 轮复验的反例:等价的结论、具体说法的可靠性 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.REGRESSIONS_R2, + ids=[f"{n}-{d}" for n, d, _, _ in sql_pairs.REGRESSIONS_R2]) +def test_round_two_counterexamples_are_never_equivalent(name, dialect, a, b): + """第 2 轮复验在 PostgreSQL / MySQL 上跑出结果不同的对:常量折叠(Decimal 和浮点数)、取两次负、HAVING 挪进 WHERE + (不分大小写的排序规则下同组的值各不一样)、ORDER BY 里有聚合的 EXISTS / IN、列名表、合并的各支调换、BETWEEN + SYMMETRIC、有并列的取行、DISTINCT ON。不管给不给表结构、谁先谁后,一律不判等价。""" + for schema in (None, sql_pairs.schema_cache()): + for x, y in ((a, b), (b, a)): + diff = explain_sql_diff(x, y, dialect, schema=schema) + check_shape(diff) + assert not diff.equivalent, name + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.DIALECT_SAME_R2, + ids=[f"{n}-{d}" for n, d, _, _ in sql_pairs.DIALECT_SAME_R2]) +def test_round_two_rewrites_are_recognized(name, dialect, a, b): + """FETCH FIRST 和 LIMIT、MySQL 列名的大小写、各方言 IS NOT NULL / NOT LIKE 的不同解析形状、MySQL 的 IF 和 CASE: + 认得出是一回事(以前把 FETCH 读成「不限行数」)。""" + diff = explain_sql_diff(a, b, dialect) + check_shape(diff) + assert diff.equivalent, (name, diff.items) + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.KNOWN_UNRECOGNIZED, + ids=[f"{n}-{d}" for n, d, _, _ in sql_pairs.KNOWN_UNRECOGNIZED]) +def test_equivalences_left_unrecognized_on_purpose_get_only_the_fallback(name, dialect, a, b): + """意思一样、规范化故意不认的(要假设「相等就是一样」才认得出):只给兜底说明,不说具体的不同。""" + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, dialect, schema=schema) + check_shape(diff) + assert diff.equivalent or all(d.aspect == "other" for d in diff.items), (name, diff.items) + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b", [ + ("SELECT COUNT(*) FROM orders o WHERE o.paid_at >= '2026-09-01' AND EXISTS " + "(SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id " + "AND o.paid_at >= '2026-09-01')"), + ("SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND o.status = 'paid')"), + ("SELECT COUNT(*) FROM orders WHERE status = 'paid' AND id IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE id IN (SELECT order_id FROM refunds WHERE status = 'paid')"), + ("SELECT COUNT(*) FROM users u WHERE u.city = 'BJ' AND EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)", + "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id AND u.city = 'BJ')"), +], ids=["exists-time", "exists-status", "in-outer-column", "exists-city"]) +def test_a_condition_moved_into_a_correlated_subquery_is_not_called_missing(dialect, a, b): + """条件挪进了相关子查询(引用的还是外层那一行):两条意思一样,B 是有这个条件的。以前说「B 没有按时间筛选」 + 「B 没有这个条件」,现在说不清就只给兜底说明。""" + for schema in (None, sql_pairs.schema_cache()): + for x, y in ((a, b), (b, a)): + diff = explain_sql_diff(x, y, dialect, schema=schema) + check_shape(diff) + assert all(d.aspect == "other" for d in diff.items), diff.items + + +@pytest.mark.parametrize("dialect", DIALECTS) +def test_time_filters_written_another_way_are_never_called_no_time_filter(dialect): + """时间条件换了写法(拆成两段、IN 和 OR、多一个下界、LIKE 和区间):另一边是按时间筛了的,不能说「没有按时间筛选」, + 也不能说成两个范围不同。""" + base = "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'" + for other in ("SELECT COUNT(*) FROM orders WHERE (paid_at >= '2026-09-01' AND paid_at < '2026-09-16') OR " + "(paid_at >= '2026-09-16' AND paid_at < '2026-10-01')", + "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01' " + "AND paid_at > '2026-08-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at LIKE '2026-09%'"): + diff = explain_sql_diff(base, other, dialect) + check_shape(diff) + assert not any("没有按时间筛选" in d.text or d.aspect == "time_range" for d in diff.items), diff.items + diff = explain_sql_diff("SELECT COUNT(*) FROM orders WHERE paid_at IN ('2026-09-01', '2026-09-30')", + "SELECT COUNT(*) FROM orders WHERE paid_at = '2026-09-01' OR paid_at = '2026-09-30'", dialect) + assert all(d.aspect == "other" for d in diff.items) + + +def test_except_with_its_sides_swapped_is_not_one_time_difference(): + """EXCEPT 两边对调,外加一边换了时间列:以前只说时间列不同、还说全了(会写进数据目录)。EXCEPT 按先后逐支比,对不上 + 就只给兜底说明;子查询、CTE、IN 子查询里套着的也一样。""" + a = "SELECT user_id FROM orders WHERE paid_at >= '2026-09-01' EXCEPT SELECT user_id FROM refunds" + b = "SELECT user_id FROM refunds EXCEPT SELECT user_id FROM orders WHERE created_at >= '2026-09-01'" + for x, y in ((a, b), (f"SELECT COUNT(*) FROM ({a}) t", f"SELECT COUNT(*) FROM ({b}) t"), + (f"WITH e AS ({a}) SELECT COUNT(*) FROM e", f"WITH e AS ({b}) SELECT COUNT(*) FROM e"), + (f"SELECT COUNT(*) FROM users WHERE id IN ({a})", f"SELECT COUNT(*) FROM users WHERE id IN ({b})")): + for dialect in ("sqlite", "postgres"): + diff = explain_sql_diff(x, y, dialect) + check_shape(diff) + assert diff.fallback and all(d.aspect == "other" for d in diff.items), (x, diff.items) + + +@pytest.mark.parametrize("a,b", [ + (sql_pairs.PAIRS_BY_ID["diff:r2-in-not-in-swapped"].a, sql_pairs.PAIRS_BY_ID["diff:r2-in-not-in-swapped"].b), + (sql_pairs.PAIRS_BY_ID["diff:r2-exists-not-exists-swapped"].a, + sql_pairs.PAIRS_BY_ID["diff:r2-exists-not-exists-swapped"].b), + (sql_pairs.PAIRS_BY_ID["diff:r2-in-left-column-swapped"].a, sql_pairs.PAIRS_BY_ID["diff:r2-in-left-column-swapped"].b), +], ids=["in-not-in", "exists-not-exists", "left-column"]) +def test_subqueries_swapped_between_in_and_not_in_are_not_explained_both_ways(a, b): + """IN 和 NOT IN(EXISTS 和 NOT EXISTS、id IN 和 referrer_id IN)两个子查询对调:以前配错了对,说出一对互相矛盾的话、 + 还把 NOT IN 说成「IN 子查询」。子查询要外面的写法(含 NOT、IN 左边的列)一模一样才配对,配不上就只给兜底。""" + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, "sqlite", schema=schema, labels=LABELS) + check_shape(diff) + assert all(d.aspect == "other" for d in diff.items), diff.items + + +def test_a_difference_inside_a_not_in_subquery_says_not_in(): + """NOT IN 子查询里的条件不同,说的是「NOT IN 的子查询」(以前说成「IN 子查询」,读的人会以为是保留这些用户)。子查询 + 里不带表名的列,没给表结构时分不清是子查询里的表的还是外层的(users 也可能有 status),不说具体的;给了表结构、 + 或者写了表名,就说得出。""" + a = ("SELECT COUNT(*) FROM users WHERE id NOT IN (SELECT user_id FROM orders WHERE status = 'void' " + "AND user_id IS NOT NULL)") + b = a.replace("'void'", "'open'") + expected = [("filter", "原流程在 id NOT IN 的子查询里按 status = 'void' 筛选,盲算在 id NOT IN 的子查询里按 " + "status = 'open' 筛选")] + diff = explain_sql_diff(a, b, "sqlite", labels=LABELS, schema=sql_pairs.schema_cache()) + check_shape(diff) + assert [(d.aspect, d.text) for d in diff.items] == expected + assert explain_sql_diff(a, b, "sqlite", labels=LABELS).items == [FacetDiff("other", None, None, FALLBACK)] + qa, qb = a.replace("FROM orders WHERE status", "FROM orders o WHERE o.status"), \ + b.replace("FROM orders WHERE status", "FROM orders o WHERE o.status") + assert [(d.aspect, d.text) for d in explain_sql_diff(qa, qb, "sqlite", labels=LABELS).items] == expected + diff = explain_sql_diff(a.replace("NOT IN", "IN"), b.replace("NOT IN", "IN"), "sqlite", labels=LABELS, + schema=sql_pairs.schema_cache()) + assert "id IN 的子查询" in diff.items[0].text and "NOT IN" not in diff.items[0].text + + +def test_a_swapped_table_keeps_each_sides_own_columns(): + """换了一张表:盲算的 created_at 是 users 的(注册时间),以前把 B 的表改名成 A 的表再比,中文名和 detail 都归到了 + orders。现在两边各用各的表说。""" + names = {"orders.created_at": "下单时间", "users.created_at": "注册时间", "orders.paid_at": "支付时间"} + diff = explain_sql_diff("SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM users WHERE created_at >= '2026-09-01'", "sqlite", names=names, + labels=LABELS) + check_shape(diff) + assert [(d.aspect, d.text) for d in diff.items] == [ + ("tables", "原流程查的是 orders 表,盲算查的是 users 表"), + ("time", "原流程按支付时间 paid_at 筛时间,盲算按注册时间 created_at")] + assert diff.items[1].detail == {"a": [{"table": "orders", "column": "paid_at"}], + "b": [{"table": "users", "column": "created_at"}]} + diff = explain_sql_diff("SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id WHERE u.city = 'SH'", + "sqlite") + check_shape(diff) + assert [d.text for d in diff.items] == [ + "A 查的是 stores 表,B 查的是 users 表", + "A 关联的条件是 orders.store_id = stores.id,B 是 orders.user_id = users.id"] + + +def test_ordering_that_decides_which_row_distinct_on_keeps_is_not_just_ordering(): + """DISTINCT ON 留哪一行由 ORDER BY 决定:排序不同就是留下的行不同,不能说成「只是排序不同」(aspect=order)。""" + a = "SELECT DISTINCT ON (user_id) user_id, status FROM orders ORDER BY user_id, created_at DESC NULLS LAST, id" + b = "SELECT DISTINCT ON (user_id) user_id, status FROM orders ORDER BY user_id, paid_at DESC NULLS LAST, id" + diff = explain_sql_diff(a, b, "postgres") + check_shape(diff) + assert not diff.equivalent and all(d.aspect != "order" for d in diff.items) + + +def test_a_table_name_differing_only_in_case_is_not_called_another_table_in_mysql(): + """MySQL 的表名在 Linux 上分大小写、在 macOS / Windows 上不分:BILLS 和 bills 不判等价,也不说「查的是另一张表」。""" + diff = explain_sql_diff("SELECT store_id, SUM(amount) AS s FROM orders WHERE is_deleted = 0 GROUP BY store_id", + "select STORE_ID, sum( AMOUNT ) as s from ORDERS where IS_DELETED = 0 group by store_id", + "mysql") + check_shape(diff) + assert all(d.aspect == "other" for d in diff.items) + + +def test_redundant_or_constant_conditions_are_not_claimed(): + """多写一个本来就成立的条件(1 = 1、已经被 > 筛掉的空值、关联键上的 IS NOT NULL、分组后 COUNT(*) > 0):两条意思一样, + 不说「A 只保留 … 的行,B 没有这个条件」。1 = 1 直接认成等价。""" + assert explain_sql_diff("SELECT COUNT(*) FROM orders WHERE 1 = 1 AND status = 'paid'", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'", "sqlite").equivalent + for dialect in DIALECTS: + for a, b in (("SELECT SUM(tax) FROM orders WHERE tax > 1 AND tax IS NOT NULL", "SELECT SUM(tax) FROM orders WHERE tax > 1"), + ("SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id WHERE o.user_id IS NOT NULL", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id"), + ("SELECT store_id, SUM(amount) FROM orders GROUP BY store_id HAVING COUNT(*) > 0", + "SELECT store_id, SUM(amount) FROM orders GROUP BY store_id"), + ("SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1", + "SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) >= 2"), + ("SELECT COUNT(tax) FROM orders", "SELECT COUNT(*) FROM orders WHERE tax IS NOT NULL"), + ("SELECT COUNT(r.id) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(r.id) FROM orders o JOIN refunds r ON r.order_id = o.id")): + diff = explain_sql_diff(a, b, dialect) + check_shape(diff) + assert all(d.aspect == "other" for d in diff.items), (dialect, a, diff.items) + + +def test_single_item_in_list_becomes_equality_only_for_a_constant(): + """IN 列表只有一项:是常量才写成 =;是一列的话 SQLite 里 a IN (b) 是 a = +b(右边不带类型亲和),和 a = b 不同。""" + assert explain_sql_diff("SELECT COUNT(*) FROM orders WHERE status IN ('paid')", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'", "sqlite").equivalent + assert not explain_sql_diff("SELECT id FROM t WHERE a IN (b)", "SELECT id FROM t WHERE a = b", "sqlite").equivalent + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b", [ + ("SELECT id FROM orders ORDER BY amount DESC LIMIT 5 OFFSET 10", "SELECT id FROM orders ORDER BY amount DESC LIMIT 20 OFFSET 0"), + ("WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders WHERE amount > 10 GROUP BY user_id) " + "SELECT COUNT(*) FROM p WHERE total > 100", + "WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders WHERE amount > 50 GROUP BY user_id) " + "SELECT COUNT(*) FROM p WHERE total > 999"), + ("SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2 AND status IN ('paid', 'done')", + "SELECT COUNT(*) FROM orders WHERE amount = 0.5 + 0.25 AND status IN ('open', 'void')"), + ("SELECT COUNT(*) FROM users WHERE id NOT IN (SELECT user_id FROM orders WHERE created_at >= '2026-09-01')", + "SELECT COUNT(*) FROM users WHERE id NOT IN (SELECT user_id FROM orders WHERE created_at >= '2026-10-01')"), +], ids=["limit-offset", "cte-column-list", "arith-and-in-list", "not-in"]) +def test_only_literals_changed_stays_the_same_template_after_the_rewrite(dialect, a, b): + """normalize_sql「只改了参数」的判断不受这次改动影响:只改字面量的照旧同一个模板(等价判定照样说不等价)。""" + na, nb = normalize_sql(a, dialect), normalize_sql(b, dialect) + assert na.parse_error is None and na.same_shape(nb) and na.literals != nb.literals + assert not explain_sql_diff(a, b, dialect).equivalent + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b", [ + ("WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) SELECT COUNT(*) FROM p WHERE total > 1", + "WITH p(total, uid) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) SELECT COUNT(*) FROM p WHERE total > 1"), + ("SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM orders)", + "SELECT COUNT(*) FROM users WHERE id NOT IN (SELECT user_id FROM orders)"), + ("SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2", "SELECT COUNT(*) FROM orders WHERE amount = 0.3"), + ("SELECT u.city, COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id GROUP BY u.city", + "SELECT u.city, COUNT(*) FROM users u LEFT JOIN orders o ON o.user_id = u.id GROUP BY u.city"), +], ids=["cte-column-list-order", "in-vs-not-in", "folded-constant", "join-kind"]) +def test_a_structural_change_is_still_a_different_template(dialect, a, b): + """结构变了的(列名表的先后、IN 和 NOT IN、常量写成一个数、连接类型)照旧不是同一个模板。""" + assert not normalize_sql(a, dialect).same_shape(normalize_sql(b, dialect)) From 02b08f9ad4e77126bd5844b59513b33082109840 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 12:11:37 +0800 Subject: [PATCH 6/6] =?UTF-8?q?SQL=20=E5=B7=AE=E5=BC=82=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E9=87=8D=E5=81=9A=EF=BC=88=E7=AC=AC=202=20=E8=BD=AE=E8=A1=A5?= =?UTF-8?q?=E5=85=85=EF=BC=89=EF=BC=9A=E5=9C=A8=E7=9C=9F=E5=AE=9E=E7=9A=84?= =?UTF-8?q?=20PostgreSQL=20/=20MySQL=20=E4=B8=8A=E5=A4=8D=E6=9F=A5?= =?UTF-8?q?=EF=BC=8C=E6=B5=AE=E7=82=B9=E6=95=B0=E6=B1=82=E5=92=8C=E7=9A=84?= =?UTF-8?q?=E4=B8=80=E6=AE=B5=E4=B8=8D=E8=B0=83=E6=8D=A2=E5=85=88=E5=90=8E?= =?UTF-8?q?=E3=80=81=E5=A4=96=E5=B1=82=E6=9D=A1=E4=BB=B6=E5=8F=AA=E6=8A=8A?= =?UTF-8?q?=E8=81=9A=E5=90=88=E7=BB=93=E6=9E=9C=E5=92=8C=E3=80=8C=E7=9B=B8?= =?UTF-8?q?=E7=AD=89=E5=B0=B1=E6=98=AF=E4=B8=80=E6=A0=B7=E3=80=8D=E7=9A=84?= =?UTF-8?q?=E5=88=86=E7=BB=84=E9=94=AE=E6=8C=AA=E8=BF=9B=20HAVING=EF=BC=9B?= =?UTF-8?q?=E6=8E=92=E5=88=97=E6=95=B0=E5=85=88=E7=AE=97=E5=86=8D=E5=B1=95?= =?UTF-8?q?=E5=BC=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:第 2 轮的主体提交只在 SQLite 上执行比对。这次私起了 PostgreSQL 18 和 MySQL 8.0(复验者语料的那份数据), 拿复验者的全部语料和自己针对新改写造的 22 对重跑,抓到两处在真实的库上结果不同、却判了等价的: - 套在子查询里的 UNION ALL 调换各支,外面 SUM:浮点数加的先后变了,1902.2 对 1902.2000000000003(PG、MySQL)。 - 外层 WHERE BINARY t.city = 'SH' 挪进分组子查询的 HAVING:MySQL 自己会把外层只涉及分组键的条件推进子查询的 WHERE 先筛行,不分大小写时和 HAVING 结果不同。 做了什么(backend/app/data/sql_semantics.py): - _Sensitivity:PostgreSQL、MySQL 里对不知道是不是精确数(整数、定点小数)的值 SUM / AVG / STDDEV / VARIANCE,这一段 算「结果看行出来的先后」,连同给它供行的子查询、CTE 不调换来源、不排分组键、不并子查询。SQLite 不算(它用 Kahan-Babuska-Neumaier 求和,关联先后也不跟着 FROM 变;语料 120 对在 7 组数据上按精确值比没有差别)。 - 外连接按 WHERE 变成内连接、内连接的条件挪进 WHERE:三种库的优化器自己也做,执行先后不变,看先后的一段也照做; 只有 SQLite 写了 CROSS JOIN 的不挪。 - 外层 WHERE 挪进分组子查询的 HAVING 只挪聚合结果和「相等就是一样」的分组键上的条件;引用了好几次的简单 CTE 有 一处在看先后的一段里就不拆开(PostgreSQL 会先把它算出来存着)。只筛行选列的子查询照样并进求浮点数和的外层:三种库 自己也会并(PostgreSQL 的 subquery pull-up、MySQL 的 derived_merge、SQLite 的 flattening),执行先后一样(在两个 库上核对过结果一字不差)。 - 同名来源的排列先算个数再展开:十几个同名表时以前先把十几的阶乘个排列都列出来(一条 12 张表自连接的语句跑了两分钟 以上),现在超了上限直接按排好的先后。 - 模块说明照实写清上面几条和 SQLite 求和的假设。 测试:REGRESSIONS_R2 补这两处(PG、MySQL 的 UNION ALL 求和、MySQL 的分组键条件);DIALECT_SAME 里两对需要在求和的 一段调换关联先后的 PG / MySQL 改写移到 KNOWN_UNRECOGNIZED(只给兜底说明)。 验证(最终代码):tests/test_sql_semantics.py + tests/test_verify_agent.py 601 passed,tests/test_sql_equivalence.py 375 passed,全量 -n 8:7856 passed、11 skipped。等价改写 124 对认出 120 对(97%),不给表结构 117 对(94%)。复验者 vsadv 全部语料(928 次判定,SQLite / PG / MySQL 执行)无 blocker、判等价的没有一对执行结果不同,自己针对新改写的 32 对 同样;advfp 180 对等价对上具体说法 0 条;vsprobe 变异压测(种子 9、10)约 1.6 万对无误判等价、无异常;自己的压测 约 1.65 万条可执行,规范形式相同的组执行结果全一致。 --- backend/app/data/sql_semantics.py | 136 +++++++++++++++++++++------- backend/tests/fixtures/sql_pairs.py | 27 ++++-- 2 files changed, 124 insertions(+), 39 deletions(-) diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index d1a45006..3c4d937a 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -47,18 +47,22 @@ canonicalize(给日期比较加 CAST)、pushdown_projections;HAVING 里的 'SH'、'sh' 在同一组,HAVING BINARY city = 'SH' 看的是这一组留下的那个值);只有一项、又不是常量的 IN 列表写成 = (SQLite 里 a IN (b) 是 a = +b)。 -**结果看行出来的先后的一段(见 _Sensitivity)不调换来源的先后、不把 CROSS JOIN 写成逗号连接、不排 GROUP BY 的键、 -不改 DISTINCT、不往里并子查询**:取行(LIMIT / OFFSET / FETCH,ORDER BY 有并列时取哪几行也看先后)、DISTINCT ON、 -窗口函数、拼串拼数组的聚合(GROUP_CONCAT)、聚合里既不分组也不聚合的列、当成标量用的子查询(SQLite 取第一行),以及 -去重、分组、MIN / MAX、去重的 UNION 留下的值「相等却不一样」的时候(MySQL 默认不分大小写的字符串、PostgreSQL numeric -的 2.0 和 2.00、SQLite 没有类型亲和的列、算出来的表达式:留下的是先扫到的那个);给这样一段供行的子查询、CTE 也算。 -SQLite 里逗号连接和 CROSS JOIN 解析成同一个节点,按原文分开(_mark_comma_joins:SQLite 不调换 CROSS JOIN 两边)。 +**结果看行出来的先后的一段(见 _Sensitivity)不调换来源的先后、不排 GROUP BY 的键、不改 DISTINCT、不并子查询、不把 +引用了好几次的 CTE 拆开**:取行(LIMIT / OFFSET / FETCH,ORDER BY 有并列时取哪几行也看先后)、DISTINCT ON、窗口函数、 +拼串拼数组的聚合(GROUP_CONCAT)、聚合里既不分组也不聚合的列、当成标量用的子查询(SQLite 取第一行),去重、分组、 +MIN / MAX、去重的 UNION 留下的值「相等却不一样」的时候(MySQL 默认不分大小写的字符串、PostgreSQL numeric 的 2.0 和 +2.00、SQLite 没有类型亲和的列、算出来的表达式:留下的是先扫到的那个),以及 PostgreSQL、MySQL 里对不知道是不是精确数 +的值求和、求平均(浮点数加的先后不同,结果的最后几位就不同:1902.2 和 1902.2000000000003,在真实的库上跑出来过); +给这样一段供行的子查询、CTE 也算。外连接按 WHERE 变成内连接、内连接的条件挪进 WHERE 这两步三种库的优化器自己也做, +执行起来先后不变,这样的一段也照做——只有 SQLite 里写了 CROSS JOIN 的不挪(SQLite 不调换 CROSS JOIN 两边、会调换逗号 +连接;两种写法解析成同一个节点,按原文分开,见 _mark_comma_joins)。 **假设(做不到的)**:SQLite 没给表结构时按列有声明的类型、默认的 BINARY 排序规则算——声明了 COLLATE NOCASE 的列、 没有类型亲和的列(能同时存 1 和 1.0)表结构里看不出来,这样的列上去重、分组再调换关联先后可能留下不同的代表值; -比较两边都是列时对调两边(SQLite 比较两列用左边那列声明的排序规则,两边都写了 COLLATE 的不对调)。PostgreSQL、 -MySQL 没给表结构时一律不当成「相等就是一样」,去重、分组、MIN / MAX 的一段不调换来源。MySQL 的表名按 Linux 上分 -大小写算(BILLS 和 bills 不判等价)。 +比较两边都是列时对调两边(SQLite 比较两列用左边那列声明的排序规则,两边都写了 COLLATE 的不对调)。SQLite 的求和 +当成和先后无关(它用 Kahan-Babuska-Neumaier 求和,关联先后也不跟着 FROM 的先后变;语料在 7 组数据上按精确值比也没有 +差别)。PostgreSQL、MySQL 没给表结构时一律不当成「相等就是一样」、也不当成精确数,去重、分组、MIN / MAX、求和的一段 +不调换来源。MySQL 的表名按 Linux 上分大小写算(BILLS 和 bills 不判等价)。 等价的结论另有执行比对守门(tests/test_sql_equivalence.py):虚构库上五组随机数据、一组极端数据、一组专挑写法差异的 数据,判等价的两条每组结果都得一样,结果不一样的绝不能判等价。原文里有 sqlglot 解析时会丢掉、语义却不同的写法 @@ -97,7 +101,7 @@ MySQL 没给表结构时一律不当成「相等就是一样」,去重、分 SQLite 里套在子查询里的 UNION、三项以上的加乘换结合、和方言有关的同义写法(DATE(x) 和 CAST(x AS DATE))、LATERAL、 递归 CTE、实表别名带列名、USING 用在第一个连接以外的地方;没给表结构时 WHERE、GROUP BY、HAVING 里用列别名的、 SELECT 列表里和前面的列别名同名的;结果看行出来的先后的一段调换关联先后、DISTINCT 改 GROUP BY;PostgreSQL、MySQL - 没给表结构时去重、分组的一段调换关联先后。 + 没给表结构时去重、分组、求和的一段调换关联先后、并子查询。 - 说不出具体差异、只给兜底的:自己关联自己的方向或角色不同、几个派生来源互相交错、相关子查询内外层的条件、 另一边别处(别的子查询、CTE)也用到同一列筛选的、读的来源对不上、CTE 和子查询里选出来的列不同、条件或算法写法 不同又判断不了是不是真不一样的、EXCEPT 两边对调、几处不同交错在一起的。 @@ -2287,6 +2291,9 @@ _ORDERED_AGGS = frozenset({"groupconcat", "stringagg", "arrayagg", "arrayconcata "jsonobjectagg", "jsonagg", "jsonbagg", "jsongrouparray", "jsongroupobject", "listagg", "xmlagg", "anyvalue", "first", "last", "firstvalue", "lastvalue", "nthvalue", "jsonbobjectagg", "jsonobjectaggstrict", "arrayaggdistinct", "bitstringagg"}) +#: 浮点数求和类的聚合:浮点数加法不满足结合律,加的先后不同结果的最后几位就不同(1902.2 和 1902.2000000000003) +_FLOAT_AGGS = frozenset({"sum", "avg", "stddev", "stddevpop", "stddevsamp", "variance", "variancepop", "varpop", + "varsamp", "covarpop", "covarsamp", "corr", "total"}) #: 拿空值做比较、算术都得空值(外连接没关联上的那一侧整行是空值) _STRICT_UNARY = ("Neg", "Cast", "Lower", "Upper", "Length", "Abs") _STRICT_BINARY = ("Add", "Sub", "Mul", "Div", "Mod") @@ -2541,13 +2548,13 @@ def _optimized(root: Any, read: str | None, info: _Schema) -> _Form: if qualified is None: raise _Unsupported("认不全列") tree = qualified - _split_simple_ctes(tree) + _split_simple_ctes(tree, read, info) _inline_ctes(tree) # 写回的 CTE 定义各是一份副本,里面的别名一样:同一个 CTE 写回外层和相关子查询两处,并进去之后子查询里本来指外层 # 的列就被里层同名的别名遮住了。写回之后再把整条语句的别名改成互不相同的 _unique_aliases(tree) _merge_derived(tree, read, info) - _push_into_groups(tree) + _push_into_groups(tree, read, info) _drop_unused_ctes(tree) return _finish(tree, read, info, level=_OPTIMIZER, qualified=True) @@ -2589,8 +2596,8 @@ def _finish(tree: Any, read: str | None, info: _Schema, *, level: str, qualified sens = _Sensitivity(tree, read, info) _right_to_left(tree, sens) sens = _Sensitivity(tree, read, info) - _reject_nulls(tree, sens, read) - _flatten_prefix(tree, sens) + _reject_nulls(tree, read) + _flatten_prefix(tree, sens, read) _in_subquery_distinct(tree, sens) _distinct_to_group(tree, sens) _count_star(tree) @@ -3033,7 +3040,9 @@ def _merge_derived(tree: Any, read: str | None, info: _Schema) -> None: for _ in range(64): resolver = _Resolver(tree) - sens = _Sensitivity(tree, read, info, resolver) + # 浮点数求和这一条不挡并子查询:只筛行选列的子查询三种库自己也会并进外层(PostgreSQL 的 subquery pull-up、 + # MySQL 的 derived_merge、SQLite 的 flattening),并不并执行起来先后一样 + sens = _Sensitivity(tree, read, info, resolver, floats=False) done = False for select in reversed(list(tree.find_all(exp.Select))): if sens.of(select): @@ -3107,21 +3116,24 @@ def _merge_derived(tree: Any, read: str | None, info: _Schema) -> None: return -def _split_simple_ctes(tree: Any) -> None: +def _split_simple_ctes(tree: Any, read: str | None, info: "_Schema") -> None: """引用了好几次、只从几张表里筛行选列的 CTE(见 _mergeable),每处引用各写回一份 FROM 里的子查询(别名照旧): 这样的 CTE 是确定的(有不确定函数的语句不走这一级),算几遍结果一样,写回之后每处都能并进外层。语句里还有别的 - WITH 的不动(名字可能被里层的 CTE 遮住)。""" + WITH 的不动(名字可能被里层的 CTE 遮住);有一处引用在结果看行出来先后的一段里的不动(PostgreSQL 把引用了好几次 + 的 CTE 先算出来存着,写回之后扫描的先后可能就变了)。""" from sqlglot import exp with_ = tree.args.get("with_") if with_ is None or with_.args.get("recursive") or _nested_with(tree): return resolver = _Resolver(tree) + sens = _Sensitivity(tree, read, info, resolver) for cte in list(with_.expressions): if not _mergeable(cte.this): continue refs = [t for t in tree.find_all(exp.Table) if resolver.cte_of.get(id(t)) is cte] - if len(refs) < 2 or any(r.find_ancestor(exp.CTE) is not None for r in refs): + if len(refs) < 2 or any(r.find_ancestor(exp.CTE) is not None for r in refs) or \ + any(sens.of(resolver.owner[id(r)]) for r in refs): continue for ref in refs: ref.replace(exp.Subquery(this=cte.this.copy(), alias=exp.TableAlias(this=exp.to_identifier(ref.alias_or_name)))) @@ -3144,13 +3156,16 @@ def _grouped(expression: Any, keys: set[str]) -> bool: return True -def _push_into_groups(tree: Any) -> None: +def _push_into_groups(tree: Any, read: str | None, info: "_Schema") -> None: """(认列之后)外层 WHERE 里只涉及一个分组子查询(FROM 或内连接进来的)选出来的列的条件,换成子查询里对应的表达式、 挪进它的 HAVING:子查询的每一行就是一组,外层按这一行的值筛和 HAVING 按这一组的值筛,用的是同一个值(分组键取的 - 是同一个代表值,聚合是同一个结果)。子查询有窗口函数、取行、去重、ROLLUP 这些的不挪;HAVING 里的条件不往 WHERE - 挪(相等不等于一样时,组里的行各是各的值)。""" + 是同一个代表值,聚合是同一个结果)。条件涉及分组键时,分组键得是「相等就是一样」的(见 _Sensitivity._identity_safe): + MySQL 会把外层只涉及分组键的条件推进子查询的 WHERE 里先筛行,不分大小写时 BINARY city = 'SH' 推进去和放在 HAVING + 里结果不同(在真实的 MySQL 8.0 上跑出来过)。子查询有窗口函数、取行、去重、ROLLUP 这些的不挪;HAVING 里的条件 + 不往 WHERE 挪(相等不等于一样时,组里的行各是各的值)。""" from sqlglot import exp + sens = _Sensitivity(tree, read, info) for select in list(tree.find_all(exp.Select)): where = select.args.get("where") joins = select.args.get("joins") or [] @@ -3172,7 +3187,9 @@ def _push_into_groups(tree: Any) -> None: cols = list(cond.find_all(exp.Column)) if not cols or cond.find(exp.Query, exp.Subquery) is not None or \ any(c.table != alias or c.name not in outputs or not _grouped(outputs[c.name], keys) - for c in cols): + for c in cols) or \ + any(k is not None and not sens._identity_safe(k, body) + for c in cols for k in [outputs[c.name] if _fingerprint(outputs[c.name]) in keys else None]): kept.append(cond) continue new = cond.copy() @@ -3260,14 +3277,17 @@ class _Sensitivity: 取任意一个这类聚合(GROUP_CONCAT);聚合里既不分组也不聚合的列(SQLite、MySQL 取组里任意一行的值);当成标量 用的子查询(SQLite 取第一行),除非是不分组的聚合(只出一行);去重、分组、MIN / MAX、去重的 UNION 留下的是先 出现的那个值,而「相等」不等于「写法一样」的时候(MySQL 默认不分大小写的排序规则下的 'SH' 和 'sh'、PostgreSQL - numeric 的 2.0 和 2.00、SQLite 没有类型亲和的列里的 1 和 1.0、算出来的表达式)看先后,见 _identity_safe。 + numeric 的 2.0 和 2.00、SQLite 没有类型亲和的列里的 1 和 1.0、算出来的表达式)看先后,见 _identity_safe; + PostgreSQL、MySQL 里对不知道是不是精确数的值求和、求平均(浮点数加的先后不同,结果的最后几位就不同,见 _exact)。 给看先后的一段供行的(它 FROM 里的子查询、引用的 CTE,层层往里)也看先后。""" - def __init__(self, tree: Any, read: str | None, info: _Schema, resolver: _Resolver | None = None) -> None: + def __init__(self, tree: Any, read: str | None, info: _Schema, resolver: _Resolver | None = None, + floats: bool = True) -> None: from sqlglot import exp self.read = read self.info = info + self.floats = floats try: self.resolver = resolver or _Resolver(tree) except _Unsupported: @@ -3321,6 +3341,40 @@ class _Sensitivity: not self._identity_safe(n.this.expressions[0] if isinstance(n.this, exp.Distinct) and n.this.expressions else n.this, select): return True + if self.floats and self.read != "sqlite" and _float_aggregate(n) and not self._exact(n.this, select): + return True + return False + + def _exact(self, node: Any, select: Any, depth: int = 0) -> bool: + """node 的取值是不是精确的数(整数、定点小数):浮点数求和的先后不同结果就不同(PostgreSQL、MySQL 按扫到的先后 + 一个个加;SQLite 用 Kahan-Babuska-Neumaier 求和、关联先后也不跟着 FROM 的先后变,当成精确的)。不知道类型的不算。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, exp.Distinct) and len(node.expressions) == 1: + node = node.expressions[0] + if isinstance(node, exp.Count) or (isinstance(node, exp.Literal) and node.is_int): + return True + if isinstance(node, (exp.Add, exp.Sub, exp.Mul)): + return self._exact(node.left, select, depth) and self._exact(node.right, select, depth) + if not isinstance(node, exp.Column) or node.is_star or depth > 16 or self.resolver is None: + return False + item = self.resolver.source_of.get(id(node)) + if item is None: + items = _items_of(select) + item = items[0] if len(items) == 1 and not node.table else None + if item is None: + return False + body = self.resolver.body(item) + if body is None: + columns = self.info.types.get(item.name.translate(_ASCII_LOWER)) if isinstance(item, exp.Table) else None + declared = (columns or {}).get(node.name.translate(_ASCII_LOWER), "").lower() + return any(declared.startswith(w) for w in ("int", "integer", "bigint", "smallint", "tinyint", "mediumint", + "serial", "bigserial", "decimal", "numeric", "dec")) + if isinstance(body, exp.Select): + for p in body.expressions: + if not _is_star(p) and p.alias_or_name == node.name: + return self._exact(p.unalias(), body, depth + 1) return False def _identity_safe(self, node: Any, select: Any, depth: int = 0) -> bool: @@ -3399,6 +3453,18 @@ class _Sensitivity: "timestamp", "time", "bool", "uuid")) +def _float_aggregate(node: Any) -> bool: + """浮点数求和类的聚合(不在窗口函数里的)。""" + from sqlglot import exp + + if isinstance(node.parent, exp.Window): + return False + name = type(node).__name__.lower() + if isinstance(node, exp.Anonymous): + name = str(node.name).lower().replace("_", "") + return name in _FLOAT_AGGS + + def _order_dependent(node: Any) -> bool: from sqlglot import exp @@ -3598,15 +3664,16 @@ def _rejects(cond: Any, alias: str, read: str | None) -> bool: return False -def _reject_nulls(tree: Any, sens: _Sensitivity, read: str | None) -> None: +def _reject_nulls(tree: Any, read: str | None) -> None: """LEFT JOIN 关联进来的那一侧,同一段 WHERE 里有把它整行为空的行筛掉的条件(见 _rejects):没关联上的行反正留 - 不下来,这次外连接就是内连接。有 RIGHT / FULL JOIN 的段、结果看行出来的先后的段不动。""" + 不下来,这次外连接就是内连接。三种库的优化器自己也做这一步(outer join reduction),执行起来扫描的先后一样,结果 + 看行出来先后的一段也照做。有 RIGHT / FULL JOIN 的段不动。""" from sqlglot import exp for select in list(tree.find_all(exp.Select)): joins = select.args.get("joins") or [] where = select.args.get("where") - if where is None or sens.of(select) or any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + if where is None or any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): continue conds = _conjuncts(where.this) for join in joins: @@ -3619,19 +3686,23 @@ def _reject_nulls(tree: Any, sens: _Sensitivity, read: str | None) -> None: join.this.meta[_REJECTED] = "RIGHT JOIN" if join.meta.get(_FLIPPED) else "LEFT JOIN" -def _flatten_prefix(tree: Any, sens: _Sensitivity) -> None: +def _flatten_prefix(tree: Any, sens: _Sensitivity, read: str | None) -> None: """开头那组内连接(FROM 和紧跟着的内连接、CROSS JOIN、逗号连接,到第一个外连接为止)的 ON 条件挪进 WHERE、写成 逗号连接:内连接的条件写在 ON 还是 WHERE 一样;后面的 LEFT JOIN 对左边每一行各自关联,左边先筛后筛一样。后面有 RIGHT / FULL JOIN 的不动:没关联上的那一侧整行空值,挪到 WHERE 的条件会把它们筛掉(PostgreSQL、MySQL 里逗号还比 - JOIN 结合得松,a, b RIGHT JOIN c 是 a 乘上 b RIGHT JOIN c)。结果看行出来的先后的一段也不动(SQLite 不调换 CROSS - JOIN 两边、会调换逗号连接)。""" + JOIN 结合得松,a, b RIGHT JOIN c 是 a 乘上 b RIGHT JOIN c)。内连接的条件写在 ON 还是 WHERE,优化器一视同仁,结果 + 看行出来先后的一段也照做;只有 SQLite 里结果看先后、又写了 CROSS JOIN 的一段不动(SQLite 不调换 CROSS JOIN 两边、 + 会调换逗号连接)。""" from sqlglot import exp for select in list(tree.find_all(exp.Select)): joins = select.args.get("joins") or [] - if not joins or select.args.get("from_") is None or sens.of(select) or \ + if not joins or select.args.get("from_") is None or \ any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): continue + if read in (None, "sqlite") and sens.of(select) and \ + any(str(j.kind or "").upper() == "CROSS" or j.meta.get("sql_semantics_join") for j in joins): + continue moved: list[Any] = [] for join in joins: if not _inner(join): @@ -3641,7 +3712,8 @@ def _flatten_prefix(tree: Any, sens: _Sensitivity) -> None: moved += _conjuncts(on) join.set("on", None) join.set("kind", None) - join.meta.pop("sql_semantics_join", None) + if not sens.of(select): + join.meta.pop("sql_semantics_join", None) if moved: where = select.args.get("where") select.set("where", exp.Where(this=_and_all((_conjuncts(where.this) if where else []) + moved))) diff --git a/backend/tests/fixtures/sql_pairs.py b/backend/tests/fixtures/sql_pairs.py index 452071fb..056b1491 100644 --- a/backend/tests/fixtures/sql_pairs.py +++ b/backend/tests/fixtures/sql_pairs.py @@ -739,18 +739,12 @@ DIALECT_SAME: list[tuple[str, str, str, str]] = [ ("names_given_to_derived_tables_are_no_difference", "mysql", "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) r ON r.order_id = o.id WHERE r.n > 1", "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) `退款` ON o.id = `退款`.order_id WHERE `退款`.n > 1"), - ("the_same_derived_column_written_differently_is_no_difference", "mysql", - "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x", - "SELECT SUM(y.total) FROM (SELECT a.user_id, a.total FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) b JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) a ON a.user_id = b.user_id) y"), ("derived_tables_reading_the_same_table_written_differently_ar", "mysql", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", "WITH settled AS (SELECT * FROM orders WHERE status = 'paid'), pending AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(s.amount) FROM settled s JOIN pending q ON q.user_id = s.user_id"), ("names_given_to_derived_tables_are_no_difference", "postgres", "SELECT SUM(amount) FROM orders WHERE user_id IN (WITH t AS (SELECT id FROM users WHERE grade >= 3) SELECT id FROM t)", "SELECT SUM(amount) FROM orders WHERE user_id IN (WITH g AS (SELECT id FROM users WHERE grade >= 3) SELECT g.id FROM g)"), - ("derived_tables_reading_the_same_table_written_differently_ar", "postgres", - "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", - "SELECT SUM(x.amount) FROM (SELECT * FROM orders WHERE status = 'open') y JOIN (SELECT * FROM orders WHERE status = 'paid') x ON x.user_id = y.user_id"), ("the_same_output_computation_written_differently_is_no_differ", "postgres", "SELECT order_id, RANK() OVER (PARTITION BY user_id ORDER BY amount DESC) AS k FROM orders", "SELECT o.order_id, RANK() OVER (PARTITION BY o.user_id ORDER BY o.amount DESC) FROM orders o"), @@ -2149,6 +2143,17 @@ REGRESSIONS_R2: list[tuple[str, str, str, str]] = [ ("float_folding", "postgres", "SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2", "SELECT COUNT(*) FROM orders WHERE amount = 0.3"), ("double_negation", "mysql", "SELECT id, -(-name) FROM users", "SELECT id, name FROM users"), + # 本轮自己在私起的 PostgreSQL 18、MySQL 8.0 上跑出来的:调换 UNION ALL 各支,浮点数加的先后变了 + # (1902.2 对 1902.2000000000003);MySQL 把外层只涉及分组键的条件推进子查询的 WHERE,和 HAVING 结果不同 + ("float_sum_union_branch_order", "postgres", + "SELECT SUM(x) FROM (SELECT amount AS x FROM orders UNION ALL SELECT amount AS x FROM refunds) t", + "SELECT SUM(x) FROM (SELECT amount AS x FROM refunds UNION ALL SELECT amount AS x FROM orders) t"), + ("float_sum_union_branch_order", "mysql", + "SELECT SUM(x) FROM (SELECT amount AS x FROM orders UNION ALL SELECT amount AS x FROM refunds) t", + "SELECT SUM(x) FROM (SELECT amount AS x FROM refunds UNION ALL SELECT amount AS x FROM orders) t"), + ("outer_where_on_group_key_vs_having", "mysql", + "SELECT COUNT(*) FROM (SELECT city, COUNT(*) AS n FROM users GROUP BY city) t WHERE BINARY t.city = 'SH'", + "SELECT COUNT(*) FROM (SELECT city, COUNT(*) AS n FROM users GROUP BY city HAVING BINARY city = 'SH') t"), ] #: 意思一样、规范化认得出的(PostgreSQL / MySQL 的写法) @@ -2167,8 +2172,16 @@ DIALECT_SAME_R2: list[tuple[str, str, str, str]] = [ "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE 0 END) FROM orders"), ] -#: 意思一样、规范化故意不认的(认了就要假设「相等就是一样」,见模块说明):只给兜底说明,不给具体说法 +#: 意思一样、规范化故意不认的(认了就要假设「相等就是一样」、或者 PostgreSQL / MySQL 里浮点数求和的先后不变,见模块 +#: 说明):只给兜底说明,不给具体说法 KNOWN_UNRECOGNIZED: list[tuple[str, str, str, str]] = [ + # PostgreSQL / MySQL 里对浮点数求和的一段调换关联先后:加的先后变了,结果的最后几位可能不同 + ("the_same_derived_column_written_differently_is_no_difference", "mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x", + "SELECT SUM(y.total) FROM (SELECT a.user_id, a.total FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) b JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) a ON a.user_id = b.user_id) y"), + ("derived_tables_reading_the_same_table_written_differently_ar", "postgres", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(x.amount) FROM (SELECT * FROM orders WHERE status = 'open') y JOIN (SELECT * FROM orders WHERE status = 'paid') x ON x.user_id = y.user_id"), ("having_on_group_key_vs_where", "sqlite", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 100 GROUP BY user_id) SELECT SUM(total) FROM t", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(total) FROM t WHERE user_id > 100"),