diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index 613f5e41..3c4d937a 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -1,40 +1,124 @@ """SQL 语义:两条 SQL 算的是不是一回事,不是的话差在哪。 -双盲复算要回答「盲算和原流程的 SQL 差在哪」(A 按支付时间筛、B 按下单时间筛;A 排除了作废的行、B 没有),运行 -对比要回答「这一期和上一期的 SQL 是不是同一个模板」(定时运行每次的日期不同,不能算模板变了),反方审稿要看自己 -的查询和原查询的口径是否一致。三处共用这里: +双盲复算要回答「盲算和原流程的 SQL 差在哪」(A 按支付时间筛、B 按下单时间筛;A 排除了作废的行、B 没有),人据此选 +A / B,只差一个方面、又对得上数据目录里的事实(时间归属、有效行过滤、去重)时选择会写进数据目录;运行对比要回答 +「这一期和上一期的 SQL 是不是同一个模板」(定时运行每次的日期不同,不能算模板变了);反方审稿要看自己的查询和原 +查询的口径是否一致。几处共用这里: - normalize_sql:字面量换成占位符之后的规范文本,加上字面量本身(按规范文本里的顺序)。两条 SQL 只有字面量不同, - 规范文本就相同;关键字大小写、空白、注释、不加引号的名字的大小写都不算不同。 -- facets:把一条查询拆成几个方面——用到的表、连接(类型、关联进来的表、ON 条件)、WHERE 的各个合取式、其中的 - 时间条件(列、运算符、取值)、分组、聚合(函数、参数、是否去重)、HAVING、取哪几行(LIMIT、OFFSET、取前几行 - 时的排序)。别名一律换成真实表名(条件里套着的子查询也是),等式、比较的两边排好顺序:o.paid_at >= '2026-09-01' - 和 '2026-09-01' <= orders.paid_at 是同一个条件。 -- diff_facets:逐个方面比两份 facets,每处不同给一句中文说明。 + 规范文本就相同;关键字大小写、空白、注释、不加引号的名字的大小写都不算不同;结构变了的一定不同。 +- explain_sql_diff:两条 SQL 是不是等价(SqlDiff.equivalent),不是的话差在哪(items、text_diff)。 +- diff_facets:照旧收两份 facets,交给 explain_sql_diff 比两份记下的原文,返回它的 items。 +- facets:把一条查询拆成表、连接、WHERE 合取式、时间条件、分组、聚合、算出来的列、HAVING、取哪几行(给调用方取 + 结构用;CTE、子查询按位置编号,见 _Statement)。facets 和 normalize_sql 不受下面等价判定的改动影响。 -**逐段收集。** CTE、子查询里的条件、聚合和取行也算进来:盲算可能把有效记录条件写在 CTE 里,原流程写在外层,按 -「整条语句里有哪些条件」比才不会把同一个口径报成不同。逗号连接(以及 INNER JOIN 之外又写在 WHERE 里的等值关联) -写在 WHERE 里的列等式算作关联条件,不算筛选。条件里套着的子查询(EXISTS、IN (SELECT …))既单算一段,也按它自己 -那一段的来源整个写进外层那个条件:别名在子查询里定义,拿外层的来源去认,同一个条件只因别名不同就会被报成不同。 +**等价怎么判。** 过去几轮按方面归纳任意 SQL,每轮修好一批又冒出一批(还回退过),收不拢;现在先判等价、再说差在哪。 +两边各自规范化,规范化的每一步都只做保持语义的变换,同一级的两个规范形式一字不差(比语法树逐个节点的类型和参数, +见 _fingerprint,括号不算;不比生成的 SQL:sqlglot 按方言生成时会把不同的写法写成一样,SQLite 的 CAST(x AS DATE) +生成 DATE(x))才算等价,否则一律不等价、条目一定不为空。规范形式分三级(见 _canonical_forms),哪一级对上都算: -**各自起的名字不进比较。** 表的别名换成真实表名;CTE、FROM 里的子查询没有真实表名可换,比较时按它在语句里的位置 -编号(@1、@2…,从外往里、按第一次用到的先后),给人看时照写名字;条件里的子查询(IN、EXISTS、标量子查询)选出来 -的列名不影响取值,比较时去掉。这些名字都是各自起的,双盲复算的盲算更是:按名字比,同一个口径会被说成换了口径。 -编号只看位置,一边多出一个排在前面的 CTE、子查询,后面的编号就错开了:diff_facets 先按编号比,比不上的再按不写 -编号的写法一对一地配,配上的不算不同。仍按名字比的只有 CTE、子查询选出来、外层按名字引用的列(一边叫 total、 -一边叫 sum_amt,算两列)。 +- optimizer:先按方言统一名字(_normalize_names:SQLite 不带表名、带双引号、表结构里又没有的列名原样留着——找不到 + 这一列时 SQLite 把它当字符串),CTE、子查询的列名表写成里面那段的列别名,USING 写成 ON,FETCH FIRST 写成 LIMIT, + IS NOT NULL、NOT LIKE 这类带 negate 的写法统一成 NOT (…);然后 sqlglot 的 qualify 认列(给了数据源的 schema_cache + 就按它认、展开 SELECT *);再把整条语句的来源别名改成互不相同的,引用了好几次、只筛行选列的 CTE 每处写回一份,只 + 引用一次的 CTE 写回用到它的地方,只从几张表筛行选列的子查询并进外层(_merge_derived:只并内连接一侧的),外层 + WHERE 里只涉及分组子查询选出来的列的条件挪进它的 HAVING(同一个值),去掉没用的 CTE。认不全列(没给表结构、几张表 + 又都没写表名)、WHERE / GROUP BY / HAVING 里有和列别名同名又可能是表的列的名字、SELECT 列表里有和前面列别名同名的 + 名字、不确定的函数(RANDOM)的不出这一级。 +- light:不认列,只做上面不需要认列的几步。 +- parsed:解析后只统一名字的大小写、引号。 -**和 sqlcheck 一样宁可漏报。** 解析不了(语法错、方言特性、多条语句、不是查询)就带 parse_error 返回,从不抛 -异常。没加表前缀、又有好几张表可能有这一列的,照写列名(给了表结构才去对)。只读语法树,不执行 SQL。 +两级共用的几步(见 _finish):原样透出子查询全部列的一段换成那个子查询;两个来源的 RIGHT JOIN 写成 LEFT JOIN;WHERE +把外连接可空一侧没关联上的行筛掉了的写成内连接;开头那组内连接的条件挪进 WHERE(后面有 RIGHT / FULL JOIN 的不挪); +IN 子查询里的 DISTINCT 和不带聚合的 ORDER BY 去掉、SELECT DISTINCT 写成按选出来的列 GROUP BY、已经按选出来的列分组的 +DISTINCT 去掉;COUNT(1) 写成 COUNT(*)、MIN / MAX 里的 DISTINCT 去掉;SQLite 的 IIF、MySQL 的 IF 写成 CASE,CASE 的 +ELSE NULL 去掉;WHERE / HAVING / ON 里的 TRUE、1 = 1 去掉;EXISTS 子查询写成 SELECT 1(只取前 n 行的也去掉取行); +IN 列表去重排序、只有一个常量的写成 =;BETWEEN 写成两个比较(SYMMETRIC 不走这两级);NOT 套比较写成反过来的比较、 +NOT NOT 条件去掉;ORDER BY 的 ASC、OFFSET 0 去掉;最外层 ORDER BY 引用的别名换成表达式,最外层和条件里的子查询选出来 +的列名去掉;最后起规范的名字(CTE 叫 cte1…、CTE 和子查询选出来的列叫 c1…、来源叫表名、内层带 @层数、同名第二个起带 +#序号,同名来源谁先谁后排列着试、取写出来最小的),交换律成立的地方(AND、OR、比较两边、一个节点的加乘两边、IN 列表、 +GROUP BY)排好序;合并只调换整条语句最外层不取行的 UNION ALL 各支(PostgreSQL、MySQL 另加:按位置排序的,和套在 +里面、各支列名一样、外面不看先后的)。 + +**故意不做的改写**(第 2 轮复验在这些地方抓到了不保持语义的例子):sqlglot 的 simplify(数字常量按 Decimal 折叠, +SQLite 按浮点数算 0.1 + 0.2 ≠ 0.3;把 -(-x) 化成 x,丢了取负时的数值转换;三项以上的加乘拍平再合常量;BETWEEN +SYMMETRIC 改写时丢了 SYMMETRIC)、unnest_subqueries(OR 里的标量子查询改成 CROSS JOIN,子查询没有行时整条都没了)、 +canonicalize(给日期比较加 CAST)、pushdown_projections;HAVING 里的条件挪进 WHERE(MySQL 默认不分大小写的排序规则下 +'SH'、'sh' 在同一组,HAVING BINARY city = 'SH' 看的是这一组留下的那个值);只有一项、又不是常量的 IN 列表写成 = +(SQLite 里 a IN (b) 是 a = +b)。 + +**结果看行出来的先后的一段(见 _Sensitivity)不调换来源的先后、不排 GROUP BY 的键、不改 DISTINCT、不并子查询、不把 +引用了好几次的 CTE 拆开**:取行(LIMIT / OFFSET / FETCH,ORDER BY 有并列时取哪几行也看先后)、DISTINCT ON、窗口函数、 +拼串拼数组的聚合(GROUP_CONCAT)、聚合里既不分组也不聚合的列、当成标量用的子查询(SQLite 取第一行),去重、分组、 +MIN / MAX、去重的 UNION 留下的值「相等却不一样」的时候(MySQL 默认不分大小写的字符串、PostgreSQL numeric 的 2.0 和 +2.00、SQLite 没有类型亲和的列、算出来的表达式:留下的是先扫到的那个),以及 PostgreSQL、MySQL 里对不知道是不是精确数 +的值求和、求平均(浮点数加的先后不同,结果的最后几位就不同:1902.2 和 1902.2000000000003,在真实的库上跑出来过); +给这样一段供行的子查询、CTE 也算。外连接按 WHERE 变成内连接、内连接的条件挪进 WHERE 这两步三种库的优化器自己也做, +执行起来先后不变,这样的一段也照做——只有 SQLite 里写了 CROSS JOIN 的不挪(SQLite 不调换 CROSS JOIN 两边、会调换逗号 +连接;两种写法解析成同一个节点,按原文分开,见 _mark_comma_joins)。 + +**假设(做不到的)**:SQLite 没给表结构时按列有声明的类型、默认的 BINARY 排序规则算——声明了 COLLATE NOCASE 的列、 +没有类型亲和的列(能同时存 1 和 1.0)表结构里看不出来,这样的列上去重、分组再调换关联先后可能留下不同的代表值; +比较两边都是列时对调两边(SQLite 比较两列用左边那列声明的排序规则,两边都写了 COLLATE 的不对调)。SQLite 的求和 +当成和先后无关(它用 Kahan-Babuska-Neumaier 求和,关联先后也不跟着 FROM 的先后变;语料在 7 组数据上按精确值比也没有 +差别)。PostgreSQL、MySQL 没给表结构时一律不当成「相等就是一样」、也不当成精确数,去重、分组、MIN / MAX、求和的一段 +不调换来源。MySQL 的表名按 Linux 上分大小写算(BILLS 和 bills 不判等价)。 + +等价的结论另有执行比对守门(tests/test_sql_equivalence.py):虚构库上五组随机数据、一组极端数据、一组专挑写法差异的 +数据,判等价的两条每组结果都得一样,结果不一样的绝不能判等价。原文里有 sqlglot 解析时会丢掉、语义却不同的写法 +(一元加号、0x10 和 x'10' 这类字面量、换了名字的类型名——SQLite 的 CAST(x AS STRING) 是数值亲和,解析成和 TEXT +一样的节点,见 _lost_in_parsing)的,不在语法树上判等价。 + +**具体说法的门槛(见 _Explainer)。** 在两边的规范形式上逐段比,只把有把握、确实不同的地方写成具体的中文句子: + +- 来源对得上(一模一样,或者只差一处说得清的:换了一张表、一边多关联了一张表、同一张表的关联类型或条件不同、 + 外连接的主表对调)才往下比;换了表的那一边,列照它自己的表说(中文名、detail 都归到它自己的表)。FROM 里的子查询 + 只有一处不同时往里比(「在 t 里」)。 +- 说「X 不一样」之前先确认真的不一样:同一个对象只和常量比的几个条件(=、<>、大小比较、IN 常量、IS NULL、BETWEEN, + AND / OR / NOT 连起来)按取值逐段算一遍(_Range),留下的行一样的不说(IN 和 OR、多写一个 IS NOT NULL、一段时间拆成 + 两段、多一个被覆盖的下界、分组后 COUNT(*) > 1 和 >= 2);关联条件只是两列相等的,比相等关系分出的组;聚合只在参数 + 是同一列、或者各是一列时说(IIF 和 CASE、COALESCE 和 CASE 这类写法不同的算法不说);两列被关联条件定成相等的, + 选哪一列不说。 +- 说「另一边没有这个条件 / 没有按时间筛选」之前,先确认另一边整条语句(含子查询、CTE、相关子查询里引用外层的列、 + CASE 里、判空时连聚合的参数)都没用这些列筛选(_Usage;合并里别的支不算),没有像时间的列、像日期的常量。 +- 条件里的子查询:两边除了子查询之外写法一模一样(含 NOT、IN 左边的列)的才配成一对往里比,位置照实说(「id NOT IN + 的子查询」「NOT EXISTS 子查询」);子查询里不带表名的列没给表结构时分不清是里层还是外层的,不拿它说具体的。 +- 只是排序不同(order)只在整条语句最外层、没有取行、DISTINCT ON、窗口函数、不是只出一行时说;取行前的排序算 + limit;子查询里的排序不说(它可能决定 GROUP_CONCAT 的先后)。取行在标量子查询、EXISTS 里、只出一行时不说。 +- LEFT JOIN 对 INNER JOIN 只在整条语句最外层、WHERE 里没用到可空一侧的列、结果不只是可空一侧的列的聚合(那样两种 + 写法结果一样)时说;EXCEPT 按先后逐支比,两边对调了就对不上;一边 DISTINCT、一边按同样的列 GROUP BY 不说成一边 + 去重一边没有;MySQL 表名只差大小写不说成换了表。 +- 说的都是「这一段的这一项两边各写的是什么」;两边给人看的写法一样、和已经说过的一句正好反过来的不说。 + +**兜底。** 规范形式不一致、一条具体说法都没有时,给一条 aspect="other" 的「两条 SQL 的写法不同,未能归纳成具体差异, +请对照原文」;具体说法只覆盖了一部分时追加「另有未能归纳的差异,请对照原文」;fallback 为真。text_diff 给出规范化后 +不同的行(difflib 比两边 sqlglot pretty 的输出,最多 40 行)。 + +仍做不到的(只给兜底说明,或者认不出等价): +- 认不出的等价改写:IN 子查询和 EXISTS、NOT IN 和 NOT EXISTS 互换(空值时本来就不等价)、IN 子查询和关联同一张表、 + 条件挪进相关子查询、IN 和 OR、冗余的条件、COALESCE 和 CASE 这类写法不同的等价表达式、窗口函数的 PARTITION BY 换 + 先后、HAVING 里分组键上的条件和 WHERE(只认外层 WHERE 挪进 HAVING 那个方向)、去重的 UNION / INTERSECT 调换各支、 + SQLite 里套在子查询里的 UNION、三项以上的加乘换结合、和方言有关的同义写法(DATE(x) 和 CAST(x AS DATE))、LATERAL、 + 递归 CTE、实表别名带列名、USING 用在第一个连接以外的地方;没给表结构时 WHERE、GROUP BY、HAVING 里用列别名的、 + SELECT 列表里和前面的列别名同名的;结果看行出来的先后的一段调换关联先后、DISTINCT 改 GROUP BY;PostgreSQL、MySQL + 没给表结构时去重、分组、求和的一段调换关联先后、并子查询。 +- 说不出具体差异、只给兜底的:自己关联自己的方向或角色不同、几个派生来源互相交错、相关子查询内外层的条件、 + 另一边别处(别的子查询、CTE)也用到同一列筛选的、读的来源对不上、CTE 和子查询里选出来的列不同、条件或算法写法 + 不同又判断不了是不是真不一样的、EXCEPT 两边对调、几处不同交错在一起的。 + +**和 sqlcheck 一样宁可漏报。** 解析不了(语法错、方言特性、多条语句、不是查询)就只给一条 aspect="parse";从不抛异常, +规范化或说明本身出了意外就当作这一级没有、说不清,记日志。只读语法树,不执行 SQL。 """ from __future__ import annotations +import hashlib +import itertools import logging +import math import re from collections import deque from collections.abc import Iterable, Iterator, Mapping -from dataclasses import dataclass, fields, is_dataclass -from itertools import zip_longest +from dataclasses import dataclass, field, fields, is_dataclass, replace from typing import Any from app.data import catalog @@ -53,8 +137,12 @@ _SEGMENT = "sql_semantics_segment" #: 派生来源(引用 CTE 的表和 CTE 的定义、FROM / JOIN 里的子查询)在语法树上记比较用编号的 meta 键(见 #: _Statement._number_derived)。复制出来的条件里套着子查询时,靠它把 CTE 名、子查询的别名换成编号 _DERIVED = "sql_semantics_derived" -#: 比较用的写法里不写编号时,派生来源一律写成这个(见 Part.anon) +#: 不写编号的写法(render 的 anon 模式)里派生来源一律写成这个:排比较的两边谁在前时先看实表的列还是派生来源的列 _ANON = "@" +#: render 里比较用的写法(见 render):名字按 name_key,别名、各自起的名字都去掉 +_COMPARING = frozenset({"key", "loose", "anon", "shape", "pierce"}) +#: 外连接写在 WHERE 里会把可空那一侧没关联上的行(那一侧的列都是空)筛掉的条件:拿空值比都不成立 +_NULL_REJECTING = ("EQ", "NEQ", "GT", "GTE", "LT", "LTE", "Like", "ILike", "Between", "In") # ========================================================================== @@ -214,14 +302,27 @@ ColumnRef = tuple[str | None, str] class Part: """一个条件、一个分组键、一个聚合参数…… - key 用来比较:别名换成真实表名(CTE、子查询写编号),表名、列名按 name_key,等式和比较的两边排好顺序;loose 同 - key 但不写表名,两边查的表换了一张(orders 换成 orders_archive)时拿它比其余各方面,免得同一个条件只因表名不同 - 被报成不同;anon 同 key,但 CTE、子查询一律写「@」、不写编号(loose 也不写),编号错开时拿它再配一遍(见 - _Compare.unmatched)。text 给人看:好几张表都有的列写「表.列」(真实表名,不是别名),其余只写列名;full 一律 - 写表名,两边的 text 一样、key 却不同时改用它说明。excludes 只在 WHERE 的「排除」写法上有:<>、NOT、NOT IN、 - IS NOT NULL 去掉否定之后排除的是什么(IS NOT NULL 写成「列 为空」)。column_keys 和 columns 一一对应,是各列 - 比较用的写法(实表.列;CTE、子查询的列写成「@.列」):两边给 CTE、子查询起的名字不同、编号错开,也认得出两个 - 条件说的是同一列,配成一对说。 + key 用来比较:别名换成真实表名,CTE、子查询写位置编号(见 _Statement._number_derived),表名、列名按 + name_key,等式和比较的两边排好顺序;loose 同 key 但不写实表的表名(派生来源的编号照写:它不是表名,换了表也还是 + 那一个),两边查的表换了一张(orders 换成 orders_archive)时拿它 + 比其余各方面,免得同一个条件只因表名不同被报成不同。text 给人看:好几张表都有的列写「表.列」(真实表名,不是 + 别名),其余只写列名;full 一律写表名,两边的 text 一样、key 却不同时改用它说明。excludes 只在 WHERE 的「排除」 + 写法上有:<>、NOT、NOT IN、IS NOT NULL 去掉否定之后排除的是什么(IS NOT NULL 写成「列 为空」)。column_keys 和 + columns 一一对应,是各列比较用的写法(实表.列;CTE、子查询的列写成「编号.列」):两边对同一列的不同条件靠它配成 + 一对说。相关子查询里引用外层的列,比较用的写法(key、loose、column_keys)在列名后面带「^」(往外几层带几个,见 + _Statement._depth):内外读同一张表时,c.parent_id = o.id 和 c.id = o.parent_id 才分得开。 + + scope 是这一项在哪一层(条件、分组键、HAVING 上有,见 _Statement._layers):空串是最外层,「|2」是最外层 UNION + 的第 2 支,「@3」是派生来源 @3 里(「@3|2」是它的 UNION 第 2 支),「?」是条件、SELECT 列表里的子查询。facets 按 + (scope, key)去重:同一个条件在外层和子查询里各写了一次是两项。 + + 后面四项不进比较(两个 Part 相等只看前面几项),只用来把不同说清楚: + origins 是这一项用到的派生来源的列各自的来历 [(「t1.total」, 「SUM(orders.amount)」)](见 _Statement._origin): + 两边写法一模一样(都是 t1.total > 100),各自的 t1 却不是同一个时,带上它才分得清。pierce 是一路透过派生来源写到 + 实表的 key,只在中途被挡住时才有(这一项按派生来源的列写,见 _Statement.through);stage 说是被什么挡住的(「取 t + 的前 10 行」「算 t 的窗口函数」):两边 pierce 一样、只有一边被挡住,是同一个条件筛的先后不同。roles 是用到的自己 + 关联自己的别名各是哪一个 [(别名, 在关联条件里出的列, 来源的名字)](见 _Statement._roles):关联的方向反了,两边都 + 写 a.amount,各自的 a 却不是同一个时,带上它才分得清。 """ key: str @@ -231,58 +332,132 @@ class Part: loose: str = "" full: str = "" column_keys: tuple[str, ...] = () - anon: str = "" + scope: str = "" + origins: tuple[tuple[str, str], ...] = field(default=(), compare=False) + pierce: str = field(default="", compare=False) + stage: str = field(default="", compare=False) + roles: tuple[tuple[str, str, str], ...] = field(default=(), compare=False) @staticmethod def plain(key: str, text: str) -> "Part": """没有列的一项(表名、COUNT(*) 的「*」):比较用的写法都是 key,给人看的都是 text。""" - return Part(key=key, text=text, loose=key, full=text, anon=key) + return Part(key=key, text=text, loose=key, full=text) @dataclass(frozen=True) class JoinFacet: """一次连接。kind:inner / left / right / full / cross / semi / anti / natural。table 是关联进来的那一侧,other 是另一侧:关联条件里出现的别的表(给人看的真实表名);没有条件的(CROSS JOIN)是在它之前出现的各个来源。on 是 - 关联条件的合取式排好序后用 AND 连起来的写法,没有条件是 None。""" + 关联条件的合取式排好序后用 AND 连起来的写法,没有条件是 None。 + + sides 是这次连接涉及的来源(比较用的写法,排好序):关联进来的那一侧,加上关联条件里直接引用的这一段的来源(不 + 透过 CTE 找实表);没有条件的加上它之前出现的各个来源。内连接、CROSS JOIN 两边换个先后意思不变(FROM paid JOIN + refunded 和 FROM refunded JOIN paid),比它们时按 sides 和条件比,不按哪一侧是关联进来的。 + + keep、nullable 只在 LEFT / RIGHT JOIN 上有:保留全部行的主表那一侧、关联不上时记空的那一侧(比较用的写法,排好 + 序)。LEFT JOIN 关联进来的那一侧可空,关联条件里的其余来源(没有就是它之前的各个来源)是主表;RIGHT JOIN 反过来。 + orders LEFT JOIN refunds 和 refunds RIGHT JOIN orders 是同一回事,和 refunds LEFT JOIN orders 不是:按主表比, + 不按写的是 LEFT 还是 RIGHT。rejects 是同一段 WHERE 里把可空那一侧没关联上的行筛掉的条件(那一侧的列 + 拿来比较、IN、LIKE……,空值都不成立):有它这次外连接实际等于内连接。scope 是这次连接在哪一层(见 Part.scope): + 外层关联了 refunds,和 EXISTS 子查询里同样关联了 refunds,是两次连接。""" kind: str table: Part on: Part | None = None other: tuple[str, ...] = () + sides: tuple[str, ...] = () + keep: tuple[str, ...] = () + nullable: tuple[str, ...] = () + rejects: Part | None = None + scope: str = "" @dataclass(frozen=True) class TimePredicate: """WHERE 里的一个时间条件。op:= / <> / > / >= / < / <= / between / in。value 是另一侧给人看的写法: - BETWEEN 是「低 AND 高」,IN 是「(…)」。""" + BETWEEN 是「低 AND 高」,IN 是「(…)」。 + + column 是按哪一列筛时间:条件写的是 CTE、子查询算出来的列(WITH t AS (SELECT DATE(paid_at) AS d …) … WHERE + d >= …)时,它在里面是同一列套着日期函数、CAST、MIN / MAX 的,换成里面那一列,一直换到实表(paid_at):按支付 + 时间还是下单时间筛,日期表达式写在 CTE 里也认得出。via 是路过的派生来源的列(「编号.列」,不进比较):时间那一面 + 已经报了按哪一列筛不同,派生来源那一列的定义不同就不再另报一遍。table 是这一列一直换到实表的 (表, 列)(不进 + 比较):读同一张表的几个派生来源同在一段时,column 照写派生来源的列(已付那份的 created_at、未付那份的 created_at + 是两列,见 _Statement._seal),给调用方对照数据目录的还是 orders.created_at。""" column: Part op: str value: str condition: Part + via: tuple[str, ...] = field(default=(), compare=False) + table: ColumnRef | None = field(default=None, compare=False) @dataclass(frozen=True) class Aggregate: """一个聚合。func 是函数名的小写(sum / count / avg / min / max …);arg 是参数,COUNT(*)、COUNT(1) 的 arg - 是「*」,去重聚合是 DISTINCT 里面的表达式;text 是整个聚合给人看的写法。窗口函数不算。""" + 是「*」,去重聚合是 DISTINCT 里面的表达式;text 是整个聚合给人看的写法。窗口函数不算(它们按所在的那一列比,见 + Computed)。""" func: str arg: Part distinct: bool text: str + scope: str = "" # 在哪一层(见 Part.scope):外层算的和标量子查询里算的同一个聚合是两项 + + +@dataclass(frozen=True) +class Computed: + """选出来的一列是怎么算的。两种: + + - 最外层(UNION 的各支)选出来的列,不是光一列(分组那一面比)、也不是字面量的,比如 SUM(p.total) - SUM(r.total)、 + SUM(amount) OVER (PARTITION BY user_id)、CASE …;ROUND、CAST、COALESCE 给默认值这类外壳先剥掉(见 + _unwrap_value)。光一个聚合也收:另一边可能拿同一个合计再算了一下(SUM(amount) 对 SUM(amount) / 100)。scope、 + scope_key 为 None,name 为空串。 + - 派生来源(CTE、FROM 里的子查询)里被外面用到、又不是原样透出一列的列:它在里面的定义(amount * 0.9、 + DATE(paid_at)、ROW_NUMBER() OVER (…)、p.total - r.total)。外面按「t.net」引用它,两边写的都是 t.net,算法却可能 + 不同,只比外面的写法看不出来。scope 是派生来源给人看的名字,scope_key 是它的位置编号, + name 是列名的 name_key,shown 是列名的原样写法。 + + expr 是定义(按它那一段的来源写的 Part;UNION 的各支用「|」连起来)。template 是把里面的聚合换成占位符之后的写法、 + aggregates 是里面的聚合(按出现的先后):两边只差在聚合上(SUM(amount) * 100 对 SUM(paid_amount) * 100)、聚合 + 那一面又已经报了,这里不再报一遍;聚合的集合一样、只是位置换了(SUM(a) - SUM(b) 对 SUM(b) - SUM(a))才在这里报。 + """ + + scope: str | None + scope_key: str | None + name: str + shown: str + expr: Part + template: Part + aggregates: tuple[Aggregate, ...] = () @dataclass(frozen=True) class LimitFacet: """一段查询取哪几行。count、offset 是行数和跳过行数的写法(没写为 None,OFFSET 0 也算没写);order 是这一段的 排序键 (Part, 是否降序):只取前几行时,按什么排决定了取到的是哪几行。scope 是哪一段:None 是最外层,CTE 写它的 - 名字,FROM 里的子查询写它的别名(没有别名写「子查询」)。条件里套着的子查询不单算:它整个写在那个条件里比。""" + 名字,FROM 里的子查询写它的别名(没有别名写「子查询」)。条件里套着的子查询不单算:它整个写在那个条件里比。 + scope_key 是这一段所属派生来源的编号(和 Part.key 里的同一套编号;最外层、UNION 的某一支这类不是派生来源整段的 + 为 None)。""" count: str | None offset: str | None = None order: tuple[tuple[Part, bool], ...] = () scope: str | None = None + scope_key: str | None = None + + +@dataclass(frozen=True) +class SetOpFacet: + """一段查询把几支 SELECT 合起来(UNION、UNION ALL、INTERSECT、EXCEPT)。ops 是各处合并的写法,按从左到右的先后 + (「UNION」「UNION ALL」…);branches 是一共几支。scope 是哪一段:None 是最外层,CTE、FROM 里的子查询写它的名字; + scope_key 是那一段所在的层(见 Part.scope:最外层是空串,派生来源是它的位置编号)。条件里的子查询不单算: + 它整个写在那个条件里比。""" + + ops: tuple[str, ...] + branches: int + scope: str | None = None + scope_key: str = "" @dataclass(frozen=True) @@ -291,6 +466,24 @@ class Facets: limit 是最外层 LIMIT 的行数(写法);limits 是各段取哪几行(最外层在前,见 LimitFacet),只有写了 LIMIT 或 OFFSET 的段才有。 + + sources 是各段 FROM / JOIN 里的来源(实表、CTE、子查询;key 是比较用的写法,派生来源是编号,text 是给人看的 + 名字),按出现的先后、按 key 去重;条件里套着的子查询(IN (SELECT … FROM users))的来源不算,它们只筛行,不是 + 连接的一头。外连接的主表、可空那一侧按它写名字。 + + expressions 是算出来的列(见 Computed)。derived 是用到了的派生来源(key 是编号,text 是名字,含条件里套着的子 + 查询用到的 CTE):两边写法一样、同名的派生来源却不是同一个时,拿它说清「盲算的 t1 对应原流程的 t2」。 + + where、group_by、having、joins、aggregates、time 的每一项都记着在哪一层(Part.scope、JoinFacet.scope、 + Aggregate.scope),按(层, key)去重;scopes 是这条语句里有的各层(key 是层的标记,text 是给人看的说法:「外层」 + 「t」「t 的 UNION 第 2 支」「条件的子查询」),给人说是哪一层时用。group_by 里也有当分组用的 + SELECT DISTINCT 选出来的各项(这一段没有 GROUP BY、没有聚合时,和按它们分组出的行一样),distinct 是这样收进来的 + 那几层:说「去重」不说「分组」。set_ops 是各段的 UNION、INTERSECT、EXCEPT(见 SetOpFacet)。grain 是最后出来 + 的行按什么分(key 为「group:分组键…」「one」「rows」,见 _Statement._grain;最外层是 UNION 的为 None)。 + + 只收用到了的段:没用到的 CTE(定义了、外面没引用)里的条件、聚合、表不影响结果,不收(见 _Statement._enliven)。 + + _origin 是 facets 记下的 (原文, 表结构),不进比较、不进 as_dict:diff_facets 拿它交给 explain_sql_diff 比原文。 """ dialect: str | None = None @@ -303,7 +496,15 @@ class Facets: having: tuple[Part, ...] = () limit: str | None = None limits: tuple[LimitFacet, ...] = () + sources: tuple[Part, ...] = () + expressions: tuple[Computed, ...] = () + derived: tuple[Part, ...] = () + set_ops: tuple[SetOpFacet, ...] = () + distinct: tuple[str, ...] = () + grain: Part | None = None + scopes: tuple[Part, ...] = () parse_error: str | None = None + _origin: tuple[str, Mapping[str, Any] | None] | None = field(default=None, compare=False, repr=False) def as_dict(self) -> dict[str, Any]: return _jsonable(self) @@ -311,7 +512,7 @@ class Facets: def _jsonable(value: Any) -> Any: if is_dataclass(value) and not isinstance(value, type): - return {f.name: _jsonable(getattr(value, f.name)) for f in fields(value)} + return {f.name: _jsonable(getattr(value, f.name)) for f in fields(value) if not f.name.startswith("_")} if isinstance(value, (list, tuple)): return [_jsonable(v) for v in value] return value @@ -322,13 +523,15 @@ def facets(sql: str, dialect: str | None = None, *, schema: Mapping[str, Any] | schema 是数据源的 schema_cache 形状({"tables": {表名: {"columns": [{"name": …}]}}}),可以不给。给了的话, 几张表连在一起、列又没加表前缀时按它认列是哪张表的;不给就照写列名,比较时可能把同一列当成两列。 + + 返回的 Facets 记着原文和表结构(_origin):diff_facets 拿它交给 explain_sql_diff 比原文。 """ read = _read(dialect) root, problem = _parse(sql, read) if root is None: return Facets(dialect=read, parse_error=problem) try: - return _Statement(root, read, schema).facets() + return replace(_Statement(root, read, schema).facets(), _origin=(sql, schema)) except Exception: # noqa: BLE001 - 分析本身的缺陷不能连累调用方:当作分析不了,记下来修 logger.warning("SQL 语义分析失败", exc_info=True) return Facets(dialect=read, parse_error="SQL 的结构无法分析") @@ -339,9 +542,6 @@ def facets(sql: str, dialect: str | None = None, *, schema: Mapping[str, Any] | #: 比较:(运算符的写法, 两边对调之后的类型) _COMPARE: dict[str, tuple[str, str]] = {"EQ": ("=", "EQ"), "NEQ": ("<>", "NEQ"), "GT": (">", "LT"), "GTE": (">=", "LTE"), "LT": ("<", "GT"), "LTE": ("<=", "GTE")} -#: 连接类型的写法(diff_facets 里用) -_JOIN_WORDS = {"inner": "INNER JOIN", "left": "LEFT JOIN", "right": "RIGHT JOIN", "full": "FULL JOIN", - "cross": "CROSS JOIN", "semi": "SEMI JOIN", "anti": "ANTI JOIN", "natural": "NATURAL JOIN"} #: 日期、时间函数(sqlglot 把各方言的写法归成这些类型:strftime、DATE_FORMAT 都是 TimeToStr) _DATE_FUNCS = ("CurrentDate", "CurrentDatetime", "CurrentTimestamp", "CurrentTime", "Date", "Datetime", "DateAdd", "DateSub", "DateDiff", "DateTrunc", "TimestampTrunc", "DatetimeTrunc", "TimeToStr", "StrToDate", @@ -376,6 +576,13 @@ def _own_columns(node: Any) -> list[Any]: return [c for c in found if c.name and not c.is_star] +def _is_star(item: Any) -> bool: + """SELECT 列表里的一项是 * 或 t.*。""" + from sqlglot import exp + + return isinstance(item, exp.Star) or (isinstance(item, exp.Column) and item.is_star) + + def _inner_queries(query: Any, skip: Iterable[Any] = ()) -> list[Any]: """query 里直接套着的查询(不再往里找),按出现的先后:条件、SELECT 列表里的子查询,UNION 的各支,子查询里面 那段。WITH 里的 CTE 定义不算(在用到它的地方才算),skip 里的节点整个跳过。""" @@ -423,6 +630,36 @@ def _derived_body(item: Any) -> Any: return item.this if isinstance(item, exp.Subquery) else None +def _branches(query: Any) -> list[Any]: + """一段查询的各支 SELECT,按出现的先后:UNION 这类拆开,外面套的括号去掉;认不出的返回空列表。""" + from sqlglot import exp + + while isinstance(query, exp.Subquery): + query = query.this + if isinstance(query, exp.SetOperation): + return _branches(query.this) + _branches(query.expression) + return [query] if isinstance(query, exp.Select) else [] + + +def _output_names(query: Any) -> list[str | None]: + """派生来源选出来的各列叫什么(外面按这个名字引用它们):按第一支 SELECT 的位置,* 为 None。CTE、子查询自己 + 写了列名的(WITH t(n) AS …、(…) AS t(a, b),递归 CTE 常这么写)按写的列名。""" + from sqlglot import exp + + branches = _branches(query) + if not branches: + return [] + names = [None if _is_star(item) else item.alias_or_name for item in branches[0].selects] + holder = query.parent + if isinstance(holder, exp.Subquery) and isinstance(holder.parent, exp.Lateral) and not holder.alias: + holder = holder.parent # LATERAL 的别名(连同列名)挂在 LATERAL 上 + alias = holder.args.get("alias") if isinstance(holder, (exp.CTE, exp.Subquery, exp.Lateral)) else None + columns = [c.name for c in alias.columns] if isinstance(alias, exp.TableAlias) else [] + for i, column in enumerate(columns[:len(names)]): + names[i] = column + return names + + def _own_nodes(select: Any) -> Iterator[Any]: """一段 SELECT 自己的节点:不进 CTE、子查询(它们是别的段)。""" from sqlglot import exp @@ -453,6 +690,68 @@ def _datey(node: Any) -> bool: return False +def _value_column(node: Any) -> Any: + """派生来源一列的定义是不是「还是那一列」:光一列,或者同一列套着日期函数、CAST、MIN / MAX(按支付时间算的日期、 + 每人最后一次付款的时间);是的话返回那一列,不是返回 None。窗口函数(ROW_NUMBER() OVER (ORDER BY paid_at) 是排名, + 不是时间)、别的聚合、好几列算出来的、套着子查询的都不算。""" + from sqlglot import exp + + n = _unparen(node) + if isinstance(n, exp.Column): + return n if n.name and not n.is_star else None + if any(isinstance(x, (exp.Window, exp.Select, exp.Subquery)) or + (isinstance(x, exp.AggFunc) and not isinstance(x, (exp.Min, exp.Max))) for x in n.walk()): + return None + cols = _own_columns(n) + if len(cols) != 1: + return None + if _datey(n) or _bare_column(n) is not None or any(isinstance(x, (exp.Min, exp.Max)) for x in n.walk()): + return cols[0] + return None + + +def _unwrap_value(node: Any) -> Any: + """剥掉只改写法、不改是哪个数的外壳(括号、类型转换、ROUND、COALESCE / IFNULL 给默认值):最外层的 + ROUND(SUM(amount), 2)、COALESCE(SUM(amount), 0) 还是那个合计,只是显示上的取舍。""" + from sqlglot import exp + + while True: + node = _unparen(node) + if isinstance(node, (exp.Cast, exp.TryCast, exp.Round)) and node.this is not None: + node = node.this + elif isinstance(node, exp.Coalesce) and node.expressions and \ + all(isinstance(_unparen(e), exp.Literal) for e in node.expressions): + node = node.this + else: + return node + + +def _as_date(node: Any) -> Any: + """「取日期」的几种写法统一成 CAST(… AS DATE)(比较用,给人看的照原样):DATE(paid_at)、paid_at::date、 + TRY_CAST(paid_at AS DATE),MySQL 的 DATE() 读成 TsOrDsToDate。各方言、各人写法不同,取的都是同一天;带时区偏移、 + 格式的(DATE(paid_at, '+8 hours'))不是同一天,不动。""" + from sqlglot import exp + + def to_date(n: Any) -> bool: + return isinstance(n, exp.DataType) and n.this == exp.DataType.Type.DATE + + if isinstance(node, exp.Date) and node.this is not None and not node.args.get("zone") \ + and not node.args.get("expressions"): + return exp.Cast(this=node.this, to=exp.DataType.build("DATE")) + if isinstance(node, exp.TsOrDsToDate) and node.this is not None and not node.args.get("format"): + return exp.Cast(this=node.this, to=exp.DataType.build("DATE")) + if isinstance(node, exp.TryCast) and to_date(node.args.get("to")): + return exp.Cast(this=node.this, to=exp.DataType.build("DATE")) + return node + + +def _atomic(node: Any) -> bool: + """算式里的一项不是别的加减乘除:两边对调不会改变怎么结合(a * b / c 换成 b / c * a 就不一样了)。""" + from sqlglot import exp + + return not isinstance(_unparen(node), (exp.Add, exp.Sub, exp.Mul, exp.Div, exp.Mod, exp.IntDiv)) + + def _temporal_name(name: str) -> bool: """列名像日期、时间:英文按 catalog.is_date_column 的命名规则(paid_at、order_date),中文看结尾(支付日期)。""" return catalog.is_date_column(name, None) or bool(_CN_TIME.search(name)) @@ -496,6 +795,14 @@ class _Source: key: str # 比较用:实表是 name_key,派生来源是编号(见 _Statement._number_derived) idents: tuple[Any, ...] = () # 表名各段的 Identifier:给人看的写法照原样写回去 query: Any = None # derived:里面那段查询 + role: str = "" # 同一个来源在这一段里出现了不止一次时,这一次是哪一个(见 _Statement._roles) + alias: str = "" # 这一段里引用它的名字(别名,没有别名是表名、CTE 名):给人看时分内外层用 + opaque: bool = False # derived:同一段里别的来源透过去读的也是同一个来源,不透过去(见 _Statement._seal) + + @property + def ref(self) -> str: + """比较用的「哪一个来源」:key,自己关联自己时带上 role(orders#parent_id)。""" + return f"{self.key}#{self.role}" if self.role else self.key class _Statement: @@ -518,15 +825,6 @@ class _Statement: if cte.alias: self.ctes.setdefault(name_key(cte.alias), cte.this) self._cte_defs.setdefault(name_key(cte.alias), cte) - self._number_derived() - tables: dict[str, _Source] = {} - for table in root.find_all(exp.Table, bfs=False): - if isinstance(table.this, exp.Identifier) and table.name and not self._is_cte(table): - source = self._table(table) - tables.setdefault(source.key, source) - self.tables = list(tables.values()) - #: 整条语句只用到一张实表:给人看的写法一律不写表名 - self.single = len(self.tables) <= 1 self.columns: dict[str, set[str]] = {} for name, meta in ((schema or {}).get("tables") or {}).items(): if isinstance(meta, Mapping): @@ -534,12 +832,61 @@ class _Statement: if isinstance(c, Mapping) and c.get("name")} self._memo: dict[int, _Sources] = {} self._shared: set[str] | None = None + #: 各派生来源的结构标识 {位置编号: 摘要}(见 _signature)、各派生来源一列的来历 {(里面那段的 id, 列名): 来历} + #: (见 _origin):都和编号无关,重新编号不用清 + self._signatures: dict[int, str] = {} + self._origins: dict[tuple[int, str], str | None] = {} + #: 派生来源各列的名字 {里面那段的 id: [列名]}、每段外面筛和里面筛一不一样 {(那段的 id, 列名): 说法}:只看 + #: 语法树,算一次记下来(透视、认列时反复要用) + self._names_memo: dict[int, list[str | None]] = {} + self._blocks_memo: dict[tuple[int, str], str | None] = {} + self._number_derived() + self._enliven(()) # ---- 来源 def _is_cte(self, table: Any) -> bool: return not table.text("db") and not table.text("catalog") and name_key(table.name) in self.ctes + def _reach(self, extra: Iterable[str] = ()) -> set[str]: + """从最外层那段查询出发(再加上 extra 里的 CTE)一路引用到的 CTE 名:FROM / JOIN 里的、条件和 SELECT 列表里的 + 子查询里的都算,用到的 CTE 里面再引用的也算。CTE 的定义本身不算引用:没人用的 CTE 里引用的别的 CTE 也没用到。""" + from sqlglot import exp + + live = {name for name in extra if name in self.ctes} + stack = [self.root, *(self.ctes[name] for name in live)] + while stack: + for node in stack.pop().walk(prune=lambda n: isinstance(n, exp.CTE)): + if isinstance(node, exp.Table) and isinstance(node.this, exp.Identifier) and node.name \ + and self._is_cte(node) and (name := name_key(node.name)) not in live: + live.add(name) + stack.append(self.ctes[name]) + return live + + def _enliven(self, extra: Iterable[str]) -> None: + """定下哪些段算数:最外层那段查询,加上从它一路引用到的 CTE(extra 里的 CTE 也当作用到了)。没 + 用到的 CTE 里的条件、聚合、表不影响结果:收进来的话,盲算定义了只筛已付的 CTE 却直接查 orders,会被当成也筛了 + 已付;定义了没用的 CTE 里多一个条件,又会被报成多筛了一遍。用到的实表(给人看时要不要写表名)、共有的列名跟着 + 重算。""" + from sqlglot import exp + + live = self._reach(extra) + #: 没用到的 CTE 里的节点(id):facets 只收别的段 + self._dead = {id(n) for name, cte in self._cte_defs.items() if name not in live for n in cte.this.walk()} + tables: dict[str, _Source] = {} + for table in self._alive(exp.Table): + if isinstance(table.this, exp.Identifier) and table.name and not self._is_cte(table): + source = self._table(table) + tables.setdefault(source.key, source) + self.tables = list(tables.values()) + #: 整条语句只用到一张实表:给人看的写法一律不写表名 + self.single = len(self.tables) <= 1 + self._shared = None + + def _alive(self, *types: Any) -> Iterator[Any]: + """用到了的段里这些类型的节点(深度优先、按出现的先后),没用到的 CTE 里的不算(见 _enliven)。""" + return (node for node in self.root.find_all(*types, bfs=False) if id(node) not in self._dead) + @staticmethod def _items(select: Any) -> list[Any]: """一段 SELECT 的 FROM、JOIN 项,按出现的先后。""" @@ -552,14 +899,21 @@ class _Statement: CTE 名、子查询的别名是各自起的(双盲复算的盲算更是),拿它作 key,同一个口径只因起的名字不同就被报成不同; 拿里面那段查询的写法作 key 也不行:里面差一个条件,外层按它的列写的条件就全跟着变,同一处不同报好几遍,还会 - 出现两边各说一句「只保留 total > 100 的行」的自相矛盾。所以只按位置编:从最外层往里一层一层地编,同一层按 - 出现的先后;CTE 在第一次用到的地方编号,在哪儿用到都是这一个编号。写成 CTE 还是 FROM 里的子查询、CTE 定义 - 的先后都不影响编号,里层多一段、少一段也不挪动外层的编号。 + 出现两边各说一句「只保留 total > 100 的行」的自相矛盾。所以单看一条语句时按位置编:从最外层往里一层一层地 + 编,同一层按出现的先后;CTE 在第一次用到的地方编号,在哪儿用到都是这一个编号。写成 CTE 还是 FROM 里的子查询、 + CTE 定义的先后都不影响编号,里层多一段、少一段也不挪动外层的编号。这套编号只给 facets 用;比两条语句不靠 + 它(见 explain_sql_diff:两边各自规范化,CTE 按定义起名)。 """ from sqlglot import exp - ctes: dict[str, str] = {} + ctes: dict[str, int] = {} count = 0 + #: 编了号的派生来源 [(位置编号, 起的名字, 里面那段查询, CTE 的 name_key 或 None)],按编号排:facets 按它们 + #: 找各列的定义(见 _expressions) + self._derived: list[tuple[int, str, Any, str | None]] = [] + + def mark(node: Any, number: int) -> None: + node.meta[_DERIVED] = f"@{number}" def visit(select: Any) -> list[Any]: """给这一段还没编号的 FROM / JOIN 项编号,返回要接着往里看的查询(子查询、第一次用到的 CTE 的定义)。""" @@ -570,17 +924,19 @@ class _Statement: if body is not None: if _DERIVED not in item.meta: count += 1 - item.meta[_DERIVED] = f"@{count}" + mark(item, count) + self._derived.append((count, _alias(item), body, None)) inner.append(body) elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) and item.name \ and self._is_cte(item): name = name_key(item.name) if name not in ctes: count += 1 - ctes[name] = f"@{count}" - self._cte_defs[name].meta[_DERIVED] = ctes[name] + ctes[name] = count + mark(self._cte_defs[name], count) + self._derived.append((count, self._cte_defs[name].alias, self.ctes[name], name)) inner.append(self.ctes[name]) - item.meta[_DERIVED] = ctes[name] + mark(item, ctes[name]) return inner queue: deque[Any] = deque([self.root]) @@ -594,6 +950,40 @@ class _Statement: # 没走到的(没用到的 CTE 里的、表函数参数里的)按出现的先后补上:每个派生来源都得有编号 for select in self.root.find_all(exp.Select, bfs=False): visit(select) + # 没用到的 CTE 也编上号(排在最后):每个派生来源都得有编号 + for name, cte in self._cte_defs.items(): + if name not in ctes: + count += 1 + ctes[name] = count + mark(cte, count) + self._derived.append((count, cte.alias, self.ctes[name], name)) + #: {位置编号: 里面那段查询}、{位置编号: 现在比较用的编号}、反过来 {比较用的编号: 位置编号} + self._bodies = {number: body for number, _, body, _ in self._derived} + self._labels = {number: f"@{number}" for number in self._bodies} + self._numbers = {label: number for number, label in self._labels.items()} + + def column_key(self, source: _Source, column: str) -> str: + """比较用的列名:按 name_key。""" + return name_key(column) + + def _shape(self, node: Any, sources: dict[str, _Source]) -> str: + """按结构写(render 的 shape 模式,比较的两边也按结构排):配对用,和名字、编号无关。""" + return self.render(self.canonical(node, sources, by="shape"), sources, "shape") + + def _signature(self, number: int) -> str: + """派生来源的结构标识(shape 写法里引用它的地方写这个):里面那段查询按 shape 写法写出来再取摘要。里面再套着 + 的派生来源也写它们各自的结构标识,所以付款合计、退款合计两个 CTE 的同名列 total 在外面一层也分得开,和起的 + 名字、编号无关;取摘要是免得一层套一层越写越长。递归 CTE 引用自己时写「@」。""" + if number not in self._signatures: + self._signatures[number] = _ANON + text = self._shape(self._bodies[number], {}) + self._signatures[number] = "@" + hashlib.sha1(text.encode("utf-8")).hexdigest()[:12] + return self._signatures[number] + + def _signature_of(self, label: str) -> str: + """比较用的编号 → 那个派生来源的结构标识;认不出的编号写「@」。""" + number = self._numbers.get(label) + return self._signature(number) if number is not None else _ANON @staticmethod def _table(table: Any) -> _Source: @@ -611,6 +1001,7 @@ class _Statement: if id(select) in self._memo: return self._memo[id(select)] out = _Sources() + aliases: dict[str, str] = {} for i, node in enumerate(self._items(select)): alias = _alias(node) slot = name_key(alias) if alias else f"#{i}" @@ -625,22 +1016,96 @@ class _Statement: (exp.to_identifier(alias),) if alias else (), body) else: out[slot] = _Source("other", alias, slot) + out[slot].alias = alias + aliases[slot] = alias + self._roles(select, out, aliases) self._memo[id(select)] = out parent = select.parent while parent is not None and not isinstance(parent, exp.Select): parent = parent.parent out.outer = self.sources(parent) if parent is not None else None + self._seal(out) return out + def _base(self, source: _Source) -> str: + """一路透过「只读一个来源」的派生来源(through 能透过去的那一路),最后读的是哪个来源:「种类:key」。""" + from sqlglot import exp + + for _ in range(8): + if source.kind != "derived" or not isinstance(source.query, exp.Select): + break + inner = self.sources(source.query) + if len(inner) != 1: + break + source = next(iter(inner.values())) + return f"{source.kind}:{source.key}" + + def _seal(self, sources: dict[str, _Source]) -> None: + """同一段里有几个来源透过去读的是同一个来源时,其中的派生来源都不透过去(就地标 opaque,见 through)。 + + 两个原样透出的 CTE 都读 orders(已付的一份、未付的一份):透过去 p.amount、o.amount 都成了 orders.amount,外层 + 用的是哪一份就分不出来了(SUM(p.amount) 对 SUM(o.amount)、p.amount > 100 对 o.amount > 100 都成了一样);CTE 和 + 它自己读的那张表关联(父单、子单)、同一个原样透出的 CTE 关联两次也一样,自己关联自己时 _roles 设的 role 透过去 + 也就丢了。这时这几个派生来源的列照写它们自己的编号:用的是已付那份 + 还是未付那份分得开。只有一个来源读那张表时照旧透过去:先在 CTE 里筛一遍再查和直接查 + 是同一列。""" + groups: dict[str, list[_Source]] = {} + for source in sources.values(): + groups.setdefault(self._base(source), []).append(source) + for found in groups.values(): + if len(found) > 1: + for source in found: + if source.kind == "derived": + source.opaque = True + + @staticmethod + def _roles(select: Any, sources: dict[str, _Source], aliases: Mapping[str, str]) -> None: + """同一个来源在这一段里出现了不止一次(自己关联自己:orders o1 JOIN orders o2、base b1 JOIN base b2)时,各自 + 按在关联条件(ON、WHERE 里两列相等)里出的是哪几列区分,记在 role 上(就地改):比较用的写法写「来源#列」, + 和别名、先后都无关。b2.user_id = b1.referrer_id 里 b1 出 referrer_id、b2 出 user_id,b1.total > b2.total 写反了 + 就认得出来;不然两边都写成 base.total,写反了也一样。出的列一样(b1.user_id = b2.user_id)分不出,也不用分: + 两个对调过来意思一样。给人看的写法这时写别名:都写 orders.amount 读不出是哪一个。""" + from sqlglot import exp + + groups: dict[str, list[str]] = {} + for slot, source in sources.items(): + if source.kind in ("table", "derived"): + groups.setdefault(source.key, []).append(slot) + repeated = [slots for slots in groups.values() if len(slots) > 1] + if not repeated: + return + conds = [j.args["on"] for j in select.args.get("joins") or [] if j.args.get("on") is not None] + if (where := select.args.get("where")) is not None: + conds.append(where.this) + joined: dict[str, set[str]] = {} + for cond in conds: + for eq in _conjuncts(cond): + n = _unparen(eq) + if not isinstance(n, exp.EQ): + continue + cols = [_unparen(n.this), _unparen(n.expression)] + slots = [name_key(c.table) if isinstance(c, exp.Column) and c.table else None for c in cols] + if None in slots or slots[0] == slots[1] or not all(s in sources for s in slots): + continue + for col, slot in zip(cols, slots): + joined.setdefault(slot, set()).add(name_key(col.name)) + for slots in repeated: + roles = {slot: "+".join(sorted(joined.get(slot, ()))) for slot in slots} + for slot in slots: + if roles[slot] and list(roles.values()).count(roles[slot]) == 1: + sources[slot].role = roles[slot] + if aliases.get(slot): + sources[slot].idents = (exp.to_identifier(aliases[slot]),) + def _exposes(self, source: _Source, column: str) -> bool: """这个来源可能有这一列:实表看表结构(没给表结构就当可能有),CTE、子查询看它选出来的列。""" from sqlglot import exp if source.kind == "table": return source.key not in self.columns or name_key(column) in self.columns[source.key] - if source.kind == "derived" and isinstance(source.query, exp.Select): - return any(isinstance(s, exp.Star) or (isinstance(s, exp.Column) and s.is_star) - or name_key(s.alias_or_name) == name_key(column) for s in source.query.selects) + if source.kind == "derived" and isinstance(source.query, exp.Query): + return any(called is None or name_key(called) == name_key(column) + for called in self.names(source.query)) return True def locate(self, column: Any, sources: dict[str, _Source]) -> tuple[str | None, _Source | None]: @@ -660,27 +1125,131 @@ class _Statement: hits = [(slot, s) for slot, s in sources.items() if self._exposes(s, column.name)] return hits[0] if len(hits) == 1 else (None, None) - def through(self, source: _Source | None, column: str, depth: int = 0) -> tuple[_Source | None, str]: - """CTE、子查询只是把一张表的列原样透出来(SELECT *,或者直接选出这一列)时,换成那张表的列: - 原流程直接查 orders、盲算先在 CTE 里筛一遍再查,算的是同一列。""" + @staticmethod + def _depth(column: Any, sources: dict[str, _Source]) -> int: + """列是往外第几层那段的来源的(相关子查询引用外层的别名 o.id 是 1);这一段的、认不出的是 0。 + + 比较用的写法里外层的列另带「^」(orders."id^"):相关子查询读的表和外层是同一张时(orders 里找子单、父单), + c.parent_id = o.id 和 c.id = o.parent_id 不带标记都成了 orders.id = orders.parent_id,分不出来。标记按列所在 + 那一段算,从哪一段开始写都一样。""" + qualifier = column.table + if not qualifier: + return 0 + slot = name_key(qualifier) + if slot in sources: + return 0 + outer, depth = getattr(sources, "outer", None), 1 + while outer is not None: + if slot in outer: + return depth + outer, depth = outer.outer, depth + 1 + return 0 + + def _pass(self, source: _Source | None, column: str) -> tuple[_Source, str] | None: + """派生来源只读一个来源、把它的这一列原样透出来(直接选出这一列,或者 SELECT *)时,里面一层的 (来源, 列名); + 不是原样透出(算出来的列、读好几个来源)返回 None。先认写了名字的列:SELECT *, amount * 0.9 AS net 里的 net + 是算出来的,不是里面那张表的 net。""" from sqlglot import exp - if source is None or source.kind != "derived" or depth > 3 or not isinstance(source.query, exp.Select): - return source, column + if source is None or source.kind != "derived" or not isinstance(source.query, exp.Select): + return None inner = self.sources(source.query) if len(inner) != 1: - return source, column + return None only = next(iter(inner.values())) - for item in source.query.selects: - if isinstance(item, exp.Star) or (isinstance(item, exp.Column) and item.is_star): - return self.through(only, column, depth + 1) - target = _unparen(item.this if isinstance(item, exp.Alias) else item) - if name_key(item.alias_or_name) == name_key(column) and isinstance(target, exp.Column): - return self.through(only, target.name, depth + 1) + names = self.names(source.query) + named = [item for item, called in zip(source.query.selects, names) + if called is not None and name_key(called) == name_key(column)] + if named: + target = _unparen(named[0].this if isinstance(named[0], exp.Alias) else named[0]) + return (only, target.name) if isinstance(target, exp.Column) else None + return (only, column) if None in names else None + + def names(self, query: Any) -> list[str | None]: + """派生来源选出来的各列叫什么(见 _output_names),算一次记下来。""" + if id(query) not in self._names_memo: + self._names_memo[id(query)] = _output_names(query) + return self._names_memo[id(query)] + + def _blocks(self, select: Any, column: str) -> str | None: + """见 _blocking,算一次记下来。""" + memo = (id(select), name_key(column)) + if memo not in self._blocks_memo: + self._blocks_memo[memo] = self._blocking(select, column) + return self._blocks_memo[memo] + + @staticmethod + def _blocking(select: Any, column: str) -> str | None: + """在这一段外面按 column 筛和在它里面筛结果不同时,说它做了什么(「取前几行」「窗口函数」);一样返回 None。 + + 取前几行(LIMIT、OFFSET):先取前 10 行再筛已付,和先筛已付再取前 10 行,取到的不是同一批。窗口函数(含 + DISTINCT ON):先算每人的第一单再筛已付(第一单就是已付的人),和先筛已付再算(有已付订单的人)不是一回事; + 只有 column 是每个窗口的分区键时一样(按分区键筛掉的是整个分区,分区里的排名、合计不受影响)。""" + from sqlglot import exp + + if select.args.get("limit") is not None or select.args.get("offset") is not None: + return "limit" + distinct = select.args.get("distinct") + if isinstance(distinct, exp.Distinct) and distinct.args.get("on") is not None: + keys = {name_key(_unparen(e).name) for e in distinct.args["on"].expressions + if isinstance(_unparen(e), exp.Column)} + if name_key(column) not in keys: + return "window" + for node in _own_nodes(select): + if isinstance(node, exp.Window): + keys = {name_key(_unparen(e).name) for e in node.args.get("partition_by") or [] + if isinstance(_unparen(e), exp.Column)} + if name_key(column) not in keys: + return "window" + return None + + def through(self, source: _Source | None, column: str, *, pierce: bool = False, + unseal: bool = False) -> tuple[_Source | None, str]: + """CTE、子查询只是把一张表的列原样透出来(SELECT *,或者直接选出这一列)时,换成那张表的列: + 原流程直接查 orders、盲算先在 CTE 里筛一遍再查,算的是同一列。 + + 这一段取前几行、或者算了窗口函数时不透过去(见 _blocks),照写这个派生来源的列:外面的 t.status = 'paid' + 和写在 t 里面的 status = 'paid' 不是同一个条件,透过去就都成了 orders.status = 'paid',分不出来。pierce 时 + 不管这些,一直透到底:拿来认出两边是不是同一个条件、只是筛的先后不同(见 Part.pierce)。 + + 同一段里别的来源透过去读的也是同一个来源时(opaque,见 _seal)不透过去,pierce 也不透:透过去就分不出是 + 哪一份了。unseal 时连这个也不管,只拿来给人指出最后读的是哪张表的哪一列(时间条件的 detail)。""" + for _ in range(4): + if source is not None and source.opaque and not unseal: + break + found = self._pass(source, column) + if found is None or (not pierce and self._blocks(source.query, found[1])): + break + source, column = found return source, column - def resolve(self, column: Any, sources: dict[str, _Source]) -> tuple[_Source | None, str]: - return self.through(self.locate(column, sources)[1], column.name) + def resolve(self, column: Any, sources: dict[str, _Source], *, pierce: bool = False, + unseal: bool = False) -> tuple[_Source | None, str]: + return self.through(self.locate(column, sources)[1], column.name, pierce=pierce, unseal=unseal) + + def _definition(self, source: _Source | None, column: str) -> list[tuple[Any, _Sources]] | None: + """派生来源一列在里面的定义:[(定义的表达式, 它那一支的来源)],UNION 每支一项(按列的位置)。只有 SELECT * + 透出来的列,定义就是里面那一支的同名列。认不出(不是派生来源、没有这一列、某一支在那个位置是 *)返回 None。""" + from sqlglot import exp + + if source is None or source.kind != "derived" or source.query is None: + return None + branches = _branches(source.query) + if not branches: + return None + names = [None if called is None else name_key(called) for called in self.names(source.query)] + if name_key(column) not in names: + if None not in names: + return None + return [(exp.column(column), self.sources(branch)) for branch in branches] + position = names.index(name_key(column)) + out = [] + for branch in branches: + if position >= len(branch.selects) or _is_star(branch.selects[position]): + return None + item = branch.selects[position] + out.append((item.this if isinstance(item, exp.Alias) else item, self.sources(branch))) + return out def shared(self) -> set[str]: """好几张表都有的列名(name_key):这条语句里出现时挂在不止一张表上(orders.order_id、refunds.order_id), @@ -689,13 +1258,13 @@ class _Statement: if self._shared is None: owners: dict[str, set[str]] = {} - for select in self.root.find_all(exp.Select, bfs=False): + for select in self._alive(exp.Select): sources = self.sources(select) for node in _own_nodes(select): if isinstance(node, exp.Column) and node.name and not node.is_star: source, name = self.resolve(node, sources) if source is not None: - owners.setdefault(name_key(name), set()).add(source.key) + owners.setdefault(name_key(name), set()).add(source.ref) used = [t.key for t in self.tables] self._shared = {name for name, keys in owners.items() if len(keys) > 1 or sum(1 for t in used if name in self.columns.get(t, ())) > 1} @@ -721,55 +1290,77 @@ class _Statement: return default if owner is None else self.sources(self.segments[owner.meta[_SEGMENT]]) def render(self, node: Any, sources: dict[str, _Source], mode: str = "text") -> str: - """五种写法(见 Part):key、anon、loose 比较用,名字按 name_key,key、anon 写真实表名、loose 不写(CTE、子查询 - 只有 key 写编号);text、full 给人看,text 只在好几张表都有这一列时写真实表名,full 一律写(关联条件说的就是 - 两张表各出哪一列,用 full)。 + """七种写法。比较用的五种(_COMPARING),名字按 name_key:key、loose 见 Part,key 写实表的表名,loose 不写, + CTE、子查询都写编号;pierce 同 key,只是一路透过取前几行、算窗口函数的派生来源(见 through),拿来认出同一个 + 条件筛的先后不同;anon 写实表的表名、派生来源一律写「@」(排比较的两边谁在前时先分实表的列和派生来源的列); + shape 写实表的表名、派生来源写它的结构标识(见 _signature:配对用,和名字、编号都无关)。text、full 给人看 + (见 Part),text 只在好几张表都有这一列时写真实表名,full 一律写(关联条件说的就是两张表各出哪一列,用 full)。 node 里套着的子查询逐段改写:每一列按它所在那一段的来源认,子查询里的表去掉别名(列已经写成了真实表名)。 - 子查询里拿两张表的列互相比的(相关子查询),给人看时和关联条件一样写表名。比较用的写法里各自起的名字也去掉 - (见 _drop_names)。 + 子查询里拿两张表的列互相比的(相关子查询),给人看时和关联条件一样写表名;内外读的是同一个来源(orders 里找 + 子单)时写表名分不出谁是谁,照写别名、留着表别名。比较用的写法里外层的列在列名后面带「^」(见 _depth),各自 + 起的名字去掉(见 _drop_names)。 """ from sqlglot import exp copy = node.copy() + if mode in _COMPARING: + copy = copy.transform(_as_date, copy=False) groups: dict[int, tuple[Any, list[Any]]] = {} for col in copy.find_all(exp.Column): if col.name and not col.is_star: owner = self._owner(col) groups.setdefault(id(owner), (owner, []))[1].append(col) + #: 给人看时照写别名的段(id):里面的列和外层的列读的是同一个来源(见下) + keep: set[int] = set() for owner, cols in groups.values(): scope = sources if owner is None else self.sources(self.segments[owner.meta[_SEGMENT]]) - found = [(col, *self.resolve(col, scope)) for col in cols] + found = [(col, *self.resolve(col, scope, pierce=mode == "pierce"), self._depth(col, scope)) + for col in cols] here = mode - if owner is not None and mode == "text" and len({s.key for _, s, _ in found if s is not None}) > 1: + seen = {(s.ref, depth) for _, s, _, depth in found if s is not None} + # 同一个来源既是这一段的、又是外层的(相关子查询读的表和外层是同一张):写表名分不出谁是谁,写别名 + clash = {ref for ref, _ in seen if sum(1 for other, _ in seen if other == ref) > 1} + if owner is not None and mode == "text" and len(seen) > 1: here = "full" - for col, source, name in found: - self._write(col, source, name, here) + for col, source, name, depth in found: + alias = None + if mode not in _COMPARING and source is not None and source.ref in clash: + located = self.locate(col, scope)[1] + alias = col.args["table"].copy() if col.table else \ + exp.to_identifier(located.alias) if located is not None and located.alias else None + keep.add(id(owner)) + self._write(col, source, name, here, depth=depth, alias=alias) for table in copy.find_all(exp.Table): - if isinstance(table.this, exp.Identifier) and self._owner(table) is not None: - table.set("alias", None) - if mode in ("key", "loose", "anon"): + if isinstance(table.this, exp.Identifier) and (owner := self._owner(table)) is not None: + if id(owner) not in keep: + table.set("alias", None) + if mode in _COMPARING: for arg in ("catalog", "db", "this"): ident = table.args.get(arg) if isinstance(ident, exp.Identifier): table.set(arg, exp.to_identifier(name_key(ident.name))) - if mode in ("key", "loose", "anon"): - self._drop_names(copy, numbered=mode == "key") + if mode in _COMPARING: + self._drop_names(copy, mode) return copy.sql(dialect=self.read, comments=False) - @staticmethod - def _drop_names(copy: Any, *, numbered: bool) -> None: + def _drop_names(self, copy: Any, mode: str) -> None: """比较用的写法里抹掉套着的子查询里各自起的名字(就地改):引用 CTE 的表、FROM 里的子查询(含 LATERAL)、 - 子查询里自带的 CTE 定义写编号(numbered 为假时写「@」;引用它们的列已经照这样写了);只看取值的子查询(IN、 - EXISTS、标量子查询)去掉选出来的列名。FROM 里的子查询、CTE 定义选出来的列名留着:外层按这个名字引用它们, - 去掉了就分不出外层用的是哪一列。""" + 子查询里自带的 CTE 定义写编号(anon 写「@」,shape 里引用 CTE 的表写它的结构标识、别的写「@」;引用它们的列 + 已经照这样写了);只看取值的子查询(IN、EXISTS、标量子查询)去掉选出来的列名。FROM 里的子查询、CTE 定义选出来 + 的列名留着:外层按这个名字引用它们,去掉了就分不出外层用的是哪一列。复制出来的那段本身没有父节点,认不出是 + FROM 里的子查询,它选出来的列名也会去掉。""" from sqlglot import exp for node in copy.find_all(exp.Table, exp.Subquery, exp.Lateral, exp.CTE): derived = node.meta.get(_DERIVED) if not derived: continue - mark = exp.to_identifier(derived if numbered else _ANON) + if mode == "shape": + # 引用 CTE 的表写它是哪个(结构标识);FROM 里的子查询、CTE 定义本身整段写在这里,不用再标 + mark = exp.to_identifier(self._signature_of(derived) if isinstance(node, exp.Table) else _ANON) + else: + mark = exp.to_identifier(_ANON if mode == "anon" else derived) if isinstance(node, exp.Table): for arg in ("catalog", "db", "alias"): node.set(arg, None) @@ -782,22 +1373,39 @@ class _Statement: if not _named_output(select): select.set("expressions", [e.this if isinstance(e, exp.Alias) else e for e in select.expressions]) - def _write(self, col: Any, source: _Source | None, name: str, mode: str) -> None: - """按 render 的写法改写一列(就地改)。""" + def _write(self, col: Any, source: _Source | None, name: str, mode: str, *, depth: int = 0, + alias: Any = None) -> None: + """按 render 的写法改写一列(就地改)。depth 是它属于往外第几层的来源(见 _depth):比较用的写法里另带 + 「^」。alias 是给人看时照写的别名(内外层读同一个来源时,见 render):这时列名也照原样写,不写透过去的。""" from sqlglot import exp quoted = isinstance(col.this, exp.Identifier) and bool(col.this.args.get("quoted")) - if mode in ("key", "loose", "anon"): - owner = None if source is None or mode == "loose" else \ - _ANON if mode == "anon" and source.kind == "derived" else source.key - col.set("this", exp.to_identifier(name_key(name))) + if mode in _COMPARING: + if source is None: + owner = None + elif source.kind == "derived": + owner = _ANON if mode == "anon" else source.ref if mode != "shape" else \ + self._signature_of(source.key) + (f"#{source.role}" if source.role else "") + else: + owner = None if mode == "loose" else source.ref + # 外层的列在列名后面带标记(不带在表名上:表名带了标记要加引号,排比较的两边时外层的列就都排到前面去了, + # 只该在两边表名一样时才影响先后) + column = self.column_key(source, name) if source is not None else name_key(name) + col.set("this", exp.to_identifier(column + "^" * depth)) col.set("table", exp.to_identifier(owner) if owner else None) col.set("db", None) col.set("catalog", None) return + if alias is not None: + col.set("table", alias) + col.set("db", None) + col.set("catalog", None) + return col.set("this", exp.Identifier(this=name, quoted=quoted or not _PLAIN_NAME.match(name))) - if source is None or (source.kind != "other" and mode != "full" - and (self.single or name_key(name) not in self.shared())): + # 自己关联自己的(有 role)一律写别名:只写列名读不出是哪一个。不透过去的派生来源(opaque)不算「整条语句只用到 + # 一张实表」:两个 CTE 都读 orders,好几个都有的列(shared)得写出是哪一个的 + if source is None or (source.kind != "other" and mode != "full" and not source.role + and ((self.single and not source.opaque) or name_key(name) not in self.shared())): col.set("table", None) col.set("db", None) col.set("catalog", None) @@ -807,19 +1415,22 @@ class _Statement: col.set("db", heads[-1].copy() if heads else None) col.set("catalog", heads[-2].copy() if len(heads) > 1 else None) - def canonical(self, node: Any, sources: dict[str, _Source]) -> Any: - """比较的两边排好顺序(列在左、字面量在右;两边都有列的按 _rank 排),IN 的取值排好序。返回新的树。 - 套着的子查询里的比较按它那一段的来源排:拿外层的来源去认子查询的别名,排出来的先后就取决于别名怎么起。""" + def canonical(self, node: Any, sources: dict[str, _Source], *, by: str = "key") -> Any: + """比较的两边排好顺序(列在左、字面量在右;两边都有列的按 _rank 排),IN 的取值排好序;加法、乘法的两项也按 + 这个规矩排(amount * 0.9 和 0.9 * amount 是同一个算法),只排两项都不是别的加减乘除的(见 _atomic)。返回新的 + 树。套着的子查询里的比较按它那一段的来源排:拿外层的来源去认子查询的别名,排出来的先后就取决于别名怎么起。""" from sqlglot import exp def fix(n: Any) -> Any: found = _COMPARE.get(type(n).__name__) - if found is not None and isinstance(n, exp.Binary): + swap = (type(n).__name__, type(n).__name__) if isinstance(n, (exp.Add, exp.Mul)) and \ + _atomic(n.this) and _atomic(n.expression) else found + if swap is not None and isinstance(n, exp.Binary): left, right = n.this, n.expression lc, rc = bool(_own_columns(left)), bool(_own_columns(right)) here = self.scope(n, sources) if lc and rc else sources - if (rc and not lc) or (lc and rc and self._rank(left, here) > self._rank(right, here)): - return getattr(exp, found[1])(this=right, expression=left) + if (rc and not lc) or (lc and rc and self._rank(left, here, by) > self._rank(right, here, by)): + return getattr(exp, swap[1])(this=right, expression=left) elif isinstance(n, exp.In) and n.expressions and not n.args.get("query") \ and not any(_own_columns(e) for e in n.expressions): n.set("expressions", sorted(n.expressions, key=lambda e: e.sql(dialect=self.read))) @@ -827,24 +1438,108 @@ class _Statement: return node.transform(fix) - def _rank(self, node: Any, sources: dict[str, _Source]) -> tuple[str, str]: - """比较的两边谁排前面:先按不写编号的写法,分不出(同名的列)再按编号。只按编号排的话,编号错开的两条语句里 - 同一个比较会排成两个方向,按不写编号的写法也配不上了。""" - return self.render(node, sources, "anon"), self.render(node, sources, "key") + def _rank(self, node: Any, sources: dict[str, _Source], by: str = "key") -> tuple[str, str]: + """比较的两边谁排前面:先按不写编号的写法(实表的列、派生来源的列分得开),分不出(两个派生来源的同名列) + 再按 by 的写法分。 + + by 是 key 时按位置编号排;算结构标识时 by 是 shape,按结构标识排:位置编号决定不了先后。""" + return self.render(node, sources, "anon"), self.render(node, sources, by) def part(self, node: Any, sources: dict[str, _Source], *, excludes: bool = False) -> Part: - refs: dict[tuple[str | None, str], str] = {} + #: 用到的各列 {(哪一个来源, 列): ((给人看的表名, 列), 比较用的写法)}:按「哪一个来源」分,自己关联自己的两个别名 + #: 是两列(o1.amount、o2.amount) + refs: dict[tuple[str, str], tuple[ColumnRef, str]] = {} + origins: list[tuple[str, str]] = [] + roles: list[tuple[str, str, str]] = [] + stage = "" for col in _own_columns(node): - source, name = self.resolve(col, sources) - owner = "" if source is None else _ANON if source.kind == "derived" else source.key - refs.setdefault((source.name if source is not None else None, name), f"{owner}.{name_key(name)}") + located = self.locate(col, sources)[1] + # 外层的列另带标记(见 _depth):相关子查询里 c.amount 和外层的 o.amount 是两列 + mark = "^" * self._depth(col, sources) + source, name = self.through(located, col.name) + refs.setdefault((source.ref if source is not None else "", name_key(name) + mark), + ((source.name if source is not None else None, name), + f"{source.ref}.{self.column_key(source, name)}{mark}" if source is not None + else f".{name_key(name)}{mark}")) + if source is not None and source.role and source.idents: + if (note := (source.idents[-1].name, source.role, source.name)) not in roles: + roles.append(note) + if source is None or source.kind != "derived": + continue + # 透视在这个派生来源上停住了,一路透到底却还能往里走:是它取前几行、算窗口函数挡住的 + if not stage and self.through(located, col.name, pierce=True)[0] is not source: + stage = self._stage(source, name) + if (origin := self._origin(source, name)) is not None: + if (pair := (self.render(col, sources, "full"), origin)) not in origins: + origins.append(pair) full = self.render(node, sources, "full") + key = self.render(node, sources, "key") + pierce = self.render(node, sources, "pierce") if stage else "" + if pierce == key: + pierce = stage = "" + columns = tuple(ref for ref, _ in refs.values()) # 拿两张表的列互相比的(相关子查询里的 o.id = r.order_id)和关联条件一样,给人看也写表名 - across = len({table for table, _ in refs}) > 1 - return Part(key=self.render(node, sources, "key"), text=full if across else self.render(node, sources), - columns=tuple(refs), excludes=self._excludes(node, sources) if excludes else None, - loose=self.render(node, sources, "loose"), full=full, column_keys=tuple(refs.values()), - anon=self.render(node, sources, "anon")) + across = len({table for table, _ in columns}) > 1 + return Part(key=key, text=full if across else self.render(node, sources), + columns=columns, excludes=self._excludes(node, sources) if excludes else None, + loose=self.render(node, sources, "loose"), full=full, + column_keys=tuple(written for _, written in refs.values()), + origins=tuple(origins), pierce=pierce, stage=stage, roles=tuple(roles)) + + def _stage(self, source: _Source, column: str) -> str: + """挡住了透视的派生来源做了什么(给人看,见 _blocks):「取 t 的前 10 行」「算 t 的窗口函数」。""" + from sqlglot import exp + + query = source.query + if self._blocks(query, column) == "limit": + limit = query.args.get("limit") + count = limit.expression if isinstance(limit, exp.Limit) else limit.args.get("count") \ + if limit is not None else None + written = count.sql(dialect=self.read, comments=False) if count is not None else "" + return _say("取", source.name, "的前", written, "行") if written.isdigit() else \ + _say("按", "LIMIT / OFFSET", "取", source.name, "的行") + return _say("算", source.name, "的窗口函数") + + def _origin(self, source: _Source, column: str, depth: int = 0) -> str | None: + """派生来源一列的来历(给人看):它在里面怎么算的,里面再用到的派生来源的列也换成各自的来历,一直换到实表的 + 列,一律写表名(SUM(orders.amount))。两边同名的派生来源其实不是同一个时(盲算的 t1 是原流程的 t2),光写 + 「t1.total」两边一样,带上来历才说得清各自是付款合计还是退款合计。UNION 只写第一支;认不出定义、套得太深 + 返回 None。""" + if source.kind != "derived" or source.query is None or depth > 4: + return None + memo = (id(source.query), name_key(column)) + if memo not in self._origins: + self._origins[memo] = None # 递归 CTE 绕回自己时到这里为止 + found = self._definition(source, column) + self._origins[memo] = self._expand(*found[0], depth + 1) if found else None + return self._origins[memo] + + def _expand(self, node: Any, sources: dict[str, _Source], depth: int) -> str: + """把 node 写成给人看的 full 写法,里面用到的派生来源的列换成各自的来历(见 _origin)。换进去的是一段 + 写好的文字:先用占位符占住位置,整段写好之后再换回来;放在运算符里的加括号,免得优先级变了。""" + from sqlglot import exp + + bare = _unparen(node) + if isinstance(bare, exp.Column): + source, name = self.resolve(bare, sources) + inner = self._origin(source, name, depth) if source is not None else None + return inner if inner is not None else self.render(bare, sources, "full") + copy = node.copy() + subs: dict[str, str] = {} + for col in _own_columns(copy): + source, name = self.resolve(col, sources) + inner = self._origin(source, name, depth) if source is not None else None + if inner is None: + continue + mark = exp.Placeholder(this=f"sqlsem{len(subs)}") + wrap = " " in inner and isinstance(col.parent, (exp.Binary, exp.Unary)) and \ + not isinstance(col.parent, exp.Paren) + subs[mark.sql(dialect=self.read)] = f"({inner})" if wrap else inner + col.replace(mark) + text = self.render(copy, sources, "full") + for mark, inner in subs.items(): + text = text.replace(mark, inner) + return text def _excludes(self, node: Any, sources: dict[str, _Source]) -> str | None: """排除写法去掉否定之后排除的是什么:a <> 1 → a = 1;NOT IN、NOT LIKE、NOT BETWEEN 同理;IS NOT NULL → 「a 为空」。 @@ -868,43 +1563,264 @@ class _Statement: # ---- 各个方面 - def facets(self) -> Facets: + def facets(self, extra: Iterable[str] = ()) -> Facets: + """各个方面。extra 是没用到、也照样收进来的 CTE(name_key),别的没用到的 CTE 不收(见 _enliven)。 + 条件、分组键、连接、聚合、HAVING 都记下在哪一层(见 _layers),按(层, key)去重。""" from sqlglot import exp + self._enliven(extra) + tags = self._layers(extra) joins: list[JoinFacet] = [] where: list[Part] = [] time: list[TimePredicate] = [] group: list[Part] = [] + distinct: list[str] = [] aggregates: list[Aggregate] = [] having: list[Part] = [] - for select in self.root.find_all(exp.Select, bfs=False): + used: list[Part] = [] + for select in self._alive(exp.Select): sources = self.sources(select) + # 没走到的段(条件、SELECT 列表里的子查询,以及只在它们里面用到的派生来源)在「?」那一层 + layer = tags.get(id(select), "?") + if select is self.root or not self._in_condition(select): + used += [Part.plain(s.ref, s.name) for s in sources.values()] drafts = self._join_drafts(select, sources) clause = select.args.get("where") for cond in _conjuncts(clause.this) if clause is not None else []: canon = self.canonical(cond, sources) if self._joins_on_where(canon, sources, drafts): continue - where.append(self.part(canon, sources, excludes=True)) + part = replace(self.part(canon, sources, excludes=True), scope=layer) + where.append(part) + self._rejects(canon, part, sources, drafts) if found := self._time(canon, sources): - time.append(found) - joins += [self._join(d, sources) for d in drafts] - group += self._group(select, sources) - aggregates += self._aggregates(select, sources) - having += self._having(select, sources) + time.append(replace(found, condition=replace(found.condition, scope=layer))) + joins += [replace(self._join(d, sources), scope=layer) for d in drafts] + keys = self._group(select, sources) + if not keys and (keys := self._distinct_keys(select, sources)): + distinct.append(layer) + group += [replace(p, scope=layer) for p in keys] + aggregates += [replace(g, scope=layer) for g in self._aggregates(select, sources)] + having += [replace(p, scope=layer) for p in self._having(select, sources)] + derived = [Part.plain(self._labels[number], name or "子查询") + for number, name, body, _ in self._derived if id(body) not in self._dead] return Facets( dialect=self.read, tables=tuple(Part.plain(s.key, s.name) for s in self.tables), - joins=_dedupe(joins, lambda j: (j.kind, j.table.key, j.on.key if j.on else None)), - where=_dedupe(where, lambda p: p.key), - time=_dedupe(time, lambda t: t.condition.key), - group_by=_dedupe(group, lambda p: p.key), - aggregates=_dedupe(aggregates, lambda a: (a.func, a.arg.key, a.distinct)), - having=_dedupe(having, lambda p: p.key), + joins=_dedupe(joins, lambda j: (j.scope, j.kind, j.table.key, j.on.key if j.on else None)), + where=_dedupe(where, lambda p: (p.scope, p.key)), + time=_dedupe(time, lambda t: (t.condition.scope, t.condition.key)), + group_by=_dedupe(group, lambda p: (p.scope, p.key)), + aggregates=_dedupe(aggregates, lambda a: (a.scope, a.func, a.arg.key, a.distinct)), + having=_dedupe(having, lambda p: (p.scope, p.key)), limit=self._limit(), limits=tuple(self._limits()), + sources=_dedupe(used, lambda p: p.key), + expressions=_dedupe(self._expressions(), lambda c: (c.scope_key, c.name, c.expr.key)), + derived=_dedupe(derived, lambda p: p.key), + set_ops=tuple(self._set_ops(tags)), + distinct=tuple(dict.fromkeys(distinct)), + grain=self._grain(self.root), + scopes=self._scope_parts(tags), ) + # ---- 层 + + def _layers(self, extra: Iterable[str] = ()) -> dict[int, str]: + """每段 SELECT 在哪一层:{段的 id: 层的标记}(见 Part.scope)。 + + 最外层是空串,UNION 的第 k 支(k ≥ 2)在后面加「|k」。FROM / JOIN 里的派生来源(CTE、子查询、LATERAL 子查询) + 是它自己的位置编号;只从一个来源里筛出一部分行的(见 _transparent) + 和用到它的那一段算同一层:写在这种 CTE 里的条件和写在外层是一回事。读同一个来源的几个派生来源同在一段时 + (见 _seal)各算各的层:两个 CTE 里各有一个 is_deleted = 0 是两个条件。派生来源在哪一层按在 FROM / JOIN 里 + 第一次用到它的地方定,从外往里一层一层地走。没走到的段(条件、SELECT 列表里的子查询,以及只在它们里面用到 + 的派生来源)由 facets 记成「?」。extra 是没用到、也照样收进来的 CTE,在它自己的编号那一层。""" + from sqlglot import exp + + tags: dict[int, str] = {} + queue: deque[tuple[Any, str]] = deque() + + def push(query: Any, tag: str) -> None: + for i, branch in enumerate(_branches(query)): + queue.append((branch, tag if i == 0 else f"{tag}|{i + 1}")) + + def drain() -> None: + while queue: + select, tag = queue.popleft() + if id(select) in tags: + continue + tags[id(select)] = tag + sources = self.sources(select) + for i, item in enumerate(self._items(select)): + body, label = _derived_body(item), item.meta.get(_DERIVED) + if body is None and isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) \ + and item.name and self._is_cte(item): + body = self.ctes[name_key(item.name)] + if body is None or not label: + continue + alias = _alias(item) + source = sources.get(name_key(alias) if alias else f"#{i}") + lifted = source is not None and not source.opaque and self._transparent(item, body) + push(body, tag if lifted else label) + + push(self.root, "") + drain() + for name in extra: + if name in self._cte_defs and (label := self._cte_defs[name].meta.get(_DERIVED)): + push(self.ctes[name], label) + drain() + return tags + + def _transparent(self, item: Any, body: Any) -> bool: + """派生来源只是从一个来源里筛出一部分行(不分组、不聚合、不去重、不是 UNION,也不是 LATERAL):写在它里面的 + 条件和写在用到它的那一段是一回事,算同一层(见 _layers)。各列怎么算不影响有哪些行;取前几行、算窗口函数的 + 也算同一层,筛的先后另由 Part.pierce / stage 说。""" + from sqlglot import exp + + if isinstance(item, exp.Lateral) or not isinstance(body, exp.Select) or len(self._items(body)) != 1: + return False + if any(body.args.get(k) is not None for k in ("group", "having", "distinct")): + return False + return not self._aggregating(body) + + @staticmethod + def _aggregating(select: Any) -> bool: + """这一段自己算了聚合(窗口函数、套着的子查询里的不算)。""" + from sqlglot import exp + + return any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) for n in _own_nodes(select)) + + def _scope_parts(self, tags: Mapping[int, str]) -> tuple[Part, ...]: + """各层给人看的说法(Part.plain(层的标记, 说法)):最外层「外层」,派生来源写它的名字,UNION 的各支写「t 的 + UNION 第 2 支」,条件里的子查询「条件的子查询」。""" + names = {self._labels[number]: name or "子查询" for number, name, _, _ in self._derived} + unions = {self._labels[number] for number, _, body, _ in self._derived if len(_branches(body)) > 1} + root_union = len(_branches(self.root)) > 1 + out: dict[str, Part] = {} + for tag in dict.fromkeys([*tags.values(), "?"]): + head, _, branch = tag.partition("|") + if head == "?": + text = "条件的子查询" + elif head == "": + text = f"UNION 第 {branch or 1} 支" if root_union else "外层" + else: + name = names.get(head, "子查询") + text = _say(name, "的 UNION 第", branch or "1", "支") if head in unions else name + out[tag] = Part.plain(tag, text) + return tuple(out.values()) + + def _set_ops(self, tags: Mapping[int, str]) -> list[SetOpFacet]: + """各段的 UNION、INTERSECT、EXCEPT(见 SetOpFacet),条件里的子查询里的不算。""" + from sqlglot import exp + + out = [] + for node in self._alive(exp.SetOperation): + parent = node.parent + while isinstance(parent, exp.Subquery): + parent = parent.parent + branches = _branches(node) + if isinstance(parent, exp.SetOperation) or not branches: + continue # 只看一串合并最上面的那一个 + tag = tags.get(id(branches[0]), "?") + if tag != "?": + out.append(SetOpFacet(ops=tuple(self._ops(node)), branches=len(branches), + scope=self._scope_name(node), scope_key=tag)) + return out + + @classmethod + def _ops(cls, node: Any) -> list[str]: + """一串合并从左到右各处的写法:UNION、UNION ALL、INTERSECT、EXCEPT ALL……""" + from sqlglot import exp + + while isinstance(node, exp.Subquery): + node = node.this + if not isinstance(node, exp.SetOperation): + return [] + word = {"Union": "UNION", "Intersect": "INTERSECT", "Except": "EXCEPT"}.get(type(node).__name__, + type(node).__name__.upper()) + if not node.args.get("distinct"): + word += " ALL" + return cls._ops(node.this) + [word] + cls._ops(node.expression) + + def _distinct_keys(self, select: Any, sources: dict[str, _Source]) -> list[Part]: + """SELECT DISTINCT(不带 ON)选出来的各项。没有 GROUP BY、没有聚合时,对这几项去重和按它们分组出的行一模一样, + 当这一段的分组键收(见 Facets.distinct):SELECT DISTINCT user_id 和 GROUP BY user_id 不算不同,一边在子查询里 + 先去重、另一边没有照报。DISTINCT ON 是每组取一行,按窗口函数算(见 _blocking)。""" + from sqlglot import exp + + distinct = select.args.get("distinct") + if not isinstance(distinct, exp.Distinct) or distinct.args.get("on") is not None \ + or select.args.get("group") is not None or self._aggregating(select): + return [] + out = [] + for item in select.selects: + if _is_star(item): + out.append(Part.plain("*", "*")) + continue + target = item.this if isinstance(item, exp.Alias) else item + out.append(self.part(self.canonical(target, sources), sources)) + return out + + def _grain(self, query: Any, depth: int = 0) -> Part | None: + """最后出来的行按什么分:分组(GROUP BY、当分组用的 DISTINCT)是「group:各分组键」,没分组却算了聚合是 + 「one」(只出一行),都没有、只从一个派生来源取数的看那个派生来源,别的是「rows」(逐行)。最外层是 UNION + 的不算(返回 None)。分组键挪了一层(先在子查询里按 user_id 分组、外层数用户数,对外层按 user_id 分组)按 + 整条语句比分组键看不出来:外层出一行还是每个用户一行,看它。""" + from sqlglot import exp + + while isinstance(query, exp.Subquery): + query = query.this + if not isinstance(query, exp.Select) or depth > 6: + return None + sources = self.sources(query) + keys = self._group(query, sources) + if not keys and self._aggregating(query): + return Part.plain("one", "") + keys = keys or self._distinct_keys(query, sources) + if keys: + return Part(key="group:" + ", ".join(sorted({p.key for p in keys})), + text="、".join(dict.fromkeys(p.text for p in keys)), + loose="group:" + ", ".join(sorted({p.loose for p in keys})), + full="、".join(dict.fromkeys(p.full for p in keys))) + items = self._items(query) + if len(items) == 1 and not isinstance(items[0], exp.Lateral): + item, body = items[0], _derived_body(items[0]) + if body is None and isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier) \ + and item.name and self._is_cte(item): + body = self.ctes[name_key(item.name)] + if body is not None: + return self._grain(body, depth + 1) + return Part.plain("rows", "") + + def _rejects(self, cond: Any, part: Part, sources: dict[str, _Source], drafts: list[dict[str, Any]]) -> None: + """同一段 WHERE 里的条件把外连接可空那一侧没关联上的行筛掉了的,记在那次连接上(见 JoinFacet.rejects):LEFT + JOIN 关联进来的那一侧、RIGHT JOIN 之前的各个来源可空,那一侧的列直接拿来比较、IN、LIKE、BETWEEN,或者写 IS NOT + NULL,空值都不成立。IS NULL、OR、套在 COALESCE / IFNULL / CASE 里的不认(宁可漏报)。""" + from sqlglot import exp + + n = _unparen(cond) + if isinstance(n, exp.Not): + n = _unparen(n.this) + if isinstance(n, exp.Is): + if not isinstance(_unparen(n.expression), exp.Null): + return + n = n.this # IS NOT NULL + elif type(n).__name__ not in _NULL_REJECTING: + return + elif type(n).__name__ not in _NULL_REJECTING: + return + hit = set() + for col in _own_columns(n): + if col is not n and any(isinstance(p, (exp.Coalesce, exp.Case, exp.If)) for p in _parents(col, stop=n)): + continue + if (slot := self.locate(col, sources)[0]) is not None: + hit.add(slot) + for draft in drafts: + nullable = {draft["slot"]} if draft["kind"] == "left" else \ + set(draft["before_slots"]) if draft["kind"] == "right" else set() + if draft.get("rejects") is None and hit & nullable: + draft["rejects"] = part + def _join_drafts(self, select: Any, sources: dict[str, _Source]) -> list[dict[str, Any]]: from sqlglot import exp @@ -922,8 +1838,10 @@ class _Statement: conds = [self.canonical(exp.EQ(this=exp.column(ident.name, table=first_alias), expression=exp.column(ident.name, table=alias)), sources) for ident in join.args["using"]] + before = [(key, s) for key, s in list(sources.items())[:i + 1] if key != slot] drafts.append({"kind": _join_kind(join), "slot": slot, "source": sources.get(slot), "conds": conds, - "before": [s.name for key, s in list(sources.items())[:i + 1] if key != slot]}) + "before": [s.name for _, s in before], "before_keys": [s.ref for _, s in before], + "before_slots": [key for key, _ in before]}) return drafts def _joins_on_where(self, cond: Any, sources: dict[str, _Source], drafts: list[dict[str, Any]]) -> bool: @@ -950,17 +1868,33 @@ class _Statement: def _join(self, draft: dict[str, Any], sources: dict[str, _Source]) -> JoinFacet: source = draft["source"] table = Part.plain("?", "?") if source is None else \ - Part(key=source.key, text=source.name, loose=source.key, full=source.name, - anon=_ANON if source.kind == "derived" else source.key) + Part(key=source.ref, text=source.name, loose=source.ref, full=source.name) forms = sorted((self.render(c, sources, "key"), self.render(c, sources, "loose"), - self.render(c, sources, "full"), self.render(c, sources, "anon")) for c in draft["conds"]) + self.render(c, sources, "full")) for c in draft["conds"]) on = Part(key=" AND ".join(f[0] for f in forms), text=" AND ".join(f[2] for f in forms), - loose=" AND ".join(sorted(f[1] for f in forms)), full=" AND ".join(f[2] for f in forms), - anon=" AND ".join(sorted(f[3] for f in forms))) if forms else None + loose=" AND ".join(sorted(f[1] for f in forms)), full=" AND ".join(f[2] for f in forms)) \ + if forms else None mentioned = {table for c in draft["conds"] for table, _ in self.part(c, sources).columns} other = [name for name in dict.fromkeys(t for t in mentioned if t) if name != table.text] if draft["conds"] \ else draft["before"] - return JoinFacet(kind=draft["kind"], table=table, on=on, other=tuple(sorted(other))) + # 涉及的来源按条件里直接引用的算(slot 不为 None 的是这一段的来源):透过 CTE 找到的实表两边可能一样, + # 关联的却是两个不同的 CTE + sides = {source.ref} if source is not None else set() + if draft["conds"]: + sides |= {found.ref for c in draft["conds"] for col in _own_columns(c) + for slot, found in [self.locate(col, sources)] if slot is not None and found is not None} + else: + sides |= set(draft["before_keys"]) + # 外连接的主表、可空那一侧:LEFT JOIN 关联进来的那一侧可空,RIGHT JOIN 反过来;另一侧按关联条件里出现的来源 + # 算(没有就是它之前的各个来源),和内连接的 sides 一样不受前面别的连接先后的影响 + keep: set[str] = set() + nullable: set[str] = set() + if draft["kind"] in ("left", "right") and source is not None: + near, far = {source.ref}, (sides - {source.ref}) or set(draft["before_keys"]) + keep, nullable = (far, near) if draft["kind"] == "left" else (near, far) + return JoinFacet(kind=draft["kind"], table=table, on=on, other=tuple(sorted(other)), + sides=tuple(sorted(sides)), keep=tuple(sorted(keep)), nullable=tuple(sorted(nullable)), + rejects=draft.get("rejects")) def _time(self, cond: Any, sources: dict[str, _Source]) -> TimePredicate | None: """时间条件:一边恰好一列(可以套着日期函数、CAST),另一边没有列;列名像日期,或者用了日期函数、像日期的字符串。""" @@ -983,13 +1917,42 @@ class _Statement: cols = [column] if column is not None else _own_columns(subject) if len(cols) != 1 or any(isinstance(x, (exp.AggFunc, exp.Select)) for x in subject.walk()): return None - if not (_temporal_name(cols[0].name) or _datey(subject) or any(_datey(v) for v in values)): + base, scope, via, datey = self._time_column(cols[0], sources) + if not (_temporal_name(base.name) or _temporal_name(cols[0].name) or datey or _datey(subject) + or any(_datey(v) for v in values)): return None shown = [self.render(v, sources) for v in values] value = f"{shown[0]} AND {shown[1]}" if op == "between" else f"({', '.join(shown)})" if op == "in" \ else shown[0] - return TimePredicate(column=self.part(cols[0], sources), op=op, value=value, - condition=self.part(cond, sources, excludes=True)) + # 给调用方对照数据目录的那一列一直换到实表:不透过去的派生来源(见 _seal)比较时照写它自己,可它的这一列 + # 读的还是那张表的那一列 + real, real_scope, _, _ = self._time_column(cols[0], sources, unseal=True) + found = self.resolve(real, real_scope, pierce=True, unseal=True) + table = found[0].name if found[0] is not None and found[0].kind == "table" else None + return TimePredicate(column=self.part(base, scope), op=op, value=value, + condition=self.part(cond, sources, excludes=True), via=via, + table=(table, found[1]) if table else None) + + def _time_column(self, column: Any, sources: dict[str, _Source], *, + unseal: bool = False) -> tuple[Any, _Sources, tuple[str, ...], bool]: + """时间条件里的那一列是 CTE、子查询算出来的、在里面是同一列套着日期函数、CAST、MIN / MAX 时(见 _value_column), + 换成里面那一列,一直换到实表:(那一列, 它那一段的来源, 路过的派生来源的列「编号.列」, 路上有没有日期函数)。 + 原样透出的列 resolve 已经透过去了;这里补上算了一下的。不透过去的派生来源(见 _seal)到它为止:两个 CTE 都读 + orders 时,按已付那份的 created_at 筛还是按未付那份的筛,正是要分清的;unseal 时不管这个(见 through)。""" + via: list[str] = [] + datey = False + for _ in range(4): + source, name = self.resolve(column, sources, unseal=unseal) + if source is not None and source.opaque and not unseal: + break + found = self._definition(source, name) if source is not None else None + inner = _value_column(found[0][0]) if found and len(found) == 1 else None + if inner is None: + break + via.append(f"{source.key}.{self.column_key(source, name)}") + datey = datey or _datey(found[0][0]) + column, sources = inner, found[0][1] + return column, sources, tuple(via), datey @staticmethod def _aliases(query: Any) -> dict[str, Any]: @@ -1049,27 +2012,157 @@ class _Statement: def _aggregates(self, select: Any, sources: dict[str, _Source]) -> list[Aggregate]: from sqlglot import exp - out = [] - for node in _own_nodes(select): - if not isinstance(node, exp.AggFunc) or isinstance(node.parent, exp.Window): + return [self._aggregate(node, sources) for node in _own_nodes(select) + if isinstance(node, exp.AggFunc) and not isinstance(node.parent, exp.Window)] + + def _aggregate(self, node: Any, sources: dict[str, _Source]) -> Aggregate: + from sqlglot import exp + + try: + func = str(node.sql_name()).lower() + except Exception: # noqa: BLE001 - 没有固定函数名的聚合:按类型名 + func = node.key + arg = node.this + distinct = isinstance(arg, exp.Distinct) + if distinct: + items = list(arg.expressions) + arg = items[0] if len(items) == 1 else exp.Tuple(expressions=[i.copy() for i in items]) + if arg is None or isinstance(arg, exp.Star) or \ + (func == "count" and not distinct and isinstance(_unparen(arg), exp.Literal)): + part = Part.plain("*", "*") # COUNT(*)、COUNT(1):数的是行数 + else: + part = self.part(self.canonical(arg, sources), sources) + return Aggregate(func=func, arg=part, distinct=distinct, text=self.render(node, sources)) + + # ---- 算出来的列 + + def _expressions(self) -> list[Computed]: + """算出来的列(见 Computed):最外层(UNION 的各支)选出来、不是光一列也不是字面量的;用到了的派生来源里 + 被外面用到、又不是原样透出一列的列(原样透出的,外面的写法 resolve 已经透成了里面那一列,不用再比定义)。""" + from sqlglot import exp + + out: list[Computed] = [] + for select in _branches(self.root): + sources = self.sources(select) + for item in select.selects: + target = _unwrap_value(item.this if isinstance(item, exp.Alias) else item) + if not self._plain_output(target): + out.append(self._computed([(target, sources)], None, None, "", "")) + used = self._used() + for number, name, body, _ in self._derived: + label = self._labels[number] + if id(body) in self._dead or label not in used: continue - try: - func = str(node.sql_name()).lower() - except Exception: # noqa: BLE001 - 没有固定函数名的聚合:按类型名 - func = node.key - arg = node.this - distinct = isinstance(arg, exp.Distinct) - if distinct: - items = list(arg.expressions) - arg = items[0] if len(items) == 1 else exp.Tuple(expressions=[i.copy() for i in items]) - if arg is None or isinstance(arg, exp.Star) or \ - (func == "count" and not distinct and isinstance(_unparen(arg), exp.Literal)): - part = Part.plain("*", "*") # COUNT(*)、COUNT(1):数的是行数 - else: - part = self.part(self.canonical(arg, sources), sources) - out.append(Aggregate(func=func, arg=part, distinct=distinct, text=self.render(node, sources))) + source = _Source("derived", name or "子查询", label, (), body) + columns = [called for called in self.names(body) if called] + columns += sorted(c for c in used[label] if c not in {name_key(x) for x in columns}) + for column in dict.fromkeys(columns): + if name_key(column) not in used[label] or self.through(source, column)[0] is not source: + continue + if found := self._definition(source, column): + out.append(self._computed(found, source.name, label, self.column_key(source, column), column)) return out + @staticmethod + def _plain_output(target: Any) -> bool: + """最外层选出来的一项不用按算法比:光一列(分组那一面比)、*、字面量。光一个聚合照收:另一边可能是拿同一个 + 聚合再算了一下(SUM(amount) 对 SUM(amount) / 100),得配成一对说;两边只差聚合本身的,聚合那一面已经报了。""" + from sqlglot import exp + + return isinstance(_unparen(target), (exp.Column, exp.Star, exp.Literal, exp.Null, exp.Boolean)) + + def _computed(self, branches: list[tuple[Any, _Sources]], scope: str | None, scope_key: str | None, name: str, + shown: str) -> Computed: + """一列的定义(UNION 每支一个)→ Computed:定义本身、聚合换成占位符之后的写法、里面的聚合。""" + from sqlglot import exp + + exprs: list[Part] = [] + templates: list[Part] = [] + aggregates: list[Aggregate] = [] + for node, sources in branches: + canon = self.canonical(node, sources) + exprs.append(self.part(canon, sources)) + # 这一段自己的聚合(窗口函数、套着的子查询里的不算),按出现的先后 + own = [n for n in canon.walk(prune=lambda x: isinstance(x, (exp.Select, exp.Subquery))) + if isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) + and not any(isinstance(p, exp.AggFunc) for p in _parents(n, stop=None))] + aggregates += [self._aggregate(n, sources) for n in own] + if own and own[0] is canon: + templates.append(Part.plain(PLACEHOLDER, PLACEHOLDER)) # 光一个聚合 + continue + for n in own: + n.replace(exp.Placeholder()) + templates.append(Part(key=self.render(canon, sources, "key"), text="", + loose=self.render(canon, sources, "loose"))) + + def joined(parts: list[Part]) -> Part: + if len(parts) == 1: + return parts[0] + return Part(key=" | ".join(p.key for p in parts), text=" | ".join(p.text for p in parts), + loose=" | ".join(p.loose for p in parts), full=" | ".join(p.full for p in parts), + columns=tuple(c for p in parts for c in p.columns), + column_keys=tuple(k for p in parts for k in p.column_keys), + origins=tuple(o for p in parts for o in p.origins)) + + return Computed(scope=scope, scope_key=scope_key, name=name, shown=shown, expr=joined(exprs), + template=joined(templates), aggregates=tuple(aggregates)) + + def _used(self) -> dict[str, set[str]]: + """用到了的段里引用了派生来源的哪些列:{派生来源的编号: {列名的 name_key}}。原样透出的列连同里面那一层也算 + 用到了;最外层的 SELECT *(t.*)用到了那个来源的每一列;USING 的列两边都算。""" + from sqlglot import exp + + used: dict[str, set[str]] = {} + + def mark(source: _Source | None, column: str) -> None: + for _ in range(8): + if source is None or source.kind != "derived": + return + used.setdefault(source.key, set()).add(name_key(column)) + if (found := self._pass(source, column)) is None: + return + source, column = found + + def mark_all(source: _Source | None, depth: int = 0) -> None: + if source is None or source.kind != "derived" or depth > 4: + return + for branch in _branches(source.query)[:1]: + inner = self.sources(branch) + for item, called in zip(branch.selects, self.names(source.query)): + if called is None: + for found in self._starred(item, inner): + mark_all(found, depth + 1) + else: + mark(source, called) + + outermost = {id(s) for s in _branches(self.root)} + for select in self._alive(exp.Select): + sources = self.sources(select) + for node in _own_nodes(select): + if isinstance(node, exp.Column) and node.name and not node.is_star: + mark(self.locate(node, sources)[1], node.name) + if id(select) in outermost: + for item in select.selects: + if _is_star(item): + for found in self._starred(item, sources): + mark_all(found) + for join in select.args.get("joins") or []: + for ident in join.args.get("using") or []: + for found in sources.values(): + if self._exposes(found, ident.name): + mark(found, ident.name) + return used + + @staticmethod + def _starred(item: Any, sources: dict[str, _Source]) -> list[_Source]: + """SELECT 列表里的 *、t.* 选的是哪几个来源。""" + from sqlglot import exp + + if isinstance(item, exp.Column) and item.table: + found = sources.get(name_key(item.table)) + return [found] if found is not None else [] + return list(sources.values()) + def _limit(self) -> str | None: from sqlglot import exp @@ -1085,7 +2178,7 @@ class _Statement: from sqlglot import exp out = [] - for query in self.root.find_all(exp.Select, exp.SetOperation, bfs=False): + for query in self._alive(exp.Select, exp.SetOperation): if query is not self.root and self._in_condition(query): continue if (found := self._limit_of(query)) is not None: @@ -1119,7 +2212,21 @@ class _Statement: skip = None # OFFSET 0 等于没写 if limit is None and skip is None: return None - return LimitFacet(count=count, offset=skip, order=self._order(query), scope=self._scope_name(query)) + return LimitFacet(count=count, offset=skip, order=self._order(query), scope=self._scope_name(query), + scope_key=self._scope_key(query)) + + @staticmethod + def _scope_key(query: Any) -> str | None: + """query 整段是哪个派生来源(CTE 的定义、FROM 里的子查询、LATERAL 子查询)的编号;最外层、UNION 的某一支、 + SELECT 列表里的子查询这类不是派生来源整段的返回 None。编号记在 CTE 定义、FROM / JOIN 项上,往外找一两层。""" + from sqlglot import exp + + node = query + while isinstance(node.parent, (exp.Subquery, exp.Lateral, exp.CTE)): + node = node.parent + if found := node.meta.get(_DERIVED): + return found + return None def _order(self, query: Any) -> tuple[tuple[Part, bool], ...]: """这一段的排序键 (Part, 是否降序)。ORDER BY 1、ORDER BY 别名换成 SELECT 里的表达式(UNION 按第一支的)。""" @@ -1145,22 +2252,2306 @@ class _Statement: if query is self.root: return None parent = query.parent - if isinstance(parent, (exp.CTE, exp.Subquery)) and parent.alias: + if isinstance(parent, exp.Subquery) and not parent.alias and isinstance(parent.parent, exp.Lateral): + parent = parent.parent # LATERAL 的别名挂在 LATERAL 上,不在里面的子查询上 + if isinstance(parent, (exp.CTE, exp.Subquery, exp.Lateral)) and parent.alias: return parent.alias return "子查询" # ========================================================================== -# 比较 +# 等价判定:只做保持语义的变换,比规范形式 +# ========================================================================== + +#: 规范形式的几级(见 _canonical_forms)。等价的结论只在同一级的两个规范形式一字不差时才下 +_OPTIMIZER, _LIGHT, _PARSED = "optimizer", "light", "parsed" +#: 同一段里同名的几个来源(自己关联自己)谁先谁后逐个排列着试,取写出来最小的那个;排列数超过这个就只按原来的 +#: 先后(认得出的等价改写少一些,不会错判) +_MAX_ORDERINGS = 64 +#: 一条语句里一共试多少种排列(里层的子查询在外层每试一种时都要重排一遍):用完了就都只按原来的先后 +_ORDERING_BUDGET = 2000 +#: 原来的名字(CTE 名、FROM 里子查询的别名)记在里面那段查询的 meta 里:起了规范的名字之后,差异说明还要照原样说 +_ORIGINAL = "sql_semantics_name" +#: 写的是 LEFT / RIGHT JOIN、WHERE 把没关联上的行筛掉了、按内连接比的来源(记在那张表上,值是原来写的连接):差异 +#: 说明里照实说「写的是 LEFT JOIN,实际是 INNER JOIN」 +_REJECTED = "sql_semantics_rejected" +#: 从 RIGHT JOIN 改写成的 LEFT JOIN(记在 JOIN 上):差异说明里不说「用 LEFT JOIN」,说以哪一侧为主表 +_FLIPPED = "sql_semantics_flipped" +#: GROUP BY 是从 SELECT DISTINCT 改写来的:差异说明里说「去重」 +_FROM_DISTINCT = "sql_semantics_distinct" +#: 认不准是哪一列的列(记在列名的标识符上):SQLite 里不带表名、带双引号、表结构里又没有的名字(找不到这一列时被当成 +#: 字符串);条件、标量子查询里不带表名的列(没给表结构时 qualify 归给子查询里的表,可它也可能是外层的列)。差异说明 +#: 不拿它说具体的 +_UNSURE = "sql_semantics_unsure" +#: 不确定的函数:每次求值结果都可能不同,并子查询、合并定义一样的 CTE 会改变求值的次数,不走优化器那一级 +_VOLATILE = frozenset({"random", "rand", "randn", "newid", "uuid", "gen_random_uuid", "uuid_generate_v4", "random_uuid", + "randomblob", "setseed", "nextval", "sys_guid", "changes", "last_insert_rowid", "total_changes"}) +#: 结果看行出来的先后的聚合:拼成串、拼成数组、取任意一个、取第一个 / 最后一个(类名和函数名都按小写去下划线比) +_ORDERED_AGGS = frozenset({"groupconcat", "stringagg", "arrayagg", "arrayconcatagg", "arrayuniqueagg", "jsonarrayagg", + "jsonobjectagg", "jsonagg", "jsonbagg", "jsongrouparray", "jsongroupobject", "listagg", + "xmlagg", "anyvalue", "first", "last", "firstvalue", "lastvalue", "nthvalue", + "jsonbobjectagg", "jsonobjectaggstrict", "arrayaggdistinct", "bitstringagg"}) +#: 浮点数求和类的聚合:浮点数加法不满足结合律,加的先后不同结果的最后几位就不同(1902.2 和 1902.2000000000003) +_FLOAT_AGGS = frozenset({"sum", "avg", "stddev", "stddevpop", "stddevsamp", "variance", "variancepop", "varpop", + "varsamp", "covarpop", "covarsamp", "corr", "total"}) +#: 拿空值做比较、算术都得空值(外连接没关联上的那一侧整行是空值) +_STRICT_UNARY = ("Neg", "Cast", "Lower", "Upper", "Length", "Abs") +_STRICT_BINARY = ("Add", "Sub", "Mul", "Div", "Mod") +#: 解析时只留下值、丢了写法的字面量:SQLite 里 0x10 是整数 16、x'10' 是 BLOB,解析出来是同一个节点;N'…'、E'…'、 +#: $$…$$ 这些也一样,带了就不在语法树上判等价 +_LOSSY_LITERALS = frozenset({"HEX_STRING", "BIT_STRING", "BYTE_STRING", "NATIONAL_STRING", "UNICODE_STRING", + "HEREDOC_STRING", "RAW_STRING"}) +#: 类型名的同义写法:解析后重新生成换了个名字、在这个方言里却是同一个类型(SQLite 按名字里的字样定类型亲和:INT 和 +#: INTEGER 都是整数亲和,VARCHAR 和 TEXT 都是文本亲和)。不在这里的换了名字就算解析丢了写法(SQLite 的 STRING 是数值 +#: 亲和、TEXT 是文本亲和,sqlglot 解析成同一个类型) +_TYPE_SYNONYMS = { + "sqlite": {"INT": "INTEGER", "VARCHAR": "TEXT", "CHAR": "TEXT", "NVARCHAR": "TEXT", "CLOB": "TEXT", + "DOUBLE": "REAL", "FLOAT": "REAL"}, + "postgres": {"INTEGER": "INT", "INT4": "INT", "INT8": "BIGINT", "INT2": "SMALLINT", "FLOAT8": "DOUBLE", + "FLOAT4": "REAL", "BOOL": "BOOLEAN", "NUMERIC": "DECIMAL", "CHARACTER": "CHAR"}, + "mysql": {"INTEGER": "INT", "DEC": "DECIMAL", "NUMERIC": "DECIMAL", "BOOL": "BOOLEAN"}, +} +#: FROM 列表到这些词就结束了(之后的逗号不再是连接) +_CLAUSE_ENDS = frozenset({"WHERE", "GROUP_BY", "HAVING", "ORDER_BY", "LIMIT", "OFFSET", "FETCH", "UNION", "INTERSECT", + "EXCEPT", "WINDOW", "QUALIFY", "SEMICOLON", "RETURNING", "FOR", "CONNECT_BY", "START_WITH", + "CLUSTER_BY", "DISTRIBUTE_BY", "SORT_BY", "PREWHERE", "LOCK"}) +_ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") +_FLIP = {"GT": "LT", "LT": "GT", "GTE": "LTE", "LTE": "GTE", "EQ": "EQ", "NEQ": "NEQ"} +_NEGATE = {"EQ": "NEQ", "NEQ": "EQ", "GT": "LTE", "LTE": "GT", "GTE": "LT", "LT": "GTE"} + + +class _Unsupported(Exception): + """这一级规范化处理不了这条语句的结构(递归 CTE、LATERAL、表函数……),这一级不出规范形式。""" + + +#: 一元加号前面可能是这些词(不是一个值的结尾) +_OPERAND_KEYWORDS = frozenset({"SELECT", "WHERE", "AND", "OR", "NOT", "ON", "WHEN", "THEN", "ELSE", "HAVING", "BY", "IN", + "IS", "LIKE", "GLOB", "BETWEEN", "CASE", "AS", "DISTINCT", "ALL", "SET", "VALUES", + "LIMIT", "OFFSET", "RETURN", "ESCAPE", "FILTER", "OVER", "PARTITION", "USING", "IF", + "EXISTS", "ANY", "SOME", "FROM", "JOIN", "WITH", "ORDER", "GROUP"}) + + +def _type_words(tokens: list[Any], read: str | None) -> list[str]: + from sqlglot.parser import Parser + + synonyms = _TYPE_SYNONYMS.get(read or "", {}) + words = [t.text.upper() for t in tokens if t.token_type in Parser.TYPE_TOKENS] + return sorted(synonyms.get(w, w) for w in words) + + +def _lost_in_parsing(text: str, read: str | None, tree: Any) -> bool: + """原文里有 sqlglot 解析时丢掉、语义却不一样的写法,有就不在语法树上判等价: + + - 一元加号:SQLite 里 +a 去掉了列的类型亲和,+a = '10' 和 a = '10' 结果不同,解析出来却是同一棵树; + - 只留下值、丢了写法的字面量(0x10 和 x'10',见 _LOSSY_LITERALS); + - 类型名:解析后重新生成换了名字、又不是这个方言里的同义写法(SQLite 的 CAST(x AS STRING) 是数值亲和,解析成 + 和 CAST(x AS TEXT) 一样的节点)。""" + try: + tokens = _tokens(text, read) + except Exception: # noqa: BLE001 - 切词都不行的,解析也过不了,那边会说 + return False + prev = None + for token in tokens: + kind = token.token_type.name + if kind in _LOSSY_LITERALS: + return True + if kind == "PLUS": + if prev is None: + return True + pk, word = prev.token_type.name, prev.text + operand = pk in ("VAR", "IDENTIFIER", "NUMBER", "R_PAREN", "R_BRACKET", "NULL", "TRUE", "FALSE", "END", + "STAR") or pk.endswith("STRING") or ( + _PLAIN_NAME.match(word) is not None and word.upper() not in _OPERAND_KEYWORDS) + if not operand: + return True + prev = token + try: + again = _tokens(tree.sql(dialect=read, comments=False), read) + except Exception: # noqa: BLE001 - 重新生成不了:当成丢了写法 + return True + return _type_words(tokens, read) != _type_words(again, read) + + +def _mark_comma_joins(tree: Any, text: str, read: str | None) -> None: + """SQLite 里逗号连接和 CROSS JOIN 解析成同一个节点(都是 kind=CROSS),可 SQLite 不调换 CROSS JOIN 两边的先后、 + 逗号连接会调换:结果看行出来的先后时(GROUP_CONCAT、有并列的 LIMIT)两种写法不一样。按原文认出哪些是逗号连接, + 去掉它们的 CROSS(和别的方言解析出来的一样)。认不出时整条语句的 CROSS 连接标成认不出(见 _fingerprint)。""" + from sqlglot import exp + + joins = [j for j in tree.find_all(exp.Join) if str(j.args.get("kind") or "").upper() == "CROSS"] + if not joins: + return + try: + tokens = _tokens(text, read) + except Exception: # noqa: BLE001 + tokens = [] + marks: list[tuple[int, bool]] = [] + depth = 0 + in_from: set[int] = set() + for tok in tokens: + name = tok.token_type.name + if name == "L_PAREN": + depth += 1 + elif name == "R_PAREN": + in_from.discard(depth) + depth -= 1 + elif name == "FROM": + in_from.add(depth) + elif name in _CLAUSE_ENDS: + in_from.discard(depth) + elif name == "COMMA" and depth in in_from: + marks.append((tok.start, True)) + elif name in ("JOIN", "STRAIGHT_JOIN"): + marks.append((tok.start, False)) + for join in joins: + starts = [i.meta["start"] for i in join.this.find_all(exp.Identifier) if isinstance(i.meta.get("start"), int)] + before = [m for m in marks if starts and m[0] < min(starts)] + if not before: + join.meta["sql_semantics_join"] = f"unknown:{id(tree)}" + continue + comma = max(before)[1] + if comma: + join.set("kind", None) + + +@dataclass +class _Form: + """一条语句在某一级的规范形式。key 一样才算等价,text 是生成的 SQL(给人看);pre 是起规范名字之前的树。""" + + level: str + text: str + tree: Any + pre: Any = None + qualified: bool = False + + @property + def key(self) -> str: + """比等价用的写法:语法树逐个节点的类型和参数(见 _fingerprint)。不拿生成的 SQL 比:sqlglot 按方言生成时 + 会把不同的写法写成同一个样子(SQLite 的 CAST(x AS DATE) 生成 DATE(x),两者在 SQLite 里结果不同)。""" + return _fingerprint(self.tree) + + +def _fingerprint(node: Any) -> str: + """语法树的确定写法:每个节点写类名和全部参数(按参数名排序,字符串原样),不同的树一定写得不一样。括号不算 + (语法树的结构已经定了怎么结合:SQLite 里 (a) 和 a 的类型亲和也一样);类型标注、注释、meta 不算,只有认不出 + 是逗号还是 CROSS JOIN 的连接带上标记(见 _mark_comma_joins)。""" + from sqlglot import exp + + if isinstance(node, exp.Paren): + return _fingerprint(node.this) + if isinstance(node, exp.Expr): + parts = [f"{k}={_fingerprint(v)}" for k, v in sorted(node.args.items()) + if v is not None and v is not False and v != []] + mark = node.meta.get("sql_semantics_join") if isinstance(node, exp.Join) else None + if mark: + parts.append(f"@join={mark}") + return f"{type(node).__name__}({','.join(parts)})" + if isinstance(node, list): + return "[" + ",".join(_fingerprint(v) for v in node) + "]" + return repr(node) + + +def _key(node: Any, read: str | None = None) -> tuple[str, str]: + """排序、去重用的键:只看语法树(见 _fingerprint),括号写没写不影响先后。""" + return _fingerprint(node), "" + + +@dataclass +class _Schema: + """数据源的表结构:mapping 给 qualify(只要列名),types 记各列声明的类型(判断「相等的值是不是写法也一样」,见 + _Sensitivity),columns 是所有表的列名(小写)。""" + + mapping: dict[str, dict[str, str]] | None = None + types: dict[str, dict[str, str]] = field(default_factory=dict) + columns: set[str] = field(default_factory=set) + + +def _schema_info(schema: Mapping[str, Any] | None) -> _Schema: + """schema_cache → _Schema。qualify 用的类型一律记 UNKNOWN:只要列名(认列属于哪张表、展开 SELECT *);按类型做 + 的改写和方言有关,不做。""" + tables = schema.get("tables") if isinstance(schema, Mapping) else None + if not isinstance(tables, Mapping): + return _Schema() + mapping: dict[str, dict[str, str]] = {} + types: dict[str, dict[str, str]] = {} + for name, meta in tables.items(): + columns = meta.get("columns") if isinstance(meta, Mapping) else None + cols = [c for c in columns or [] if isinstance(c, Mapping) and isinstance(c.get("name"), str) and c["name"]] + if cols: + mapping[str(name)] = {c["name"]: "UNKNOWN" for c in cols} + types[str(name).translate(_ASCII_LOWER)] = { + c["name"].translate(_ASCII_LOWER): str(c.get("type") or "") for c in cols} + return _Schema(mapping or None, types, {c for cols in types.values() for c in cols}) + + +def _canonical_forms(root: Any, read: str | None, info: _Schema) -> dict[str, _Form]: + """一条语句的几级规范形式: + + - optimizer:认列(sqlglot 的 qualify:认清每一列属于哪个来源,给了表结构就展开 SELECT *)之后的全套改写(见 + _optimized、_finish)。认不全列、有和列别名同名又可能是表的列的名字、不确定的函数、递归 CTE、LATERAL 之类的 + 不出这一级。 + - light:不认列,只做不需要认列的改写(见 _light)。 + - parsed:解析后只统一名字的大小写、引号(见 _normalize_names),总是有。 + + 每一级的每一步都只做保持语义的变换,某一级出不来就没有这一级,从不抛异常。""" + out: dict[str, _Form] = {} + for level, build in ((_OPTIMIZER, _optimized), (_LIGHT, _light)): + try: + out[level] = build(root, read, info) + except _Unsupported as e: + logger.debug("SQL 规范化(%s)跳过:%s", level, e) + except Exception as e: # noqa: BLE001 - sqlglot 认不了列、写法不支持:这一级不出,别的级照样比 + from sqlglot.errors import SqlglotError + + if isinstance(e, SqlglotError): + logger.debug("SQL 规范化(%s)跳过:%s", level, e) + else: + logger.warning("SQL 规范化(%s)失败", level, exc_info=True) + try: + tree = root.copy() + _normalize_names(tree, read, info) + out[_PARSED] = _Form(_PARSED, tree.sql(dialect=read, comments=False), tree) + except Exception: # noqa: BLE001 + logger.warning("SQL 重新生成失败", exc_info=True) + return out + + +def _optimized(root: Any, read: str | None, info: _Schema) -> _Form: + """认列之后的规范形式:统一名字 → 列名表写进定义 → USING 写成 ON → FETCH 写成 LIMIT → qualify 认列(给了表结构 + 先按表结构,表结构里没有的列再不给表结构认一遍)→ 来源别名改成整条语句里互不相同的 → 只引用一次的 CTE 写回用到 + 它的地方 → 只筛行、透列的子查询并进外层 → 去掉没用的 CTE → 两级共用的几步(_finish)。""" + from sqlglot.errors import OptimizeError + from sqlglot.optimizer.qualify import qualify + + tree = root.copy() + why = _unsupported(tree, read) or ("不确定的函数" if _volatile(tree) else None) + if why: + raise _Unsupported(why) + _normalize_names(tree, read, info) + _remember_names(tree) + _mark_unsure(tree) + _push_column_lists(tree) + _expand_using(tree) + _qualify_stars(tree) + _fetch_to_limit(tree) + _normalize_negation(tree) + qualified = None + for mapping in ([info.mapping, None] if info.mapping else [None]): + if _alias_or_column(tree, read, mapping): + continue + try: + qualified = qualify(tree.copy(), dialect=read, schema=mapping, validate_qualify_columns=True, + quote_identifiers=False, identify=False, expand_stars=True, expand_alias_refs=True) + break + except OptimizeError as e: + logger.debug("SQL 认列失败:%s", e) + if qualified is None: + raise _Unsupported("认不全列") + tree = qualified + _split_simple_ctes(tree, read, info) + _inline_ctes(tree) + # 写回的 CTE 定义各是一份副本,里面的别名一样:同一个 CTE 写回外层和相关子查询两处,并进去之后子查询里本来指外层 + # 的列就被里层同名的别名遮住了。写回之后再把整条语句的别名改成互不相同的 + _unique_aliases(tree) + _merge_derived(tree, read, info) + _push_into_groups(tree, read, info) + _drop_unused_ctes(tree) + return _finish(tree, read, info, level=_OPTIMIZER, qualified=True) + + +def _light(root: Any, read: str | None, info: _Schema) -> _Form: + """不认列的规范形式:统一名字、列名表写进定义、USING 写成 ON、FETCH 写成 LIMIT、只引用一次的 CTE 写回用到它的 + 地方,再走 _finish 里不需要认列的几步。""" + tree = root.copy() + why = _unsupported(tree, read) + if why: + raise _Unsupported(why) + _normalize_names(tree, read, info) + _remember_names(tree) + _mark_unsure(tree) + _push_column_lists(tree) + _expand_using(tree) + _qualify_stars(tree) + _fetch_to_limit(tree) + _normalize_negation(tree) + _inline_ctes(tree) + _drop_unused_ctes(tree) + return _finish(tree, read, info, level=_LIGHT, qualified=False) + + +def _finish(tree: Any, read: str | None, info: _Schema, *, level: str, qualified: bool) -> _Form: + """两级共用的几步,都只做保持语义的变换(为什么保持语义见各函数的说明): + + 1. 动来源的:两个来源的 RIGHT JOIN 写成 LEFT JOIN;WHERE 把外连接可空一侧没关联上的行筛掉了的写成内连接;开头那组 + 内连接的条件挪进 WHERE、写成逗号连接。结果看行出来的先后的一段(见 _Sensitivity)都不做。 + 0. 原样选出 FROM 里一个子查询全部列、什么别的都不做的一段换成那个子查询(见 _unwrap_passthrough)。 + 1.5 去重的:SELECT DISTINCT 写成按选出来的列 GROUP BY、已经按选出来的列分组的 DISTINCT 去掉、IN 子查询里的 + DISTINCT 和 ORDER BY 去掉(见各函数:只在「相等」就是「一样」、不看行出来的先后时做)。 + 2. 和来源无关的:COUNT(1) 写成 COUNT(*)、MIN / MAX 里的 DISTINCT 去掉;EXISTS 子查询写成 SELECT 1;IN 列表去重 + 排序、只有一个常量的写成 =;BETWEEN 写成两个比较;NOT 套比较写成反过来的比较、NOT NOT 去掉;不影响结合的括号、 + ORDER BY 的 ASC、OFFSET 0 去掉;最外层 ORDER BY 引用的别名换成表达式,最外层和条件里的子查询选出来的列名去掉。 + 3. 起规范的名字、排好交换律成立的地方(见 _named)。 + """ + tree = _unwrap_passthrough(tree, read) + sens = _Sensitivity(tree, read, info) + _right_to_left(tree, sens) + sens = _Sensitivity(tree, read, info) + _reject_nulls(tree, read) + _flatten_prefix(tree, sens, read) + _in_subquery_distinct(tree, sens) + _distinct_to_group(tree, sens) + _count_star(tree) + _minmax_distinct(tree) + _case_forms(tree, read) + _drop_true_conjuncts(tree) + _exists_subqueries(tree) + _in_lists(tree) + _expand_between(tree) + _fold_not(tree) + _strip_parens(tree) + _strip_asc(tree) + _zero_offset(tree) + _order_by_aliases(tree) + _drop_output_aliases(tree) + pre = tree.copy() + sens = _Sensitivity(tree, read, info) + tree = _named(tree, read, qualified, sens) + return _Form(level, tree.sql(dialect=read, comments=False), tree, pre, qualified) + + +def _named(tree: Any, read: str | None, qualified: bool, sens: "_Sensitivity", hint: "_Hint | None" = None) -> Any: + """起规范的名字、排好交换律成立的地方,最后统一引号(只给需要的标识符加)。""" + from sqlglot.optimizer.qualify_columns import quote_identifiers + + _Namer(tree, read, qualified, sens, hint).canon(tree, 0, {}) + return quote_identifiers(tree, dialect=read, identify=False) + + +# ---- 守门:哪些结构不走规范化的两级 + +def _volatile(tree: Any) -> bool: + from sqlglot import exp + + for node in tree.walk(): + if isinstance(node, (exp.Rand, exp.Randn)) or type(node).__name__.lower() in _VOLATILE: + return True + if isinstance(node, exp.Anonymous) and str(node.name).lower() in _VOLATILE: + return True + return False + + +def _unsupported(tree: Any, read: str | None) -> str | None: + """规范化的两级不处理的结构:递归 CTE(写了 RECURSIVE,或者 CTE 引用自己、引用后面的 CTE——SQLite 里不写 + RECURSIVE 也是递归)、LATERAL、UNNEST、PIVOT、表函数、VALUES、SEMI / ANTI / ASOF / NATURAL 这些连接、实表别名 + 带列名(PostgreSQL 的 orders AS o(a, b) 给列改了名)、BETWEEN SYMMETRIC、FETCH … WITH TIES / PERCENT、ROLLUP / + CUBE / GROUPING SETS、SELECT INTO、加锁。""" + from sqlglot import exp + + for node in tree.walk(): + if isinstance(node, exp.With): + if node.args.get("recursive"): + return "递归 CTE" + names = [c.alias_or_name for c in node.expressions] + for i, cte in enumerate(node.expressions): + later = set(names[i:]) + if any(t.name in later and not t.args.get("db") for t in cte.this.find_all(exp.Table)): + return "CTE 引用自己或后面的 CTE" + if isinstance(node, (exp.Lateral, exp.Unnest, exp.Pivot, exp.TableSample, exp.Values, exp.Into, exp.Lock)): + return type(node).__name__ + if isinstance(node, exp.Table): + if not isinstance(node.this, exp.Identifier): + return "表函数" + alias = node.args.get("alias") + if isinstance(alias, exp.TableAlias) and alias.columns: + return "实表别名带列名" + if isinstance(node, exp.Join) and (node.args.get("method") or str(node.kind or "").upper() in + ("SEMI", "ANTI", "ASOF", "POSITIONAL", "STRAIGHT_JOIN", "ARRAY")): + return "连接方式" + if isinstance(node, exp.Between) and node.args.get("symmetric"): + return "BETWEEN SYMMETRIC" + if isinstance(node, exp.Fetch): + options = node.args.get("limit_options") + if options is not None and (options.args.get("percent") or options.args.get("with_ties")): + return "FETCH WITH TIES / PERCENT" + if isinstance(node, exp.Group) and any(node.args.get(k) for k in ("rollup", "cube", "grouping_sets", "totals")): + return "ROLLUP / CUBE / GROUPING SETS" + return None + + +# ---- 名字 + +def _identifier_role(ident: Any) -> str: + """一个标识符在语法树里是什么:column(列名)/ column_alias(列的别名、CTE 和子查询的列名表)/ table(表名、 + 列前面的表名或别名)/ table_alias(表、子查询、CTE 的别名)/ other。""" + from sqlglot import exp + + parent, key = ident.parent, ident.arg_key + if isinstance(parent, exp.Column): + return "column" if key == "this" else "table" + if isinstance(parent, exp.Table): + return "table" + if isinstance(parent, exp.TableAlias): + return "column_alias" if key == "columns" else "table_alias" + if isinstance(parent, exp.Alias): + return "column_alias" + return "other" + + +def _normalize_names(tree: Any, read: str | None, info: _Schema) -> None: + """按方言的规则统一名字的大小写和引号,只做库里认成同一个名字的改写;处理过的标识符标成 case_sensitive,后面 + qualify 不再按 sqlglot 的规则改(sqlglot 认为 SQLite 带引号的名字也不分大小写,可 SQLite 找不到这一列时把双引号 + 里的当成字符串,'Paid' 和 'paid' 就不同了)。只认 ASCII 字母的大小写(库里也只折 ASCII)。 + + - SQLite:不带引号的名字、带引号的表名和别名不分大小写,统一成小写;条件、SELECT 列表里不带表名、带引号的列名 + ("Paid")原样留着,除非表结构里确实有这一列(有这一列就不会被当成字符串)。 + - PostgreSQL(和认不出的方言):不带引号的名字折成小写;带引号的里面没有大写字母就和不带引号的是同一个名字。 + - MySQL:列名、列的别名不分大小写,统一成小写;表名、表的别名在 Linux 上分大小写,原样。 + """ + from sqlglot import exp + + for ident in list(tree.find_all(exp.Identifier)): + name = ident.this + if not isinstance(name, str): + continue + role = _identifier_role(ident) + quoted = bool(ident.args.get("quoted")) + lower = name.translate(_ASCII_LOWER) + if read == "sqlite": + keep = quoted and role == "column" and not ident.parent.args.get("table") and lower not in info.columns + if keep: + ident.meta[_UNSURE] = True + else: + name, quoted = lower, False + elif read == "mysql": + if role in ("column", "column_alias"): + name = lower + quoted = False + elif not quoted or lower == name: + name, quoted = lower, False + if quoted or not _PLAIN_NAME.match(name): + quoted = True + ident.set("this", name) + ident.set("quoted", quoted) + ident.meta["case_sensitive"] = True + + +def _mark_unsure(tree: Any) -> None: + """条件里的子查询(IN、EXISTS、标量)里不带表名的列标成认不准(见 _UNSURE):它可能是子查询里的表的列,也可能是 + 外层的列,表结构里没有就分不清。只影响差异说明说不说具体的,不影响规范形式。""" + from sqlglot import exp + + for col in tree.find_all(exp.Column): + if col.table or not isinstance(col.this, exp.Identifier): + continue + node = col.parent + while node is not None: + if isinstance(node, exp.Select) and (_scalar_context(node) or _condition_query(node)): + col.this.meta[_UNSURE] = True + break + node = node.parent + + +def _remember_names(tree: Any) -> None: + """CTE、FROM 里的子查询原来的名字记在里面那段查询上(见 _ORIGINAL)。""" + from sqlglot import exp + + for cte in tree.find_all(exp.CTE): + cte.this.meta.setdefault(_ORIGINAL, cte.alias_or_name) + for sub in tree.find_all(exp.Subquery): + if isinstance(sub.parent, (exp.From, exp.Join)) and sub.alias: + sub.this.meta.setdefault(_ORIGINAL, sub.alias) + + +def _push_column_lists(tree: Any) -> None: + """CTE、FROM 里的子查询自己写了列名表的(WITH p(uid, total) AS (…)、(…) AS t(x, y)),列名写成里面那段(UNION 是 + 第一支)选出来的列的别名,列名表去掉:外面按这些名字引用,两种写法是一回事。里面那段选的是 *(不知道有几列)、 + 列数对不上、本段 ORDER BY / GROUP BY / HAVING 还按原来的别名引用的,没法改写,不走规范化的两级。""" + from sqlglot import exp + + for holder in list(tree.find_all(exp.CTE, exp.Subquery)): + alias = holder.args.get("alias") + if not isinstance(alias, exp.TableAlias) or not alias.columns: + continue + body = holder.this + target = _leaves(body)[0] if isinstance(body, exp.SetOperation) else body + names = [c.name for c in alias.columns] + if not isinstance(target, exp.Select) or _starry(target) or len(target.expressions) != len(names): + raise _Unsupported("列名表对不上选出来的列") + own = {p.alias for p in target.expressions if isinstance(p, exp.Alias) and p.alias} + clauses = [target.args.get(k) for k in ("order", "group", "having")] + if isinstance(body, exp.SetOperation): + if body.args.get("order") is not None: + raise _Unsupported("合并的结果按名字排序") + for clause in clauses: + if clause is not None and any(not c.table and c.name in own for c in clause.find_all(exp.Column)): + raise _Unsupported("列名表改名之后本段的引用对不上") + target.set("expressions", [exp.alias_(p.unalias().copy() if isinstance(p, exp.Alias) else p.copy(), n, + copy=False) for p, n in zip(target.expressions, names)]) + alias.set("columns", None) + + +def _expand_using(tree: Any) -> None: + """JOIN … USING (c) 写成 ON 左表.c = 右表.c,这一段里不带表名的 c 写成左表.c(内连接、LEFT JOIN 时合并出来的 c + 就是左表的 c)。只改得了「FROM 只有一个来源、第一个连接就是 USING、SELECT 里没有 *」的写法:别的(左边好几个 + 来源时 c 是哪一张的、SELECT * 列的先后、RIGHT / FULL JOIN 合并出来的值、子查询里引用 c)没法照原样改写,不走 + 规范化的两级。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + if not any(j.args.get("using") for j in joins): + continue + if len(joins) < 1 or not joins[0].args.get("using") or any(j.args.get("using") for j in joins[1:]): + raise _Unsupported("USING") + join = joins[0] + if str(join.side or "").upper() not in ("", "LEFT") or join.args.get("on") is not None or \ + any(_is_star(p) for p in select.expressions): + raise _Unsupported("USING") + left, right = select.args["from_"].this, join.this + la, ra = left.alias_or_name, right.alias_or_name + if not la or not ra: + raise _Unsupported("USING") + names = [u.name for u in join.args["using"]] + for sub in _nested_queries(select): + if any(not c.table and c.name in names for c in sub.find_all(exp.Column)): + raise _Unsupported("子查询里引用 USING 的列") + for node in list(_block_nodes(select)): + if isinstance(node, exp.Column) and not node.table and node.name in names: + node.set("table", exp.to_identifier(la)) + conds = [exp.EQ(this=exp.column(n, table=la), expression=exp.column(n, table=ra)) for n in names] + join.set("using", None) + join.set("on", _and_all(conds)) + + +def _qualify_stars(tree: Any) -> None: + """几个来源的 SELECT * 写成按 FROM 的先后逐个来源的 t.*:两种写法是一回事(没有 USING / NATURAL 时 * 就是各来源的 + 全部列依次排开)。sqlglot 的 qualify 展开 * 时把实表的列排在 FROM 里的子查询前面,和库里的先后不一样,先写成 t.* + 它就按写的先后展开。有来源没有名字的不动(这时也不走规范化的两级)。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + if not any(isinstance(p, exp.Star) for p in select.expressions): + continue + items = _items_of(select) + if len(items) < 2 or any(not i.alias_or_name for i in items): + continue + stars = [exp.Column(this=exp.Star(), table=exp.to_identifier(i.alias_or_name)) for i in items] + new: list[Any] = [] + for p in select.expressions: + new += [c.copy() for c in stars] if isinstance(p, exp.Star) else [p] + select.set("expressions", new) + + +def _fetch_to_limit(tree: Any) -> None: + """FETCH FIRST n ROWS ONLY(不带 WITH TIES / PERCENT)就是 LIMIT n(sqlglot 把 FETCH 放在 limit 这个参数上)。""" + from sqlglot import exp + + for query in list(tree.find_all(exp.Select, exp.SetOperation)): + for key in ("limit", "fetch"): + fetch = query.args.get(key) + if not isinstance(fetch, exp.Fetch): + continue + count = fetch.args.get("count") or exp.Literal.number(1) + query.set(key, None) + query.set("limit", exp.Limit(expression=count.copy())) + + +# ---- 认列属于哪个来源 + +class _Resolver: + """按别名认清每一列属于哪个来源(相关子查询里的列到外层找):refs[id(来源)] 是引用它的列,cte_of[id(表)] 是它 + 引用的 CTE,owner[id(来源)] 是来源所在的那段 SELECT。不带表名的列不认(轻量一级)。""" + + def __init__(self, tree: Any) -> None: + self.refs: dict[int, list[Any]] = {} + self.cte_of: dict[int, Any] = {} + self.owner: dict[int, Any] = {} + self.source_of: dict[int, Any] = {} + self._walk(tree, [], {}) + + def _walk(self, query: Any, frames: list[dict[str, Any]], ctes: dict[str, Any]) -> None: + from sqlglot import exp + + if isinstance(query, exp.Subquery): + self._walk(query.this, frames, ctes) + return + with_ = query.args.get("with_") + if with_ is not None: + ctes = dict(ctes) + for cte in with_.expressions: + self._walk(cte.this, frames, ctes) + ctes[cte.alias_or_name] = cte + if isinstance(query, exp.SetOperation): + self._walk(query.left, frames, ctes) + self._walk(query.right, frames, ctes) + return + if not isinstance(query, exp.Select): + raise _Unsupported(type(query).__name__) + frame: dict[str, Any] = {} + for item in _items_of(query): + if isinstance(item, exp.Subquery): + self._walk(item.this, frames, ctes) + elif isinstance(item, exp.Table) and isinstance(item.this, exp.Identifier): + if not item.args.get("db") and item.name in ctes: + self.cte_of[id(item)] = ctes[item.name] + else: + raise _Unsupported(type(item).__name__) + name = item.alias_or_name + if not name or name in frame: + raise _Unsupported("来源没有名字或重名") + frame[name] = item + self.owner[id(item)] = query + self.refs.setdefault(id(item), []) + stack = frames + [frame] + for node in _block_nodes(query): + if isinstance(node, exp.Column) and node.table: + for f in reversed(stack): + if node.table in f: + self.refs[id(f[node.table])].append(node) + self.source_of[id(node)] = f[node.table] + break + for sub in _nested_queries(query): + self._walk(sub, stack, ctes) + + def body(self, item: Any) -> Any: + """派生来源里面那段查询(FROM 里的子查询、引用的 CTE);实表是 None。""" + from sqlglot import exp + + if isinstance(item, exp.Subquery): + return item.this + cte = self.cte_of.get(id(item)) + return cte.this if cte is not None else None + + +def _unique_aliases(tree: Any) -> None: + """(认列之后)整条语句里每个来源的别名改成互不相同的(_s1、_s2…),引用它的列跟着改:按原来的别名认清每一列 + 属于哪个来源之后再改,相关子查询里引用外层的列不会认错;之后并子查询、写回 CTE 时名字不会撞上。""" + from sqlglot import exp + + resolver = _Resolver(tree) + taken = {t.name for t in tree.find_all(exp.Table)} | {c.alias_or_name for c in tree.find_all(exp.CTE)} + n = 0 + for item in [i for s in tree.find_all(exp.Select) for i in _items_of(s)]: + n += 1 + name = f"_s{n}" + while name in taken: + name += "_" + alias = item.args.get("alias") + if isinstance(alias, exp.TableAlias): + alias.set("this", exp.to_identifier(name)) + else: + item.set("alias", exp.TableAlias(this=exp.to_identifier(name))) + for col in resolver.refs.get(id(item), []): + col.set("table", exp.to_identifier(name)) + + +def _nested_with(tree: Any) -> bool: + from sqlglot import exp + + top = tree.args.get("with_") + return any(isinstance(n, exp.With) and n is not top for n in tree.walk()) + + +def _inline_ctes(tree: Any) -> None: + """只引用一次的非递归顶层 CTE 写回引用它的地方,成为 FROM 里的子查询(别名照旧):写成 CTE 还是子查询是同一个 + 意思。语句里还有别的 WITH 的不动(名字可能被里层的 CTE 遮住)。""" + from sqlglot import exp + + with_ = tree.args.get("with_") + if with_ is None or with_.args.get("recursive") or _nested_with(tree): + return + changed = True + while changed and with_.expressions: + changed = False + resolver = _Resolver(tree) + for cte in list(with_.expressions): + refs = [t for t in tree.find_all(exp.Table) if resolver.cte_of.get(id(t)) is cte] + if len(refs) != 1: + continue + ref = refs[0] + sub = exp.Subquery(this=cte.this.copy(), alias=exp.TableAlias(this=exp.to_identifier(ref.alias_or_name))) + ref.replace(sub) + cte.pop() + changed = True + break + if not with_.expressions: + tree.set("with_", None) + + +def _drop_unused_ctes(tree: Any) -> None: + """没有被引用的非递归 CTE 去掉:查询语句里没用到的 CTE 不影响结果。""" + from sqlglot import exp + + changed = True + while changed: + changed = False + resolver = _Resolver(tree) + used = {id(c) for c in resolver.cte_of.values()} + for with_ in list(tree.find_all(exp.With)): + if with_.args.get("recursive"): + continue + for cte in list(with_.expressions): + if id(cte) not in used: + cte.pop() + changed = True + if not with_.expressions and isinstance(with_.parent, exp.Expr): + with_.parent.set("with_", None) + + +def _mergeable(body: Any) -> bool: + """FROM 里的子查询能不能并进外层:只从几张实表(内连接)里筛行、选列(没有分组、聚合、去重、排序、取行、窗口函数、 + UNION、WITH,选出来的列里没有子查询),并进去之后每一行、每个值都一样。""" + from sqlglot import exp + + if not isinstance(body, exp.Select): + return False + if {k for k, v in body.args.items() if v} - {"expressions", "from_", "where", "joins"}: + return False + sources = _items_of(body) + if not sources or any(not isinstance(i, exp.Table) for i in sources): + return False + if any(not _inner(j) for j in body.args.get("joins") or []): + return False + stars = [p for p in body.expressions if _is_star(p)] + if stars and (len(body.expressions) != 1 or not isinstance(body.expressions[0], exp.Star) or len(sources) != 1): + return False + for p in body.expressions: + if any(isinstance(n, (exp.AggFunc, exp.Window, exp.Query, exp.Subquery)) for n in p.walk()): + return False + names = [p.alias_or_name for p in body.expressions] + if not stars and (not all(names) or len(set(names)) != len(names)): + return False + where = body.args.get("where") + return where is None or not any(isinstance(n, (exp.AggFunc, exp.Window)) for n in where.walk()) + + +def _merge_derived(tree: Any, read: str | None, info: _Schema) -> None: + """(认列之后)只筛行、选列的子查询(见 _mergeable)并进外层:外层引用它的列换成里面的定义,里面的 WHERE 和外层的 + WHERE(和它里面内连接的条件)用 AND 连起来,来源换成里面那几张表。只并内连接一侧(FROM 或内连接进来的)的:外连接 + 可空一侧没关联上的行该是空值,并进来就成了算出来的值。结果看行出来的先后的外层(见 _Sensitivity)不并。 + 外层有 SELECT * 的:子查询选的就是 *(换成那张表,列一样)或者它是唯一的来源(* 换成它选出来的列)才并。""" + from sqlglot import exp + + def substitute(col: Any, expr: Any) -> None: + new = expr.copy() + if not isinstance(new, (exp.Column, exp.Literal, exp.Null, exp.Boolean)): + new = exp.Paren(this=new) + col.replace(new) + + for _ in range(64): + resolver = _Resolver(tree) + # 浮点数求和这一条不挡并子查询:只筛行选列的子查询三种库自己也会并进外层(PostgreSQL 的 subquery pull-up、 + # MySQL 的 derived_merge、SQLite 的 flattening),并不并执行起来先后一样 + sens = _Sensitivity(tree, read, info, resolver, floats=False) + done = False + for select in reversed(list(tree.find_all(exp.Select))): + if sens.of(select): + continue + joins = select.args.get("joins") or [] + if any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + items = _items_of(select) + for index, item in enumerate(items): + if not isinstance(item, exp.Subquery) or not _mergeable(item.this): + continue + if index and not _inner(joins[index - 1]): + continue + body = item.this + sources = _items_of(body) + src = sources[0] + star_body = isinstance(body.expressions[0], exp.Star) + defs = {} if star_body else {p.alias_or_name: p.unalias() for p in body.expressions} + refs = resolver.refs.get(id(item), []) + if not star_body and any(c.name not in defs for c in refs): + continue + stars = [p for p in select.expressions if _is_star(p)] + bare = [p for p in stars if isinstance(p, exp.Star)] + if bare and not star_body and len(items) != 1: + continue + alias = item.alias_or_name + for col in refs: + if star_body: + col.set("table", exp.to_identifier(src.alias_or_name)) + else: + substitute(col, defs[col.name]) + if not star_body: + expanded = [exp.alias_(e.copy(), n, copy=False) for n, e in defs.items()] + new_list = [] + for p in select.expressions: + if isinstance(p, exp.Star) or (isinstance(p, exp.Column) and p.is_star and p.table == alias): + new_list += [e.copy() for e in expanded] + else: + new_list.append(p) + select.set("expressions", new_list) + else: + for p in stars: + if isinstance(p, exp.Column) and p.table == alias: + p.set("table", exp.to_identifier(src.alias_or_name)) + conds = [] + inner_where = body.args.get("where") + if inner_where is not None: + conds += _conjuncts(inner_where.this) + for join in body.args.get("joins") or []: + if join.args.get("on") is not None: + conds += _conjuncts(join.args["on"]) + if index: + own = joins[index - 1] + if own.args.get("on") is not None: + conds += _conjuncts(own.args["on"]) + position = index - 1 + new_joins = joins[:position] + [exp.Join(this=s.copy()) for s in sources] + joins[position + 1:] + select.set("joins", new_joins) + else: + select.args["from_"].set("this", src.copy()) + select.set("joins", [exp.Join(this=s.copy()) for s in sources[1:]] + list(joins) or None) + outer = select.args.get("where") + conds += _conjuncts(outer.this) if outer is not None else [] + if conds: + select.set("where", exp.Where(this=_and_all([c.copy() for c in conds]))) + done = True + break + if done: + break + if not done: + return + + +def _split_simple_ctes(tree: Any, read: str | None, info: "_Schema") -> None: + """引用了好几次、只从几张表里筛行选列的 CTE(见 _mergeable),每处引用各写回一份 FROM 里的子查询(别名照旧): + 这样的 CTE 是确定的(有不确定函数的语句不走这一级),算几遍结果一样,写回之后每处都能并进外层。语句里还有别的 + WITH 的不动(名字可能被里层的 CTE 遮住);有一处引用在结果看行出来先后的一段里的不动(PostgreSQL 把引用了好几次 + 的 CTE 先算出来存着,写回之后扫描的先后可能就变了)。""" + from sqlglot import exp + + with_ = tree.args.get("with_") + if with_ is None or with_.args.get("recursive") or _nested_with(tree): + return + resolver = _Resolver(tree) + sens = _Sensitivity(tree, read, info, resolver) + for cte in list(with_.expressions): + if not _mergeable(cte.this): + continue + refs = [t for t in tree.find_all(exp.Table) if resolver.cte_of.get(id(t)) is cte] + if len(refs) < 2 or any(r.find_ancestor(exp.CTE) is not None for r in refs) or \ + any(sens.of(resolver.owner[id(r)]) for r in refs): + continue + for ref in refs: + ref.replace(exp.Subquery(this=cte.this.copy(), alias=exp.TableAlias(this=exp.to_identifier(ref.alias_or_name)))) + cte.pop() + if not with_.expressions: + tree.set("with_", None) + + +def _grouped(expression: Any, keys: set[str]) -> bool: + """分组的子查询选出来的一列在每组里只有一个值:整个是分组键,或者每一列都在聚合函数里、或者本身是分组键。""" + from sqlglot import exp + + if _fingerprint(expression) in keys: + return True + if any(isinstance(n, (exp.Window, exp.Query, exp.Subquery)) for n in expression.walk()): + return False + for col in expression.find_all(exp.Column): + if _fingerprint(col) not in keys and col.find_ancestor(exp.AggFunc) is None: + return False + return True + + +def _push_into_groups(tree: Any, read: str | None, info: "_Schema") -> None: + """(认列之后)外层 WHERE 里只涉及一个分组子查询(FROM 或内连接进来的)选出来的列的条件,换成子查询里对应的表达式、 + 挪进它的 HAVING:子查询的每一行就是一组,外层按这一行的值筛和 HAVING 按这一组的值筛,用的是同一个值(分组键取的 + 是同一个代表值,聚合是同一个结果)。条件涉及分组键时,分组键得是「相等就是一样」的(见 _Sensitivity._identity_safe): + MySQL 会把外层只涉及分组键的条件推进子查询的 WHERE 里先筛行,不分大小写时 BINARY city = 'SH' 推进去和放在 HAVING + 里结果不同(在真实的 MySQL 8.0 上跑出来过)。子查询有窗口函数、取行、去重、ROLLUP 这些的不挪;HAVING 里的条件 + 不往 WHERE 挪(相等不等于一样时,组里的行各是各的值)。""" + from sqlglot import exp + + sens = _Sensitivity(tree, read, info) + for select in list(tree.find_all(exp.Select)): + where = select.args.get("where") + joins = select.args.get("joins") or [] + if where is None or any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + for index, item in enumerate(_items_of(select)): + if not isinstance(item, exp.Subquery) or (index and not _inner(joins[index - 1])): + continue + body = item.this + if not isinstance(body, exp.Select) or body.args.get("group") is None or \ + any(body.args.get(k) for k in ("limit", "offset", "distinct", "qualify", "windows", "with_")) or \ + any(n.find(exp.Window) for n in body.expressions): + continue + outputs = {p.alias_or_name: p.unalias() for p in body.expressions if not _is_star(p)} + keys = {_fingerprint(k) for k in body.args["group"].expressions} + alias = item.alias_or_name + moved, kept = [], [] + for cond in _conjuncts(where.this): + cols = list(cond.find_all(exp.Column)) + if not cols or cond.find(exp.Query, exp.Subquery) is not None or \ + any(c.table != alias or c.name not in outputs or not _grouped(outputs[c.name], keys) + for c in cols) or \ + any(k is not None and not sens._identity_safe(k, body) + for c in cols for k in [outputs[c.name] if _fingerprint(outputs[c.name]) in keys else None]): + kept.append(cond) + continue + new = cond.copy() + for col in list(new.find_all(exp.Column)): + col.replace(exp.Paren(this=outputs[col.name].copy())) + moved.append(new) + if moved: + having = body.args.get("having") + body.set("having", exp.Having(this=_and_all(([having.this] if having else []) + moved))) + select.set("where", exp.Where(this=_and_all(kept)) if kept else None) + where = select.args.get("where") + if where is None: + break + + +def _distinct_to_group(tree: Any, sens: "_Sensitivity") -> None: + """SELECT DISTINCT a, b(没有分组、聚合、窗口函数)和 GROUP BY a, b 出的行一样(空值也算一组)——只在选出来的都是 + 「相等就是一样」的列(见 _Sensitivity._identity_safe)、这一段不看行出来的先后(DISTINCT 按扫到的先后出行,GROUP BY + 按分组键排好了出)时改:不然两种写法留下的代表值、取到的行可能不同。ORDER BY 用到没选出来的列时不改(两种写法对 + 这种排序的规矩不一样)。已经按选出来的列分组的 DISTINCT 去掉(每组一行、各行的分组键都不同)。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + distinct = select.args.get("distinct") + if distinct is None or distinct.args.get("on") or sens.of(select) or _starry(select): + continue + group = select.args.get("group") + if group is not None: + selected = {_fingerprint(p.unalias()) for p in select.expressions} + if group.expressions and all(_fingerprint(k) in selected for k in group.expressions) and \ + not any(group.args.get(k) for k in ("rollup", "cube", "grouping_sets")) and \ + not any(p.find(exp.Window) for p in select.expressions): + select.set("distinct", None) + continue + if select.args.get("having") is not None or _aggregating(select) or \ + any(p.find(exp.Window, exp.Query, exp.Subquery) for p in select.expressions): + continue + keys = [p.unalias() for p in select.expressions] + if not all(isinstance(k, exp.Column) and sens._identity_safe(k, select) for k in keys): + continue + order = select.args.get("order") + if order is not None: + allowed = {_fingerprint(k) for k in keys} + aliases = {p.alias for p in select.expressions if p.alias} + if any(_fingerprint(o.this) not in allowed and not ( + isinstance(o.this, exp.Column) and not o.this.table and o.this.name in aliases) + for o in order.expressions): + continue + new = exp.Group(expressions=[k.copy() for k in keys]) + new.meta[_FROM_DISTINCT] = True + select.set("distinct", None) + select.set("group", new) + + +def _in_subquery_distinct(tree: Any, sens: "_Sensitivity") -> None: + """IN (子查询) 只看取值的集合:里面不取行时,没有聚合的 ORDER BY 不影响(有聚合的 ORDER BY 让它成了聚合查询,只出 + 一行);选出来的那一列「相等就是一样」时 DISTINCT 也不影响(MySQL 默认不分大小写,DISTINCT 留下 'SH' 之后 + BINARY 'sh' 就找不到了)。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.In)): + query = node.args.get("query") + sub = query.this if isinstance(query, exp.Subquery) else query + if not isinstance(sub, exp.Select) or any(sub.args.get(k) for k in ("limit", "offset", "fetch")): + continue + if any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) for n in _block_nodes(sub)) or \ + sub.args.get("group") is not None or sub.args.get("having") is not None: + continue + if not any(n.find(exp.Window) for n in sub.expressions): + sub.set("order", None) + distinct = sub.args.get("distinct") + if distinct is not None and not distinct.args.get("on") and len(sub.expressions) == 1 and \ + isinstance(sub.expressions[0].unalias(), exp.Column) and \ + sens._identity_safe(sub.expressions[0].unalias(), sub): + sub.set("distinct", None) + + +# ---- 结果看不看行出来的先后 + +class _Sensitivity: + """哪几段 SELECT 的结果看行出来的先后:这样的一段不调换来源的先后、不把 CROSS JOIN 写成逗号连接、不排 GROUP BY + 的键、不把 RIGHT JOIN 写成 LEFT JOIN,不往里并子查询——这些改写都会改变库里扫描的先后,实际取到的行、值就不同了。 + + 本段自己看先后的:取行(LIMIT / OFFSET / FETCH,有 ORDER BY 也可能有并列);DISTINCT ON;窗口函数;拼串、拼数组、 + 取任意一个这类聚合(GROUP_CONCAT);聚合里既不分组也不聚合的列(SQLite、MySQL 取组里任意一行的值);当成标量 + 用的子查询(SQLite 取第一行),除非是不分组的聚合(只出一行);去重、分组、MIN / MAX、去重的 UNION 留下的是先 + 出现的那个值,而「相等」不等于「写法一样」的时候(MySQL 默认不分大小写的排序规则下的 'SH' 和 'sh'、PostgreSQL + numeric 的 2.0 和 2.00、SQLite 没有类型亲和的列里的 1 和 1.0、算出来的表达式)看先后,见 _identity_safe; + PostgreSQL、MySQL 里对不知道是不是精确数的值求和、求平均(浮点数加的先后不同,结果的最后几位就不同,见 _exact)。 + 给看先后的一段供行的(它 FROM 里的子查询、引用的 CTE,层层往里)也看先后。""" + + def __init__(self, tree: Any, read: str | None, info: _Schema, resolver: _Resolver | None = None, + floats: bool = True) -> None: + from sqlglot import exp + + self.read = read + self.info = info + self.floats = floats + try: + self.resolver = resolver or _Resolver(tree) + except _Unsupported: + self.resolver = None + selects = list(tree.find_all(exp.Select)) + marked = {id(s) for s in selects if self._own(s)} + for op in tree.find_all(exp.SetOperation): + if op.args.get("distinct") or isinstance(op, (exp.Intersect, exp.Except)): + for leaf in _leaves(op): + if isinstance(leaf, exp.Select) and \ + not all(self._identity_safe(p.unalias(), leaf) for p in leaf.expressions): + marked.add(id(leaf)) + by_id = {id(s): s for s in selects} + stack = [by_id[i] for i in marked if i in by_id] + while stack: + select = stack.pop() + for item in _items_of(select): + body = self.resolver.body(item) if self.resolver is not None else ( + item.this if isinstance(item, exp.Subquery) else None) + for leaf in _leaves(body) if body is not None else []: + if isinstance(leaf, exp.Select) and id(leaf) not in marked: + marked.add(id(leaf)) + stack.append(leaf) + self.marked = marked + + def of(self, select: Any) -> bool: + return id(select) in self.marked + + def _own(self, select: Any) -> bool: + from sqlglot import exp + + if any(select.args.get(k) for k in ("limit", "offset", "fetch")): + return True + distinct = select.args.get("distinct") + if distinct is not None and distinct.args.get("on"): + return True + own = list(_block_nodes(select)) + if any(isinstance(n, exp.Window) or _order_dependent(n) for n in own): + return True + if _scalar_context(select) and not (_aggregating(select) and select.args.get("group") is None): + return True + if _loose_columns(select): + return True + if distinct is not None and not all(self._identity_safe(p.unalias(), select) for p in select.expressions): + return True + group = select.args.get("group") + if group is not None and not all(self._identity_safe(k, select) for k in group.expressions): + return True + for n in own: + if isinstance(n, (exp.Min, exp.Max)) and not n.expressions and \ + not self._identity_safe(n.this.expressions[0] if isinstance(n.this, exp.Distinct) and + n.this.expressions else n.this, select): + return True + if self.floats and self.read != "sqlite" and _float_aggregate(n) and not self._exact(n.this, select): + return True + return False + + def _exact(self, node: Any, select: Any, depth: int = 0) -> bool: + """node 的取值是不是精确的数(整数、定点小数):浮点数求和的先后不同结果就不同(PostgreSQL、MySQL 按扫到的先后 + 一个个加;SQLite 用 Kahan-Babuska-Neumaier 求和、关联先后也不跟着 FROM 的先后变,当成精确的)。不知道类型的不算。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, exp.Distinct) and len(node.expressions) == 1: + node = node.expressions[0] + if isinstance(node, exp.Count) or (isinstance(node, exp.Literal) and node.is_int): + return True + if isinstance(node, (exp.Add, exp.Sub, exp.Mul)): + return self._exact(node.left, select, depth) and self._exact(node.right, select, depth) + if not isinstance(node, exp.Column) or node.is_star or depth > 16 or self.resolver is None: + return False + item = self.resolver.source_of.get(id(node)) + if item is None: + items = _items_of(select) + item = items[0] if len(items) == 1 and not node.table else None + if item is None: + return False + body = self.resolver.body(item) + if body is None: + columns = self.info.types.get(item.name.translate(_ASCII_LOWER)) if isinstance(item, exp.Table) else None + declared = (columns or {}).get(node.name.translate(_ASCII_LOWER), "").lower() + return any(declared.startswith(w) for w in ("int", "integer", "bigint", "smallint", "tinyint", "mediumint", + "serial", "bigserial", "decimal", "numeric", "dec")) + if isinstance(body, exp.Select): + for p in body.expressions: + if not _is_star(p) and p.alias_or_name == node.name: + return self._exact(p.unalias(), body, depth + 1) + return False + + def _identity_safe(self, node: Any, select: Any, depth: int = 0) -> bool: + """node 的取值里「相等」是不是就是「一样」:COUNT 是整数;光一列的话看它是哪张表的哪一列、声明的什么类型。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, exp.Count): + return True + if not isinstance(node, exp.Column) or node.is_star or depth > 16: + return False + item = self.resolver.source_of.get(id(node)) if self.resolver is not None else None + if item is None: + items = _items_of(select) + item = items[0] if len(items) == 1 and not node.table else None + if item is None: + return self._safe_type(None) + body = self.resolver.body(item) if self.resolver is not None else ( + item.this if isinstance(item, exp.Subquery) else None) + if body is None: + if not isinstance(item, exp.Table): + return False + columns = self.info.types.get(item.name.translate(_ASCII_LOWER)) + declared = columns.get(node.name.translate(_ASCII_LOWER)) if columns is not None else None + return self._safe_type(declared) + if not isinstance(body, exp.Select): + return False + for p in body.expressions: + if not _is_star(p) and p.alias_or_name == node.name: + return self._identity_safe(p.unalias(), body, depth + 1) + sources = _items_of(body) + if any(isinstance(p, exp.Star) for p in body.expressions) and len(sources) == 1: + return self._through_star(sources[0], node.name, depth + 1) + return False + + def _through_star(self, item: Any, name: str, depth: int) -> bool: + """SELECT * 原样透出来的一列:到它唯一的来源里找同名的列。""" + from sqlglot import exp + + body = self.resolver.body(item) if self.resolver is not None else None + if body is None: + if not isinstance(item, exp.Table): + return False + columns = self.info.types.get(item.name.translate(_ASCII_LOWER)) + return self._safe_type(columns.get(name.translate(_ASCII_LOWER)) if columns is not None else None) + if not isinstance(body, exp.Select) or depth > 16: + return False + for p in body.expressions: + if not _is_star(p) and p.alias_or_name == name: + return self._identity_safe(p.unalias(), body, depth + 1) + sources = _items_of(body) + if any(isinstance(p, exp.Star) for p in body.expressions) and len(sources) == 1: + return self._through_star(sources[0], name, depth + 1) + return False + + def _safe_type(self, declared: str | None) -> bool: + """声明的类型下「相等」是不是就是「一样」。SQLite 没给表结构时按列有声明的类型、默认的排序规则算(假设); + 有类型的话 BLOB、没写类型(没有类型亲和)的列可能混着 1 和 1.0。PostgreSQL、MySQL 不知道类型就不算:numeric + 的 2.0 和 2.00、MySQL 默认不分大小写的字符串都是相等写法却不一样。""" + if self.read == "sqlite": + if declared is None: + return True + upper = declared.upper() + if "INT" in upper or any(w in upper for w in ("CHAR", "CLOB", "TEXT", "REAL", "FLOA", "DOUB")): + return True + return bool(upper) and "BLOB" not in upper + if declared is None: + return False + lower = declared.lower().strip() + if self.read == "mysql": + return any(lower.startswith(w) for w in ("int", "integer", "bigint", "smallint", "tinyint", "mediumint", + "date", "datetime", "timestamp", "time", "year", "decimal", + "numeric", "bit", "bool")) + return any(lower.startswith(w) for w in ("int", "integer", "bigint", "smallint", "serial", "bigserial", + "text", "varchar", "character", "char", "bpchar", "date", + "timestamp", "time", "bool", "uuid")) + + +def _float_aggregate(node: Any) -> bool: + """浮点数求和类的聚合(不在窗口函数里的)。""" + from sqlglot import exp + + if isinstance(node.parent, exp.Window): + return False + name = type(node).__name__.lower() + if isinstance(node, exp.Anonymous): + name = str(node.name).lower().replace("_", "") + return name in _FLOAT_AGGS + + +def _order_dependent(node: Any) -> bool: + from sqlglot import exp + + name = type(node).__name__.lower() + if isinstance(node, exp.Anonymous): + name = str(node.name).lower().replace("_", "") + return name in _ORDERED_AGGS + + +def _aggregating(select: Any) -> bool: + """这一段是聚合查询:有 GROUP BY,或者自己的 SELECT 列表、HAVING、ORDER BY 里有聚合函数(不算窗口函数)。""" + from sqlglot import exp + + if select.args.get("group") is not None or select.args.get("having") is not None: + return True + return any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) for n in _block_nodes(select) + if not isinstance(n.parent, exp.Window)) + + +def _scalar_context(query: Any) -> bool: + """这段查询当成一个值用(标量子查询):不在 FROM / JOIN、IN、EXISTS、ANY / ALL、CTE 里,也不是整条语句。""" + from sqlglot import exp + + node = query + while True: + parent = node.parent + if parent is None: + return False + if isinstance(parent, exp.Subquery): + node = parent + continue + if isinstance(parent, exp.SetOperation): + node = parent + continue + if isinstance(parent, (exp.From, exp.Join, exp.Exists, exp.CTE, exp.Any, exp.All)): + return False + if isinstance(parent, exp.In) and node.arg_key == "query": + return False + return True + + +def _loose_columns(select: Any) -> bool: + """聚合的一段里既不在聚合函数里、也不是分组键(的一部分)的列:SQLite、MySQL 取组里任意一行的值。""" + from sqlglot import exp + + if not _aggregating(select): + return False + group = select.args.get("group") + keys = {_fingerprint(k) for k in (group.expressions if group is not None else [])} + + def loose(node: Any) -> bool: + if isinstance(node, (exp.Query, exp.Subquery)) or _fingerprint(node) in keys: + return False + if isinstance(node, exp.AggFunc) and not isinstance(node.parent, exp.Window): + return False + if isinstance(node, exp.Window): + return False + if isinstance(node, exp.Column): + return not node.is_star + return any(loose(c) for c in node.iter_expressions()) + + clauses = [c for c in (*select.expressions, select.args.get("having"), select.args.get("order")) if c is not None] + return any(loose(c) for c in clauses) + + +# ---- 动来源的改写 + +def _unwrap_passthrough(tree: Any, read: str | None) -> Any: + """原样选出 FROM 里唯一一个子查询的全部列、什么别的都不做的一段(SELECT * FROM (…) t、SELECT t.a, t.b FROM (…) t), + 换成那个子查询:出的行一样,列的先后也一样。最外层和条件里的子查询列名不影响取值,列名可以不同;别的地方(FROM 里 + 的子查询、CTE)外面按名字引用,名字也得一样。不换的:子查询自己带 ORDER BY / LIMIT / OFFSET(套一层之后顺序、取行 + 不一定照旧);子查询是 UNION 这类合并、库是 SQLite(和不知道是哪种库时)——SQLite 从子查询里读合并的结果时,按第一支 + 那一列的类型亲和转换别的支的值(REAL 列在前,另一支的整数 2 读出来成了 2.0),套不套这一层结果不一样。""" + from sqlglot import exp + + def outputs(query: Any) -> list[str] | None: + if any(query.args.get(k) is not None for k in ("order", "limit", "offset", "fetch")): + return None + if isinstance(query, exp.SetOperation): + if read in (None, "sqlite"): + return None + leaves = _leaves(query) + return outputs(leaves[0]) if leaves and isinstance(leaves[0], exp.Select) else None + if not isinstance(query, exp.Select) or _starry(query) or query.args.get("with_") is not None: + return None + names = [p.alias_or_name for p in query.expressions] + return names if all(names) and len(set(names)) == len(names) else None + + changed = True + while changed: + changed = False + for select in list(tree.find_all(exp.Select)): + if select.args.get("joins") or select.args.get("from_") is None or \ + {k for k, v in select.args.items() if v} - {"expressions", "from_", "with_"} or \ + (select.args.get("with_") is not None and select is not tree): + continue + item = select.args["from_"].this + if not isinstance(item, exp.Subquery): + continue + body = item.this + names = outputs(body) + if names is None: + continue + alias = item.alias_or_name + star = len(select.expressions) == 1 and isinstance(select.expressions[0], exp.Star) + cols = [p.unalias() for p in select.expressions] + if not star and (len(cols) != len(names) or any( + not isinstance(c, exp.Column) or c.is_star or c.name != n or (c.table and c.table != alias) + for c, n in zip(cols, names))): + continue + renamed = not star and any(p.alias_or_name != n for p, n in zip(select.expressions, names)) + parent = select.parent + loose = select is tree or isinstance(parent, (exp.In, exp.Exists)) or ( + isinstance(parent, exp.Subquery) and not isinstance(parent.parent, (exp.From, exp.Join, exp.CTE))) + if renamed and not loose: + continue + new = body.copy() + if select is tree: + if select.args.get("with_") is not None: + if new.args.get("with_") is not None: + continue + new.set("with_", select.args["with_"]) + tree = new + else: + select.replace(new) + changed = True + break + return tree + + +def _bare_star(select: Any) -> bool: + from sqlglot import exp + + return any(isinstance(p, exp.Star) for p in select.expressions) + + +def _right_to_left(tree: Any, sens: _Sensitivity) -> None: + """只有两个来源的 a RIGHT JOIN b ON p 就是 b LEFT JOIN a ON p。选的是 * 时列的先后跟着变、结果看行出来的先后的 + 一段扫描的先后跟着变,不改。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + if len(joins) != 1 or str(joins[0].side or "").upper() != "RIGHT" or not _plain_join(joins[0]) or \ + joins[0].args.get("on") is None or _bare_star(select) or sens.of(select): + continue + first = select.args["from_"] + left, right = first.this, joins[0].this + first.set("this", right.copy()) + joins[0].set("this", left.copy()) + joins[0].set("side", "LEFT") + joins[0].set("kind", None) + joins[0].meta[_FLIPPED] = True + for join in tree.find_all(exp.Join): + if join.side and str(join.kind or "").upper() == "OUTER": + join.set("kind", None) + + +def _null_propagating(node: Any, alias: str, read: str | None) -> bool: + """alias 那一侧整行是空值时,node 一定是空值:那一侧的列,或者对它做比较、算术、取大小写、转类型。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, exp.Column): + return bool(node.table) and node.table == alias + if type(node).__name__ in _STRICT_UNARY: + return _null_propagating(node.this, alias, read) + if type(node).__name__ in _STRICT_BINARY or (isinstance(node, exp.DPipe) and read != "mysql"): + return _null_propagating(node.left, alias, read) or _null_propagating(node.right, alias, read) + return False + + +def _rejects(cond: Any, alias: str, read: str | None) -> bool: + """alias 那一侧整行是空值时 cond 不成立(是空值或假):比较、LIKE、IN、BETWEEN 的一边一定是空值,IS NOT NULL, + NOT 套着这样的比较,AND 里有一项不成立、OR 里每一项都不成立。IS NULL、COALESCE、CASE、<=> 之类不算。""" + from sqlglot import exp + + cond = _unparen(cond) + comparisons = (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.Like, exp.ILike) + if isinstance(cond, comparisons): + return _null_propagating(cond.left, alias, read) or _null_propagating(cond.right, alias, read) + if isinstance(cond, (exp.In, exp.Between)): + return _null_propagating(cond.this, alias, read) + if isinstance(cond, exp.Not): + inner = _unparen(cond.this) + if isinstance(inner, exp.Is) and isinstance(inner.expression, exp.Null): + return _null_propagating(inner.this, alias, read) + if isinstance(inner, comparisons): + return _null_propagating(inner.left, alias, read) or _null_propagating(inner.right, alias, read) + if isinstance(inner, (exp.In, exp.Between)): + return _null_propagating(inner.this, alias, read) + return False + if isinstance(cond, exp.And): + return _rejects(cond.left, alias, read) or _rejects(cond.right, alias, read) + if isinstance(cond, exp.Or): + return _rejects(cond.left, alias, read) and _rejects(cond.right, alias, read) + return False + + +def _reject_nulls(tree: Any, read: str | None) -> None: + """LEFT JOIN 关联进来的那一侧,同一段 WHERE 里有把它整行为空的行筛掉的条件(见 _rejects):没关联上的行反正留 + 不下来,这次外连接就是内连接。三种库的优化器自己也做这一步(outer join reduction),执行起来扫描的先后一样,结果 + 看行出来先后的一段也照做。有 RIGHT / FULL JOIN 的段不动。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + where = select.args.get("where") + if where is None or any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + conds = _conjuncts(where.this) + for join in joins: + if str(join.side or "").upper() != "LEFT" or not _plain_join(join): + continue + alias = join.this.alias_or_name + if alias and any(_rejects(c, alias, read) for c in conds): + join.set("side", None) + join.set("kind", None) + join.this.meta[_REJECTED] = "RIGHT JOIN" if join.meta.get(_FLIPPED) else "LEFT JOIN" + + +def _flatten_prefix(tree: Any, sens: _Sensitivity, read: str | None) -> None: + """开头那组内连接(FROM 和紧跟着的内连接、CROSS JOIN、逗号连接,到第一个外连接为止)的 ON 条件挪进 WHERE、写成 + 逗号连接:内连接的条件写在 ON 还是 WHERE 一样;后面的 LEFT JOIN 对左边每一行各自关联,左边先筛后筛一样。后面有 + RIGHT / FULL JOIN 的不动:没关联上的那一侧整行空值,挪到 WHERE 的条件会把它们筛掉(PostgreSQL、MySQL 里逗号还比 + JOIN 结合得松,a, b RIGHT JOIN c 是 a 乘上 b RIGHT JOIN c)。内连接的条件写在 ON 还是 WHERE,优化器一视同仁,结果 + 看行出来先后的一段也照做;只有 SQLite 里结果看先后、又写了 CROSS JOIN 的一段不动(SQLite 不调换 CROSS JOIN 两边、 + 会调换逗号连接)。""" + from sqlglot import exp + + for select in list(tree.find_all(exp.Select)): + joins = select.args.get("joins") or [] + if not joins or select.args.get("from_") is None or \ + any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins): + continue + if read in (None, "sqlite") and sens.of(select) and \ + any(str(j.kind or "").upper() == "CROSS" or j.meta.get("sql_semantics_join") for j in joins): + continue + moved: list[Any] = [] + for join in joins: + if not _inner(join): + break + on = join.args.get("on") + if on is not None: + moved += _conjuncts(on) + join.set("on", None) + join.set("kind", None) + if not sens.of(select): + join.meta.pop("sql_semantics_join", None) + if moved: + where = select.args.get("where") + select.set("where", exp.Where(this=_and_all((_conjuncts(where.this) if where else []) + moved))) + + +# ---- 和来源无关的改写 + +def _items_of(select: Any) -> list[Any]: + first = select.args.get("from_") + return ([first.this] if first is not None else []) + [j.this for j in select.args.get("joins") or []] + + +def _and_all(items: list[Any]) -> Any: + from sqlglot import exp + + if not items: + return None + return exp.and_(*items, copy=False) if len(items) > 1 else items[0] + + +def _plain_join(join: Any) -> bool: + return not (join.args.get("using") or join.args.get("method") or join.args.get("natural")) + + +def _comma(join: Any) -> bool: + """开头那组内连接里的一项:写成逗号连接、条件都在 WHERE 里(见 _flatten_prefix)。""" + return _plain_join(join) and not join.side and not join.args.get("kind") and join.args.get("on") is None + + +def _inner(join: Any) -> bool: + return _plain_join(join) and not join.side and str(join.kind or "").upper() in ("", "INNER", "CROSS") + + +def _constant(node: Any) -> bool: + from sqlglot import exp + + return isinstance(node, (exp.Literal, exp.Null, exp.Boolean)) or ( + isinstance(node, exp.Neg) and isinstance(node.this, exp.Literal)) + + +def _strip_asc(tree: Any) -> None: + from sqlglot import exp + + for o in tree.find_all(exp.Ordered): + if o.args.get("desc") is False: + o.set("desc", None) + + +def _normalize_negation(tree: Any) -> None: + """x IS NOT NULL、x NOT LIKE y 这类带 negate 的写法统一写成 NOT (x IS NULL)、NOT (x LIKE y):各方言解析出来的形状 + 不一样(PostgreSQL 的 IS NOT NULL 是 Is(negate=True),SQLite 的是 Not(Is)),意思一样。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.Predicate)): + if node.args.get("negate") is True: + node.set("negate", None) + copy = node.copy() + node.replace(exp.Not(this=exp.Paren(this=copy))) + + +def _case_forms(tree: Any, read: str | None) -> None: + """CASE … ELSE NULL END 的 ELSE NULL 去掉(不写 ELSE 就是空值);SQLite 的 IIF(c, a, b)、MySQL 的 IF(c, a, b) 写成 + CASE WHEN c THEN a ELSE b END(SQLite 文档:两者生成同样的字节码;MySQL 的 IF 在 c 为真时取 a,否则取 b)。""" + from sqlglot import exp + + if read in ("sqlite", "mysql"): + for node in list(tree.find_all(exp.If)): + if isinstance(node.parent, exp.Case) or node.args.get("true") is None: + continue + case = exp.Case(ifs=[exp.If(this=node.this.copy(), true=node.args["true"].copy())]) + if node.args.get("false") is not None: + case.set("default", node.args["false"].copy()) + node.replace(case) + for case in tree.find_all(exp.Case): + if isinstance(case.args.get("default"), exp.Null): + case.set("default", None) + + +def _drop_true_conjuncts(tree: Any) -> None: + """WHERE、HAVING、ON 里恒为真的项(TRUE、1 = 1 这种同一个常量和自己相等)去掉:x AND TRUE 就是 x。""" + from sqlglot import exp + + def true(node: Any) -> bool: + node = _unparen(node) + if isinstance(node, exp.Boolean): + return bool(node.this) + if isinstance(node, (exp.EQ, exp.GTE, exp.LTE)): + left, right = _unparen(node.this), _unparen(node.expression) + return isinstance(left, exp.Literal) and isinstance(right, exp.Literal) and \ + _fingerprint(left) == _fingerprint(right) + return False + + holders = [(w, "this") for w in tree.find_all(exp.Where, exp.Having)] + \ + [(j, "on") for j in tree.find_all(exp.Join) if j.args.get("on") is not None] + for holder, key in holders: + cond = holder.args.get(key) + parts = _conjuncts(cond) + kept = [p for p in parts if not true(p)] + if len(kept) == len(parts): + continue + if kept: + holder.set(key, _and_all([p.copy() for p in kept])) + elif isinstance(holder, exp.Join): + holder.set(key, exp.true()) + else: + holder.parent.set(holder.arg_key, None) + + +def _count_star(tree: Any) -> None: + """COUNT(1)、COUNT('x'):数的是不为空的常量,就是行数。COUNT(DISTINCT 1)(有行就是 1)、COUNT(NULL) 不是。""" + from sqlglot import exp + + for c in list(tree.find_all(exp.Count)): + if isinstance(c.this, exp.Literal) and not c.args.get("distinct"): + c.set("this", exp.Star()) + + +def _minmax_distinct(tree: Any) -> None: + """MIN(DISTINCT x)、MAX(DISTINCT x) 就是 MIN(x)、MAX(x):去不去重最小最大的都是那个值。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.Min, exp.Max)): + arg = node.this + if not node.expressions and isinstance(arg, exp.Distinct) and len(arg.expressions) == 1 and \ + not arg.args.get("on"): + node.set("this", arg.expressions[0]) + + +def _simple_value(node: Any) -> bool: + """只是列、常量、* 和它们的加减乘除(没有函数调用:PostgreSQL 里 SELECT 列表的集合返回函数会改变行数)。""" + from sqlglot import exp + + return all(isinstance(n, (exp.Column, exp.Literal, exp.Null, exp.Boolean, exp.Star, exp.Identifier, exp.Paren, + exp.Neg, exp.Add, exp.Sub, exp.Mul, exp.Div, exp.Alias)) for n in node.walk()) + + +def _exists_subqueries(tree: Any) -> None: + """EXISTS 只看有没有行:选什么、去不去重、怎么排都不影响,统一写成 SELECT 1;只取前 n 行(n ≥ 1、不跳过行)也 + 不影响有没有行。动之前先确认这一段不是聚合查询(SELECT、HAVING、ORDER BY 里有聚合函数时总出一行,ORDER BY + COUNT(*) 去掉就成了普通查询)、不跳过行(DISTINCT 在 OFFSET 之前算,去掉 DISTINCT 行数就变了)、选的只是列和 + 常量(PostgreSQL 的集合返回函数会改变行数)。""" + from sqlglot import exp + + for ex in list(tree.find_all(exp.Exists)): + sub = ex.this.this if isinstance(ex.this, exp.Subquery) else ex.this + if not isinstance(sub, exp.Select) or _aggregating(sub) and sub.args.get("group") is None: + continue + if sub.args.get("having") is not None or sub.args.get("offset") is not None or \ + sub.args.get("fetch") is not None: + continue + if any(isinstance(n, exp.AggFunc) and not isinstance(n.parent, exp.Window) for n in _block_nodes(sub)): + continue + distinct = sub.args.get("distinct") + if distinct is not None and distinct.args.get("on"): + continue + limit = sub.args.get("limit") + if limit is not None: + value = limit.expression + if not (isinstance(value, exp.Literal) and value.is_int and int(value.this) >= 1): + continue + if not all(_simple_value(p) for p in sub.expressions): + continue + sub.set("limit", None) + sub.set("distinct", None) + sub.set("order", None) + sub.set("expressions", [exp.Literal.number(1)]) + + +def _in_lists(tree: Any) -> None: + """IN 列表是个集合:一模一样的项去掉、排序;只有一个常量的写成 =(NOT IN 跟着成了 <>,空值的结果一样)。只有 + 一项、又不是常量的不改:SQLite 里 a IN (b) 是 a = +b,右边那一列不带类型亲和,和 a = b 结果不同。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.In)): + if not node.expressions or node.args.get("query") or node.args.get("unnest") or node.args.get("field"): + continue + uniq = {_fingerprint(i): i for i in node.expressions} + ordered = [uniq[k] for k in sorted(uniq)] + if len(ordered) == 1 and _constant(ordered[0]): + node.replace(exp.EQ(this=node.this, expression=ordered[0])) + else: + node.set("expressions", ordered) + + +def _expand_between(tree: Any) -> None: + """x BETWEEN a AND b 就是 x >= a AND x <= b(SQLite 文档原话,PostgreSQL、MySQL 也是)。x 里有子查询、不确定的 + 函数的不改(BETWEEN 只算一次 x)。""" + from sqlglot import exp + + for node in list(tree.find_all(exp.Between)): + x = node.this + if node.args.get("symmetric") or x is None or x.find(exp.Query, exp.Subquery) is not None or _volatile(x): + continue + pair = exp.And(this=exp.GTE(this=x.copy(), expression=node.args["low"].copy()), + expression=exp.LTE(this=x.copy(), expression=node.args["high"].copy())) + node.replace(exp.Paren(this=pair)) + + +def _predicate(node: Any) -> bool: + """取值只会是真、假、空值的条件(比较、IS、IN、LIKE、EXISTS、BETWEEN,和它们的 AND / OR / NOT)。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.Predicate, exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.Is, exp.In, + exp.Like, exp.ILike, exp.Exists, exp.Between)): + return True + if isinstance(node, (exp.And, exp.Or)): + return _predicate(node.left) and _predicate(node.right) + if isinstance(node, exp.Not): + return _predicate(node.this) + return False + + +def _fold_not(tree: Any) -> None: + """NOT 套比较写成反过来的比较(NOT a = b 就是 a <> b,NOT a > b 就是 a <= b:两边有空值时都是空值);NOT NOT p 写成 p + (p 是条件时;NOT NOT 5 是 1,不是 5)。""" + from sqlglot import exp + + changed = True + while changed: + changed = False + for node in list(tree.find_all(exp.Not)): + if node.parent is None and node is not tree: + continue + inner = _unparen(node.this) + name = type(inner).__name__ + if name in _NEGATE and not isinstance(inner.this, (exp.Any, exp.All)) and \ + not isinstance(inner.expression, (exp.Any, exp.All)): + node.replace(getattr(exp, _NEGATE[name])(this=inner.this, expression=inner.expression)) + changed = True + elif isinstance(inner, exp.Not) and _predicate(inner.this): + node.replace(inner.this) + changed = True + if changed: + break + + +#: 括号里是这些(运算符的一部分)时,外面是比较运算的话括号可以去掉:算术比比较结合得紧 +_ARITHMETIC = ("Add", "Sub", "Mul", "Div", "Mod") +_COMPARISON = ("EQ", "NEQ", "GT", "GTE", "LT", "LTE") + + +def _strip_parens(tree: Any) -> None: + """不影响结合的括号去掉(比较时本来就不看括号,见 _fingerprint;这里只为生成的写法好读):括号外面不是运算符 + (函数参数、别名、SELECT 列表、ORDER BY、GROUP BY、WHERE 整个条件、IN 列表的一项……)、括号里是一列一个常量一次 + 函数调用,或者括号里是算术、外面是比较。""" + from sqlglot import exp + + for paren in list(tree.find_all(exp.Paren)): + inner, parent = paren.this, paren.parent + if inner is None or parent is None or isinstance(inner, (exp.Query, exp.Subquery, exp.Tuple)): + continue + atomic = isinstance(inner, (exp.Column, exp.Literal, exp.Null, exp.Boolean, exp.Paren)) or ( + isinstance(inner, exp.Func) and not isinstance(inner, (exp.Binary, exp.Unary, exp.Predicate))) + outside = isinstance(parent, (exp.Alias, exp.Distinct, exp.Select, exp.Ordered, exp.Where, exp.Having, + exp.Group, exp.Window)) or ( + isinstance(parent, exp.Func) and not isinstance(parent, (exp.Binary, exp.Unary, exp.Predicate))) + loose = type(inner).__name__ in _ARITHMETIC and type(parent).__name__ in _COMPARISON + if isinstance(parent, exp.In) and paren.arg_key == "expressions": + outside = True + if atomic or outside or loose: + paren.replace(inner) + + +def _zero_offset(tree: Any) -> None: + from sqlglot import exp + + for offset in list(tree.find_all(exp.Offset)): + value = offset.expression + if isinstance(value, exp.Literal) and value.is_int and int(value.this) == 0: + offset.pop() + + +def _order_by_aliases(tree: Any) -> None: + """最外层 ORDER BY 引用 SELECT 的别名的,换成那个表达式:别名接下来要去掉。不换的:别名指的是数字常量(ORDER BY 1 + 是按第一列排);别名指的是一个不带表名、又和某个别名同名的列(ORDER BY 里单独一个名字先认别名:SELECT user_id AS + amount, amount AS user_id … ORDER BY amount 换成 ORDER BY user_id 就成了按第二列排)。""" + from sqlglot import exp + + if not isinstance(tree, exp.Select) or tree.args.get("order") is None: + return + aliases = {p.alias: p for p in tree.expressions if p.alias} + for o in tree.args["order"].expressions: + key = o.this + if not (isinstance(key, exp.Column) and not key.table and key.name in aliases): + continue + target = aliases[key.name].unalias() + if target.is_number or (isinstance(target, exp.Column) and not target.table and target.name in aliases): + continue + o.set("this", target.copy()) + + +def _drop_aliases(query: Any) -> None: + from sqlglot import exp + + if not isinstance(query, exp.Select): + return + inside = {id(n) for p in query.expressions for n in p.walk()} + used = {n.name for n in _block_nodes(query) if isinstance(n, exp.Column) and not n.table and id(n) not in inside} + query.set("expressions", [p.unalias() if isinstance(p, exp.Alias) and p.alias not in used else p + for p in query.expressions]) + + +def _drop_output_aliases(tree: Any) -> None: + """最外层(UNION 不按名字排序时各支都是)和条件里的子查询选出来的列名不影响任何取值,去掉;本段里还按名字引用 + 的留着。""" + from sqlglot import exp + + if isinstance(tree, exp.SetOperation): + if tree.args.get("order") is None: + for leaf in _leaves(tree): + _drop_aliases(leaf) + else: + _drop_aliases(tree) + for node in tree.find_all(exp.Exists, exp.In): + query = node.this if isinstance(node, exp.Exists) else node.args.get("query") + _drop_aliases(query.this if isinstance(query, exp.Subquery) else query) + for sub in tree.find_all(exp.Subquery): + if not isinstance(sub.parent, (exp.From, exp.Join, exp.Exists, exp.In, exp.CTE)) and not sub.alias: + _drop_aliases(sub.this) + + +def _leaves(query: Any) -> list[Any]: + """一棵 UNION / INTERSECT / EXCEPT 树的各支(套着括号的也拆开)。""" + from sqlglot import exp + + if isinstance(query, exp.Subquery): + return _leaves(query.this) + if isinstance(query, exp.SetOperation): + return _leaves(query.left) + _leaves(query.right) + return [query] + + +def _block_nodes(select: Any) -> Iterator[Any]: + """这一段自己的节点:不进套着的查询;FROM / JOIN 的来源不算(ON 条件算)。""" + from sqlglot import exp + + stack = [v for k, value in select.args.items() if k not in ("with_", "from_") + for v in (value if isinstance(value, list) else [value]) if isinstance(v, exp.Expr)] + while stack: + node = stack.pop() + if isinstance(node, (exp.Query, exp.Subquery)): + continue + if isinstance(node, exp.Join): + if node.args.get("on") is not None: + stack.append(node.args["on"]) + continue + yield node + stack += [v for value in node.args.values() for v in (value if isinstance(value, list) else [value]) + if isinstance(v, exp.Expr)] + + +def _nested_queries(select: Any) -> list[Any]: + """这一段自己的表达式里直接套着的查询(条件、SELECT 列表、ON 里的子查询),不含 FROM 里的子查询。""" + from sqlglot import exp + + out: list[Any] = [] + stack = [v for k, value in select.args.items() if k not in ("with_", "from_") + for v in (value if isinstance(value, list) else [value]) if isinstance(v, exp.Expr)] + while stack: + node = stack.pop() + if isinstance(node, exp.Subquery): + out.append(node.this) + elif isinstance(node, exp.Query): + out.append(node) + elif isinstance(node, exp.Join): + if node.args.get("on") is not None: + stack.append(node.args["on"]) + else: + stack += [v for value in node.args.values() for v in (value if isinstance(value, list) else [value]) + if isinstance(v, exp.Expr)] + return out + + +def _starry(select: Any) -> bool: + from sqlglot import exp + + return any(isinstance(p, exp.Star) or (isinstance(p, exp.Column) and p.is_star) for p in select.expressions) + + +def _alias_or_column(tree: Any, read: str | None, schema: dict[str, dict[str, str]] | None) -> bool: + """有不带表名、又和这一段的列别名同名的名字,qualify 会把它当成别名展开,库里认的却可能是表的列,这一遍不认列: + + - WHERE、GROUP BY、HAVING 里:库里认的是列还是别名各方言不一样(SQLite 的 GROUP BY、HAVING 都先认列:HAVING + amount > 10 里的 amount 是组里任意一行的 amount,不是 SUM(amount) AS amount),qualify 却一律当成别名(GROUP BY + created_at 成了按 DATE(created_at) 分组)。表结构里有这一段的每张表、这个名字又不是其中哪张表的列时,它只能是 + 别名,照常认。 + - SELECT 列表里引用前面列的别名:SQLite、PostgreSQL、MySQL 都不认,名字指的是表的列(SELECT user_id AS amount, + amount * 2 里的 amount 是表的 amount 列),不是表的列就报错;qualify 没有表结构时却当成别名展开成 user_id * 2, + 有表结构时不是表的列的也展开。表结构里有这一段的每张表、这个名字又是其中的列时,qualify 先认列,照常认。""" + from sqlglot import exp + + columns = {name.lower(): {c.lower() for c in cols} for name, cols in (schema or {}).items()} + for select in tree.find_all(exp.Select): + aliases = {p.alias.lower() for p in select.expressions if isinstance(p, exp.Alias) and p.alias} + if not aliases: + continue + items = _items_of(select) + known = all(isinstance(i, exp.Table) and i.name.lower() in columns for i in items) + real = set().union(*(columns[i.name.lower()] for i in items)) if known and items else set() + for key in ("where", "group", "having"): + node = select.args.get(key) + if node is None: + continue + names = {c.name.lower() for c in node.find_all(exp.Column) if not c.table and not _nested(c, node)} + if (names & aliases) - (set() if not known else (aliases - real)): + return True + earlier: set[str] = set() + for projection in select.expressions: + names = {c.name.lower() for c in projection.find_all(exp.Column) + if not c.table and (c is projection or not _nested(c, projection))} + if (names & earlier) - real: + return True + if isinstance(projection, exp.Alias) and projection.alias: + earlier.add(projection.alias.lower()) + return False + + +# ---- 起规范的名字 + +@dataclass +class _Hint: + """按另一边起名(差异说明用):定义一样的 CTE 叫另一边的名字;names 是这一边原来的 CTE 名 → 另一边的名字;taken + 是另一边用过的名字,配不上的另起名字时避开。""" + + bodies: dict[str, str] = field(default_factory=dict) + names: dict[str, str] = field(default_factory=dict) + taken: set[str] = field(default_factory=set) + + +class _Namer: + """给一条语句起规范的名字(见 _finish 第 3 步)。 + + 先按原来的别名认清每一列属于哪个来源(见 _Resolver);之后改来源的别名时把这些列一起改掉,不再按名字找,换名不会 + 把列认错。别名带层数:内外层的别名不会同名,相关子查询里引用外层的列不会被里层遮住;同一层的兄弟子查询各自看不见 + 对方,可以重名。""" + + def __init__(self, tree: Any, read: str | None, qualified: bool, sens: _Sensitivity, + hint: _Hint | None = None) -> None: + from sqlglot import exp + + self.tree = tree + self.read = read + self.qualified = qualified + self.sens = sens + self.hint = hint + resolver = _Resolver(tree) + self.refs = resolver.refs + self.cte_of = resolver.cte_of + self.base_names = {t.name.lower() for t in tree.find_all(exp.Table) if t.name} + self.budget = _ORDERING_BUDGET + self.starred = self._starred_sources(tree) + + def _starred_sources(self, tree: Any) -> set[int]: + """被 SELECT * 读着的来源(* 没展开):它选出来的列名、先后会原样传到外面,不能改名、重排。""" + from sqlglot import exp + + out: set[int] = set() + for select in tree.find_all(exp.Select): + for p in select.expressions: + if isinstance(p, exp.Star): + out |= {id(i) for i in _items_of(select)} + elif isinstance(p, exp.Column) and p.is_star: + out |= {id(i) for i in _items_of(select) if i.alias_or_name == p.table} + return out + + def _text(self, node: Any) -> tuple[str, str]: + return _key(node, self.read) + + def _alias(self, base: str, depth: int, k: int) -> str: + name = base if depth == 0 else f"{base}@{depth}" + if k > 1: + name = f"{name}#{k}" + while (depth or k > 1) and name.lower() in self.base_names: + name += "#" + return name + + def canon(self, query: Any, depth: int, cte_names: dict[int, str]) -> tuple[str, str]: + from sqlglot import exp + + if isinstance(query, exp.Subquery): + return self.canon(query.this, depth, cte_names) + with_ = query.args.get("with_") + if with_ is not None: + cte_names = dict(cte_names) + self._name_ctes(with_, depth, cte_names) + if isinstance(query, exp.SetOperation): + self._canon_setop(query, depth, cte_names) + return self._text(query) + return self._canon_select(query, depth, cte_names) + + def _name_ctes(self, with_: Any, depth: int, cte_names: dict[int, str]) -> None: + """CTE 按依赖分层,同一层按规范化后的定义排序起名;定义一模一样的合成一个(引用它们的都改叫同一个名字)。""" + from sqlglot import exp + + hint = self.hint if depth == 0 else None + pending = list(with_.expressions) + kept: list[Any] = [] + by_text: dict[tuple[str, str], str] = {} + prefix = "cte" if depth == 0 else f"cte@{depth}_" + n = 0 + while pending: + names = {c.alias_or_name for c in pending} + ready = [c for c in pending if not any(t.name in names and t.name != c.alias_or_name and not t.args.get("db") + for t in c.this.find_all(exp.Table))] + if not ready: + raise _Unsupported("CTE 互相引用") + texts = [] + for cte in ready: + original = cte.alias_or_name + body = self.canon(cte.this, depth + 1 if depth else 0, cte_names) + refs = [t for t in self.tree.find_all(exp.Table) if self.cte_of.get(id(t)) is cte] + if self._rename_outputs(cte.this, refs): + body = self._text(cte.this) + texts.append((body, original, cte)) + texts.sort(key=lambda x: x[0]) + for body, original, cte in texts: + pending.remove(cte) + if body in by_text: + cte_names[id(cte)] = by_text[body] + continue + name = (hint.names.get(original) or hint.bodies.get(body[0])) if hint else None + while name is None or name.lower() in self.base_names or name in by_text.values(): + n += 1 + name = f"{prefix}{n}" + if hint and name in hint.taken: + name = None + cte_names[id(cte)] = name + by_text[body] = name + cte.set("alias", exp.TableAlias(this=exp.to_identifier(name))) + kept.append(cte) + with_.set("expressions", kept) + + def _rename_outputs(self, body: Any, holders: list[Any]) -> bool: + """(认列之后)CTE、FROM 里的子查询选出来的列改叫 c1、c2…,引用它们的列跟着改:外面按名字引用,名字是各自起的 + (total 还是 sum_amt)。没有被 SELECT * 原样读着(* 没展开时列名、先后会传到外面)的,先按定义排序:外面按名字 + 引用,先后不影响。UNION、选了 *、列名重复、本段里还按名字引用自己的列、GROUP BY / ORDER BY 按位置引用的不改。""" + from sqlglot import exp + + if not self.qualified or not isinstance(body, exp.Select) or any(id(h) in self.starred for h in holders): + return False + projections = body.expressions + names = [p.alias_or_name for p in projections] + if not projections or any(not n or _is_star(p) for n, p in zip(names, projections)) or \ + len(set(names)) != len(names): + return False + inside = {id(n) for p in projections for n in p.walk()} + if {n.name for n in _block_nodes(body) if isinstance(n, exp.Column) and not n.table and id(n) not in inside} & \ + set(names): + return False + for clause in ("group", "order"): + node = body.args.get(clause) + keys = [k.this if isinstance(k, exp.Ordered) else k for k in (node.expressions if node is not None else [])] + if any(k.is_number for k in keys): + return False + ordered = sorted(projections, key=lambda p: self._text(p.unalias())) + mapping = {p.alias_or_name: f"c{i}" for i, p in enumerate(ordered, 1)} + body.set("expressions", [exp.alias_(p.unalias().copy(), mapping[p.alias_or_name], copy=False) + for p in ordered]) + for holder in holders: + for col in self.refs.get(id(holder), []): + if col.name in mapping: + col.set("this", exp.to_identifier(mapping[col.name])) + return True + + def _canon_setop(self, query: Any, depth: int, cte_names: dict[int, str]) -> None: + """连着同一种合并写的几支各自规范化;调换先后不改变结果的(见 _reorderable)再按规范化后的写法排序。""" + kind = (type(query), bool(query.args.get("distinct"))) + branches: list[Any] = [] + + def collect(node: Any) -> None: + from sqlglot import exp + + if isinstance(node, exp.SetOperation) and (type(node), bool(node.args.get("distinct"))) == kind and \ + not any(node.args.get(k) for k in ("order", "limit", "offset", "with_")): + collect(node.left) + collect(node.right) + else: + branches.append(node) + + collect(query.left) + collect(query.right) + texts = [(self.canon(b, depth, cte_names), i, b) for i, b in enumerate(branches)] + if len(branches) < 2 or not self._reorderable(query): + return + ordered = [b for _, _, b in sorted(texts, key=lambda x: (x[0], x[1]))] + node = ordered[0] + for b in ordered[1:-1]: + node = type(query)(this=node, expression=b, distinct=query.args.get("distinct")) + query.set("this", node) + query.set("expression", ordered[-1]) + + def _reorderable(self, query: Any) -> bool: + """这一处合并的各支调换先后,结果(行、值)一样不一样。只认整条语句最外层、不排序不取行的 UNION ALL: + + - UNION(去重)、INTERSECT 留先出现的那个值,相等不等于一样(SQLite 里 REAL 列的 2.0 和 INTEGER 列的 2、MySQL + 默认不分大小写的 'SH' 和 'sh');EXCEPT 本来就分左右。 + - 带 LIMIT / OFFSET 的取的是先出来的几行;ORDER BY 按名字排的看第一支的列名;SQLite 给合并的结果排序时用最左 + 一支那列声明的排序规则。PostgreSQL、MySQL 按位置排序(ORDER BY 1)的可以排。 + - 套在 CTE、子查询里的:SQLite 从子查询读合并的结果时按第一支的类型亲和、排序规则转换别的支,不排;别的方言 + 只在各支选出来的列名、先后都一样(外面按列名引用,列名取第一支的)、外面不看行出来的先后(不去重、不取行, + 见 _Sensitivity)时排。""" + from sqlglot import exp + + if not isinstance(query, exp.Union) or query.args.get("distinct"): + return False + if any(query.args.get(k) is not None for k in ("limit", "offset", "fetch")): + return False + sqlite = self.read in (None, "sqlite") + order = query.args.get("order") + if order is not None and (sqlite or any(not (isinstance(o.this, exp.Literal) and o.this.is_int) + for o in order.expressions)): + return False + top = query + while isinstance(top.parent, exp.Subquery): + top = top.parent + if top is self.tree: + return True + if sqlite or order is not None: + return False + leaves = _leaves(query) + if any(not isinstance(leaf, exp.Select) or self.sens.of(leaf) or _starry(leaf) for leaf in leaves): + return False + names = [[p.alias_or_name for p in leaf.expressions] for leaf in leaves] + return all(n == names[0] for n in names) and all(names[0]) + + def _signature(self, item: Any, cte_names: dict[int, str], subs: dict[int, tuple[str, str]]) -> tuple[str, str]: + """来源的 (排序用的标识, 起别名用的名字):实表是表名,CTE 是它的规范名,FROM 里的子查询是规范化后的写法。""" + from sqlglot import exp + + if isinstance(item, exp.Table): + cte = self.cte_of.get(id(item)) + if cte is not None: + name = cte_names.get(id(cte), cte.alias_or_name) + return f"cte:{name}", name + return "table:" + ".".join(p for p in (item.catalog, item.db, item.name) if p), item.name + return f"sub:{subs[id(item)][0]}", "sub" + + def _canon_select(self, select: Any, depth: int, cte_names: dict[int, str]) -> tuple[str, str]: + from sqlglot import exp + + items = _items_of(select) + subs: dict[int, tuple[str, str]] = {} + for item in items: + if isinstance(item, exp.Subquery): + subs[id(item)] = self.canon(item.this, depth + 1, cte_names) + if self._rename_outputs(item.this, [item]): + subs[id(item)] = self._text(item.this) + elif id(item) in self.cte_of: + cte = self.cte_of[id(item)] + item.set("this", exp.to_identifier(cte_names.get(id(cte), cte.alias_or_name))) + keys = {id(i): self._signature(i, cte_names, subs) for i in items} + joins = select.args.get("joins") or [] + n_prefix = 0 + if items: + n_prefix = 1 + for join in joins: + if _comma(join): + n_prefix += 1 + else: + break + prefix, rest = items[:n_prefix], joins[max(n_prefix - 1, 0):] if items else joins + # 后面有 RIGHT / FULL JOIN 时开头那组来源不调换:PostgreSQL、MySQL 里 a, b RIGHT JOIN c 是 a 乘上 (b RIGHT JOIN c), + # 换成 b, a 就成了 b 乘上 (a RIGHT JOIN c) + movable = not _bare_star(select) and not self.sens.of(select) and \ + not any(str(j.side or "").upper() in ("RIGHT", "FULL") for j in joins) + free_lefts = movable and bool(rest) and self._independent_lefts(prefix, rest) + + def groups(xs: list[Any], key: Any) -> list[list[Any]]: + xs = sorted(xs, key=key) + return [list(g) for _, g in itertools.groupby(xs, key=key)] + + prefix_groups = groups(prefix, lambda s: keys[id(s)][0]) if movable else [[s] for s in prefix] + rest_groups = groups(rest, lambda j: keys[id(j.this)][0]) if free_lefts else [[j] for j in rest] + total = 1 + for g in prefix_groups + rest_groups: + total *= math.factorial(len(g)) + if total > _MAX_ORDERINGS: + break + if total > _MAX_ORDERINGS or total > self.budget: + # 排列太多(十几个同名来源):先算个数再展开,超了就只按排好的先后 + options = [[g] for g in prefix_groups + rest_groups] + total = 1 + else: + options = [[list(p) for p in itertools.permutations(g)] if len(g) > 1 else [g] + for g in prefix_groups + rest_groups] + self.budget -= total + combos = list(itertools.product(*options)) + best: tuple[tuple[str, str], int] | None = None + for index, combo in enumerate(combos): + text = self._apply(select, depth, cte_names, keys, combo, len(prefix_groups)) + if best is None or text < best[0]: + best = (text, index) + if best is None: + return self._text(select) + if len(combos) > 1: + self._apply(select, depth, cte_names, keys, combos[best[1]], len(prefix_groups)) + return best[0] + + @staticmethod + def _independent_lefts(prefix: list[Any], rest: list[Any]) -> bool: + """跟在一组内连接后面的几个 LEFT JOIN,各自的 ON 只引用这组来源和自己(不引用别的 LEFT JOIN 进来的表、没有 + 子查询):它们之间谁先谁后结果一样。""" + from sqlglot import exp + + names = {s.alias_or_name for s in prefix} + for join in rest: + if str(join.side or "").upper() != "LEFT" or not _plain_join(join): + return False + on = join.args.get("on") + if on is None or on.find(exp.Select, exp.Subquery) is not None: + return False + if not {c.table for c in on.find_all(exp.Column)} <= names | {join.this.alias_or_name}: + return False + return True + + def _apply(self, select: Any, depth: int, cte_names: dict[int, str], keys: dict[int, tuple[str, str]], + combo: tuple[list[Any], ...], n_prefix_groups: int) -> tuple[str, str]: + from sqlglot import exp + + prefix = [s for g in combo[:n_prefix_groups] for s in g] + rest = [j for g in combo[n_prefix_groups:] for j in g] + if prefix: + select.args["from_"].set("this", prefix[0]) + select.set("joins", [exp.Join(this=s) for s in prefix[1:]] + rest or None) + counts: dict[str, int] = {} + for item in _items_of(select): + _, base = keys[id(item)] + counts[base] = counts.get(base, 0) + 1 + name = self._alias(base, depth, counts[base]) + item.set("alias", exp.TableAlias(this=exp.to_identifier(name))) + for col in self.refs.get(id(item), []): + col.set("table", exp.to_identifier(name)) + for sub in _nested_queries(select): + self.canon(sub, depth + 1, cte_names) + _sort_own(select, self.read, self.sens.of(select)) + return self._text(select) + + +def _fixed(node: Any) -> bool: + """不涉及任何列、子查询的表达式(常量、常量算出来的日期):排比较、加乘的两边时放在右边,好读。""" + from sqlglot import exp + + return node.find(exp.Column, exp.Select, exp.Subquery) is None + + +def _sort_own(select: Any, read: str | None, sensitive: bool = False) -> None: + """这一段自己的表达式里交换律成立的地方排好(里层先排):AND、OR(各项都是条件时去掉一模一样的项:x AND x 在 + SELECT 列表里不是 x)、比较(不涉及列的一边在右,两边都涉及列时排好、大于小于跟着换方向;两边都写了 COLLATE 的 + 不换,SQLite 用左边那个)、两项的加和乘(只换这一个节点的两边,不跨项挪动:浮点数加法不满足结合律)、IN 列表、 + GROUP BY(结果看行出来的先后时不排:SQLite 按分组键排好了出行)。""" + from sqlglot import exp + + def key(node: Any) -> tuple[str, str]: + return _key(node, read) + + def depth(node: Any) -> int: + d = 0 + while node is not select and node.parent is not None: + node = node.parent + d += 1 + return d + + for node in sorted(_block_nodes(select), key=depth, reverse=True): + if node is not select and node.parent is None: + continue + if isinstance(node, (exp.And, exp.Or)): + if isinstance(_unparen_parent(node), type(node)): + continue + parts = _flat(node, type(node)) + if all(_predicate(p) for p in parts): + uniq: dict[tuple[str, str], Any] = {} + for part in parts: + uniq.setdefault(key(part), part) + ordered = [uniq[k] for k in sorted(uniq)] + else: + ordered = sorted(parts, key=key) + new = ordered[0] if len(ordered) == 1 else ( + exp.and_(*ordered, copy=False) if isinstance(node, exp.And) else exp.or_(*ordered, copy=False)) + node.replace(new) + elif type(node).__name__ in _FLIP: + left, right = node.this, node.expression + if isinstance(left, (exp.Any, exp.All)) or isinstance(right, (exp.Any, exp.All)): + continue + if left.find(exp.Collate) is not None and right.find(exp.Collate) is not None: + continue + if (_fixed(left) and not _fixed(right)) or (_fixed(left) == _fixed(right) and key(left) > key(right)): + node.replace(getattr(exp, _FLIP[type(node).__name__])(this=right, expression=left)) + elif isinstance(node, (exp.Add, exp.Mul)): + left, right = node.this, node.expression + if (_fixed(left) and not _fixed(right)) or (_fixed(left) == _fixed(right) and key(left) > key(right)): + node.set("this", right) + node.set("expression", left) + elif isinstance(node, exp.Group): + if node.expressions and not sensitive: + node.set("expressions", sorted(node.expressions, key=key)) + elif isinstance(node, exp.In) and node.expressions: + uniq = {key(i): i for i in node.expressions} + node.set("expressions", [uniq[k] for k in sorted(uniq)]) + + +def _flat(node: Any, cls: type) -> list[Any]: + """AND / OR 连着的各项(穿过括号:BETWEEN 改写出来的 (x >= a AND x <= b) 也拆开)。""" + out: list[Any] = [] + stack = [node] + while stack: + n = stack.pop() + inner = _unparen(n) + if isinstance(inner, cls): + stack += [inner.expression, inner.this] + else: + out.append(n) + return out + + +def _unparen_parent(node: Any) -> Any: + from sqlglot import exp + + parent = node.parent + while isinstance(parent, exp.Paren): + parent = parent.parent + return parent + + +# ========================================================================== +# 差异说明 # ========================================================================== @dataclass(frozen=True) class FacetDiff: - """一处不同。aspect:time(时间列)/ time_range(同一时间列、范围不同)/ filter / aggregate / join / tables / - group_by / having / limit(取哪几行:行数、跳过的行数、取前几行时的排序)/ parse。a、b 是两边各自的写法(给人看, - 没有就是 None),text 是一句中文说明。 - detail 只在 time 上有:两边的时间列 [{table, column}],调用方据此对照数据目录(业务日期)。""" + """一处不同。aspect:time(时间列)/ time_range(同一时间列、范围不同)/ filter(筛选条件)/ aggregate(聚合函数、 + 去重计数)/ expression(选出来的列)/ join(关联的表、关联类型、关联条件、外连接的主表)/ tables(查的是哪张表)/ + set_op(几支查询怎么合并)/ group_by(分组键、去重)/ having / limit(取哪几行,含取行前的排序)/ order(只是最终 + 结果的排序不同,不影响取到哪些行)/ parse(解析不了)/ other(兜底:未能归纳成具体差异,见 explain_sql_diff)。a、b + 是两边各自的写法(给人看,没有就是 None),text 是一句中文说明。detail 只在 time 上有:两边的时间列 + [{table, column}](各是各的表),调用方据此对照数据目录(业务日期);在 filter 上有 {"columns": [{table, column}]}: + 条件只涉及的实表列,对照有效行过滤用。""" aspect: str a: Any @@ -1175,13 +4566,43 @@ class FacetDiff: return out +@dataclass(frozen=True) +class SqlDiff: + """两条 SQL 差在哪(explain_sql_diff 的结果)。 + + equivalent:两边经过只保持语义的变换之后,规范形式一字不差(level 是哪一级的规范形式,见 _canonical_forms)。 + items:具体差异,等价时为空;不等价时一定不为空——归纳不出具体差异时是一条 aspect="other" 的兜底说明。 + fallback:用了兜底说明(一条具体说法都没有,或者具体说法只覆盖了一部分)。 + text_diff:规范化之后两条 SQL 不同的行(difflib 比两边 sqlglot pretty 的输出,「- 」开头是 A 的、「+ 」开头是 + B 的,最多 40 行),给界面高亮;等价时为空。""" + + equivalent: bool + items: list[FacetDiff] = field(default_factory=list) + fallback: bool = False + text_diff: tuple[str, ...] = () + level: str | None = None + + def as_dict(self) -> dict[str, Any]: + return {"equivalent": self.equivalent, "items": [d.as_dict() for d in self.items], "fallback": self.fallback, + "text_diff": list(self.text_diff), "level": self.level} + + +#: 兜底说明 +_FALLBACK = "两条 SQL 的写法不同,未能归纳成具体差异,请对照原文" +_FALLBACK_REST = "另有未能归纳的差异,请对照原文" +#: text_diff 最多列几行 +_MAX_DIFF_LINES = 40 + _CJK_PUNCT = frozenset(",。;:、!?「」『』()《》【】…—") #: 聚合函数的说法 _VERBS = {"sum": "求和", "count": "计数", "avg": "求平均", "min": "取最小值", "max": "取最大值"} +#: 只取日期的字符串(2026-09-01):PostgreSQL、MySQL 里拿它和时间列比,按字符串比的先后和按日期比的一样 +_DATE_ONLY = re.compile(r"^\d{4}-\d{2}-\d{2}$") +_COMPARE_OPS = ("EQ", "NEQ", "GT", "GTE", "LT", "LTE") def _wide(ch: str) -> bool: - return "⺀" <= ch <= "鿿" or "豈" <= ch <= "﫿" or "＀" <= ch <= "￯" + return "⺀" <= ch <= "鿿" or "豈" <= ch <= "﫿" or "＀" <= ch <= "￯" def _say(*parts: str) -> str: @@ -1202,8 +4623,8 @@ def _say(*parts: str) -> str: def catalog_labels(notes: Mapping[str, Mapping[str, Any]] | None) -> dict[str, str]: - """数据目录({表名: notes})里列的中文名 → diff_facets 的 names:{「表.列」: 中文名};只在一张表里有、或者各表 - 叫法一致的列另记 {列: 中文名}。被驳回的项不算。""" + """数据目录({表名: notes})里列的中文名 → diff_facets / explain_sql_diff 的 names:{「表.列」: 中文名};只在一张 + 表里有、或者各表叫法一致的列另记 {列: 中文名}。被驳回的项不算。""" out: dict[str, str] = {} bare: dict[str, tuple[str, set[str]]] = {} for table, raw in (notes or {}).items(): @@ -1231,16 +4652,1601 @@ def _labeler(names: Mapping[str, str] | None) -> Any: return label -def _single_column(part: Part, *, loose: bool) -> str | None: - """只涉及一列的条件:那一列(Part.column_keys:实表.列,CTE、子查询的列写「@.列」,两边起的名字不同、编号错开 - 也配得上;loose 时只看列名),用来把两边对同一列的不同写法配成一对。""" - if len(part.columns) != 1 or len(part.column_keys) != 1: +# ---- 同一个对象上的条件:留下的取值一样不一样 + +def _literal_value(node: Any) -> tuple[str, Any] | None: + """常量的值:("n", Decimal) 或 ("s", 字符串);不是数字、字符串常量返回 None。""" + from decimal import Decimal, InvalidOperation + + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, exp.Neg): + inner = _literal_value(node.this) + return ("n", -inner[1]) if inner is not None and inner[0] == "n" else None + if isinstance(node, exp.Literal): + if node.is_string: + return "s", str(node.this) + try: + return "n", Decimal(str(node.this)) + except (InvalidOperation, ValueError): + return None + return None + + +class _Range: + """一个对象(一列、一个聚合、一个函数)上只和常量比的条件(=、<>、<、<=、>、>=、IN 常量列表、IS [NOT] NULL、 + BETWEEN,用 AND / OR / NOT 连起来),在对象的各段取值上成立不成立:用到的常量把取值切成一段段(小于最小的、 + 每个常量本身、相邻两个之间、大于最大的),每段里条件的真假都一样,各取一个代表算一遍,再算对象是空值时。两组 + 条件在每一段、空值上都同真同不真,留下的行就一样。 + + 字符串的大小先后:SQLite 按默认的 BINARY 排序规则;PostgreSQL(语言排序规则下 'a' < 'B')、MySQL(默认不分 + 大小写)只认等不等,除非常量都是只取日期的字符串(2026-09-01,按字符串比和按日期比先后一样)。MySQL 的字符串 + 等不等不分大小写。COUNT 的取值是整数(分组时至少是 1)。""" + + def __init__(self, read: str | None) -> None: + self.read = read + + def subject(self, cond: Any) -> tuple[str, Any, list[tuple[str, Any]], bool] | None: + """cond 只是同一个对象和常量比时,返回 (对象的指纹, 对象, 常量, 有没有比大小);不然 None。""" + from sqlglot import exp + + subjects: dict[str, Any] = {} + literals: list[tuple[str, Any]] = [] + ranged = [False] + + def side(subj: Any, lits: list[Any]) -> bool: + if subj is None or _literal_value(subj) is not None or isinstance(_unparen(subj), exp.Null) or \ + subj.find(exp.Query, exp.Subquery) is not None or _volatile(subj): + return False + subjects[_fingerprint(subj)] = subj + literals.extend(lits) + return True + + def walk(n: Any) -> bool: + n = _unparen(n) + if isinstance(n, (exp.And, exp.Or)): + return walk(n.left) and walk(n.right) + if isinstance(n, exp.Not): + return walk(n.this) + name = type(n).__name__ + if name in _COMPARE_OPS: + lv, rv = _literal_value(n.expression), _literal_value(n.this) + if name not in ("EQ", "NEQ"): + ranged[0] = True + if lv is not None and rv is None: + return side(n.this, [lv]) + if rv is not None and lv is None: + return side(n.expression, [rv]) + return False + if isinstance(n, exp.In): + if n.args.get("query") or n.args.get("unnest") or n.args.get("field") or not n.expressions: + return False + lits = [] + for item in n.expressions: + if isinstance(_unparen(item), exp.Null): + continue + value = _literal_value(item) + if value is None: + return False + lits.append(value) + return side(n.this, lits) + if isinstance(n, exp.Is): + return isinstance(_unparen(n.expression), exp.Null) and side(n.this, []) + if n.args.get("negate"): + return False + if isinstance(n, exp.Between): + low, high = _literal_value(n.args.get("low")), _literal_value(n.args.get("high")) + ranged[0] = True + return not n.args.get("symmetric") and low is not None and high is not None and \ + side(n.this, [low, high]) + return False + + if not walk(cond) or len(subjects) != 1: + return None + kinds = {k for k, _ in literals} + if len(kinds) > 1: + return None + if ranged[0] and kinds == {"s"} and self.read != "sqlite" and \ + not all(_DATE_ONLY.match(v) for _, v in literals): + return None + fp, subj = next(iter(subjects.items())) + return fp, subj, literals, ranged[0] + + def _norm(self, value: tuple[str, Any]) -> Any: + kind, v = value + if kind == "s" and self.read == "mysql": + return v.casefold() + return v + + def _pieces(self, subject: Any, literals: list[tuple[str, Any]], ranged: bool, grouped: bool) -> list[Any]: + """代表各段取值的值(不含空值)。""" + from decimal import Decimal + + from sqlglot import exp + + values = sorted({self._norm(v) for v in literals}) + kind = literals[0][0] if literals else "n" + if not values: + return [Decimal(0)] if kind == "n" else ["x"] + if not ranged: + other = values[-1] + 1 if kind == "n" else values[-1] + "\x00~" + return values + [other] + out: list[Any] = [] + if kind == "n": + integral = isinstance(_unparen(subject), exp.Count) + low = Decimal(1) if integral and grouped and isinstance(_unparen(subject).this, exp.Star) else ( + Decimal(0) if integral else None) + candidates = [values[0] - 1] + [x for i, v in enumerate(values) + for x in ([v] + ([(v + values[i + 1]) / 2] if i + 1 < len(values) else []))] + candidates.append(values[-1] + 1) + if integral: + ints: set[Decimal] = set() + for i, v in enumerate(values): + ints |= {v.to_integral_value(rounding="ROUND_FLOOR") + d for d in (-1, 0, 1)} + ints |= {v.to_integral_value(rounding="ROUND_CEILING") + d for d in (-1, 0, 1)} + candidates = sorted(ints) + for c in candidates: + if low is not None and c < low: + continue + out.append(c) + return out or [low or Decimal(0)] + if values[0] != "": + out.append("") + for i, v in enumerate(values): + out.append(v) + between = v + "\x00" + if i + 1 >= len(values) or between < values[i + 1]: + out.append(between) + return out + + def _truth(self, n: Any, v: Any) -> str: + """n 在对象取 v(None 是空值)时是真(T)、假(F)还是空值(N)。""" + from sqlglot import exp + + n = _unparen(n) + if isinstance(n, exp.And): + a, b = self._truth(n.left, v), self._truth(n.right, v) + return "F" if "F" in (a, b) else ("N" if "N" in (a, b) else "T") + if isinstance(n, exp.Or): + a, b = self._truth(n.left, v), self._truth(n.right, v) + return "T" if "T" in (a, b) else ("N" if "N" in (a, b) else "F") + if isinstance(n, exp.Not): + t = self._truth(n.this, v) + return {"T": "F", "F": "T"}.get(t, "N") + if isinstance(n, exp.Is): + hit = v is None + return "T" if hit != bool(n.args.get("negate")) else "F" + if v is None: + return "N" + name = type(n).__name__ + if name in _COMPARE_OPS: + right = _literal_value(n.expression) + if right is not None: + lit, op = self._norm(right), name + else: + lit, op = self._norm(_literal_value(n.this)), _FLIP[name] + if type(v) is not type(lit): + return "N" + result = {"EQ": v == lit, "NEQ": v != lit, "GT": v > lit, "GTE": v >= lit, "LT": v < lit, + "LTE": v <= lit}[op] + return "T" if result else "F" + if isinstance(n, exp.In): + hit = any(not isinstance(_unparen(i), exp.Null) and self._norm(_literal_value(i)) == v + for i in n.expressions) + if hit: + return "T" + return "N" if any(isinstance(_unparen(i), exp.Null) for i in n.expressions) else "F" + if isinstance(n, exp.Between): + low, high = self._norm(_literal_value(n.args["low"])), self._norm(_literal_value(n.args["high"])) + return "T" if low <= v <= high else "F" + return "N" + + def compare(self, conds_a: list[Any], conds_b: list[Any], grouped: bool = False) -> bool | None: + """两组条件(各自 AND 连,可以一边为空 = 不筛)留下的取值是不是一样:一样 True,不一样 False,认不出 None。""" + parsed = [self.subject(c) for c in conds_a + conds_b] + if any(p is None for p in parsed) or len({p[0] for p in parsed}) != 1: + return None + subject = parsed[0][1] + literals = [lit for p in parsed for lit in p[2]] + if len({k for k, _ in literals}) > 1: + return None + ranged = any(p[3] for p in parsed) + from sqlglot import exp + + values = self._pieces(subject, literals, ranged, grouped) + if not isinstance(_unparen(subject), exp.Count): + values.append(None) + for v in values: + ta = all(self._truth(c, v) == "T" for c in conds_a) + tb = all(self._truth(c, v) == "T" for c in conds_b) + if ta != tb: + return False + return True + + +# ---- 一边用到了哪些列(判断「另一边没有」能不能说) + +class _Usage: + """一条语句里每一列(按定义找到实表列,认不出来源的只记列名)用在了哪儿: + + - cond:WHERE、ON、HAVING 里,条件里的子查询(IN、EXISTS、标量)选出来的列; + - case:CASE / IF / FILTER 里(SUM(CASE WHEN status = 'paid' THEN amount END) 等于先按 status 筛再求和); + - agg:聚合函数、窗口函数的参数(COUNT(x)、SUM(x) 跳过空值,等于先筛掉 x 为空的行)。 + + 光选出来、分组、排序的列不记。说「另一边没有这个条件」「另一边没有按时间筛选」之前,先看另一边在这一段能影响到的 + 地方(整条语句,除了合并里别的支)用没用到这些列。""" + + def __init__(self, tree: Any) -> None: + from sqlglot import exp + + self.nodes: dict[tuple[str | None, str], dict[int, str]] = {} + self.temporal = False + try: + self.resolver: _Resolver | None = _Resolver(tree) + except _Unsupported: + self.resolver = None + for col in tree.find_all(exp.Column): + if col.is_star or not col.name: + continue + kind = self._kind(col) + if kind is None: + continue + bases = self._bases(col, 0) + if isinstance(col.this, exp.Identifier) and col.this.meta.get(_UNSURE): + bases.append((None, col.name)) + for table, column in bases: + self.nodes.setdefault((table, column), {})[id(col)] = kind + if kind != "agg" and _temporal_name(column): + self.temporal = True + for node in tree.find_all(exp.Where, exp.Having, exp.Join): + target = node.args.get("on") if isinstance(node, exp.Join) else node.this + if target is not None and _datey(target): + self.temporal = True + + @staticmethod + def _owner(node: Any) -> tuple[Any, str | None, bool, bool]: + """node 属于哪一段 SELECT、在它的哪个子句里、路上有没有 CASE / IF / FILTER、有没有聚合 / 窗口函数。""" + from sqlglot import exp + + case = agg = False + child, parent = node, node.parent + while parent is not None: + if isinstance(parent, (exp.Case, exp.If, exp.Filter)): + case = True + if isinstance(parent, (exp.AggFunc, exp.Window)): + agg = True + if isinstance(parent, exp.Select): + return parent, child.arg_key, case, agg + child, parent = parent, parent.parent + return None, None, case, agg + + def _kind(self, col: Any) -> str | None: + select, key, case, agg = self._owner(col) + if select is None: + return None + if key == "expressions": + if _scalar_context(select) or _condition_query(select): + return "cond" + return "case" if case else ("agg" if agg else None) + if key in ("group", "order", "distinct"): + return "case" if case else None + return "cond" + + def _bases(self, col: Any, depth: int) -> list[tuple[str | None, str]]: + from sqlglot import exp + + if depth > 16 or self.resolver is None: + return [(None, col.name)] + item = self.resolver.source_of.get(id(col)) + if item is None: + select = self._owner(col)[0] + items = _items_of(select) if select is not None else [] + if col.table or len(items) != 1: + return [(None, col.name)] + item = items[0] + return self._from_item(item, col.name, depth) + + def _from_item(self, item: Any, name: str, depth: int) -> list[tuple[str | None, str]]: + from sqlglot import exp + + body = self.resolver.body(item) if self.resolver is not None else None + if body is None: + return [(item.name, name)] if isinstance(item, exp.Table) else [(None, name)] + leaves = _leaves(body) + if not leaves or not isinstance(leaves[0], exp.Select) or depth > 16: + return [(None, name)] + names = [p.alias_or_name for p in leaves[0].expressions] + if name not in names: + sources = _items_of(leaves[0]) + if len(leaves) == 1 and any(isinstance(p, exp.Star) for p in leaves[0].expressions) and len(sources) == 1: + return self._from_item(sources[0], name, depth + 1) + return [(None, name)] + index = names.index(name) + out: list[tuple[str | None, str]] = [] + for leaf in leaves: + if not isinstance(leaf, exp.Select) or index >= len(leaf.expressions): + return [(None, name)] + expr = leaf.expressions[index].unalias() + for c in ([expr] if isinstance(expr, exp.Column) else list(expr.find_all(exp.Column))): + if not c.is_star: + out += self._bases(c, depth + 1) + return out or [(None, name)] + + def touches(self, refs: Iterable[tuple[str, str]], skip: Iterable[Any] = (), + kinds: tuple[str, ...] = ("cond", "case"), scope: Any = None) -> bool: + """这些实表列用在了 kinds 里(认不出来源的同名列也算);skip 里的条件中的那几处不算,scope 所在合并的别的支 + (不碰 scope 的行)不算。""" + from sqlglot import exp + + skipped = {id(c) for cond in skip for c in cond.find_all(exp.Column)} + node = scope + while node is not None and node.parent is not None: + parent = node.parent + if isinstance(parent, exp.SetOperation): + for branch in (parent.this, parent.expression): + if branch is not node: + skipped |= {id(c) for c in branch.find_all(exp.Column)} + node = parent + for table, column in refs: + for key in ((table, column), (None, column)): + if any(i not in skipped and k in kinds for i, k in self.nodes.get(key, {}).items()): + return True + return False + + +def _kinds_for(conds: list[Any]) -> tuple[str, ...]: + """说某个条件只在一边有之前,要看另一边哪些地方用到了这些列:条件里有判空(IS NULL / IS NOT NULL)的连聚合的 + 参数也看(COUNT(x) 本来就跳过 x 为空的行)。""" + from sqlglot import exp + + if any(isinstance(n, exp.Is) for c in conds for n in c.walk()): + return "cond", "case", "agg" + return "cond", "case" + + +def _condition_query(select: Any) -> bool: + """这段查询是条件里的子查询(IN、EXISTS、ANY / ALL):选出来的列参与比较。""" + from sqlglot import exp + + node = select + while isinstance(node.parent, (exp.Subquery, exp.SetOperation)): + node = node.parent + parent = node.parent + return isinstance(parent, (exp.Exists, exp.Any, exp.All)) or (isinstance(parent, exp.In) and + node.arg_key == "query") + + +# ---- 一段 SELECT + +@dataclass +class _Rel: + """一段里的一个来源:alias 是规范别名,name 是实表名(CTE 是规范名,子查询是 sub),kind 是 table / cte / derived, + shown 是给人看的名字(实表是表名,CTE、子查询是原来起的名字),rejected 表示写的是 LEFT JOIN、被 WHERE 变成了 + 内连接。""" + + alias: str + name: str + kind: str + shown: str + node: Any + rejected: str | None = None + + @property + def table(self) -> bool: + return self.kind == "table" + + +def _rel_of(item: Any, ctes: Mapping[str, str]) -> _Rel: + from sqlglot import exp + + alias = item.alias_or_name + if isinstance(item, exp.Subquery): + return _Rel(alias, "sub", "derived", item.this.meta.get(_ORIGINAL) or alias, item, item.meta.get(_REJECTED)) + if item.name in ctes: + return _Rel(alias, item.name, "cte", ctes[item.name], item, item.meta.get(_REJECTED)) + return _Rel(alias, item.name, "table", item.name, item, item.meta.get(_REJECTED)) + + +@dataclass +class _Block: + """一段 SELECT 拆开来比:开头那组逗号连接的来源(rels)、后面的连接(joins:LEFT JOIN、没写成逗号连接的内连接, + 按先后)、WHERE、GROUP BY、HAVING 的各项,选出来的列,排序和取行。""" + + select: Any + rels: list[_Rel] + joins: list[Any] + where: list[Any] + group: list[Any] + having: list[Any] + exprs: list[Any] + order: list[Any] + limit: str | None + offset: str | None + distinct: bool + distinct_on: bool + windowed: bool + ctes: dict[str, str] + deduped: bool = False + + @classmethod + def of(cls, select: Any, read: str | None, ctes: Mapping[str, str], root: bool = False) -> "_Block | None": + from sqlglot import exp + + if not isinstance(select, exp.Select) or (select.args.get("with_") is not None and not root): + return None + items = _items_of(select) + if not items or any(not isinstance(i, (exp.Table, exp.Subquery)) for i in items): + return None + joins = select.args.get("joins") or [] + n = 1 + for join in joins: + if _comma(join): + n += 1 + else: + break + where, group, having = select.args.get("where"), select.args.get("group"), select.args.get("having") + order, limit, offset = select.args.get("order"), select.args.get("limit"), select.args.get("offset") + distinct = select.args.get("distinct") + + def number(node: Any) -> str | None: + return node.expression.sql(dialect=read) if node is not None and node.expression is not None else None + + return cls(select, [_rel_of(i, ctes) for i in items[:n]], joins[n - 1:], + _conjuncts(where.this) if where is not None else [], + list(group.expressions) if group is not None else [], + _conjuncts(having.this) if having is not None else [], + list(select.expressions), list(order.expressions) if order is not None else [], + number(limit), number(offset), distinct is not None, + bool(distinct is not None and distinct.args.get("on")), + any(isinstance(n, exp.Window) for n in _block_nodes(select)), dict(ctes), + bool(group is not None and group.meta.get(_FROM_DISTINCT))) + + def all_rels(self) -> list[_Rel]: + return self.rels + [_rel_of(j.this, self.ctes) for j in self.joins] + + def rel(self, alias: str) -> _Rel | None: + for r in self.all_rels(): + if r.alias == alias: + return r return None - return name_key(part.columns[0][1]) if loose else part.column_keys[0] + + def names(self) -> list[str]: + return [r.name for r in self.all_rels()] -def _verb(g: Aggregate) -> str: - return ("去重" if g.distinct else "") + _VERBS.get(g.func, f"用 {g.func.upper()} 汇总") +class _Explainer: + """在两边的规范形式上逐段比,只把有把握的不同写成具体的中文句子,同时记下是不是所有不同都说到了。 + + 有把握的门槛: + - 两段读的来源对得上(一模一样,或者只差一处说得清的:换了一张表、一边多关联了一张表、同一张表的关联类型或 + 关联条件不同、外连接的主表对调)才往下比;换了表的那一边,列照它自己的表说(不借另一边的表名)。 + - 说「X 不一样」之前先确认真的不一样:同一个对象只和常量比的几个条件按取值逐段算一遍(_Range),留下的行一样的 + 不说(IN 和 OR、多写一个 IS NOT NULL、把一段时间拆成两段);关联条件只是列相等的,比相等关系分出的组。认不出 + 的写法不说。 + - 说「另一边没有」之前先确认另一边整条语句(含子查询、CTE、相关子查询里引用外层的列)的筛选里都没用到这些列 + (_Usage);说「没有按时间筛选」之前确认另一边哪儿都没有像时间的列、像日期的常量。 + - 条件里的子查询两边写法除子查询外一模一样(含 NOT、IN 左边的列)才配成一对往里比,说清是哪个子查询。 + - 只是排序不同的说法(order)只在整条语句最外层、没有取行、DISTINCT ON、窗口函数时说;取行前的排序算 limit。 + - LEFT JOIN 对 INNER JOIN 只在整条语句最外层、WHERE 里没用到可空一侧的列、结果不只是可空一侧的列的聚合(那样 + 两种写法结果一样)时说。 + 说的都是「这一段的这一项两边各写的是什么」。两边给人看的写法一样的话不说(读起来自相矛盾),说过的一对反过来再说 + 一遍的也不说。""" + + def __init__(self, read: str | None, labels: tuple[str, str], names: Mapping[str, str] | None, + info: _Schema) -> None: + self.read = read + self.la, self.lb = labels + self.label = _labeler(names) + self.info = info + self.items: list[FacetDiff] = [] + self.ctes: dict[str, str] = {} + self.range = _Range(read) + self.usage_a: _Usage | None = None + self.usage_b: _Usage | None = None + self.roots: set[int] = set() + #: 有一句话没说出口(两边给人看的写法一样,或者和前面的话矛盾):这一处不同就没说到 + self.withheld = False + + def sql(self, node: Any) -> str: + """给人看的写法:按方言生成;生成的写法读回来不是同一棵树时(SQLite 把 CAST(x AS DATE) 生成 DATE(x),两者 + 在 SQLite 里结果不同)改用 sqlglot 的通用写法,免得给人看的和实际的不是一回事。""" + import sqlglot + + text = node.sql(dialect=self.read, comments=False) + try: + if _fingerprint(sqlglot.parse_one(text, read=self.read)) == _fingerprint(node): + return text + except Exception: # noqa: BLE001 - 片段单独读不回来:用通用写法 + pass + return node.sql(comments=False) + + @staticmethod + def fp(node: Any) -> str: + return _fingerprint(node) + + # ---- 入口:整条语句 + + def statement(self, form_a: _Form, form_b: _Form) -> bool: + """比两条语句的规范形式,返回是不是全都说到了。B 先按 A 的 CTE 名重新起名(定义一样的叫同一个名字,各剩一个 + 对不上的也配成一对),两边才能逐段对齐。""" + ta, tb = form_a.tree, form_b.tree + wa = ta.args.get("with_") + names_a = {c.alias_or_name: c for c in (wa.expressions if wa is not None else [])} + if form_b.pre is not None and names_a: + tb = self._renamed(form_a, form_b, names_a) + self.usage_a, self.usage_b = _Usage(ta), _Usage(tb) + wb = tb.args.get("with_") + names_b = {c.alias_or_name: c for c in (wb.expressions if wb is not None else [])} + shown_a = {n: c.this.meta.get(_ORIGINAL) or n for n, c in names_a.items()} + shown_b = {n: c.this.meta.get(_ORIGINAL) or n for n, c in names_b.items()} + covered = set(names_a) == set(names_b) + self.ctes = {**shown_b, **shown_a} + self.roots = {id(ta), id(tb)} + covered &= self.query(ta, tb, ("", ""), "top") + for name in sorted(set(names_a) & set(names_b)): + body_a, body_b = names_a[name].this, names_b[name].this + if self.fp(body_a) != self.fp(body_b): + where = (_say("在", shown_a[name], "里"), _say("在", shown_b[name], "里")) + covered &= self.query(body_a, body_b, where, "derived") + return covered and not self.withheld + + def _renamed(self, form_a: _Form, form_b: _Form, names_a: dict[str, Any]) -> Any: + """B 起名之前的树按 A 的 CTE 名重新起名:定义一样的叫 A 的名字;两边各剩一个对不上的,B 的那个也叫 A 的。""" + from sqlglot import exp + + hint = _Hint(bodies={_fingerprint(c.this): name for name, c in names_a.items()}, taken=set(names_a)) + + def name(pre: Any) -> Any: + return _named(pre, self.read, form_b.qualified, _Sensitivity(pre, self.read, self.info), hint) + + first = name(form_b.pre.copy()) + wb = first.args.get("with_") + left_b = [c for c in (wb.expressions if wb is not None else []) if c.alias_or_name not in names_a] + used = {c.alias_or_name for c in (wb.expressions if wb is not None else [])} + left_a = [n for n in names_a if n not in used] + if len(left_a) == 1 and len(left_b) == 1: + pre = form_b.pre.copy() + original = left_b[0].this.meta.get(_ORIGINAL) + matches = [c.alias_or_name for c in pre.find_all(exp.CTE) if c.this.meta.get(_ORIGINAL) == original] + if original is not None and len(matches) == 1: + hint.names[matches[0]] = left_a[0] + return name(pre) + return first + + def query(self, qa: Any, qb: Any, where: tuple[str, str], ctx: str) -> bool: + from sqlglot import exp + + if self.fp(qa) == self.fp(qb): + return True + if isinstance(qa, exp.Select) and isinstance(qb, exp.Select): + return self.select(qa, qb, where, ctx) + if isinstance(qa, exp.SetOperation) and isinstance(qb, exp.SetOperation): + return self.setop(qa, qb, where, ctx) + return False + + # ---- 说法的拼法 + + def sides(self, where: tuple[str, str]) -> tuple[str, str]: + """两边的叫法,带上在哪一段(「A 在 paid 里」)。两边都说在哪:只说一边,「另一边没有」会读成整条语句都没有。""" + return _say(self.la, where[0]), _say(self.lb, where[1]) + + def add(self, aspect: str, a: Any, b: Any, text: str, detail: dict[str, Any] | None = None) -> None: + """记一句话。两边给人看的写法一样(「A 算的是 1,B 算的是 1」)、和已经说过的一句正好反过来(A 说 X、B 说 Y + 对上 A 说 Y、B 说 X)的不说:读起来自相矛盾,这一处算没说到。""" + if isinstance(a, str) and isinstance(b, str) and a == b: + self.withheld = True + return + for old in self.items: + if old.aspect == aspect and old.a == b and old.b == a and (a is not None or b is not None): + self.items.remove(old) + self.withheld = True + return + if old.text == text: + return + self.items.append(FacetDiff(aspect, a, b, text, detail)) + + def base_columns(self, node: Any, block: _Block) -> list[tuple[_Rel, Any]] | None: + """node 只涉及这一段的实表列(同名的表在这一段只出现一次)、不套子查询时,返回各列和它的来源;否则 None。""" + from sqlglot import exp + + if node.find(exp.Select, exp.Subquery) is not None: + return None + names = block.names() + aliases = {e.alias for e in block.exprs if isinstance(e, exp.Alias) and e.alias} + out = [] + for col in node.find_all(exp.Column): + if col.is_star or (not col.table and col.name in aliases): + return None + rel = block.rel(col.table) if col.table else (block.rels[0] if len(block.all_rels()) == 1 else None) + if rel is None or not rel.table or names.count(rel.name) != 1: + return None + if isinstance(col.this, exp.Identifier) and col.this.meta.get(_UNSURE): + columns = self.info.types.get(rel.name.translate(_ASCII_LOWER)) + if columns is None or col.name.translate(_ASCII_LOWER) not in columns: + return None + out.append((rel, col)) + return out + + def refs(self, node: Any, block: _Block) -> set[tuple[str, str]] | None: + cols = self.base_columns(node, block) + return None if cols is None else {(r.name, c.name) for r, c in cols} + + def show(self, node: Any, block: _Block, *, full: bool = False) -> str: + """给人看的写法:只有一个来源时列不写表名,否则写实表名(规范别名换回表名,CTE 换回原来的名字);选出来的 + 列去掉规范化起的别名。full 时一律写表名。""" + from sqlglot import exp + + copy = node.unalias().copy() if isinstance(node, exp.Alias) else node.copy() + single = len(block.all_rels()) == 1 and not full + for col in copy.find_all(exp.Column): + rel = block.rel(col.table) if col.table else None + if rel is not None: + col.set("table", None if single else exp.to_identifier(rel.shown)) + return self.sql(copy) + + def show_all(self, conds: list[Any], block: _Block) -> str: + """几个条件用 AND 连起来给人看:OR 套上括号(按 sqlglot 的结合规则生成)。""" + from sqlglot import exp + + if len(conds) == 1: + return self.show(conds[0], block) + return self.show(exp.and_(*[c.copy() for c in conds], copy=False), block) + + def named(self, table: str, column: str) -> str: + """列带中文名(有的话):「支付时间 paid_at」。""" + label = self.label((table, column)) + return _say(label, column) if label else column + + def what(self, rel: _Rel) -> str: + return _say(rel.shown, "表") if rel.table else _say("子查询", rel.shown) + + @staticmethod + def mapped(node: Any, mapping: Mapping[str, str]) -> Any: + """B 的一个节点按来源的对应关系(B 的别名 → A 的别名)改名后的副本:只用来和 A 比是不是同一个写法。""" + from sqlglot import exp + + if not mapping: + return node + copy = node.copy() + for col in copy.find_all(exp.Column): + if col.table in mapping: + col.set("table", exp.to_identifier(mapping[col.table])) + return copy + + def mfp(self, node: Any, mapping: Mapping[str, str]) -> str: + return self.fp(self.mapped(node, mapping)) + + # ---- 一段 SELECT + + def select(self, a: Any, b: Any, where: tuple[str, str], ctx: str) -> bool: + A = _Block.of(a, self.read, self.ctes, id(a) in self.roots) + B = _Block.of(b, self.read, self.ctes, id(b) in self.roots) + if A is None or B is None: + return False + matched = self.relations(A, B, where, ctx) + if matched is None: + return False + covered, mapping, skip_a, skip_b = matched + covered &= self.derived(A, B, mapping, where) + covered &= self.conditions(A, B, mapping, where, skip_a, skip_b) + covered &= self.outputs(A, B, mapping, where, ctx) + covered &= self.having(A, B, mapping, where) + covered &= self.rows(A, B, mapping, where, ctx) + return covered + + @staticmethod + def _join_sig(join: Any) -> tuple[str, str]: + return str(join.side or "").upper() or str(join.kind or "").upper(), join.this.alias_or_name + + def relations(self, A: _Block, B: _Block, where: tuple[str, str], ctx: str + ) -> tuple[bool, dict[str, str], set[str], set[str]] | None: + """两段读的来源对不对得上。对得上返回 (说到了没有, B 的别名 → A 的别名, A 已经说过的 WHERE 项, B 已经说过的 + WHERE 项);对不上返回 None。""" + la, lb = self.sides(where) + ra, rb = [r.alias for r in A.rels], [r.alias for r in B.rels] + ja = [self._join_sig(j) + (self.fp(j.args.get("on")) if j.args.get("on") is not None else "",) + for j in A.joins] + jb = [self._join_sig(j) + (self.fp(j.args.get("on")) if j.args.get("on") is not None else "",) + for j in B.joins] + kinds_a = {r.alias: r.kind for r in A.all_rels()} + kinds_b = {r.alias: r.kind for r in B.all_rels()} + if ra == rb and ja == jb and all(kinds_a[k] == kinds_b.get(k) for k in kinds_a): + return True, {}, set(), set() + # 换了一张表:开头那组来源只差一张实表(各自只出现一次),后面的连接一样 + if len(ra) == len(rb) and not A.joins and not B.joins: + only_a = [r for r in A.rels if r.alias not in rb] + only_b = [r for r in B.rels if r.alias not in ra] + if len(only_a) == 1 and len(only_b) == 1 and only_a[0].table and only_b[0].table and \ + A.names().count(only_a[0].name) == 1 and B.names().count(only_b[0].name) == 1: + x, y = only_a[0], only_b[0] + if x.name.translate(_ASCII_LOWER) == y.name.translate(_ASCII_LOWER): + return None + self.add("tables", x.name, y.name, _say(la, "查的是", x.name, "表,", lb, "查的是", y.name, "表")) + return True, {y.alias: x.alias}, set(), set() + # 一边多关联了一张表(内连接) + if not A.joins and not B.joins: + if set(rb) < set(ra) and len(ra) == len(rb) + 1: + return self._extra(A, B, la, lb, 0) + if set(ra) < set(rb) and len(rb) == len(ra) + 1: + return self._extra(B, A, lb, la, 1) + if ctx != "top": + return None + return self._outer(A, B, where) + + def _extra(self, big: _Block, small: _Block, lbig: str, lsmall: str, side: int + ) -> tuple[bool, dict[str, str], set[str], set[str]] | None: + from sqlglot import exp + + [rel] = [r for r in big.rels if r.alias not in {s.alias for s in small.rels}] + if not rel.table or big.names().count(rel.name) != 1: + return None + conds = [c for c in big.where if rel.alias in {col.table for col in c.find_all(exp.Column)}] + what = self.what(rel) + shown = _say("(关联的条件是", self.show_all(conds, big), ")") if conds and \ + all(self.base_columns(c, big) is not None for c in conds) else "" + used = any(t.name == rel.name and id(t) not in {id(i) for i in _items_of(small.select)} + for t in small.select.find_all(exp.Table)) + tail = ("没有关联它(只在子查询里用到了", rel.shown, ")") if used else ("没有",) + text = _say(lbig, "关联了", what, shown, ",", lsmall, *tail) + self.add("join", what if side == 0 else None, what if side == 1 else None, text) + skip = {self.fp(c) for c in conds} + return (True, {}, skip, set()) if side == 0 else (True, {}, set(), skip) + + def _nullable_used(self, block: _Block, alias: str) -> bool: + """这一段在 ON 之外(WHERE、HAVING)用到了外连接可空一侧的列:可能把没关联上的行筛掉了(COALESCE(x, 0) > 1 + 这类规范化认不出的写法),LEFT JOIN 就不一定是外连接。""" + from sqlglot import exp + + clauses = block.where + block.having + return any(c.table == alias for cond in clauses for c in cond.find_all(exp.Column)) + + def _null_blind(self, block: _Block, alias: str) -> bool: + """这一段不分组、选出来的都是只涉及可空一侧的列、会跳过空值的聚合(COUNT(x)、SUM、AVG、MIN、MAX):LEFT JOIN + 多出来的没关联上的行在这些聚合里都是空值、被跳过,和 INNER JOIN 结果一样。""" + from sqlglot import exp + + if block.group: + return False + for e in block.exprs: + agg = _aggregate(e.unalias()) + if agg is None or agg[1] is None: + return False + if any(c.table != alias for c in e.find_all(exp.Column)): + return False + return True + + def _outer(self, A: _Block, B: _Block, where: tuple[str, str] + ) -> tuple[bool, dict[str, str], set[str], set[str]] | None: + """(整条语句最外层)同一张表:LEFT JOIN 对 INNER JOIN、关联条件不同、只有一边关联了(LEFT JOIN)、外连接的 + 主表对调。""" + la, lb = self.sides(where) + + def kind(join: Any) -> str: + return str(join.side or "").lower() or "inner" + + ra, rb = [r.alias for r in A.rels], [r.alias for r in B.rels] + ja = {j.this.alias_or_name: j for j in A.joins} + jb = {j.this.alias_or_name: j for j in B.joins} + if len(A.joins) != len(ja) or len(B.joins) != len(jb) or any(not r.table for r in A.all_rels() + B.all_rels()): + return None + # 外连接的主表对调:两个来源,一边 x LEFT JOIN y,一边 y LEFT JOIN x,条件一样 + if len(ra) == len(rb) == 1 and len(ja) == len(jb) == 1 and ra != rb and set(ra) == set(jb) and \ + set(rb) == set(ja): + x, y = A.joins[0], B.joins[0] + if kind(x) == kind(y) == "left" and self.fp(x.args.get("on")) == self.fp(y.args.get("on")) and \ + not self._nullable_used(A, x.this.alias_or_name) and not self._nullable_used(B, y.this.alias_or_name): + wa = _say("以", A.rels[0].shown, "为主表外连接", B.rels[0].shown) + wb = _say("以", B.rels[0].shown, "为主表外连接", A.rels[0].shown) + self.add("join", wa, wb, _say(la, wa, ",", lb, wb)) + return True, {}, set(), set() + return None + if [r for r in ra if r not in rb and r not in jb] or [r for r in rb if r not in ra and r not in ja]: + return None + one_sided_a = [k for k in ja if k not in jb and k not in rb] + one_sided_b = [k for k in jb if k not in ja and k not in ra] + if one_sided_a and one_sided_b: + return None + skip_a: set[str] = set() + skip_b: set[str] = set() + for alias in sorted(set(ja) | set(jb)): + x, y = ja.get(alias), jb.get(alias) + if x is not None and y is not None: + if kind(x) != kind(y) or not _plain_join(x) or not _plain_join(y): + return None + if self.fp(x) != self.fp(y): + on_x, on_y = x.args.get("on"), y.args.get("on") + if on_x is None or on_y is None or self._same_condition(_conjuncts(on_x), _conjuncts(on_y), + A, B) is not False: + return None + if kind(x) == "left" and (self._nullable_used(A, alias) or self._nullable_used(B, alias)): + return None + rel = A.rel(alias) + wa, wb = self.show(on_x, A, full=True), self.show(on_y, B, full=True) + self.add("join", wa, wb, _say(la, "关联", self.what(rel), "的条件是", wa, ",", lb, "是", wb)) + continue + # 一边 LEFT JOIN,另一边内连接(在开头那组来源里,关联条件在 WHERE 里) + if x is not None and alias in rb and kind(x) == "left": + done = self._left_inner(x, A, B, alias, la, lb, 0) + if done is None: + return None + skip_b |= done + continue + if y is not None and alias in ra and kind(y) == "left": + done = self._left_inner(y, B, A, alias, la, lb, 1) + if done is None: + return None + skip_a |= done + continue + # 只有一边关联了 + join = x if x is not None else y + if join is None or kind(join) != "left" or not _plain_join(join): + return None + block, me, other = (A, la, lb) if x is not None else (B, lb, la) + if self._nullable_used(block, alias): + return None + rel = block.rel(alias) + text = _say(me, *self._outer_words(join, block), self.what(rel), ",", other, "没有") + self.add("join", rel.shown if x is not None else None, rel.shown if y is not None else None, text) + return True, {}, skip_a, skip_b + + def _left_inner(self, join: Any, left: _Block, inner: _Block, alias: str, la: str, lb: str, + side: int) -> set[str] | None: + """left 这边 LEFT JOIN alias,inner 那边同一张表是内连接:inner 的 WHERE 里有 left 的 ON 的每一项才算关联条件 + 一样,只差连接类型。left 那边 WHERE 里用到可空一侧的列(可能已经把没关联上的行筛掉了)、结果只是可空一侧 + 的列的聚合(两种写法结果一样)时不说。照原文说:left 那边原来写的是 RIGHT JOIN 的说以哪一侧为主表;inner + 那边写的是 LEFT / RIGHT JOIN、被 WHERE 里的条件变成了内连接的,说写的是什么、实际是什么。""" + if not _plain_join(join) or self._nullable_used(left, alias) or self._null_blind(left, alias): + return None + on = join.args.get("on") + conds = [self.fp(c) for c in _conjuncts(on)] if on is not None else [] + if not conds or not set(conds) <= {self.fp(c) for c in inner.where}: + return None + rel = inner.rel(alias) + what = self.what(rel) + outer = self._outer_words(join, left) + turned = [r for r in inner.rels if r.rejected] + if turned: + words = "、".join(dict.fromkeys(r.rejected for r in turned)) + names = "、".join(r.shown for r in turned) + inner_says = ("写的是", words, ",但 WHERE 里的条件把关联不上", names, "的行筛掉了,实际是 INNER JOIN") + if side == 0: + text = _say(la, *outer, what, ",", lb, *(inner_says if turned else ("用 INNER JOIN",))) + self.add("join", "OUTER JOIN", "INNER JOIN", text) + else: + first = (*inner_says, ";") if turned else ("用 INNER JOIN 关联", what, ",") + second = (*outer, what) if turned or join.meta.get(_FLIPPED) else ("用 LEFT JOIN",) + text = _say(la, *first, lb, *second) + self.add("join", "INNER JOIN", "OUTER JOIN", text) + return set(conds) + + def _outer_words(self, join: Any, block: _Block) -> tuple[str, ...]: + """一次外连接的说法:原文就是 LEFT JOIN 的说「用 LEFT JOIN 关联」;从 RIGHT JOIN 改写来的说「以谁为主表外连接」 + (照写 LEFT 会和原文对不上)。""" + if join.meta.get(_FLIPPED): + return ("以", "、".join(r.shown for r in block.rels), "为主表外连接") + return ("用 LEFT JOIN 关联",) + + def derived(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str]) -> bool: + """FROM 里的子查询:两边别名对得上、定义不同的,只有一处时往里比(说「在 t 里」,照原来起的名字)。""" + pairs = [] + for ra in A.all_rels(): + if ra.kind != "derived": + continue + rb = next((r for r in B.all_rels() if mapping.get(r.alias, r.alias) == ra.alias), None) + if rb is None or rb.kind != "derived": + return False + if self.fp(ra.node.this) != self.fp(rb.node.this): + pairs.append((ra, rb)) + if not pairs: + return True + if len(pairs) > 1: + return False + ra, rb = pairs[0] + inner = (_deeper(where[0], ra.shown, plain=True), _deeper(where[1], rb.shown, plain=True)) + return self.query(ra.node.this, rb.node.this, inner, "derived") + + # ---- WHERE + + def timed(self, cond: Any, block: _Block) -> tuple[str, str] | None: + """时间条件:一列(可以套着日期函数、CAST)和常量比较,列名像时间或者常量像日期。返回 (实表, 列)。""" + from sqlglot import exp + + if not isinstance(cond, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.In)): + return None + if isinstance(cond, exp.In): + column_side, others = cond.this, list(cond.expressions) + elif _fixed(cond.expression): + column_side, others = cond.this, [cond.expression] + elif _fixed(cond.this): + column_side, others = cond.expression, [cond.this] + else: + return None + if not others or not all(_fixed(o) for o in others): + return None + cols = self.base_columns(column_side, block) + if not cols or len({(r.alias, c.name) for r, c in cols}) != 1 or _value_column(column_side) is None: + return None + rel, col = cols[0] + if _temporal_name(col.name) or any(_datey(o) for o in others) or _datey(column_side): + return rel.name, col.name + return None + + def conditions(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str], + skip_a: set[str], skip_b: set[str]) -> bool: + la, lb = self.sides(where) + xa = [c for c in A.where if self.fp(c) not in skip_a] + xb = [c for c in B.where if self.fp(c) not in skip_b] + only_a, only_b = self._unmatched(xa, xb, mapping) + if not only_a and not only_b: + return True + covered = True + only_a, only_b, ok = self._times(A, B, mapping, la, lb, only_a, only_b) + covered &= ok + only_a, only_b, ok = self._same_subject(A, B, mapping, la, lb, only_a, only_b) + covered &= ok + only_a, only_b, ok = self._join_equalities(A, B, mapping, la, lb, only_a, only_b) + covered &= ok + fa, fb = _say(self.la, where[0]), _say(self.lb, where[1]) + for side, rest, block, other_block, other_usage, me, other in ( + (0, only_a, A, B, self.usage_b, fa, fb), (1, only_b, B, A, self.usage_a, fb, fa)): + for c in list(rest): + refs = self.refs(c, block) + if not refs or other_usage is None or \ + other_usage.touches(refs, kinds=_kinds_for([c]), scope=other_block.select): + continue + if self.range.subject(c) is not None and self.range.compare([c], []) is not False: + continue + shown = self.show(c, block) + excluded = self._excludes(c, block) + text = _say(me, "排除了", excluded, "的行,", other, "没有") if excluded else \ + _say(me, "只保留", shown, "的行,", other, "没有这个条件") + self.add("filter", shown if side == 0 else None, shown if side == 1 else None, text, + {"columns": [{"table": t, "column": col} for t, col in sorted(refs)]}) + rest.remove(c) + only_a, only_b, ok = self._subqueries(A, B, mapping, where, only_a, only_b) + covered &= ok + return covered and not only_a and not only_b + + def _unmatched(self, xs: list[Any], ys: list[Any], mapping: Mapping[str, str]) -> tuple[list[Any], list[Any]]: + rest = list(ys) + only_x = [] + for x in xs: + found = next((y for y in rest if self.mfp(y, mapping) == self.fp(x)), None) + if found is None: + only_x.append(x) + else: + rest.remove(found) + return only_x, rest + + def _on_column(self, conds: list[Any], block: _Block, key: tuple[str, str]) -> list[Any]: + return [c for c in conds if self.refs(c, block) == {key}] + + def _times(self, A: _Block, B: _Block, mapping: Mapping[str, str], la: str, lb: str, only_a: list[Any], + only_b: list[Any]) -> tuple[list[Any], list[Any], bool]: + """时间条件:两边差在按哪一列筛时间、同一列上的范围,或者只有一边按时间筛。说到了的从两边拿掉。""" + ta = [(c, t) for c in only_a if (t := self.timed(c, A)) is not None] + tb = [(c, t) for c in only_b if (t := self.timed(c, B)) is not None] + cols_a, cols_b = {t for _, t in ta}, {t for _, t in tb} + if not (ta or tb) or len(cols_a) > 1 or len(cols_b) > 1 or self.usage_a is None or self.usage_b is None: + return only_a, only_b, True + detail = {"a": [{"table": t, "column": c} for t, c in sorted(cols_a)], + "b": [{"table": t, "column": c} for t, c in sorted(cols_b)]} + all_a = {t for c in A.where if (t := self.timed(c, A)) is not None} + all_b = {t for c in B.where if (t := self.timed(c, B)) is not None} + if cols_a and cols_b and cols_a != cols_b: + (ka,), (kb,) = cols_a, cols_b + if all_a != cols_a or all_b != cols_b or self.usage_b.touches([ka], scope=B.select) or \ + self.usage_a.touches([kb], scope=A.select): + return only_a, only_b, False + text = _say(la, "按", self.named(*ka), "筛时间,", lb, "按", self.named(*kb)) + if self._shape([c for c, _ in ta]) != self._shape([c for c, _ in tb]): + text += _say("(", la, "是", self.show_all(_by_bound([c for c, _ in ta]), A), ",", lb, "是", + self.show_all(_by_bound([c for c, _ in tb]), B), ")") + self.add("time", ka[1], kb[1], text, detail) + elif cols_a == cols_b: + (key,) = cols_a + ra = _by_bound(self._on_column(A.where, A, key)) + rb = _by_bound(self._on_column(B.where, B, key)) + same = self.range.compare(ra, [self.mapped(c, mapping) for c in rb]) + if same is not False or self.usage_a.touches([key], ra, scope=A.select) or \ + self.usage_b.touches([key], rb, scope=B.select): + return only_a, only_b, False + wa, wb = self.show_all(ra, A), self.show_all(rb, B) + self.add("time_range", wa, wb, _say(la, "的时间范围是", wa, ",", lb, "是", wb)) + else: + (key,) = cols_a or cols_b + if cols_a: + if self.usage_b.temporal or self.usage_b.touches([key], scope=B.select): + return only_a, only_b, False + self.add("time", key[1], None, _say(la, "按", self.named(*key), "筛时间,", lb, "没有按时间筛选"), detail) + else: + if self.usage_a.temporal or self.usage_a.touches([key], scope=A.select): + return only_a, only_b, False + self.add("time", None, key[1], _say(la, "没有按时间筛选,", lb, "按", self.named(*key), "筛时间"), detail) + return [c for c in only_a if all(c is not x for x, _ in ta)], \ + [c for c in only_b if all(c is not x for x, _ in tb)], True + + def _same_subject(self, A: _Block, B: _Block, mapping: Mapping[str, str], la: str, lb: str, + only_a: list[Any], only_b: list[Any]) -> tuple[list[Any], list[Any], bool]: + """同一个对象只和常量比的条件:两边这个对象上的条件合起来按取值逐段算(_Range),留下的行一样的不说;不一样 + 的,两边都有没配上的条件时说「A 按 X 筛选,B 按 Y 筛选」,只有一边有时说「A 只保留 X 的行,B 没有这个条件」 + (另一边在这个对象上的条件这一边都有,多出来的确实多筛掉了行)。这一列在两边别的地方(别的条件、子查询、 + CTE)还用来筛选的不说:留下哪些行就不只是这几个条件定的。""" + covered = True + done_a: list[Any] = [] + done_b: list[Any] = [] + + def subject(c: Any, block: _Block, m: Mapping[str, str]) -> str | None: + s = self.range.subject(self.mapped(c, m)) + return s[0] if s is not None and self.refs(c, block) is not None else None + + keys_a = {id(c): subject(c, A, {}) for c in A.where} + keys_b = {id(c): subject(c, B, mapping) for c in B.where} + wanted = [k for k in dict.fromkeys([keys_a.get(id(c)) for c in only_a] + [keys_b.get(id(c)) for c in only_b]) + if k is not None] + for key in wanted: + mine = [c for c in only_a if keys_a.get(id(c)) == key] + theirs = [c for c in only_b if keys_b.get(id(c)) == key] + all_a = [c for c in A.where if keys_a.get(id(c)) == key] + all_b = [c for c in B.where if keys_b.get(id(c)) == key] + refs_a = set().union(*(self.refs(c, A) for c in all_a)) if all_a else set() + refs_b = set().union(*(self.refs(c, B) for c in all_b)) if all_b else set() + done_a += mine + done_b += theirs + same = self.range.compare(all_a, [self.mapped(c, mapping) for c in all_b]) + usage_a, usage_b = self.usage_a, self.usage_b + kinds = _kinds_for(all_a + all_b) + if same is not False or usage_a is None or usage_b is None or \ + usage_a.touches(refs_a | refs_b, all_a, kinds, A.select) or \ + usage_b.touches(refs_a | refs_b, all_b, kinds, B.select): + covered = False + continue + detail_a = {"columns": [{"table": t, "column": col} for t, col in sorted(refs_a)]} + detail_b = {"columns": [{"table": t, "column": col} for t, col in sorted(refs_b)]} + if mine and theirs: + wa, wb = self.show_all(mine, A), self.show_all(theirs, B) + if wa == wb: + wa, wb = self.show(exp_and(mine), A, full=True), self.show(exp_and(theirs), B, full=True) + self.add("filter", wa, wb, _say(la, "按", wa, "筛选,", lb, "按", wb, "筛选"), detail_a) + else: + side, conds, block, me, other, detail = (0, mine, A, la, lb, detail_a) if mine else \ + (1, theirs, B, lb, la, detail_b) + shown = self.show_all(conds, block) + excluded = self._excludes(conds[0], block) if len(conds) == 1 else None + text = _say(me, "排除了", excluded, "的行,", other, "没有") if excluded else \ + _say(me, "只保留", shown, "的行,", other, "没有这个条件") + self.add("filter", shown if side == 0 else None, shown if side == 1 else None, text, detail) + return [c for c in only_a if all(c is not d for d in done_a)], \ + [c for c in only_b if all(c is not d for d in done_b)], covered + + def _join_equalities(self, A: _Block, B: _Block, mapping: Mapping[str, str], la: str, lb: str, + only_a: list[Any], only_b: list[Any]) -> tuple[list[Any], list[Any], bool]: + """两边没配上的条件都是「两列相等」的关联条件时,比相等关系把列分成的组:分法不同就是关联的条件不同(列各是各的 + 表);分法一样(a = b AND b = c 和 a = b AND a = c)不说。""" + ea = [c for c in only_a if _column_equality(c) and self.base_columns(c, A) is not None] + eb = [c for c in only_b if _column_equality(c) and self.base_columns(c, B) is not None] + if not ea or not eb or len(ea) != len(only_a) or len(eb) != len(only_b): + return only_a, only_b, True + all_a = [c for c in A.where if _column_equality(c)] + all_b = [self.mapped(c, mapping) for c in B.where if _column_equality(c)] + if _partition(all_a) == _partition(all_b): + return [], [], False + wa, wb = self.show_all(ea, A), self.show_all(eb, B) + if wa == wb: + wa, wb = self.show(exp_and(ea), A, full=True), self.show(exp_and(eb), B, full=True) + self.add("join", wa, wb, _say(la, "关联的条件是", wa, ",", lb, "是", wb)) + return [], [], True + + def _same_condition(self, xs: list[Any], ys: list[Any], A: _Block, B: _Block) -> bool | None: + """两组条件(各自 AND 连)是不是一样:只差在两列相等的分法上、或者同一个对象只和常量比时能判断;不然 None。""" + only_x, only_y = self._unmatched(xs, ys, {}) + if not only_x and not only_y: + return True + if all(_column_equality(c) for c in only_x + only_y): + return _partition([c for c in xs if _column_equality(c)]) == \ + _partition([c for c in ys if _column_equality(c)]) + subjects = {self.range.subject(c)[0] if self.range.subject(c) else None for c in only_x + only_y} + if None in subjects or len(subjects) != 1: + return None + (fp,) = subjects + all_x = [c for c in xs if (s := self.range.subject(c)) is not None and s[0] == fp] + all_y = [c for c in ys if (s := self.range.subject(c)) is not None and s[0] == fp] + return self.range.compare(all_x, all_y) + + def _shape(self, conds: list[Any]) -> list[str]: + """时间条件把那一列换成占位符之后的写法:两边只差时间列时一样。""" + from sqlglot import exp + + out = [] + for c in conds: + copy = c.copy() + for col in list(copy.find_all(exp.Column)): + col.replace(exp.column("_time_")) + out.append(self.fp(copy)) + return sorted(out) + + def _excludes(self, cond: Any, block: _Block) -> str | None: + """「排除」的写法去掉否定之后排除的是什么:<> 写成 =,NOT x 写成 x,IS NOT NULL 写成「列 为空」。""" + from sqlglot import exp + + if isinstance(cond, exp.NEQ): + return self.show(exp.EQ(this=cond.this.copy(), expression=cond.expression.copy()), block) + if isinstance(cond, exp.Not): + inner = cond.this.unnest() if isinstance(cond.this, exp.Paren) else cond.this + if isinstance(inner, exp.Is) and isinstance(inner.expression, exp.Null): + return _say(self.show(inner.this, block), "为空") + return self.show(inner, block) + return None + + def _subqueries(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str], + only_a: list[Any], only_b: list[Any]) -> tuple[list[Any], list[Any], bool]: + """条件里套着的子查询:两边各一个、除了子查询之外写法一模一样(含 NOT、IN 左边的列)的配成一对往里比。""" + covered = True + done_a, done_b = [], [] + for c in only_a: + template = self._template(c, {}) + if template is None: + continue + mine = [x for x in only_a if self._template(x, {}) == template] + theirs = [d for d in only_b if self._template(d, mapping) == template] + if len(mine) != 1 or len(theirs) != 1: + continue + d = theirs[0] + place_a, place_b = self._place(c, A), self._place(d, B) + inner = (_deeper(where[0], place_a), _deeper(where[1], place_b)) + kind = _subquery_kind(c) + covered &= self.query(self._subquery(c), self._subquery(d), inner, kind) + done_a.append(c) + done_b.append(d) + return [c for c in only_a if all(c is not x for x in done_a)], \ + [c for c in only_b if all(c is not x for x in done_b)], covered + + def _template(self, cond: Any, mapping: Mapping[str, str]) -> str | None: + """条件里只套着一个子查询时,子查询换成占位符之后的写法;不然 None。""" + from sqlglot import exp + + tops = [n for n in cond.walk() if isinstance(n, (exp.Select, exp.SetOperation)) and + not any(isinstance(p, (exp.Select, exp.SetOperation)) for p in _ancestors(n, cond))] + if len(tops) != 1: + return None + copy = self.mapped(cond, mapping).copy() if mapping else cond.copy() + for node in copy.walk(): + if isinstance(node, (exp.Select, exp.SetOperation)): + node.replace(exp.Literal.string("")) + break + return self.fp(copy) + + @staticmethod + def _subquery(cond: Any) -> Any: + from sqlglot import exp + + return next(n for n in cond.walk() if isinstance(n, (exp.Select, exp.SetOperation))) + + def _place(self, cond: Any, block: _Block) -> str: + """子查询在条件里的位置:「users.id NOT IN 的子查询」「NOT EXISTS 子查询」「条件的子查询」。""" + from sqlglot import exp + + sub = self._subquery(cond) + node = sub + while isinstance(node.parent, (exp.Subquery, exp.Paren)): + node = node.parent + parent = node.parent + negated = isinstance(_unparen_parent(parent), exp.Not) if parent is not None else False + if isinstance(parent, exp.Exists): + return "NOT EXISTS 子查询" if negated else "EXISTS 子查询" + if isinstance(parent, exp.In): + return _say(self.show(parent.this, block), "NOT IN" if negated else "IN", "的子查询") + return "条件的子查询" + + # ---- 选出来的列、分组 + + def groups(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str]) -> bool: + """分组键:两边不同、又都只涉及实表列、多出来的键不是另一边的键算出来的时,说各按什么分组。""" + la, lb = self.sides(where) + ka = sorted(self.fp(g) for g in A.group) + kb = sorted(self.mfp(g, mapping) for g in B.group) + if ka == kb: + return True + ra = [self.refs(g, A) for g in A.group] + rb = [self.refs(g, B) for g in B.group] + if any(r is None for r in ra + rb): + return False + cols_a = set().union(*ra) if ra else set() + cols_b = set().union(*rb) if rb else set() + extra_a = [g for g, r in zip(A.group, ra) if self.fp(g) not in kb] + extra_b = [g for g, r in zip(B.group, rb) if self.mfp(g, mapping) not in ka] + if any(self.refs(g, A) <= cols_b for g in extra_a) or any(self.refs(g, B) <= cols_a for g in extra_b): + return False + + def say(block: _Block, other: _Block) -> tuple[str, ...]: + if not block.group: + return ("没有去重",) if other.deduped else ("没有分组",) + shown = "、".join(self.show(k, block) for k in block.group) + return ("对", shown, "去重") if block.deduped else ("按", shown, "分组") + + wa = "、".join(self.show(k, A) for k in A.group) or None + wb = "、".join(self.show(k, B) for k in B.group) or None + self.add("group_by", wa, wb, _say(la, *say(A, B), ",", lb, *say(B, A))) + return True + + def _dedup_swap(self, A: _Block, B: _Block, mapping: Mapping[str, str]) -> bool: + """一边 SELECT DISTINCT(不分组),另一边按选出来的那几列分组、不去重、不聚合:两种写法都是去掉重复的行, + 不能说成一边去重一边没去重。""" + def plain(block: _Block) -> bool: + return block.distinct and not block.distinct_on and not block.group and not _aggregating(block.select) + + def grouped(block: _Block, m: Mapping[str, str]) -> set[str] | None: + if block.distinct or not block.group or block.having or \ + any(_aggregate(e.unalias()) is not None for e in block.exprs): + return None + keys = {self.mfp(g, m) for g in block.group} + outs = {self.mfp(e.unalias(), m) for e in block.exprs} + return keys if keys == outs else None + + outs_a = {self.fp(e.unalias()) for e in A.exprs} + outs_b = {self.mfp(e.unalias(), mapping) for e in B.exprs} + return (plain(A) and grouped(B, mapping) == outs_a) or (plain(B) and grouped(A, {}) == outs_b) + + def outputs(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str], ctx: str) -> bool: + """分组键、选出来的列。选出来的列只在整条语句最外层、合并的一支、标量子查询里比(CTE、FROM 里的子查询选出来的 + 列规范化时改过名、排过序,和原文对不上);分组键不同时,选出来的列只差在分组键那几列上的算在分组键的不同里。 + 其余逐列比:都是一个聚合、都只是一列时才说,两列被关联条件定成相等的不说。""" + la, lb = self.sides(where) + ga = [self.fp(g) for g in A.group] + gb = [self.mfp(g, mapping) for g in B.group] + regrouped = sorted(ga) != sorted(gb) + if self._dedup_swap(A, B, mapping): + return False + covered = self.groups(A, B, mapping, where) + ea, eb = A.exprs, B.exprs + ta, tb = [self.fp(e.unalias()) for e in ea], [self.mfp(e.unalias(), mapping) for e in eb] + if ta == tb: + return covered + if ctx not in ("top", "branch", "scalar") or any(_is_star(e) for e in ea + eb): + return False + if regrouped and covered: + na = next((i for i, t in enumerate(ta) if t not in ga), len(ta)) + nb = next((i for i, t in enumerate(tb) if t not in gb), len(tb)) + if set(ta[:na]) == set(ga) and set(tb[:nb]) == set(gb) and ta[na:] == tb[nb:]: + return covered + if len(ea) != len(eb) or sorted(ta) == sorted(tb): + if any(self.base_columns(e, A) is None for e in ea) or any(self.base_columns(e, B) is None for e in eb): + return False + wa = "、".join(self.show(e, A) for e in ea) + wb = "、".join(self.show(e, B) for e in eb) + if len(ea) != len(eb): + text = _say(la, "选出", str(len(ea)), "列(", wa, "),", lb, "选出", str(len(eb)), "列(", wb, ")") + else: + text = _say("两边选出来的列一样,先后不同:", la, "是", wa, ",", lb, "是", wb) + self.add("expression", wa, wb, text) + return covered + for i, (x, y) in enumerate(zip(ea, eb), 1): + if self.fp(x.unalias()) == self.mfp(y.unalias(), mapping): + continue + if not self.output(x, y, A, B, mapping, la, lb, i, len(ea)): + covered = False + return covered + + def output(self, x: Any, y: Any, A: _Block, B: _Block, mapping: Mapping[str, str], la: str, lb: str, i: int, + n: int) -> bool: + from sqlglot import exp + + if self.base_columns(x, A) is None or self.base_columns(y, B) is None: + return False + xu, yu = x.unalias(), y.unalias() + equal = _partition([c for c in A.where if _column_equality(c)] + + [self.mapped(c, mapping) for c in B.where if _column_equality(c)]) + ga, gb = _aggregate(xu), _aggregate(yu) + if ga is not None and gb is not None: + (fa, arg_a, da), (fb, arg_b, db) = ga, gb + same_arg = (arg_a is None and arg_b is None) or ( + arg_a is not None and arg_b is not None and (self.fp(arg_a) == self.mfp(arg_b, mapping) or ( + isinstance(_unparen(arg_a), exp.Column) and isinstance(_unparen(arg_b), exp.Column) and + self.refs(arg_a, A) == self.refs(arg_b, B)))) + if not same_arg and any(arg is not None and not isinstance(_unparen(arg), exp.Column) + for arg in (arg_a, arg_b)): + return False + if same_arg and fa == fb and da == db: + return False + if not same_arg and arg_a is not None and arg_b is not None and \ + _same_class(equal, self.fp(arg_a), self.mfp(arg_b, mapping)): + return False + if fa == fb and same_arg and fa in ("min", "max"): + return False + if {fa, fb} == {"count"} and (arg_a is None) != (arg_b is None): + arg = arg_a if arg_a is not None else arg_b + block = A if arg_a is not None else B + refs = self.refs(arg, block) or set() + if self.usage_a is None or self.usage_b is None or \ + self.usage_a.touches(refs, [xu], scope=A.select) or self.usage_b.touches(refs, [yu], scope=B.select): + return False + wa = self.show(arg_a, A) if arg_a is not None else "*" + wb = self.show(arg_b, B) if arg_b is not None else "*" + if wa == wb and not same_arg: + wa, wb = self.show(arg_a, A, full=True), self.show(arg_b, B, full=True) + if same_arg and fa == fb: + text = _say(la, "对", wa, _verb(fa, da) + ",", lb, "没去重") if da else \ + _say(la, "对", wa, _verb(fa, da) + "时没去重,", lb, _verb(fb, db)) + elif same_arg and arg_a is not None: + text = _say(la, "对", wa, _verb(fa, da) + ",", lb, _verb(fb, db)) + else: + what_a = ("数的是行数",) if fa == "count" and arg_a is None else ("对", wa, _verb(fa, da)) + what_b = ("数的是行数",) if fb == "count" and arg_b is None else ("对", wb, _verb(fb, db)) + text = _say(la, *what_a, ",", lb, *what_b) + self.add("aggregate", self.show(x, A), self.show(y, B), text) + return True + if isinstance(xu, exp.Column) and isinstance(yu, exp.Column): + if _same_class(equal, self.fp(xu), self.mfp(yu, mapping)) or self.refs(xu, A) == self.refs(yu, B): + return False + wa, wb = self.show(x, A), self.show(y, B) + if wa == wb: + wa, wb = self.show(x, A, full=True), self.show(y, B, full=True) + where_col = _say("第", str(i), "列") if n > 1 else "" + self.add("expression", wa, wb, _say(la, where_col, "选的是", wa, ",", lb, "选的是", wb)) + return True + return False + + # ---- HAVING、取行 + + def having(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str]) -> bool: + """HAVING:同一个对象(聚合)只和常量比的,两边合起来逐段算,留下的组一样的不说(COUNT(*) > 1 和 >= 2、分组 + 之后 COUNT(*) > 0);只有一边有、另一边 HAVING 里没有别的条件的,说「分组后只保留」。""" + la, lb = self.sides(where) + only_a, only_b = self._unmatched(A.having, B.having, mapping) + if not only_a and not only_b: + return True + grouped = bool(A.group) and bool(B.group) + if any(self.base_columns(c, A) is None for c in only_a) or any(self.base_columns(c, B) is None for c in only_b): + return False + subjects = {s[0] for c in only_a + [self.mapped(c, mapping) for c in only_b] + if (s := self.range.subject(c)) is not None} + if len(subjects) == 1: + (fp,) = subjects + all_a = [c for c in A.having if (s := self.range.subject(c)) is not None and s[0] == fp] + all_b = [self.mapped(c, mapping) for c in B.having + if (s := self.range.subject(self.mapped(c, mapping))) is not None and s[0] == fp] + if len(all_a) != len(A.having) or len(all_b) != len(B.having) or \ + self.range.compare(all_a, all_b, grouped=grouped) is not False: + return False + elif subjects or only_a and only_b: + return False + if only_a and only_b: + wa, wb = self.show_all(only_a, A), self.show_all(only_b, B) + self.add("having", wa, wb, _say(la, "分组后只保留", wa, "的组,", lb, "只保留", wb, "的组")) + return True + side, rest, block, me, other = (0, only_a, A, la, lb) if only_a else (1, only_b, B, lb, la) + usage = self.usage_b if side == 0 else self.usage_a + refs = set().union(*(self.refs(c, block) or set() for c in rest)) + if (B.having if side == 0 else A.having) or usage is None or \ + usage.touches(refs, scope=(B if side == 0 else A).select): + return False + shown = self.show_all(rest, block) + self.add("having", shown if side == 0 else None, shown if side == 1 else None, + _say(me, "分组后只保留", shown, "的组,", other, "没有这个条件")) + return True + + def rows(self, A: _Block, B: _Block, mapping: Mapping[str, str], where: tuple[str, str], ctx: str) -> bool: + """取行、排序、去重。取行在标量子查询、EXISTS 子查询里不说(SQLite 的标量子查询本来就取第一行),不分组的聚合 + (只出一行:取行、排序、去重都不起作用)不说;排序只在整条语句最外层、没有取行、DISTINCT ON、窗口函数时说成 + order,有取行时算 limit,别的地方不说(子查询里的排序可能决定 GROUP_CONCAT 的先后),按常量排的不说。""" + la, lb = self.sides(where) + covered = True + + def single(block: _Block) -> bool: + return _aggregating(block.select) and not block.group + + if A.limit != B.limit or A.offset != B.offset: + if ctx in ("scalar", "exists") or single(A) or single(B): + covered = False + else: + if A.limit != B.limit: + self.add("limit", A.limit, B.limit, _say(la, _rows(A.limit), ",", lb, _rows(B.limit))) + if A.offset != B.offset: + self.add("limit", A.offset, B.offset, _say(la, _skip(A.offset), ",", lb, _skip(B.offset))) + if [self.fp(o) for o in A.order] != [self.mfp(o, mapping) for o in B.order]: + limited = A.limit is not None or B.limit is not None or A.offset is not None or B.offset is not None + if A.distinct_on or B.distinct_on or A.windowed or B.windowed or single(A) or single(B) or \ + any(not self.base_columns(o.this, A) for o in A.order) or \ + any(not self.base_columns(o.this, B) for o in B.order) or \ + (not limited and ctx != "top") or (limited and ctx in ("scalar", "exists")): + covered = False + else: + wa, wb = self._sorting(A.order, A), self._sorting(B.order, B) + if limited: + self.add("limit", wa, wb, _say(la, "取行前", wa, ",", lb, wb)) + else: + self.add("order", wa, wb, _say(la, wa, ",", lb, wb)) + if A.distinct != B.distinct: + if A.distinct_on or B.distinct_on or self._dedup_swap(A, B, mapping) or ctx in ("in", "exists") or \ + single(A) or single(B): + return False + block = A if A.distinct else B + keys = {self.fp(g) for g in block.group} + if keys and keys <= {self.fp(e.unalias()) for e in block.exprs}: + return False + me, other = (la, lb) if A.distinct else (lb, la) + self.add("group_by", "DISTINCT" if A.distinct else None, "DISTINCT" if B.distinct else None, + _say(me, "用 SELECT DISTINCT 去掉了重复的行,", other, "没有")) + return covered + + def _sorting(self, order: list[Any], block: _Block) -> str: + if not order: + return "没有排序" + keys = "、".join(_say(self.show(o.this, block), "降序" if o.args.get("desc") else "升序") for o in order) + return _say("按", keys, "排") + + # ---- UNION + + def setop(self, a: Any, b: Any, where: tuple[str, str], ctx: str) -> bool: + """合并:种类不同(UNION 对 INTERSECT)不说具体的;UNION 对 UNION ALL 说一句。最外层的 UNION ALL 各支排过序, + 按写法配对;别的(去重的 UNION、EXCEPT、INTERSECT、套在里面的)按先后逐支比(EXCEPT 两边对调了就对不上)。 + 只有一支不同的往里比。""" + from sqlglot import exp + + la, lb = self.sides(where) + if type(a) is not type(b): + return False + xa, xb = _set_branches(a), _set_branches(b) + covered = True + if bool(a.args.get("distinct")) != bool(b.args.get("distinct")) and isinstance(a, exp.Union): + wa, wb = _merging(a), _merging(b) + self.add("set_op", wa, wb, _say(la, "用", wa, ",", lb, "用", wb)) + if len(xa) != len(xb): + if isinstance(a, exp.Union) and bool(a.args.get("distinct")) == bool(b.args.get("distinct")): + self.add("set_op", str(len(xa)), str(len(xb)), + _say(la, "合并了", str(len(xa)), "支查询,", lb, "合并了", str(len(xb)), "支")) + return False + free = isinstance(a, exp.Union) and not a.args.get("distinct") and not b.args.get("distinct") and \ + ctx == "top" and not any(a.args.get(k) or b.args.get(k) for k in ("order", "limit", "offset")) + if free: + only_a, only_b = self._unmatched(xa, xb, {}) + pairs = list(zip(only_a, only_b)) if len(only_a) == len(only_b) == 1 else None + if only_a and pairs is None: + return False + else: + pairs = [(x, y) for x, y in zip(xa, xb) if self.fp(x) != self.fp(y)] + if len(pairs) > 1: + return False + for x, y in pairs or []: + if isinstance(a, exp.Except): + index = next(i for i, v in enumerate(xa) if v is x) + place = "EXCEPT 前面那支" if index == 0 else "EXCEPT 后面那支" + else: + place = "合并的一支" + branch = (_deeper(where[0], place), _deeper(where[1], place)) + covered &= self.query(x, y, branch, "branch" if ctx == "top" else ctx) + for key in ("order", "limit", "offset"): + va = self.fp(a.args[key]) if a.args.get(key) else "" + vb = self.fp(b.args[key]) if b.args.get(key) else "" + if va != vb: + covered = False + return covered + + +def exp_and(conds: list[Any]) -> Any: + from sqlglot import exp + + return conds[0] if len(conds) == 1 else exp.and_(*[c.copy() for c in conds], copy=False) + + +def _column_equality(cond: Any) -> bool: + from sqlglot import exp + + cond = _unparen(cond) + return isinstance(cond, exp.EQ) and isinstance(_unparen(cond.this), exp.Column) and \ + isinstance(_unparen(cond.expression), exp.Column) + + +def _partition(conds: list[Any]) -> frozenset[frozenset[str]]: + """「两列相等」的几个条件把列分成的组(传递:a = b、b = c 时 a、b、c 一组)。""" + parent: dict[str, str] = {} + + def find(x: str) -> str: + parent.setdefault(x, x) + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + for c in conds: + c = _unparen(c) + a, b = _fingerprint(_unparen(c.this)), _fingerprint(_unparen(c.expression)) + parent[find(a)] = find(b) + groups: dict[str, set[str]] = {} + for x in list(parent): + groups.setdefault(find(x), set()).add(x) + return frozenset(frozenset(g) for g in groups.values() if len(g) > 1) + + +def _same_class(partition: frozenset[frozenset[str]], x: str, y: str) -> bool: + return any(x in g and y in g for g in partition) + + +def _subquery_kind(cond: Any) -> str: + from sqlglot import exp + + sub = next(n for n in cond.walk() if isinstance(n, (exp.Select, exp.SetOperation))) + node = sub + while isinstance(node.parent, (exp.Subquery, exp.Paren)): + node = node.parent + parent = node.parent + if isinstance(parent, exp.Exists): + return "exists" + if isinstance(parent, exp.In): + return "in" + return "scalar" + + +def _deeper(where: str, place: str, plain: bool = False) -> str: + """在哪一段里再往里一层:「在 t 里」+「IN 子查询」→「在 t 的 IN 子查询里」。""" + if not where: + return _say("在", place, "里") + return _say(where[:-1] if where.endswith("里") else where, "的", place, "里") + + +def _by_bound(conds: list[Any]) -> list[Any]: + """时间条件给人看时下界在前:>=、> 在前,<、<= 在后,其余照原来的先后。""" + from sqlglot import exp + + def rank(c: Any) -> int: + return 0 if isinstance(c, (exp.GTE, exp.GT)) else 1 if isinstance(c, (exp.LT, exp.LTE)) else 2 + + return sorted(conds, key=rank) + + +def _ancestors(node: Any, stop: Any) -> Iterator[Any]: + node = node.parent + while node is not None and node is not stop: + yield node + node = node.parent + + +def _set_branches(query: Any) -> list[Any]: + """连着同一种合并方式的几支(规范形式里已经排好序)。""" + kind = (type(query), bool(query.args.get("distinct"))) + out: list[Any] = [] + + def collect(node: Any) -> None: + from sqlglot import exp + + if isinstance(node, exp.SetOperation) and (type(node), bool(node.args.get("distinct"))) == kind and \ + (node is query or not any(node.args.get(k) for k in ("order", "limit", "offset"))): + collect(node.left) + collect(node.right) + else: + out.append(node) + + collect(query) + return out + + +def _merging(query: Any) -> str: + return "UNION 合并(去掉重复的行)" if query.args.get("distinct") else "UNION ALL(重复的行都留着)" + + +def _aggregate(node: Any) -> tuple[str, Any, bool] | None: + """光一个聚合(SUM / AVG / COUNT / MIN / MAX,MIN / MAX 只有一个参数:两个参数的 max(a, b) 是取较大的那个,不是 + 聚合):(函数, 参数(COUNT(*) 是 None), 是否去重)。""" + from sqlglot import exp + + kinds = {exp.Sum: "sum", exp.Avg: "avg", exp.Count: "count", exp.Min: "min", exp.Max: "max"} + func = kinds.get(type(node)) + if func is None or node.args.get("expressions"): + return None + arg = node.this + distinct = isinstance(arg, exp.Distinct) + if distinct: + if len(arg.expressions) != 1: + return None + arg = arg.expressions[0] + if isinstance(arg, exp.Star): + arg = None + return func, arg, distinct + + +def _verb(func: str, distinct: bool) -> str: + return ("去重" if distinct else "") + _VERBS.get(func, f"用 {func.upper()} 汇总") def _rows(limit: str | None) -> str: @@ -1255,280 +6261,98 @@ def _skip(offset: str | None) -> str: return f"跳过前 {offset} 行" if offset.isdigit() else f"按 OFFSET {offset} 跳过行" -class _Compare: - """一次比较:两边的叫法、列的中文名,以及按哪种写法比。 +def _text_diff(a: _Form | None, b: _Form | None, read: str | None) -> tuple[str, ...]: + """规范化之后两条 SQL 不同的行:difflib 比两边 sqlglot pretty 的输出,「- 」是 A 的、「+ 」是 B 的。""" + import difflib - 两边各有一张只在自己这边出现、又不是关联进来的表(orders 换成了 orders_archive)时,其余各方面按不写表名的 - loose 比:表换了这件事由一条「tables」说清,同一个条件、同一个聚合不再因为表名不同被重复报一遍。 + def lines(form: _Form | None) -> list[str]: + if form is None: + return [] + try: + return form.tree.sql(dialect=read, pretty=True, comments=False).splitlines() + except Exception: # noqa: BLE001 + return form.text.splitlines() - 每个方面都先按 k 比,比不上的再按 ak(CTE、子查询不写编号)一对一地配一遍(见 unmatched)。 + out = [line for line in difflib.ndiff(lines(a), lines(b)) if line[:2] in ("- ", "+ ")] + return tuple(out[:_MAX_DIFF_LINES]) + + +def explain_sql_diff(a: str, b: str, dialect: str | None = None, *, schema: Mapping[str, Any] | None = None, + names: Mapping[str, str] | None = None, labels: tuple[str, str] = ("A", "B")) -> SqlDiff: + """两条 SQL 是不是等价、不是的话差在哪。从不抛异常。 + + 等价怎么判:两边各自规范化(见 _canonical_forms:认列之后的那一级、不认列的轻量一级、只统一名字的一级),每一步都 + 只做保持语义的变换;同一级的两个规范形式完全一样(比语法树,见 _Form.key)才算等价,否则一律不等价,items 一定 + 不为空。schema 是数据源的 schema_cache,给了就按它认列、展开 SELECT *(表结构要和库对得上)。原文里有解析时丢了 + 写法的(一元加号、0x10、换了名字的类型名,见 _lost_in_parsing)不在语法树上判等价。 + + 不等价时怎么说:在两边的规范形式上逐段比(见 _Explainer),只把有把握、确实不同的地方写成具体的中文句子;具体说法 + 一条都没有时给一条 aspect="other" 的兜底说明(「两条 SQL 的写法不同,未能归纳成具体差异,请对照原文」),只覆盖了 + 一部分时追加一条「另有未能归纳的差异」,fallback 为真;text_diff 给出规范化之后不同的行。 + + labels 是两边的叫法(缺省 A / B,双盲复算传「原流程」「盲算」);names 是列的中文名({「表.列」或「列」: 中文名}, + 可以用 catalog_labels 从数据目录取),时间列写成「支付时间 paid_at」。任一边解析不了:只给一条 aspect="parse"。 """ - - def __init__(self, a: Facets, b: Facets, labels: tuple[str, str], names: Mapping[str, str] | None) -> None: - self.a, self.b = a, b - self.la, self.lb = labels - self.label = _labeler(names) - joined = {j.table.key for j in (*a.joins, *b.joins)} - self.ta = [t for t in a.tables if t.key not in {u.key for u in b.tables} and t.key not in joined] - self.tb = [t for t in b.tables if t.key not in {u.key for u in a.tables} and t.key not in joined] - self.swapped = len(self.ta) == 1 and len(self.tb) == 1 - - def k(self, part: Part | None) -> str | None: - if part is None: - return None - return part.loose if self.swapped else part.key - - def ak(self, part: Part | None) -> str | None: - """同 k,但 CTE、子查询不写编号(见 unmatched)。""" - if part is None: - return None - return part.loose if self.swapped else (part.anon or part.key) - - @staticmethod - def unmatched(xs: Iterable[Any], ys: Iterable[Any], key: Any, anon: Any) -> tuple[list[Any], list[Any]]: - """两边各自独有的项,各按原来的先后。先按 key 比;剩下的再按 anon(CTE、子查询不写编号)一对一地配,配上的 - 也不算不同。编号只看位置,一边多出一个排在前面的 CTE、子查询,后面的编号就错开了:同一个条件不能因此被说成 - 两边各有一个。编号对得上时 key 就分得清是哪个 CTE 的列,anon 只收拾 key 比不上的;两个 CTE 算出同名的列、 - 编号又错开时可能配错一对,宁可漏报。""" - xs, ys = list(xs), list(ys) - kx, ky = {key(x) for x in xs}, {key(y) for y in ys} - only_x = [x for x in xs if key(x) not in ky] - only_y = [y for y in ys if key(y) not in kx] - paired_x: set[int] = set() - paired_y: set[int] = set() - for i, x in enumerate(only_x): - j = next((j for j, y in enumerate(only_y) if j not in paired_y and anon(y) == anon(x)), None) - if j is not None: - paired_x.add(i) - paired_y.add(j) - return [x for i, x in enumerate(only_x) if i not in paired_x], \ - [y for j, y in enumerate(only_y) if j not in paired_y] - - @staticmethod - def shown(xs: list[str], ys: list[str], full_xs: list[str], full_ys: list[str]) -> tuple[str, str]: - """两边给人看的写法;读起来一样(各自只写列名)却不是同一个东西时,改成一律写表名的写法。""" - wa, wb = "、".join(xs), "、".join(ys) - return ("、".join(full_xs), "、".join(full_ys)) if wa == wb else (wa, wb) - - def named(self, parts: list[Part], *, full: bool = False) -> str: - """时间列:有中文名的写「支付时间 paid_at」,几列用顿号隔开。""" - out = [] - for p in parts: - text = p.full if full else p.text - name = self.label(p.columns[0]) if p.columns else None - out.append(_say(name, text) if name else text) - return "、".join(out) - - def one_sided(self, side: str, other: str, part: Part, *, group: bool = False) -> str: - if group: - return _say(side, "分组后只保留", part.text, "的组,", other, "没有这个条件") - if part.excludes: - return _say(side, "排除了", part.excludes, "的行,", other, "没有") - return _say(side, "只保留", part.text, "的行,", other, "没有这个条件") - - # ---- 各个方面 - - def time(self) -> list[FacetDiff]: - a, b, la, lb = self.a, self.b, self.la, self.lb - ca = list(_dedupe((t.column for t in a.time), self.k)) - cb = list(_dedupe((t.column for t in b.time), self.k)) - if any(self.unmatched(ca, cb, self.k, self.ak)): - wa, wb = self.named(ca), self.named(cb) - if wa == wb: - wa, wb = self.named(ca, full=True), self.named(cb, full=True) - if ca and cb: - text = _say(la, "按", wa, "筛时间,", lb, "按", wb) - elif ca: - text = _say(la, "按", wa, "筛时间,", lb, "没有按时间筛选") - else: - text = _say(la, "没有按时间筛选,", lb, "按", wb, "筛时间") - detail = {side: [{"table": p.columns[0][0], "column": p.columns[0][1]} for p in parts if p.columns] - for side, parts in (("a", ca), ("b", cb))} - return [FacetDiff("time", "、".join(p.text for p in ca) or None, "、".join(p.text for p in cb) or None, - text, detail)] - if any(self.unmatched(a.time, b.time, lambda t: self.k(t.condition), lambda t: self.ak(t.condition))): - ra = " AND ".join(t.condition.text for t in a.time) - rb = " AND ".join(t.condition.text for t in b.time) - return [FacetDiff("time_range", ra, rb, _say(la, "的时间范围是", ra, ",", lb, "是", rb))] - return [] - - def filters(self) -> list[FacetDiff]: - timed = {self.k(t.condition) for t in (*self.a.time, *self.b.time)} - fa = [p for p in self.a.where if self.k(p) not in timed] - fb = [p for p in self.b.where if self.k(p) not in timed] - only_a, only_b = self.unmatched(fa, fb, self.k, self.ak) - out: list[FacetDiff] = [] - for p in list(only_a): - column = _single_column(p, loose=self.swapped) - mine = [q for q in only_a if column and _single_column(q, loose=self.swapped) == column] - theirs = [q for q in only_b if column and _single_column(q, loose=self.swapped) == column] - if len(mine) == 1 and len(theirs) == 1: - q = theirs[0] - wa, wb = self.shown([p.text], [q.text], [p.full], [q.full]) - out.append(FacetDiff("filter", p.text, q.text, _say(self.la, "按", wa, "筛选,", self.lb, "按", wb, "筛选"))) - only_a.remove(p) - only_b.remove(q) - out += [FacetDiff("filter", p.text, None, self.one_sided(self.la, self.lb, p)) for p in only_a] - out += [FacetDiff("filter", None, p.text, self.one_sided(self.lb, self.la, p)) for p in only_b] - return out - - def aggregates(self) -> list[FacetDiff]: - la, lb = self.la, self.lb - - def key(g: Aggregate) -> tuple[str, str | None, bool]: - return g.func, self.k(g.arg), g.distinct - - def anon(g: Aggregate) -> tuple[str, str | None, bool]: - return g.func, self.ak(g.arg), g.distinct - - def pair(xs: list[Aggregate], ys: list[Aggregate], same: Any) -> list[tuple[Aggregate, Aggregate]]: - pairs = [] - for x in list(xs): - y = next((y for y in ys if same(x, y)), None) - if y is not None: - pairs.append((x, y)) - xs.remove(x) - ys.remove(y) - return pairs - - ga, gb = self.unmatched(self.a.aggregates, self.b.aggregates, key, anon) - # 先配参数相同的(同一列:函数或去重不同),再配函数相同的(同一种算法:列不同);各剩一个的也配成一对 - pairs = pair(ga, gb, lambda x, y: self.k(x.arg) == self.k(y.arg)) - pairs += pair(ga, gb, lambda x, y: x.func == y.func) - if len(ga) == 1 and len(gb) == 1: - pairs.append((ga.pop(), gb.pop())) - out = [] - for x, y in pairs: - same_arg = self.k(x.arg) == self.k(y.arg) - wa, wb = self.shown([x.arg.text], [y.arg.text], [x.arg.full], [y.arg.full]) if not same_arg \ - else (x.arg.text, y.arg.text) - if same_arg and x.func == y.func: - # 只差去重:COUNT(DISTINCT order_id) 和 COUNT(order_id) - text = _say(la, "对", wa, _verb(x) + ",", lb, "没去重") if x.distinct \ - else _say(la, "对", wa, _verb(x) + "时没去重,", lb, _verb(y)) - elif same_arg and x.arg.key != "*": - # 同一列、算法不同:后半句不再重复列名 - text = _say(la, "对", wa, _verb(x) + ",", lb, _verb(y)) - else: - what_a = ["数的是行数"] if x.func == "count" and x.arg.key == "*" else ["对", wa, _verb(x)] - what_b = ["数的是行数"] if y.func == "count" and y.arg.key == "*" else ["对", wb, _verb(y)] - text = _say(la, *what_a, ",", lb, *what_b) - out.append(FacetDiff("aggregate", x.text, y.text, text)) - out += [FacetDiff("aggregate", g.text, None, _say(la, "多算了", g.text, ",", lb, "没有")) for g in ga] - out += [FacetDiff("aggregate", None, g.text, _say(lb, "多算了", g.text, ",", la, "没有")) for g in gb] - return out - - def joins(self) -> list[FacetDiff]: - """连接:先把完全一样的(类型、关联的表、条件)两边配掉,剩下的按关联的是哪张表配成一对、说类型或条件哪里 - 不同,再剩下的是只有一边关联了。直接按表配的话,编号错开时会把两个不相干的 CTE 配成一对,说成「关联条件 - 不同」;先配完全一样的,同一张表关联了两次、先后不同也不算不同。""" - la, lb = self.la, self.lb - rest_a, rest_b = self.unmatched(self.a.joins, self.b.joins, - lambda j: (j.kind, j.table.key, self.k(j.on)), - lambda j: (j.kind, j.table.anon, self.ak(j.on))) - pairs: list[tuple[JoinFacet, JoinFacet]] = [] - for table in (lambda j: j.table.key, lambda j: j.table.anon): - for x in list(rest_a): - y = next((y for y in rest_b if table(y) == table(x)), None) - if y is not None: - pairs.append((x, y)) - rest_a.remove(x) - rest_b.remove(y) - out: list[FacetDiff] = [] - for x, y in pairs: - if x.kind != y.kind: - kx, ky = _JOIN_WORDS.get(x.kind, x.kind), _JOIN_WORDS.get(y.kind, y.kind) - out.append(FacetDiff("join", kx, ky, _say(la, "用", kx, "关联", x.table.text, "表,", lb, "用", ky))) - if self.k(x.on) != self.k(y.on) and self.ak(x.on) != self.ak(y.on): - xa, ya = (x.on.text if x.on else "(无条件)"), (y.on.text if y.on else "(无条件)") - out.append(FacetDiff("join", xa, ya, _say(la, "关联", x.table.text, "表的条件是", xa, ",", lb, "是", ya))) - out += [FacetDiff("join", x.table.text, None, _say(la, "关联了", x.table.text, "表,", lb, "没有")) for x in rest_a] - out += [FacetDiff("join", None, y.table.text, _say(lb, "关联了", y.table.text, "表,", la, "没有")) for y in rest_b] - # 关联之外的表(FROM 的表、子查询里的表)只在一边出现的 - if self.swapped: - ta, tb = self.ta[0], self.tb[0] - return out + [FacetDiff("tables", ta.text, tb.text, _say(la, "查的是", ta.text, "表,", lb, "查的是", tb.text, - "表"))] - out += [FacetDiff("tables", t.text, None, _say(la, "用到了", t.text, "表,", lb, "没有")) for t in self.ta] - out += [FacetDiff("tables", None, t.text, _say(lb, "用到了", t.text, "表,", la, "没有")) for t in self.tb] - return out - - def group(self) -> list[FacetDiff]: - a, b = self.a.group_by, self.b.group_by - if not any(self.unmatched(a, b, self.k, self.ak)): - return [] - wa, wb = self.shown([p.text for p in a], [p.text for p in b], [p.full for p in a], [p.full for p in b]) - if wa and wb: - text = _say(self.la, "按", wa, "分组,", self.lb, "按", wb, "分组") - elif wa: - text = _say(self.la, "按", wa, "分组,", self.lb, "没有分组") + la, lb = labels + read = _read(dialect) + ra, ea = _parse(a, read) + rb, eb = _parse(b, read) + if ea or eb: + if ea and eb: + text = _say(la, ":", ea, ";", lb, ":", eb, ";没有逐项比较") else: - text = _say(self.la, "没有分组,", self.lb, "按", wb, "分组") - return [FacetDiff("group_by", wa or None, wb or None, text)] - - def having(self) -> list[FacetDiff]: - only_a, only_b = self.unmatched(self.a.having, self.b.having, self.k, self.ak) - return [FacetDiff("having", p.text, None, self.one_sided(self.la, self.lb, p, group=True)) for p in only_a] + \ - [FacetDiff("having", None, p.text, self.one_sided(self.lb, self.la, p, group=True)) for p in only_b] - - def limit(self) -> list[FacetDiff]: - """取哪几行:最外层两边比;CTE、子查询里的按出现的先后一一配对(两边给它起的名字可以不同),整组一样就不报。""" - def outer(f: Facets) -> LimitFacet | None: - return next((x for x in f.limits if x.scope is None), None) - - out = self.rows(outer(self.a), outer(self.b)) - ia = [x for x in self.a.limits if x.scope is not None] - ib = [x for x in self.b.limits if x.scope is not None] - if [self.lk(x) for x in ia] != [self.lk(x) for x in ib] and \ - [self.lk(x, anon=True) for x in ia] != [self.lk(x, anon=True) for x in ib]: - for x, y in zip_longest(ia, ib): - out += self.rows(x, y) - return out - - def lk(self, x: LimitFacet | None, *, anon: bool = False) -> tuple[Any, ...] | None: - """一段取哪几行的比较用写法:行数、跳过的行数、排序键。anon 时排序键按 ak(见 unmatched)。""" - if x is None: - return None - key = self.ak if anon else self.k - return x.count, x.offset, tuple((key(p), desc) for p, desc in x.order) - - def sorting(self, x: LimitFacet, *, full: bool = False) -> str: - if not x.order: - return "没有排序" - keys = "、".join(_say(p.full if full else p.text, "降序" if desc else "升序") for p, desc in x.order) - return _say("按", keys, "排") - - def rows(self, x: LimitFacet | None, y: LimitFacet | None) -> list[FacetDiff]: - """一对段(一边可以没有)取的行有什么不同:行数、跳过的行数,两边都限了行时再比排序。""" - if self.lk(x) == self.lk(y) or self.lk(x, anon=True) == self.lk(y, anon=True): - return [] - sa, sb = (x.scope if x else None), (y.scope if y else None) - la = _say(self.la, "在", sa, "里") if sa else self.la - lb = _say(self.lb, "在", sb, "里") if sb and sb != sa else self.lb - out = [] - ca, cb = (x.count if x else None), (y.count if y else None) - if ca != cb: - out.append(FacetDiff("limit", ca, cb, _say(la, _rows(ca), ",", lb, _rows(cb)))) - oa, ob = (x.offset if x else None), (y.offset if y else None) - if oa != ob: - out.append(FacetDiff("limit", oa, ob, _say(la, _skip(oa), ",", lb, _skip(ob)))) - if x is not None and y is not None and self.lk(x)[2] != self.lk(y)[2] \ - and self.lk(x, anon=True)[2] != self.lk(y, anon=True)[2]: - wa, wb = self.sorting(x), self.sorting(y) - if wa == wb: - wa, wb = self.sorting(x, full=True), self.sorting(y, full=True) - out.append(FacetDiff("limit", wa, wb, _say(la, "取行前", wa, ",", lb, wb))) - return out + side, error = (la, ea) if ea else (lb, eb) + text = _say(side, ":", error or "", ",没有逐项比较") + return SqlDiff(False, [FacetDiff("parse", ea, eb, text)]) + info = _schema_info(schema) + forms_a: dict[str, _Form] = {} + forms_b: dict[str, _Form] = {} + try: + if read == "sqlite": + _mark_comma_joins(ra, a.strip(), read) + _mark_comma_joins(rb, b.strip(), read) + forms_a = {} if _lost_in_parsing(a.strip(), read, ra) else _canonical_forms(ra, read, info) + forms_b = {} if _lost_in_parsing(b.strip(), read, rb) else _canonical_forms(rb, read, info) + except Exception: # noqa: BLE001 - 规范化本身的缺陷不能连累调用方:当作说不清,记下来修 + logger.warning("SQL 规范化失败", exc_info=True) + forms_a, forms_b = {}, {} + for level in (_OPTIMIZER, _LIGHT, _PARSED): + fa, fb = forms_a.get(level), forms_b.get(level) + if fa is not None and fb is not None and fa.key == fb.key: + return SqlDiff(True, level=level) + level = next((lv for lv in (_OPTIMIZER, _LIGHT) if lv in forms_a and lv in forms_b), None) + items: list[FacetDiff] = [] + covered = False + if level is not None: + explainer = _Explainer(read, (la, lb), names, info) + try: + covered = explainer.statement(forms_a[level], forms_b[level]) + items = explainer.items + except Exception: # noqa: BLE001 - 说明本身的缺陷不能连累调用方:当作说不清,记下来修 + logger.warning("SQL 差异说明失败", exc_info=True) + items, covered = [], False + shown = level or _PARSED + if not items: + items = [FacetDiff("other", None, None, _FALLBACK)] + fallback = True + elif not covered: + items = items + [FacetDiff("other", None, None, _FALLBACK_REST)] + fallback = True + else: + fallback = False + text_diff = _text_diff(forms_a.get(shown), forms_b.get(shown), read) + if not text_diff: + text_diff = tuple(line for line in (f"- {a.strip()}", f"+ {b.strip()}")) + return SqlDiff(False, items, fallback, text_diff, level) def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, names: Mapping[str, str] | None = None) -> list[FacetDiff]: - """逐个方面比两份 facets,每处不同一条,按对结果影响大致从大到小排:时间、筛选、聚合、连接和表、分组、HAVING、 - 取哪几行。完全一致返回空列表。 + """两份 facets 记下的原文交给 explain_sql_diff 比,返回它的 items(含兜底条目):完全一致(规范形式一字不差) + 才返回空列表,不等价时一定不为空。 - labels 是两边的叫法(缺省 A / B,双盲复算可以传「原流程」「盲算」);names 是列的中文名({「表.列」或「列」: - 中文名},可以用 catalog_labels 从数据目录取),给了的话时间列写成「支付时间 paid_at」。任一边解析不了就只返回 - 一条 parse,不逐项比较:拿半份结构去比,只会报出一堆不存在的不同。 + labels、names 见 explain_sql_diff。任一边解析不了:只返回一条 parse。不是 facets 算出来的 Facets(没记原文) + 比不了原文:同一个对象才返回空列表,否则返回一条兜底说明(facets 记的结构不全,两份一样不说明 SQL 一样)。 """ la, lb = labels if a.parse_error or b.parse_error: @@ -1538,9 +6362,13 @@ def diff_facets(a: Facets, b: Facets, labels: tuple[str, str] = ("A", "B"), *, side, error = (la, a.parse_error) if a.parse_error else (lb, b.parse_error) text = _say(side, ":", error or "", ",没有逐项比较") return [FacetDiff("parse", a.parse_error, b.parse_error, text)] - cmp = _Compare(a, b, labels, names) - return cmp.time() + cmp.filters() + cmp.aggregates() + cmp.joins() + cmp.group() + cmp.having() + cmp.limit() + if a._origin is None or b._origin is None: + return [] if a is b else [FacetDiff("other", None, None, _FALLBACK)] + (sql_a, schema_a), (sql_b, schema_b) = a._origin, b._origin + return explain_sql_diff(sql_a, sql_b, a.dialect or b.dialect, schema=schema_a or schema_b, names=names, + labels=labels).items -__all__ = ["MAX_SQL_CHARS", "PLACEHOLDER", "Aggregate", "ColumnRef", "FacetDiff", "Facets", "JoinFacet", "LimitFacet", - "NormalizedSql", "Part", "TimePredicate", "catalog_labels", "diff_facets", "facets", "normalize_sql"] +__all__ = ["MAX_SQL_CHARS", "PLACEHOLDER", "Aggregate", "ColumnRef", "Computed", "FacetDiff", "Facets", "JoinFacet", + "LimitFacet", "NormalizedSql", "Part", "SqlDiff", "TimePredicate", "catalog_labels", "diff_facets", + "explain_sql_diff", "facets", "normalize_sql"] diff --git a/backend/tests/fixtures/sql_pairs.py b/backend/tests/fixtures/sql_pairs.py new file mode 100644 index 00000000..056b1491 --- /dev/null +++ b/backend/tests/fixtures/sql_pairs.py @@ -0,0 +1,2207 @@ +"""SQL 差异说明的执行比对夹具:虚构业务库的表结构、按固定种子生成的几组数据、SQL 对的语料。 + +为什么要执行:explain_sql_diff 判「等价」靠的是两条 SQL 规范化之后的形式一字不差,规范化的每一步都必须保持语义。 +光看规范化的代码保证不了这一点(sqlglot 的 merge_subqueries 把外连接可空一侧的常量列并进外层、unnest_subqueries 把 +OR 里的标量子查询改成 CROSS JOIN,都是复验时真踩到的),所以拿真实的库来守门:判等价的两条在每组数据上结果都得 +一样,结果不一样的绝不能判等价。 + +数据刻意带上会让写法差异暴露出来的东西:空值(金额、状态、时间、关联键)、重复的行、边界值(金额正好 100、时间 +正好在月初零点、月末 23:59:59)、已删除的行、退款(一单多退、退款单关联不上订单)、悬空的外键、空表。金额都是 +2 的负幂次的倍数,不同的求和先后得到的浮点数一样;比较结果时浮点数另按 6 位小数取整(乘 0.9 这类每行都有舍入)。 + +语料(PAIRS,SQLite,拿去执行)每一项 (编号, A, B, 预期):same 是已知等价的改写(别名、CTE 改名、关联先后、条件 +左右、子查询和 CTE 互换……),编号以 hard: 开头的是要按语义推理才看得出等价的(条件挪进相关子查询、冗余条件),规范化 +不保证认得出、认不出时只许给兜底说明;diff 是不许判等价的(换时间列、少一个条件、去重与否、外连接与内连接,以及只在 +某个方言、某种排序规则下才等价的)。第 2 轮复验的反例在 _SAME_HARD、_SAME_R2、_DIFF_R2(执行)和 REGRESSIONS_R2、 +DIALECT_SAME_R2、KNOWN_UNRECOGNIZED(各方言,只查结论)里。LIMIT 都带能定出唯一顺序的排序,没有 ORDER BY 的 LIMIT 不进语料。过去复验报过的反例都 +收进来了:第 5 轮回退的「只在 IN 子查询里用到的 CTE」在 PAIRS 里;复验者留在 /private/tmp 的脚本里、在这几组数据上 +结果不同的对是 VALIDATOR_DIFF(也进 PAIRS 执行);旧测试(五轮复核修正按复验报告补的用例,各方言,用到的列这个库里 +不全有)里报过不同的对是 REGRESSIONS、意思一样的对是 DIALECT_SAME,这两份不执行,只查判没判等价。 +""" +from __future__ import annotations + +import random +import sqlite3 +from dataclasses import dataclass + +DDL = { + "users": "id INTEGER PRIMARY KEY, name TEXT, city TEXT, vip INTEGER, grade INTEGER, referrer_id INTEGER, " + "created_at TEXT, is_deleted INTEGER", + "stores": "id INTEGER PRIMARY KEY, name TEXT, city TEXT, region TEXT, owner_id INTEGER", + "orders": "id INTEGER PRIMARY KEY, user_id INTEGER, store_id INTEGER, parent_id INTEGER, referrer_id INTEGER, " + "amount REAL, discount REAL, tax REAL, status TEXT, region TEXT, created_at TEXT, paid_at TEXT, " + "is_deleted INTEGER", + "orders_archive": "id INTEGER PRIMARY KEY, user_id INTEGER, store_id INTEGER, parent_id INTEGER, " + "referrer_id INTEGER, amount REAL, discount REAL, tax REAL, status TEXT, region TEXT, " + "created_at TEXT, paid_at TEXT, is_deleted INTEGER", + "order_items": "id INTEGER PRIMARY KEY, order_id INTEGER, product_id INTEGER, qty INTEGER, price REAL", + "refunds": "id INTEGER PRIMARY KEY, order_id INTEGER, user_id INTEGER, amount REAL, state TEXT, refunded_at TEXT", + "payments": "id INTEGER PRIMARY KEY, order_id INTEGER, user_id INTEGER, amount REAL, method TEXT, paid_at TEXT", + "shipments": "id INTEGER PRIMARY KEY, order_id INTEGER, carrier TEXT, shipped_at TEXT", + # 一张叫 nums 的实表:SQLite 里不写 RECURSIVE 的递归 CTE(WITH nums(n) AS (… FROM nums …))引用的是它自己,换个名字 + # 引用的就是这张表了 + "nums": "n INTEGER", +} + + +def schema_cache() -> dict: + """数据源 schema_cache 的形状({"tables": {表: {"columns": [{"name", "type"}]}}}),explain_sql_diff 的 schema。""" + tables = {} + for name, ddl in DDL.items(): + cols = [] + for part in ddl.split(", "): + col, typ = part.split(" ")[:2] + cols.append({"name": col, "type": typ}) + tables[name] = {"columns": cols} + return {"tables": tables} + + +AMOUNTS = [0, 0.5, 10, 12.5, 99.5, 100, 100.25, 250, -5, None] +DATES = ["2026-08-31", "2026-08-31 23:59:59", "2026-09-01", "2026-09-01 08:00:00", "2026-09-15", "2026-09-30", + "2026-09-30 23:59:59", "2026-10-01", "2026-10-01 00:00:00", None] +STATUSES = ["paid", "paid", "open", "done", "void", "shipped", None] +CITIES = ["SH", "BJ", "虚构市", None] + + +def _rows(seed: int) -> dict[str, list[tuple]]: + rng = random.Random(seed) + pick = rng.choice + n_users = rng.randint(4, 12) + users = [(i, f"u{i}", pick(CITIES), pick([0, 1, 1, None]), pick([1, 2, 3, 4, 5, None]), + pick([None, None] + list(range(1, n_users + 1))), pick(DATES), pick([0, 0, 0, 1, None])) + for i in range(1, n_users + 1)] + n_stores = rng.randint(2, 5) + stores = [(i, f"s{i}", pick(CITIES), pick(["east", "west", None]), pick([None] + list(range(1, n_users + 1)))) + for i in range(1, n_stores + 1)] + user_ids = list(range(1, n_users + 1)) + [99, None] + orders: list[tuple] = [] + for i in range(1, rng.randint(8, 36) + 1): + parent = pick([None, None, None] + [o[0] for o in orders[-5:]]) + orders.append((i, pick(user_ids), pick(list(range(1, n_stores + 1)) + [None]), parent, + pick([None] + list(range(1, n_users + 1))), pick(AMOUNTS), pick([0, 0.5, 2, None]), + pick([0, 1, 2.5, None]), pick(STATUSES), pick(["east", "west", None]), pick(DATES), + pick(DATES), pick([0, 0, 1, None]))) + # 重复的行:同一个用户同一笔金额再来一单 + for o in rng.sample(orders, min(3, len(orders))): + orders.append((len(orders) + 1, *o[1:])) + archive = [o for o in orders if rng.random() < 0.5] + items, refunds, payments, shipments = [], [], [], [] + for o in orders: + for _ in range(rng.randint(0, 3)): + items.append((len(items) + 1, o[0], rng.randint(1, 4), pick([0, 1, 2, 5, None]), pick([0.5, 10, 12.5, None]))) + for _ in range(pick([0, 0, 1, 1, 2])): + refunds.append((len(refunds) + 1, o[0], pick([o[1], o[1], None]), pick(AMOUNTS), + pick(["done", "pending", "rejected", None]), pick(DATES))) + for _ in range(rng.randint(0, 2)): + payments.append((len(payments) + 1, o[0], pick([o[1], None]), pick(AMOUNTS), pick(["card", "cash", None]), + pick(DATES))) + if rng.random() < 0.6: + shipments.append((len(shipments) + 1, o[0], pick(["sf", "ems", None]), pick(DATES))) + # 关联不上订单的退款、付款 + refunds.append((len(refunds) + 1, pick([None, 999]), pick(user_ids), pick(AMOUNTS), "done", pick(DATES))) + payments.append((len(payments) + 1, None, pick(user_ids), pick(AMOUNTS), "cash", pick(DATES))) + nums = [(n,) for n in rng.sample([1, 2, 3, 4, 10, 20], rng.randint(1, 4))] + return {"users": users, "stores": stores, "orders": orders, "orders_archive": archive, "order_items": items, + "refunds": refunds, "payments": payments, "shipments": shipments, "nums": nums} + + +def _edge_rows() -> dict[str, list[tuple]]: + """一组极端数据:退款、付款、发货都是空表,订单只有几行,状态、时间全是空值或边界值。""" + users = [(1, "u1", None, None, None, None, None, None), (2, "u2", "SH", 1, 3, 1, "2026-09-01", 0)] + stores = [(1, "s1", None, None, None)] + orders = [(1, 1, 1, None, None, 100, None, None, None, None, "2026-09-01", "2026-10-01", None), + (2, 1, None, 1, 2, None, 0, 0, "paid", "east", None, None, 0), + (3, None, 1, None, None, 100, 0, 1, "paid", None, "2026-09-30 23:59:59", "2026-09-01", 1)] + return {"users": users, "stores": stores, "orders": orders, "orders_archive": [], "order_items": [], + "refunds": [], "payments": [], "shipments": [], "nums": []} + + +def _tricky_rows() -> dict[str, list[tuple]]: + """一组专挑写法差异的数据(第 2 轮复验的反例在这上面现形):金额 0.1、0.2、0.3(浮点数 0.1 + 0.2 ≠ 0.3)、 + 100 * 0.07;状态里有像数字的文本('3.0'、'12')和大小写不同的 'PAID';用户名是像数字的文本('0100' 取两次负是 + 100);城市 'SH'、'sh';id 16 的订单(0x10);nums 表有几行。""" + users = [(1, "0100", "SH", 1, 3, None, "2026-09-01", 0), (2, "5", "sh", 0, 1, 1, "2026-09-02", 0), + (3, "alice", "Sh", None, None, 2, None, 1), (4, "Bob", None, 1, 5, None, "2026-08-31 23:59:59", None)] + stores = [(1, "s1", "SH", "east", 1), (2, "s2", "sh", None, None), (3, "3.0", "12", "west", 2)] + amounts = [0.1, 0.2, 0.3, 0.3, 0.30000000000000004, 7.0, 100 * 0.07, 12.5, None, 100.0, 0, 0.5] + statuses = ["paid", "PAID", "3.0", "12", None, "open", "void", "paid"] + orders = [] + for i in range(1, 21): + orders.append((i, (i % 4) + 1 if i % 7 else None, (i % 3) + 1, i - 1 if i % 5 == 0 else None, (i % 2) + 1, + amounts[i % len(amounts)], [0, 0.5, None][i % 3], [0, 1, None][i % 3], + statuses[i % len(statuses)], ["east", "west", None][i % 3], DATES[i % len(DATES)], + DATES[(i * 3) % len(DATES)], [0, 1, None][i % 3])) + refunds = [(1, 1, 2, 0.1, "done", "2026-09-05"), (2, 16, 1, 0.2, None, None), (3, 3, None, 0.3, "done", "2026-09-30"), + (4, 99, 3, 7.0, "pending", "2026-10-01")] + payments = [(1, 1, 2, 0.3, "card", "2026-09-02"), (2, 16, 1, 0.1, "cash", None)] + items = [(1, 1, 1, 2, 0.1), (2, 16, 2, None, 0.2), (3, 3, 1, 1, None)] + shipments = [(1, 1, "sf", "2026-09-03"), (2, 16, None, None)] + return {"users": users, "stores": stores, "orders": orders, "orders_archive": orders[:5], "order_items": items, + "refunds": refunds, "payments": payments, "shipments": shipments, "nums": [(1,), (2,), (10,)]} + + +#: 生成数据用的种子(每个种子一组),外加一组极端数据、一组专挑写法差异的数据 +SEEDS = (7, 11, 23, 42, 2026) + + +def build_databases() -> list[sqlite3.Connection]: + """每组数据一个内存库。""" + out = [] + for rows in [_rows(seed) for seed in SEEDS] + [_edge_rows(), _tricky_rows()]: + conn = sqlite3.connect(":memory:") + for name, ddl in DDL.items(): + conn.execute(f"CREATE TABLE {name} ({ddl})") + if rows[name]: + marks = ", ".join("?" * len(rows[name][0])) + conn.executemany(f"INSERT INTO {name} VALUES ({marks})", rows[name]) + out.append(conn) + return out + + +def _norm(value): + return round(value, 6) if isinstance(value, float) else value + + +def run(conn: sqlite3.Connection, sql: str) -> list[tuple]: + """执行结果按行排序(行的先后不算结果的一部分),浮点数按 6 位小数取整。""" + rows = [tuple(_norm(v) for v in row) for row in conn.execute(sql).fetchall()] + return sorted(rows, key=lambda r: tuple((v is None, str(type(v)), v if v is not None else 0) for v in r)) + + +@dataclass(frozen=True) +class Pair: + id: str + a: str + b: str + #: same:已知等价的改写;diff:不许判等价——语义不同,或者只在某个方言、某种排序规则、某种列类型下才等价 + #: (比如 SQLite 里 status > 'b' AND status > 'o' 和 status > 'o' 一样,PostgreSQL 的语言排序规则下不一定) + expect: str + + +PR = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) ") + +_SAME = [ + # ---- 别名、大小写、空白、注释 + ("alias", "SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid'", + "SELECT SUM(x.amount) FROM orders AS x WHERE x.status = 'paid'"), + ("bare-vs-qualified", "SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid' AND o.paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND status = 'paid'"), + ("case-comments", "SELECT store_id, SUM(amount) FROM orders WHERE status = 'paid' GROUP BY store_id", + "select STORE_ID , -- 门店\n sum( Amount ) /* 金额 */ from ORDERS\n where status='paid' group by store_id"), + ("quoted-names", 'SELECT SUM("amount") FROM "orders" WHERE "status" = \'paid\'', + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("output-alias", "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id", + "SELECT o.user_id AS uid, SUM(o.amount) AS s FROM orders o GROUP BY o.user_id"), + # ---- 条件左右、方向、先后、括号、等价写法 + ("flip-compare", "SELECT COUNT(*) FROM orders WHERE amount > 100 AND paid_at >= '2026-09-01' AND store_id = 3", + "SELECT COUNT(*) FROM orders WHERE 3 = store_id AND '2026-09-01' <= paid_at AND 100 < amount"), + ("and-order-parens", "SELECT COUNT(*) FROM orders WHERE (status = 'paid') AND ((amount > 0)) AND is_deleted = 0", + "SELECT COUNT(*) FROM orders WHERE is_deleted = 0 AND amount > 0 AND status = 'paid'"), + ("or-order", "SELECT COUNT(*) FROM orders WHERE status = 'paid' OR status = 'done'", + "SELECT COUNT(*) FROM orders WHERE status = 'done' OR status = 'paid'"), + ("between", "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at <= '2026-09-30'"), + ("not-eq", "SELECT COUNT(*) FROM orders WHERE NOT status = 'void'", + "SELECT COUNT(*) FROM orders WHERE status <> 'void'"), + ("bang-eq", "SELECT COUNT(*) FROM orders WHERE status != 'void'", + "SELECT COUNT(*) FROM orders WHERE status <> 'void'"), + ("not-null", "SELECT COUNT(*) FROM orders WHERE paid_at IS NOT NULL", + "SELECT COUNT(*) FROM orders WHERE NOT (paid_at IS NULL)"), + ("not-gt", "SELECT COUNT(*) FROM orders WHERE NOT amount > 100", + "SELECT COUNT(*) FROM orders WHERE amount <= 100"), + ("in-order", "SELECT COUNT(*) FROM orders WHERE status IN ('paid', 'done')", + "SELECT COUNT(*) FROM orders WHERE status IN ('done', 'paid', 'done')"), + ("in-single", "SELECT COUNT(*) FROM orders WHERE status IN ('paid')", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("mul-commute", "SELECT SUM(price * qty) FROM order_items WHERE price + 1 > 10", + "SELECT SUM(qty * price) FROM order_items WHERE 1 + price > 10"), + ("paren-arith", "SELECT COUNT(*) FROM orders WHERE (amount + tax) > 10", + "SELECT COUNT(*) FROM orders WHERE amount + tax > 10"), + ("count-1", "SELECT COUNT(1) FROM orders WHERE status = 'paid'", + "SELECT count(*) FROM orders WHERE status = 'paid'"), + ("count-1-having", "SELECT store_id FROM orders GROUP BY store_id HAVING COUNT(1) > 2", + "SELECT store_id FROM orders GROUP BY store_id HAVING 2 < COUNT(*)"), + ("having-flip", "SELECT store_id FROM orders GROUP BY store_id HAVING COUNT(*) >= 5", + "SELECT store_id FROM orders GROUP BY store_id HAVING 5 <= COUNT(*)"), + ("scalar-flip", "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders WHERE is_deleted = 0)", + "SELECT SUM(amount) FROM orders WHERE (SELECT AVG(amount) FROM orders WHERE 0 = is_deleted) < amount"), + ("count-distinct-paren", "SELECT COUNT(DISTINCT(user_id)) FROM orders", + "SELECT COUNT(DISTINCT user_id) FROM orders"), + ("expr-commute", "SELECT store_id, COUNT(*) * 100 AS x FROM orders GROUP BY store_id", + "SELECT store_id, 100 * COUNT(*) AS y FROM orders GROUP BY store_id"), + # ---- 分组、排序、取行 + ("group-ordinal", "SELECT store_id, SUM(amount) AS s FROM orders GROUP BY 1", + "SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id"), + ("group-order", "SELECT region, store_id, SUM(amount) FROM orders GROUP BY region, store_id HAVING SUM(amount) > 10", + "SELECT region, store_id, SUM(amount) AS s FROM orders GROUP BY store_id, region HAVING s > 10"), + ("group-alias", "SELECT o.region, SUM(o.amount) FROM orders o GROUP BY o.region HAVING SUM(o.amount) > 100", + "SELECT region AS r, SUM(amount) AS s FROM orders GROUP BY r HAVING s > 100"), + ("group-expr-ordinal", "SELECT DATE(paid_at) AS d, SUM(amount) FROM orders GROUP BY DATE(paid_at)", + "SELECT DATE(paid_at) AS d, SUM(amount) FROM orders GROUP BY 1"), + ("limit-offset0", "SELECT id, amount FROM orders ORDER BY amount DESC, id LIMIT 3 OFFSET 0", + "SELECT o.id, o.amount FROM orders o ORDER BY o.amount DESC, o.id ASC LIMIT 3"), + ("order-alias", "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id ORDER BY total DESC, user_id LIMIT 3", + "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id ORDER BY SUM(amount) DESC, user_id LIMIT 3"), + ("order-ordinal", "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id ORDER BY 2 DESC, 1 LIMIT 3", + "SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id ORDER BY total DESC, user_id LIMIT 3"), + ("distinct-vs-group", "SELECT user_id, store_id FROM orders GROUP BY user_id, store_id", + "SELECT DISTINCT user_id, store_id FROM orders"), + ("distinct-group-redundant", "SELECT DISTINCT user_id FROM orders GROUP BY user_id", + "SELECT user_id FROM orders GROUP BY user_id"), + ("distinct-subq-vs-group", "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' GROUP BY user_id) AS u"), + ("window-alias", "SELECT o.id, ROW_NUMBER() OVER (PARTITION BY o.user_id ORDER BY o.paid_at, o.id) AS rn FROM orders o", + "SELECT x.id, ROW_NUMBER() OVER (PARTITION BY x.user_id ORDER BY x.paid_at, x.id) AS k FROM orders x"), + ("window-partition-order", "SELECT id, SUM(amount) OVER (PARTITION BY user_id, store_id) AS s FROM orders", + "SELECT id, SUM(amount) OVER (PARTITION BY store_id, user_id) AS s FROM orders"), + ("case-alias", "SELECT SUM(CASE WHEN o.status = 'paid' THEN o.amount ELSE 0 END) FROM orders o", + "SELECT SUM(CASE WHEN x.status = 'paid' THEN x.amount ELSE 0 END) FROM orders AS x"), + # ---- 连接 + ("join-swap", "SELECT s.name, SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id GROUP BY s.name", + "SELECT s.name, SUM(o.amount) FROM stores s INNER JOIN orders o ON o.store_id = s.id GROUP BY s.name"), + ("comma-join", "SELECT COUNT(*) FROM orders o, stores s WHERE o.store_id = s.id AND s.city = 'SH'", + "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'"), + ("on-vs-where", "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'", + "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id AND s.city = 'SH'"), + ("cross-where", "SELECT COUNT(*) FROM orders o CROSS JOIN stores s WHERE s.id = o.store_id", + "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id"), + ("three-way", "SELECT COUNT(*) FROM orders o JOIN stores s ON s.id = o.store_id JOIN users u ON u.id = o.user_id " + "WHERE u.vip = 1 AND s.city = 'SH'", + "SELECT COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id JOIN stores s ON o.store_id = s.id " + "WHERE s.city = 'SH' AND u.vip = 1"), + ("multi-on", "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id AND r.user_id = o.user_id", + "SELECT COUNT(*) FROM refunds r JOIN orders o ON o.user_id = r.user_id AND o.id = r.order_id"), + ("left-outer-word", "SELECT o.id, r.amount FROM orders o LEFT OUTER JOIN refunds r ON r.order_id = o.id", + "SELECT o.id, r.amount FROM orders o LEFT JOIN refunds r ON o.id = r.order_id"), + ("left-right-mirror", "SELECT o.id, r.amount FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT o.id, r.amount FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("left-chain-reorder", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id " + "LEFT JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM orders o LEFT JOIN users u ON u.id = o.user_id LEFT JOIN refunds r ON r.order_id = o.id"), + ("left-rejected-vs-inner", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.amount > 10", + "SELECT COUNT(*) FROM refunds r JOIN orders o ON o.id = r.order_id WHERE r.amount > 10"), + ("left-not-null-vs-inner", "SELECT SUM(o.amount) FROM orders o LEFT JOIN users u ON u.id = o.user_id " + "WHERE u.city IS NOT NULL", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id WHERE u.city IS NOT NULL"), + ("anti-join-alias", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.id IS NULL", + "SELECT COUNT(*) FROM orders x LEFT JOIN refunds y ON x.id = y.order_id WHERE y.id IS NULL"), + ("using-vs-on", "SELECT COUNT(*) FROM orders o JOIN refunds r USING (user_id)", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.user_id = o.user_id"), + ("count-distinct-join", "SELECT COUNT(DISTINCT o.user_id) FROM orders o JOIN stores s ON s.id = o.store_id " + "WHERE s.city = 'SH'", + "SELECT COUNT(DISTINCT a.user_id) FROM stores b JOIN orders a ON a.store_id = b.id WHERE b.city = 'SH'"), + # ---- 自连接 + ("self-join-swap", "SELECT COUNT(*) FROM orders o JOIN orders c ON c.parent_id = o.id WHERE c.amount > o.amount", + "SELECT COUNT(*) FROM orders x JOIN orders y ON x.id = y.parent_id WHERE x.amount < y.amount"), + ("self-join-names-exchanged", "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id " + "WHERE a.vip = 1 AND b.vip = 0", + "SELECT COUNT(*) FROM users b JOIN users a ON a.id = b.referrer_id WHERE b.vip = 1 AND a.vip = 0"), + ("self-join-flip-op", "SELECT COUNT(*) FROM orders a JOIN orders b ON a.user_id = b.user_id WHERE a.id < b.id", + "SELECT COUNT(*) FROM orders a JOIN orders b ON a.user_id = b.user_id WHERE b.id > a.id"), + ("self-join-symmetric", "SELECT COUNT(*) FROM orders a JOIN orders b ON a.user_id = b.user_id " + "WHERE a.status = 'paid' AND b.status = 'void'", + "SELECT COUNT(*) FROM orders b JOIN orders a ON b.user_id = a.user_id WHERE b.status = 'paid' AND a.status = 'void'"), + ("self-join-3way", "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id JOIN users c " + "ON c.id = b.referrer_id WHERE c.vip = 1 AND a.vip = 0", + "SELECT COUNT(*) FROM users x JOIN users y ON y.id = x.referrer_id JOIN users z ON z.id = y.referrer_id " + "WHERE x.vip = 0 AND z.vip = 1"), + # ---- CTE、子查询 + ("cte-rename", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT store_id, SUM(amount) FROM paid GROUP BY store_id", + "WITH p2 AS (SELECT * FROM orders WHERE status = 'paid') SELECT store_id, SUM(amount) FROM p2 GROUP BY store_id"), + ("cte-inline", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT store_id, SUM(amount) FROM paid GROUP BY store_id", + "SELECT store_id, SUM(amount) FROM orders WHERE status = 'paid' GROUP BY store_id"), + ("cte-vs-subquery", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE t.total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id) AS q WHERE q.s > 100"), + ("cte-def-order", "WITH a AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "b AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " + "SELECT SUM(a.total - b.total) FROM a JOIN b ON a.user_id = b.user_id", + "WITH b AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "a AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(a.total - b.total) FROM b JOIN a ON a.user_id = b.user_id"), + ("cte-vs-subq-swap", PR + "SELECT SUM(p.total) - SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "SELECT SUM(a.total) - SUM(b.total) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) b " + "JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) a ON a.user_id = b.user_id"), + ("cte-names-swapped", PR + "SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "refunded AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(r.total - p.total) FROM refunded r JOIN paid p ON p.user_id = r.user_id"), + ("cte-condition-flipped", PR + "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id " + "WHERE p.total - r.total > 100", + "WITH rf AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "pd AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM rf JOIN pd ON pd.user_id = rf.user_id WHERE 100 < pd.total - rf.total"), + ("cte-dead", "WITH unused AS (SELECT * FROM refunds), t AS (SELECT user_id, SUM(amount) AS s FROM orders " + "GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 10", + "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 10"), + ("cte-chain-vs-nested", "WITH a AS (SELECT * FROM orders WHERE status = 'paid'), b AS (SELECT user_id, " + "SUM(amount) AS s FROM a GROUP BY user_id) SELECT COUNT(*) FROM b WHERE s > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS s FROM (SELECT * FROM orders WHERE status = 'paid') AS a " + "GROUP BY user_id) AS b WHERE b.s > 100"), + ("cte-filter-then-group", "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), per_user AS (SELECT user_id, " + "SUM(amount) AS total FROM base GROUP BY user_id) " + "SELECT COUNT(*) FROM per_user WHERE total > 100", + "WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " + "SELECT COUNT(*) FROM totals WHERE total > 100"), + ("cte-filter-inner-vs-outer", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT store_id, SUM(amount) FROM p GROUP BY store_id", + "WITH p AS (SELECT * FROM orders) SELECT store_id, SUM(amount) FROM p WHERE status = 'paid' GROUP BY store_id"), + ("cte-join-inline", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT SUM(p.amount) FROM users u JOIN p ON p.user_id = u.id WHERE u.vip = 1", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON o.user_id = u.id WHERE o.status = 'paid' AND u.vip = 1"), + ("subq-join-inline", "SELECT COUNT(*) FROM users u JOIN (SELECT * FROM orders WHERE status = 'paid') o " + "ON o.user_id = u.id", + "SELECT COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id WHERE o.status = 'paid'"), + ("cte-group-city", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT s.city, SUM(p.amount) FROM p JOIN stores s ON s.id = p.store_id GROUP BY s.city", + "SELECT s.city, SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE o.status = 'paid' " + "GROUP BY s.city"), + ("cte-join-in-cte", "WITH t AS (SELECT o.amount, s.city FROM orders o JOIN stores s ON s.id = o.store_id) " + "SELECT city, SUM(amount) FROM t GROUP BY city", + "SELECT s.city, SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id GROUP BY s.city"), + ("cte-passthrough-group", "SELECT store_id, SUM(amount) FROM orders GROUP BY store_id", + "WITH t AS (SELECT store_id, amount FROM orders) SELECT store_id, SUM(amount) FROM t GROUP BY store_id"), + ("cte-computed-col", "WITH t AS (SELECT id, (amount * 0.5) AS net FROM orders) SELECT SUM(net) FROM t", + "WITH t AS (SELECT id, amount * 0.5 AS net FROM orders) SELECT SUM(t.net) FROM t"), + ("cte-date-inline", "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) " + "SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "SELECT SUM(amount) FROM orders WHERE DATE(paid_at) >= '2026-09-01'"), + ("cte-grouped-wrapped", "SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id", + "SELECT t.store_id, t.s FROM (SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id) t"), + ("cte-grouped-star", "WITH t AS (SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id) SELECT * FROM t", + "SELECT * FROM (SELECT store_id, SUM(amount) AS s FROM orders GROUP BY store_id) x"), + ("cte-having-vs-where", "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id " + "HAVING SUM(amount) > 100) SELECT COUNT(*) FROM t", + "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 100"), + ("cte-join-subq-mix", "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(p.total - r.total) FROM paid p JOIN (SELECT user_id, SUM(amount) AS total " + "FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "WITH refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " + "SELECT SUM(p.total - r.total) FROM refunded r JOIN (SELECT user_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id) p ON p.user_id = r.user_id"), + ("cte-three-reorder", "WITH p AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " + "r AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "v AS (SELECT id FROM users WHERE vip = 1) " + "SELECT SUM(p.total - r.total) FROM p JOIN r ON r.user_id = p.user_id JOIN v ON v.id = p.user_id", + "WITH v AS (SELECT id FROM users WHERE vip = 1), r AS (SELECT user_id, SUM(amount) AS total FROM refunds " + "GROUP BY user_id), p AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(p.total - r.total) FROM v JOIN p ON p.user_id = v.id JOIN r ON p.user_id = r.user_id"), + ("cte-self-join-swapped", "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM t a JOIN t b " + "ON b.user_id = a.referrer_id WHERE a.total > b.total", + "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) " + "SELECT COUNT(*) FROM t b JOIN t a ON a.referrer_id = b.user_id WHERE b.total < a.total"), + ("two-passthrough-ctes", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders " + "WHERE status = 'open') SELECT SUM(p.amount) - SUM(o.amount) FROM paid p JOIN open o " + "ON o.user_id = p.user_id", + "WITH op AS (SELECT * FROM orders WHERE status = 'open'), pd AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT SUM(x.amount) - SUM(y.amount) FROM op y JOIN pd x ON x.user_id = y.user_id"), + ("passthrough-cte-own-table", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) " + "FROM orders o JOIN paid p ON p.parent_id = o.id WHERE o.amount > p.amount", + "WITH pp AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM pp c JOIN orders m " + "ON c.parent_id = m.id WHERE c.amount < m.amount"), + # ---- 条件里的子查询:IN、EXISTS、标量 + ("in-cte", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)", + "WITH v AS (SELECT id FROM users WHERE vip = 1) SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM v)"), + ("in-cte-regression", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM p)", + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + ("in-alias", "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id AS uid FROM users WHERE vip = 1)"), + ("in-and-order", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1 AND city = 'SH')", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users WHERE city = 'SH' AND vip = 1)"), + ("in-join-swap", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT u.id FROM users u JOIN stores s " + "ON s.owner_id = u.id)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT u.id FROM stores s JOIN users u ON u.id = s.owner_id)"), + ("in-distinct", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT DISTINCT user_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM refunds)"), + ("exists-alias", "SELECT o.id FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT orders.id FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), + ("exists-star", "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT * FROM refunds r WHERE r.order_id = o.id)"), + ("exists-and-order", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id " + "AND c.status = 'paid')", + "SELECT COUNT(*) FROM orders AS m WHERE EXISTS (SELECT 1 FROM orders AS k WHERE k.status = 'paid' " + "AND m.id = k.parent_id)"), + ("exists-cte-vs-subq", "WITH v AS (SELECT id FROM users WHERE vip = 1) SELECT COUNT(*) FROM orders o " + "WHERE EXISTS (SELECT 1 FROM v WHERE v.id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM (SELECT id FROM users WHERE vip = 1) v " + "WHERE v.id = o.user_id)"), + ("exists-names-exchanged", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders o2 " + "WHERE o2.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o2 WHERE EXISTS (SELECT 1 FROM orders o WHERE o.parent_id = o2.id)"), + ("exists-outer-by-name", "SELECT COUNT(*) FROM orders WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = orders.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)"), + ("correlated-scalar", "SELECT COUNT(*) FROM orders o WHERE o.amount > (SELECT AVG(i.amount) FROM orders i " + "WHERE i.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders x WHERE (SELECT AVG(y.amount) FROM orders y WHERE x.user_id = y.user_id) < x.amount"), + ("correlated-select-list", "SELECT o.id, (SELECT COUNT(*) FROM refunds r WHERE r.order_id = o.id) AS n FROM orders o", + "SELECT x.id, (SELECT COUNT(*) FROM refunds y WHERE x.id = y.order_id) AS cnt FROM orders x"), + ("scalar-vs-cte", "SELECT COUNT(*) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders)", + "WITH a AS (SELECT AVG(amount) AS m FROM orders) SELECT COUNT(*) FROM orders WHERE amount > (SELECT m FROM a)"), + ("in-outer-and-sub", "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND o.user_id IN " + "(SELECT p.user_id FROM orders p WHERE p.status = 'paid' AND p.amount > 50)", + "SELECT COUNT(*) FROM orders a WHERE a.user_id IN (SELECT b.user_id FROM orders b WHERE b.amount > 50 " + "AND b.status = 'paid') AND a.status = 'paid'"), + ("cte-exists-renamed", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p " + "WHERE EXISTS (SELECT 1 FROM p p2 WHERE p2.parent_id = p.id)", + "WITH q AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM q AS m WHERE EXISTS " + "(SELECT 1 FROM q AS c WHERE c.parent_id = m.id)"), + ("cte-correlated-vs-subq", "WITH t AS (SELECT o.* FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r " + "WHERE r.order_id = o.id)) SELECT COUNT(*) FROM t", + "SELECT COUNT(*) FROM (SELECT x.* FROM orders x WHERE EXISTS (SELECT 1 FROM refunds y WHERE y.order_id = x.id)) AS q"), + ("cte-in-and-in", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM p " + "WHERE p.user_id IN (SELECT user_id FROM orders WHERE amount > 50)", + "SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid' AND o.user_id IN (SELECT user_id FROM orders " + "WHERE amount > 50)"), + ("subq-from-subq-alias", "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT s.user_id FROM " + "(SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) s WHERE s.total > 100)", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT agg.user_id FROM " + "(SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) agg WHERE agg.total > 100)"), + # ---- UNION + ("union-branch-order", "SELECT user_id, amount FROM orders WHERE amount > 1 UNION ALL SELECT user_id, amount FROM refunds", + "SELECT user_id, amount FROM refunds UNION ALL SELECT user_id, amount FROM orders WHERE 1 < amount"), + ("union-wrapped", "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT * FROM (SELECT user_id FROM orders UNION SELECT user_id FROM refunds) t"), + ("union-cte-vs-subq", "WITH t AS (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) " + "SELECT COUNT(*) FROM t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) AS q"), + ("union-aliases", "SELECT o.user_id, o.amount FROM orders o WHERE o.status = 'paid' UNION ALL " + "SELECT r.user_id, -r.amount FROM refunds r", + "SELECT x.user_id, x.amount FROM orders x WHERE x.status = 'paid' UNION ALL SELECT y.user_id, -y.amount FROM refunds y"), + ("union-output-alias", "SELECT user_id AS u FROM orders UNION SELECT user_id FROM refunds", + "SELECT user_id AS uid FROM orders UNION SELECT user_id FROM refunds"), + ("union-cte-filter", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT user_id FROM p WHERE amount > 10 " + "UNION ALL SELECT user_id FROM p WHERE amount < 0", + "SELECT user_id FROM orders WHERE status = 'paid' AND amount > 10 UNION ALL " + "SELECT user_id FROM orders WHERE status = 'paid' AND amount < 0"), + ("archive-union-reorder", "SELECT id, amount FROM orders UNION SELECT id, amount FROM orders_archive", + "SELECT id, amount FROM orders_archive UNION SELECT id, amount FROM orders"), + # ---- 过滤挪了一层、分组键上的条件 + ("group-key-filter-moved", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 3 " + "GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t AS z WHERE 100 < z.total AND z.user_id > 3"), + ("preserved-side-constant", "SELECT d.flag, s.id FROM (SELECT id, 1 AS flag FROM users) d LEFT JOIN stores s " + "ON s.owner_id = d.id", + "SELECT 1 AS flag, s.id FROM users u LEFT JOIN stores s ON s.owner_id = u.id"), + ("orders-items-join", "SELECT o.id, SUM(i.qty * i.price) FROM orders o JOIN order_items i ON i.order_id = o.id " + "GROUP BY o.id", + "SELECT x.id, SUM(y.price * y.qty) FROM order_items y JOIN orders x ON x.id = y.order_id GROUP BY x.id"), + ("payments-left", "SELECT o.id, COALESCE(SUM(p.amount), 0) FROM orders o LEFT JOIN payments p ON p.order_id = o.id " + "GROUP BY o.id", + "SELECT a.id, COALESCE(SUM(b.amount), 0) FROM orders a LEFT JOIN payments b ON a.id = b.order_id GROUP BY a.id"), + ("shipments-exists", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM shipments s WHERE s.order_id = o.id " + "AND s.carrier = 'sf')", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT * FROM shipments AS t WHERE 'sf' = t.carrier " + "AND o.id = t.order_id)"), +] + +_DIFF = [ + # ---- 时间列、时间范围 + ("time-column", "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' AND created_at < '2026-10-01'"), + ("time-one-side", "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders"), + ("time-range-end", "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'"), + ("time-range-op", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at > '2026-09-01'"), + ("time-in-cte", "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT SUM(amount) FROM t " + "WHERE d >= '2026-09-01'", + "WITH t AS (SELECT DATE(created_at) AS d, amount FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("time-date-fn", "SELECT COUNT(*) FROM orders WHERE DATE(paid_at) = '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at = '2026-09-01'"), + # ---- 有效行过滤 + ("filter-deleted", "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01'"), + ("filter-exclusion", "SELECT SUM(amount) FROM orders WHERE is_deleted <> 1", "SELECT SUM(amount) FROM orders"), + ("filter-status-value", "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status = 'done'"), + ("filter-status-in", "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')"), + ("filter-not-vs-neq-null", "SELECT COUNT(*) FROM orders WHERE status <> 'void'", + "SELECT COUNT(*) FROM orders WHERE status <> 'void' OR status IS NULL"), + ("filter-threshold", "SELECT COUNT(*) FROM orders WHERE amount > 100", "SELECT COUNT(*) FROM orders WHERE amount >= 100"), + ("filter-and-vs-or", "SELECT COUNT(*) FROM orders WHERE status = 'paid' AND amount > 10", + "SELECT COUNT(*) FROM orders WHERE status = 'paid' OR amount > 10"), + ("filter-in-cte-dropped", "WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT v.user_id) FROM v", + "SELECT COUNT(DISTINCT user_id) FROM orders"), + ("filter-cte-unused", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("filter-cte-status", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT store_id, SUM(amount) FROM p GROUP BY store_id", + "WITH p AS (SELECT * FROM orders WHERE status = 'open') SELECT store_id, SUM(amount) FROM p GROUP BY store_id"), + ("in-cte-regression", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM p)", + "WITH p AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM p)"), + ("exists-cte-regression", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM p WHERE p.user_id = u.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM p WHERE p.user_id = u.id)"), + ("scalar-cte-regression", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') " + "SELECT COUNT(*) FROM users u WHERE (SELECT COUNT(*) FROM p WHERE p.user_id = u.id) > 0", + "WITH p AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT COUNT(*) FROM users u WHERE (SELECT COUNT(*) FROM p WHERE p.user_id = u.id) > 0"), + ("in-subquery-column", "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.referrer_id FROM users u WHERE u.vip = 1)"), + ("in-subquery-filter", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT id FROM users)"), + ("not-in-vs-not-exists", "SELECT COUNT(*) FROM orders o WHERE o.id NOT IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)"), + ("in-vs-not-in", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE user_id NOT IN (SELECT user_id FROM refunds)"), + ("exists-child-vs-parent", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)"), + ("correlated-earlier", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p " + "WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id " + "AND p.created_at > o.created_at)"), + ("scalar-subquery-filter", "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders " + "WHERE is_deleted = 0)", + "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders)"), + ("unnest-or-trap", "SELECT id FROM orders WHERE status = 'paid' OR amount > (SELECT amount FROM refunds WHERE id = 999999)", + "SELECT orders.id FROM orders CROSS JOIN (SELECT amount FROM refunds WHERE id = 999999) AS q " + "WHERE orders.status = 'paid' OR orders.amount > q.amount"), + ("unnest-scalar-trap", "SELECT id FROM orders WHERE amount > (SELECT amount FROM refunds ORDER BY id LIMIT 1)", + "SELECT orders.id FROM orders CROSS JOIN (SELECT amount FROM refunds) AS q WHERE orders.amount > q.amount"), + # ---- 聚合、去重 + ("count-distinct", "SELECT COUNT(DISTINCT user_id) FROM orders", "SELECT COUNT(user_id) FROM orders"), + ("count-star-vs-col", "SELECT COUNT(*) FROM orders", "SELECT COUNT(user_id) FROM orders"), + ("count-star-vs-distinct", "SELECT COUNT(*) FROM orders", "SELECT COUNT(DISTINCT id) FROM orders WHERE amount > 0"), + ("sum-vs-avg", "SELECT SUM(amount) FROM orders", "SELECT AVG(amount) FROM orders"), + ("sum-column", "SELECT SUM(amount) FROM orders", "SELECT SUM(discount) FROM orders"), + ("sum-vs-coalesce", "SELECT SUM(amount) FROM refunds", "SELECT COALESCE(SUM(amount), 0) FROM refunds"), + ("max-vs-min", "SELECT user_id, MAX(paid_at) FROM orders GROUP BY user_id", + "SELECT user_id, MIN(paid_at) FROM orders GROUP BY user_id"), + ("expression-factor", "WITH t AS (SELECT id, amount * 0.5 AS net FROM orders) SELECT SUM(net) FROM t", + "WITH t AS (SELECT id, amount * 0.25 AS net FROM orders) SELECT SUM(net) FROM t"), + ("expression-swap-sub", PR + "SELECT SUM(p.total) - SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + PR + "SELECT SUM(r.total) - SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("cte-which-total", PR + "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > 100", + PR + "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE r.total > 100"), + ("cte-swapped-defs", PR + "SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " + "refunded AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("distinct-dropped", "SELECT DISTINCT user_id FROM orders", "SELECT user_id FROM orders"), + ("distinct-in-subq", "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders) t"), + ("count-distinct-vs-subq", "SELECT COUNT(DISTINCT user_id) FROM orders", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders) t"), + ("window-partition", "SELECT id, SUM(amount) OVER (PARTITION BY user_id) AS s FROM orders", + "SELECT id, SUM(amount) OVER (PARTITION BY store_id) AS s FROM orders"), + ("window-rank-vs-rownum", "SELECT id, RANK() OVER (ORDER BY amount) AS r FROM orders", + "SELECT id, ROW_NUMBER() OVER (ORDER BY amount, id) AS r FROM orders"), + ("window-filter-stage", "SELECT COUNT(*) FROM (SELECT id, status, ROW_NUMBER() OVER (PARTITION BY user_id " + "ORDER BY paid_at, id) AS rn FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at, id) AS rn " + "FROM orders WHERE status = 'paid') t WHERE t.rn = 1"), + ("limit-filter-stage", "SELECT COUNT(*) FROM (SELECT * FROM orders ORDER BY id LIMIT 5) t WHERE t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid' ORDER BY id LIMIT 5) t"), + ("case-default", "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE 0 END) FROM orders", + "SELECT SUM(CASE WHEN status = 'paid' THEN amount END) FROM orders"), + ("output-columns-order", "SELECT user_id, SUM(amount) FROM orders GROUP BY user_id", + "SELECT SUM(amount), user_id FROM orders GROUP BY user_id"), + ("output-extra-column", "SELECT user_id, SUM(amount) FROM orders GROUP BY user_id", + "SELECT user_id, SUM(amount), COUNT(*) FROM orders GROUP BY user_id"), + # ---- 分组、HAVING、取行 + ("group-key", "SELECT region, SUM(amount) FROM orders GROUP BY region", + "SELECT region, status, SUM(amount) FROM orders GROUP BY region, status"), + ("group-none", "SELECT SUM(amount) FROM orders", "SELECT region, SUM(amount) FROM orders GROUP BY region"), + ("group-grain", "SELECT COUNT(*) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id"), + ("having-dropped", "SELECT region, SUM(amount) FROM orders GROUP BY region HAVING SUM(amount) > 100", + "SELECT region, SUM(amount) FROM orders GROUP BY region"), + ("having-threshold", "SELECT store_id FROM orders GROUP BY store_id HAVING COUNT(*) > 2", + "SELECT store_id FROM orders GROUP BY store_id HAVING COUNT(*) > 3"), + ("limit-count", "SELECT id FROM orders ORDER BY id LIMIT 3", "SELECT id FROM orders ORDER BY id LIMIT 5"), + ("limit-vs-none", "SELECT id FROM orders ORDER BY id LIMIT 3", "SELECT id FROM orders ORDER BY id"), + ("limit-offset", "SELECT id FROM orders ORDER BY id LIMIT 3", "SELECT id FROM orders ORDER BY id LIMIT 3 OFFSET 2"), + ("limit-order-dir", "SELECT id FROM orders ORDER BY amount DESC, id LIMIT 3", + "SELECT id FROM orders ORDER BY amount, id LIMIT 3"), + ("limit-in-cte", "WITH t AS (SELECT * FROM orders ORDER BY id LIMIT 5) SELECT SUM(amount) FROM t", + "WITH t AS (SELECT * FROM orders ORDER BY id LIMIT 8) SELECT SUM(amount) FROM t"), + ("order-only", "SELECT id, amount FROM orders ORDER BY amount DESC, id", "SELECT id, amount FROM orders"), + # ---- 连接 + ("left-vs-inner", "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("left-join-key", "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.user_id = o.user_id"), + ("join-dropped", "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(amount) FROM orders"), + ("inner-join-added", "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id", + "SELECT SUM(amount) FROM orders"), + ("join-filter-city", "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'", + "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'BJ'"), + ("left-keep-side", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON r.order_id = o.id"), + ("right-keep-side", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id"), + ("left-on-vs-where", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id AND r.state = 'done'", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'"), + ("left-is-null-not-rejected", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id " + "WHERE r.amount > 5 OR r.id IS NULL", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.amount > 5 OR r.id IS NULL"), + ("left-coalesce-not-rejected", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id " + "WHERE COALESCE(r.amount, 0) = 0", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE COALESCE(r.amount, 0) = 0"), + ("nullable-side-constant", "SELECT o.id, r.flag FROM orders o LEFT JOIN (SELECT order_id, 1 AS flag FROM refunds) r " + "ON r.order_id = o.id", + "SELECT o.id, 1 AS flag FROM orders o LEFT JOIN refunds r ON r.order_id = o.id"), + ("nullable-side-coalesce", "SELECT o.id, r.a FROM orders o LEFT JOIN (SELECT order_id, COALESCE(amount, 0) AS a " + "FROM refunds) r ON r.order_id = o.id", + "SELECT o.id, COALESCE(r.amount, 0) AS a FROM orders o LEFT JOIN refunds r ON r.order_id = o.id"), + ("nullable-nested-constant", "SELECT o.id, d.flag FROM orders o LEFT JOIN (SELECT x.order_id, x.flag FROM " + "(SELECT order_id, 1 AS flag FROM refunds) x) d ON d.order_id = o.id", + "SELECT o.id, 1 AS flag FROM orders o LEFT JOIN refunds r ON r.order_id = o.id"), + ("right-side-constant", "SELECT d.flag, s.id FROM (SELECT id, 1 AS flag FROM users) d RIGHT JOIN stores s " + "ON s.owner_id = d.id", + "SELECT 1 AS flag, s.id FROM users u RIGHT JOIN stores s ON s.owner_id = u.id"), + ("full-side-constant", "SELECT d.flag, s.id FROM (SELECT id, 1 AS flag FROM users) d FULL JOIN stores s " + "ON s.owner_id = d.id", + "SELECT 1 AS flag, s.id FROM users u FULL JOIN stores s ON s.owner_id = u.id"), + ("preserved-side-constant", "SELECT d.flag, s.id FROM (SELECT id, 1 AS flag FROM users) d LEFT JOIN stores s " + "ON s.owner_id = d.id", + "SELECT d.flag, s.id FROM (SELECT id, 2 AS flag FROM users) d LEFT JOIN stores s ON s.owner_id = d.id"), + ("self-join-direction", "SELECT COUNT(*) FROM orders o JOIN orders c ON c.parent_id = o.id WHERE c.amount > o.amount", + "SELECT COUNT(*) FROM orders x JOIN orders y ON x.id = y.parent_id WHERE x.amount > y.amount"), + ("self-join-roles", "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE a.vip = 1", + "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE b.vip = 1"), + ("table-swap", "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 0 AND paid_at >= '2026-09-01'"), + ("two-columns-same-name", "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("join-multiplies", "SELECT SUM(o.amount) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("passthrough-which-cte", "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders " + "WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') " + "SELECT SUM(o.amount) FROM paid p JOIN open o ON o.user_id = p.user_id"), + ("cte-self-join-direction", "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM t a JOIN t b " + "ON b.user_id = a.referrer_id WHERE a.total > b.total", + "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) " + "SELECT COUNT(*) FROM t a JOIN t b ON b.user_id = a.referrer_id WHERE b.total > a.total"), + ("extra-cte-join", "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE total > 100", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders " + "GROUP BY user_id) SELECT COUNT(*) FROM vip JOIN t ON t.user_id = vip.id WHERE t.total > 100"), + # ---- UNION + ("union-vs-union-all", "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds"), + ("union-branch-filter", "SELECT user_id FROM orders WHERE status = 'paid' UNION ALL SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds"), + ("union-branch-dup", "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds UNION ALL SELECT user_id FROM refunds"), + ("union-distinct-branch", "SELECT DISTINCT user_id FROM orders UNION ALL SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds"), + ("except-order", "SELECT user_id FROM orders EXCEPT SELECT user_id FROM refunds", + "SELECT user_id FROM refunds EXCEPT SELECT user_id FROM orders"), + ("intersect-vs-union", "SELECT user_id FROM orders INTERSECT SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION SELECT user_id FROM refunds"), + # ---- 写法上看着一样 + ("in-vs-exists-null", "SELECT COUNT(*) FROM users u WHERE u.id NOT IN (SELECT user_id FROM orders)", + "SELECT COUNT(*) FROM users u WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"), + ("coalesce-filter", "SELECT COUNT(*) FROM orders WHERE COALESCE(is_deleted, 0) = 0", + "SELECT COUNT(*) FROM orders WHERE is_deleted = 0"), + ("integer-division", "SELECT SUM(amount) / COUNT(*) FROM orders", "SELECT AVG(amount) FROM orders"), + ("cast-date-sqlite", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= CAST('2026-09-01' AS DATE)"), + ("string-compare-redundant", "SELECT COUNT(*) FROM orders WHERE status > 'b' AND status > 'o'", + "SELECT COUNT(*) FROM orders WHERE status > 'o'"), + ("text-number-compare", "SELECT COUNT(*) FROM orders WHERE region > 10 AND region > 9", + "SELECT COUNT(*) FROM orders WHERE region > 10"), + ("arith-equality", "SELECT COUNT(*) FROM orders WHERE amount + 1 = 101", "SELECT COUNT(*) FROM orders WHERE amount = 100"), + # ---- 别名和列同名(压测撞到的):SELECT 列表里的名字指表的列,不指前面列的别名;去重的 UNION 留先出现的值 + ("select-alias-shadows-column", "SELECT user_id AS amount, amount * 2 AS twice FROM orders", + "SELECT user_id AS amount, user_id * 2 AS twice FROM orders"), + ("select-alias-shadows-column-2", "SELECT store_id AS region, region FROM orders WHERE status = 'paid'", + "SELECT store_id AS region, store_id FROM orders WHERE status = 'paid'"), + ("having-alias-shadows-column", "SELECT user_id, SUM(amount) AS amount FROM orders GROUP BY user_id " + "HAVING SUM(amount) > 10", + "SELECT user_id, SUM(amount) AS amount FROM orders GROUP BY user_id HAVING amount > 10"), + ("union-dedup-branch-order", "SELECT tax FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", + "SELECT referrer_id FROM orders UNION SELECT tax FROM orders WHERE status = 'paid'"), +] + + +# ---------------------------------------------------------------------------- 过去的反例 +# 下面三份是从旧的 tests/test_sql_semantics.py(五轮复核修正里按复验报告补的用例)和复验者留在 /private/tmp 的 +# 脚本里收集来的 SQL 对,原样收进来:名字是出处(旧测试的函数名、复验脚本:用例名)。 + +#: 旧测试里两边写法不同、意思一样的对(各方言),新做法认得出是等价的 +DIALECT_SAME: list[tuple[str, str, str, str]] = [ + ("names_given_to_derived_tables_are_no_difference", "mysql", + "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) r ON r.order_id = o.id WHERE r.n > 1", + "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) `退款` ON o.id = `退款`.order_id WHERE `退款`.n > 1"), + ("derived_tables_reading_the_same_table_written_differently_ar", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH settled AS (SELECT * FROM orders WHERE status = 'paid'), pending AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(s.amount) FROM settled s JOIN pending q ON q.user_id = s.user_id"), + ("names_given_to_derived_tables_are_no_difference", "postgres", + "SELECT SUM(amount) FROM orders WHERE user_id IN (WITH t AS (SELECT id FROM users WHERE grade >= 3) SELECT id FROM t)", + "SELECT SUM(amount) FROM orders WHERE user_id IN (WITH g AS (SELECT id FROM users WHERE grade >= 3) SELECT g.id FROM g)"), + ("the_same_output_computation_written_differently_is_no_differ", "postgres", + "SELECT order_id, RANK() OVER (PARTITION BY user_id ORDER BY amount DESC) AS k FROM orders", + "SELECT o.order_id, RANK() OVER (PARTITION BY o.user_id ORDER BY o.amount DESC) FROM orders o"), + ("a_self_join_tells_its_two_aliases_apart", "postgres", + "WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id WHERE b1.total > b2.total", + "WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM base y JOIN base x ON x.referrer_id = y.user_id WHERE y.total < x.total"), + ("the_same_outer_join_written_differently_is_no_difference", "postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM refunded r RIGHT JOIN paid p ON p.user_id = r.user_id"), + ("names_given_to_derived_tables_are_no_difference", "postgres", + "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM user_totals WHERE total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) AS agg WHERE agg.total > 100"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION ALL SELECT referrer_id FROM orders WHERE status = 'paid') t", + "WITH ids AS (SELECT user_id FROM orders WHERE status = 'paid' UNION ALL SELECT referrer_id FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM ids"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE p.total > r.total", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id WHERE p.total > r.total"), + ("the_same_outer_join_written_differently_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("the_same_outer_join_written_differently_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id LEFT JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM orders o LEFT JOIN users u ON u.id = o.user_id LEFT JOIN refunds r ON r.order_id = o.id"), + ("the_same_outer_join_written_differently_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + "SELECT COUNT(*) FROM orders AS o LEFT OUTER JOIN refunds AS r ON o.id = r.order_id WHERE o.status = 'paid'"), + ("the_same_outer_join_written_differently_is_no_difference", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done'"), + ("the_same_correlated_subquery_written_differently_is_no_diffe", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders x WHERE EXISTS (SELECT 1 FROM orders y WHERE x.id = y.parent_id)"), + ("the_same_correlated_subquery_written_differently_is_no_diffe", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), + ("a_self_join_tells_its_two_aliases_apart", "sqlite", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o1.amount > o2.amount", + "SELECT COUNT(*) FROM orders c JOIN orders p ON c.parent_id = p.id WHERE c.amount > p.amount"), + ("the_same_item_on_equivalent_layers_is_no_difference", "sqlite", + "SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT x.user_id FROM orders x WHERE x.status = 'paid') AND o.status = 'paid'"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.amount > r.amount", + "SELECT SUM(o.amount) FROM refunds r JOIN orders o ON o.id = r.order_id WHERE o.amount > r.amount"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(o2.amount) AS avg_amount FROM orders o2 WHERE o2.is_deleted = 0)", + "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders WHERE is_deleted = 0)"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted = 0", + "select sum(amount) from orders where 0 = is_deleted and '2026-09-01' <= paid_at"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT s.user_id FROM (SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) s WHERE s.total > 100)", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT agg.user_id FROM (SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) agg WHERE agg.total > 100)"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id AS uid FROM users u WHERE u.vip = 1)", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o, refunds r WHERE r.order_id = o.id", + "SELECT SUM(o.amount) FROM refunds r, orders o WHERE r.order_id = o.id"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT SUM(o.amount) FROM orders o, stores s WHERE o.store_id = s.id AND s.city = '虚构市'", + "SELECT SUM(orders.amount) FROM orders INNER JOIN stores ON stores.id = orders.store_id WHERE stores.city = '虚构市'"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10 OFFSET 0", + "SELECT o.amount FROM orders o ORDER BY o.amount DESC LIMIT 10"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "SELECT o.id FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT orders.id FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), + ("the_same_output_computation_written_differently_is_no_differ", "sqlite", + "SELECT region, SUM(amount) * 1.0 / COUNT(*) AS avg_amt FROM orders GROUP BY region", + "SELECT o.region, 1.0 * SUM(o.amount) / COUNT(*) FROM orders o GROUP BY 1"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT region, SUM(amount) FROM orders GROUP BY region", + "WITH t AS (SELECT region, SUM(amount) AS s FROM orders GROUP BY region) SELECT region, s FROM t"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT user_id FROM orders GROUP BY user_id", + "SELECT DISTINCT user_id FROM orders"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id", + "SELECT user_id, n FROM (SELECT user_id, COUNT(*) AS n FROM orders GROUP BY user_id) t"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), per_user AS (SELECT user_id, SUM(amount) AS total FROM base GROUP BY user_id) SELECT COUNT(*) FROM per_user WHERE total > 100", + "WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM totals WHERE total > 100"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH c AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), d AS (SELECT user_id, COUNT(*) AS n FROM refunds GROUP BY user_id), a AS (SELECT * FROM c WHERE total > 100), b AS (SELECT * FROM d WHERE n > 1) SELECT COUNT(*) FROM b JOIN a ON a.user_id = b.user_id", + "SELECT COUNT(*) FROM (SELECT * FROM (SELECT user_id, COUNT(*) AS n FROM refunds GROUP BY user_id) x WHERE n > 1) b JOIN (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) y WHERE total > 100) a ON a.user_id = b.user_id"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH c AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), d AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), a AS (SELECT * FROM c), b AS (SELECT * FROM d) SELECT COUNT(*) FROM b JOIN a ON a.user_id = b.user_id WHERE a.total > b.total", + "SELECT COUNT(*) FROM (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) x) b JOIN (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) y) a ON a.user_id = b.user_id WHERE a.total > b.total"), + ("derived_tables_reading_the_same_table_written_differently_ar", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM paid p JOIN refunds r ON r.order_id = p.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'"), + ("derived_tables_reading_the_same_table_written_differently_ar", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > o.amount", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM open o JOIN paid p ON p.user_id = o.user_id WHERE o.amount < p.amount"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM refunded r JOIN paid p ON p.user_id = r.user_id WHERE p.total > r.total"), + ("the_same_output_computation_written_differently_is_no_differ", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) - SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) - SUM(r.total) FROM refunded r JOIN paid p ON p.user_id = r.user_id"), + ("swapping_the_order_of_an_inner_join_is_no_difference", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > o.total", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) SELECT COUNT(*) FROM open o JOIN paid p ON p.user_id = o.user_id WHERE o.total < p.total"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", + "WITH r AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), p AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT COUNT(*) FROM p JOIN r ON p.user_id = r.user_id WHERE p.total > r.total"), + ("filtering_at_an_equivalent_stage_is_no_difference", "sqlite", + "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t WHERE status = 'paid'", + "WITH top10 AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(t.amount) FROM top10 t WHERE t.status = 'paid'"), + ("derived_tables_reading_the_same_table_written_differently_ar", "sqlite", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(y.amount) FROM t x JOIN t y ON y.referrer_id = x.user_id"), + ("derived_columns_renamed_on_one_side_are_paired", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100", + "WITH x AS (SELECT user_id, SUM(amount) AS sum_amt FROM orders GROUP BY user_id) SELECT COUNT(*) FROM x WHERE sum_amt > 100"), + ("a_shifted_number_inside_a_condition_subquery_is_no_differenc", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u WHERE u.vip = 1 AND u.id IN (SELECT user_id FROM t WHERE total > 100)", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM vip WHERE vip.id IN (SELECT user_id FROM t WHERE total > 100)"), + ("swapped_join_order_of_two_derived_tables_is_no_difference", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM t JOIN u ON u.user_id = t.user_id WHERE t.total > u.cnt", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM u JOIN t ON t.user_id = u.user_id WHERE t.total > u.cnt"), + ("derived_columns_renamed_on_one_side_are_paired", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100 AND n > 1", + "WITH t AS (SELECT user_id, SUM(amount) AS s, COUNT(*) AS c FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE s > 100 AND c > 1"), + ("the_same_derived_column_written_differently_is_no_difference", "sqlite", + "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT SUM(net) FROM t", + "WITH x AS (SELECT user_id, 0.9 * amount AS net FROM orders) SELECT SUM(x.net) FROM x"), + ("derived_columns_renamed_on_one_side_are_paired", "sqlite", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS day FROM orders) SELECT SUM(amount) FROM t WHERE day >= '2026-09-01'"), + ("the_same_derived_column_written_differently_is_no_difference", "sqlite", + "WITH t AS (SELECT user_id, amount, paid_at AS ts FROM orders) SELECT SUM(amount) FROM t", + "WITH t AS (SELECT user_id, amount, created_at AS ts FROM orders) SELECT SUM(amount) FROM t"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM user_totals WHERE total > 100", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM t WHERE t.total > 100"), + ("the_same_meaning_written_differently_is_no_difference", "sqlite", + "WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT v.order_id) FROM v", + "SELECT COUNT(DISTINCT order_id) FROM orders WHERE is_deleted = 0"), + ("the_same_item_on_equivalent_layers_is_no_difference", "sqlite", + "WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT SUM(amount) FROM v WHERE is_deleted = 0", + "SELECT SUM(amount) FROM orders WHERE is_deleted = 0"), + ("names_given_to_derived_tables_are_no_difference", "sqlite", + "WITH vip AS (SELECT id FROM users WHERE grade >= 3) SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM vip)", + "WITH good_users AS (SELECT id FROM users WHERE grade >= 3) SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT g.id FROM good_users g)"), +] + +#: 过去五轮复验、旧测试里报过的不同(各方言):绝不能判等价 +REGRESSIONS: list[tuple[str, str, str, str]] = [ + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY p.store_id", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid') t"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT id FROM orders WHERE status = 'paid' UNION ALL SELECT id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100) x", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100) x"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT referrer_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT referrer_id FROM orders) t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + ("which_side_an_outer_join_keeps_is_compared", "mysql", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "mysql", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "mysql", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND o.created_at < p.created_at)"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)"), + ("each_dialect_gives_the_same_three_findings", "mysql", + "SELECT COUNT(DISTINCT o.order_id) AS `订单数` FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted <> 1", + "SELECT COUNT(o.order_id) AS `订单数` FROM orders o WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT COUNT(t.user_id) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT COUNT(user_id) FROM orders GROUP BY user_id"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "mysql", + "SELECT DISTINCT region FROM orders", + "SELECT region FROM orders"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.referrer_id = a.user_id"), + ("the_same_output_computation_written_differently_is_no_differ", "mysql", + "SELECT SUM(amount) FROM orders", + "SELECT IFNULL(SUM(amount), 0) FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", + "SELECT AVG(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(o.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id"), + ("how_a_derived_column_is_computed_is_reported", "mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x", + "SELECT SUM(x.total) FROM (SELECT p.user_id, r.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "mysql", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "SELECT user_id, COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1) GROUP BY user_id", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1)"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id"), + ("time_columns_inside_derived_tables", "mysql", + "WITH a AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id), b AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'", + "WITH a AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id), b AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id) SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("an_outer_join_turned_inner_by_where_is_reported", "mysql", + "WITH done AS (SELECT * FROM refunds WHERE state = 'done') SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id", + "WITH done AS (SELECT * FROM refunds) SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id WHERE d.state = 'done'"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "mysql", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = p.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = p.parent_id)"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY p.store_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT p.amount - o.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT o.amount - p.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(o.amount) FROM paid p JOIN open o ON o.user_id = p.user_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "mysql", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.user_id = o.referrer_id", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.referrer_id = o.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "mysql", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id WHERE o.status = 'paid'", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id"), + ("how_a_derived_column_is_computed_is_reported", "mysql", + "WITH r AS (SELECT user_id, amount, RANK() OVER (ORDER BY amount DESC) AS k FROM orders) SELECT SUM(amount) FROM r WHERE k <= 10", + "WITH r AS (SELECT user_id, amount, ROW_NUMBER() OVER (ORDER BY amount DESC) AS k FROM orders) SELECT SUM(amount) FROM r WHERE k <= 10"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.referrer_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(b.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id"), + ("derived_tables_reading_the_same_table_keep_apart", "mysql", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM t p JOIN orders c ON c.parent_id = p.id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(c.amount) FROM t p JOIN orders c ON c.parent_id = p.id"), + ("how_a_derived_column_is_computed_is_reported", "mysql", + "WITH t AS (SELECT DATE_FORMAT(paid_at, '%Y-%m') AS m, amount FROM orders) SELECT m, SUM(amount) FROM t GROUP BY m", + "WITH t AS (SELECT DATE_FORMAT(created_at, '%Y-%m') AS m, amount FROM orders) SELECT m, SUM(amount) FROM t GROUP BY m"), + ("how_a_derived_column_is_computed_is_reported", "mysql", + "WITH t AS (SELECT user_id, CASE WHEN status = 'paid' THEN amount ELSE 0 END AS paid_amt FROM orders) SELECT SUM(paid_amt) FROM t", + "WITH t AS (SELECT user_id, CASE WHEN status IN ('paid', 'shipped') THEN amount ELSE 0 END AS paid_amt FROM orders) SELECT SUM(paid_amt) FROM t"), + ("a_time_column_computed_in_a_cte_is_seen_through", "mysql", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(created_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY p.store_id", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid') t"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(*) FROM (SELECT id FROM orders WHERE status = 'paid' UNION ALL SELECT id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100) x", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100) x"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT referrer_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT referrer_id FROM orders) t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + ("filtering_at_an_equivalent_stage_is_no_difference", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE t.rn = 1 AND t.user_id > 100", + "SELECT COUNT(*) FROM (SELECT user_id, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders WHERE user_id > 100) t WHERE t.rn = 1"), + ("how_a_derived_column_is_computed_is_reported", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id, amount - fee AS net FROM orders) a WHERE a.net > 100", + "SELECT COUNT(*) FROM (SELECT user_id, amount + fee AS net FROM orders) b WHERE b.net > 100"), + ("filtering_before_or_after_a_window_or_limit_is_reported", "postgres", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders WHERE status = 'paid') t WHERE t.rn = 1"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)"), + ("an_outer_join_turned_inner_by_where_is_reported", "postgres", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'"), + ("which_side_an_outer_join_keeps_is_compared", "postgres", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "postgres", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "postgres", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND o.created_at < p.created_at)"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)"), + ("each_dialect_gives_the_same_three_findings", "postgres", + "SELECT COUNT(DISTINCT o.order_id) AS \"订单数\" FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted <> 1", + "SELECT COUNT(o.order_id) AS \"订单数\" FROM orders o WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT COUNT(t.user_id) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT COUNT(user_id) FROM orders GROUP BY user_id"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "postgres", + "SELECT DISTINCT region FROM orders", + "SELECT region FROM orders"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.referrer_id = a.user_id"), + ("how_a_derived_column_is_computed_is_reported", "postgres", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE rn = 1", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at DESC) AS rn FROM orders) t WHERE rn = 1"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + ("an_outer_join_turned_inner_by_where_is_reported", "postgres", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE o.status = 'paid'"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", + "SELECT AVG(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)"), + ("names_given_to_derived_tables_are_no_difference", "postgres", + "SELECT SUM(o.amount) FROM orders o, LATERAL (SELECT COUNT(*) AS n FROM refunds r WHERE r.order_id = o.id) l WHERE l.n > 0", + "SELECT SUM(o.amount) FROM orders o CROSS JOIN LATERAL (SELECT COUNT(*) AS n FROM refunds r WHERE r.order_id = o.id) AS x WHERE x.n > 0"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(o.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id"), + ("how_an_output_column_is_computed_is_compared", "postgres", + "SELECT SUM(r.total) / SUM(p.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT SUM(p.total) / SUM(r.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id"), + ("time_columns_inside_derived_tables", "postgres", + "SELECT SUM(x.amount) FROM (SELECT amount, CAST(paid_at AS DATE) AS day FROM orders) x WHERE x.day BETWEEN '2026-09-01' AND '2026-09-30'", + "SELECT SUM(y.amount) FROM (SELECT amount, CAST(created_at AS DATE) AS day FROM orders) y WHERE y.day BETWEEN '2026-09-01' AND '2026-09-30'"), + ("a_lateral_subquery_is_named_by_its_alias", "postgres", + "SELECT o.id FROM orders o, LATERAL (SELECT amount FROM refunds r WHERE r.order_id = o.id ORDER BY amount DESC LIMIT 1) l", + "SELECT o.id FROM orders o CROSS JOIN LATERAL (SELECT amount FROM refunds r WHERE r.order_id = o.id ORDER BY amount DESC LIMIT 2) AS x"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "postgres", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o"), + ("a_correlated_lateral_subquery_on_the_outer_table_is_reported", "postgres", + "SELECT o.id, x.s FROM orders o CROSS JOIN LATERAL (SELECT SUM(c.amount) AS s FROM orders c WHERE c.parent_id = o.id) x", + "SELECT o.id, x.s FROM orders o CROSS JOIN LATERAL (SELECT SUM(c.amount) AS s FROM orders c WHERE c.id = o.parent_id) x"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "SELECT user_id, COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1) GROUP BY user_id", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1)"), + ("how_a_derived_column_is_computed_is_reported", "postgres", + "WITH RECURSIVE t(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM t WHERE n < 10) SELECT SUM(n) FROM t", + "WITH RECURSIVE t(n) AS (SELECT 1 UNION ALL SELECT n + 2 FROM t WHERE n < 10) SELECT SUM(n) FROM t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("a_self_join_tells_its_two_aliases_apart", "postgres", + "WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id WHERE b1.total > b2.total", + "WITH base AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT COUNT(*) FROM base b1 JOIN base b2 ON b2.user_id = b1.referrer_id WHERE b2.total > b1.total"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "postgres", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = p.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = p.parent_id)"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY p.store_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("a_time_column_of_a_pass_through_derived_table_names_the_real", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.created_at::date >= '2026-09-01'", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.created_at::date >= '2026-09-01'"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT p.amount - o.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT o.amount - p.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(o.amount) FROM paid p JOIN open o ON o.user_id = p.user_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "postgres", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("how_a_derived_column_is_computed_is_reported", "postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, p.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT COUNT(*) FROM net WHERE v > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, r.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT COUNT(*) FROM net WHERE v > 100"), + ("which_side_an_outer_join_keeps_is_compared", "postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM refunded r LEFT JOIN paid p ON p.user_id = r.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.user_id = o.referrer_id", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.referrer_id = o.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "postgres", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id WHERE o.status = 'paid'", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.referrer_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(b.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id"), + ("derived_tables_reading_the_same_table_keep_apart", "postgres", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM t p JOIN orders c ON c.parent_id = p.id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(c.amount) FROM t p JOIN orders c ON c.parent_id = p.id"), + ("time_columns_inside_derived_tables", "postgres", + "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT CAST(paid_at AS DATE) AS d, amount FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("the_same_derived_column_written_differently_is_no_difference", "postgres", + "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT d, SUM(amount) FROM t GROUP BY d", + "WITH t AS (SELECT CAST(paid_at AS DATE) AS d, amount FROM orders) SELECT d, SUM(amount) FROM t GROUP BY d"), + ("a_time_column_computed_in_a_cte_is_seen_through", "postgres", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(created_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("same_named_derived_tables_with_swapped_definitions_are_told_", "postgres", + "WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), t2 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100", + "WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), t2 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY p.store_id", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "SELECT COUNT(*) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid') t"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(*) FROM (SELECT id FROM orders WHERE status = 'paid' UNION ALL SELECT id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE p.amount > 100) x", + "SELECT COUNT(*) FROM (SELECT p.user_id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id WHERE o.amount > 100) x"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT referrer_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT referrer_id FROM orders) t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE p.total - r.total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE r.total - p.total > 100"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE p.total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id WHERE r.total > 100"), + ("other_aggregate_differences", "sqlite", + "SELECT COUNT(*) FROM orders", + "SELECT COUNT(DISTINCT order_id) FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders o2 JOIN refunds r2 ON r2.order_id = o2.id WHERE o2.user_id = o.user_id)"), + ("an_outer_join_turned_inner_by_where_is_reported", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN (SELECT * FROM refunds WHERE state = 'done') r ON r.order_id = o.id", + "SELECT COUNT(*) FROM orders o LEFT JOIN (SELECT * FROM refunds) r ON r.order_id = o.id WHERE r.state = 'done'"), + ("which_side_an_outer_join_keeps_is_compared", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id"), + ("only_null_rejecting_where_conditions_turn_an_outer_join_inne", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE COALESCE(r.amount, 0) < 100", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id"), + ("which_side_an_outer_join_keeps_is_compared", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.order_id IS NULL", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE r.order_id IS NULL"), + ("only_null_rejecting_where_conditions_turn_an_outer_join_inne", "sqlite", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done' OR r.id IS NULL", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.state = 'done' OR r.id IS NULL"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = o.parent_id)"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND o.created_at < p.created_at)"), + ("a_difference_inside_a_condition_subquery_is_said_once", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND r.state = 'done')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM orders x WHERE x.user_id = o.user_id AND x.status = 'paid' AND x.id <> o.id)"), + ("a_self_join_tells_its_two_aliases_apart", "sqlite", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o1.amount > o2.amount", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.id = o1.parent_id WHERE o2.amount > o1.amount"), + ("each_dialect_gives_the_same_three_findings", "sqlite", + "SELECT COUNT(DISTINCT o.order_id) AS \"订单数\" FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted <> 1", + "SELECT COUNT(o.order_id) AS \"订单数\" FROM orders o WHERE o.created_at >= '2026-09-01'"), + ("distinct_count_versus_plain_count", "sqlite", + "SELECT COUNT(DISTINCT order_id) FROM orders", + "SELECT COUNT(order_id) FROM orders"), + ("side_labels_in_chinese_have_no_stray_spaces", "sqlite", + "SELECT COUNT(DISTINCT order_id) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(order_id) FROM orders WHERE created_at >= '2026-09-01'"), + ("distinct_count_versus_plain_count", "sqlite", + "SELECT COUNT(order_id) FROM orders", + "SELECT COUNT(DISTINCT order_id) FROM orders"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT COUNT(t.user_id) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT COUNT(user_id) FROM orders GROUP BY user_id"), + ("dedup_union_and_how_many_rows_come_out_are_compared", "sqlite", + "SELECT DISTINCT region FROM orders", + "SELECT region FROM orders"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.referrer_id = a.user_id"), + ("how_an_output_column_is_computed_is_compared", "sqlite", + "SELECT SUM(amount) - SUM(fee) FROM orders", + "SELECT SUM(amount), SUM(fee) FROM orders"), + ("other_aggregate_differences", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT AVG(amount) FROM orders"), + ("the_same_output_computation_written_differently_is_no_differ", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT ROUND(SUM(amount), 2) FROM orders"), + ("how_an_output_column_is_computed_is_compared", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) / 100 FROM orders"), + ("time_filter_on_one_side_only", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'"), + ("an_exclusion_on_one_side", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) FROM orders WHERE refund_id IS NOT NULL"), + ("a_kept_condition_on_one_side", "sqlite", + "SELECT SUM(amount) FROM orders", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("which_rows_are_taken_is_compared", "sqlite", + "SELECT SUM(amount) FROM orders ORDER BY 1 DESC", + "SELECT SUM(amount) FROM orders"), + ("an_exclusion_on_one_side", "sqlite", + "SELECT SUM(amount) FROM orders WHERE is_deleted <> 1", + "SELECT SUM(amount) FROM orders"), + ("a_different_source_table_is_said_once", "sqlite", + "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 0 AND paid_at >= '2026-09-01'"), + ("a_different_source_table_is_said_once", "sqlite", + "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 1 AND paid_at >= '2026-09-01'"), + ("time_filter_on_one_side_only", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders"), + ("time_column_differs_with_catalog_names", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' AND created_at < '2026-10-01'"), + ("only_literals_differ_is_still_a_different_range", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-08' AND paid_at < '2026-10-08'"), + ("same_time_column_different_range", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE paid_at >= '2026-09-01')"), + ("two_conditions_on_the_same_column_are_paired", "sqlite", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT SUM(amount) FROM orders WHERE status = 'paid' AND user_id IN (SELECT user_id FROM orders WHERE status = 'paid')", + "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + ("an_extra_source_written_in_from_is_named_correctly", "sqlite", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM users u JOIN refunds r ON r.user_id = u.id JOIN orders o ON o.id = r.order_id"), + ("two_columns_that_read_the_same_are_told_apart", "sqlite", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("join_differences", "sqlite", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(amount) FROM orders"), + ("join_differences", "sqlite", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("join_differences", "sqlite", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.user_id = o.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)", + "SELECT AVG(o.amount) FROM orders o WHERE o.amount > (SELECT AVG(amount) FROM orders)"), + ("an_extra_source_written_in_from_is_named_correctly", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE o.status = 'paid'"), + ("an_extra_source_written_in_from_is_named_correctly", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT id FROM users WHERE vip = 1)", + "SELECT SUM(o.amount) FROM users u JOIN orders o ON o.user_id = u.id WHERE u.vip = 1"), + ("a_real_difference_inside_a_subquery_is_still_reported", "sqlite", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", + "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.referrer_id FROM users u WHERE u.vip = 1)"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(o.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT SUM(p.total - r.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT SUM(r.total - p.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT SUM(p.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT SUM(r.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id"), + ("other_aggregate_differences", "sqlite", + "SELECT SUM(paid_amount) FROM orders", + "SELECT SUM(amount) FROM orders"), + ("limit_difference", "sqlite", + "SELECT amount FROM orders LIMIT 10", + "SELECT amount FROM orders"), + ("limit_difference", "sqlite", + "SELECT amount FROM orders LIMIT 10", + "SELECT amount FROM orders LIMIT 100"), + ("which_rows_are_taken_is_compared", "sqlite", + "SELECT amount FROM orders LIMIT 10 OFFSET 0", + "SELECT amount FROM orders LIMIT 10 OFFSET 10"), + ("which_rows_are_taken_is_compared", "sqlite", + "SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10", + "SELECT amount FROM orders ORDER BY 1 ASC LIMIT 10"), + ("which_rows_are_taken_is_compared", "sqlite", + "SELECT amount FROM orders ORDER BY amount DESC LIMIT 10", + "SELECT amount FROM orders LIMIT 10"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "sqlite", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o"), + ("how_an_output_column_is_computed_is_compared", "sqlite", + "SELECT order_id, SUM(amount) OVER (PARTITION BY user_id) AS s FROM orders", + "SELECT order_id, SUM(amount) OVER (PARTITION BY store_id) AS s FROM orders"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "SELECT p.id FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT p.user_id FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id GROUP BY p.user_id HAVING MAX(p.total) > 10", + "SELECT p.user_id FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id GROUP BY p.user_id HAVING MAX(r.total) > 10"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "SELECT p.user_id FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id ORDER BY p.total DESC LIMIT 10", + "SELECT p.user_id FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id ORDER BY r.total DESC LIMIT 10"), + ("group_by_difference", "sqlite", + "SELECT region, SUM(amount) FROM orders GROUP BY region", + "SELECT region, city, SUM(amount) FROM orders GROUP BY region, city"), + ("having_difference", "sqlite", + "SELECT region, SUM(amount) FROM orders GROUP BY region HAVING SUM(amount) > 100", + "SELECT region, SUM(amount) FROM orders GROUP BY region"), + ("the_same_rows_written_differently_are_no_difference", "sqlite", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", + "SELECT * FROM (SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders) t"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "SELECT user_id, COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1) GROUP BY user_id", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1)"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status = 'paid'), b AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), paid AS (SELECT * FROM base WHERE status = 'paid'), open AS (SELECT * FROM base WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("a_correlated_subquery_on_the_outer_table_tells_inner_from_ou", "sqlite", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.parent_id = p.id)", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p WHERE EXISTS (SELECT 1 FROM orders c WHERE c.id = p.parent_id)"), + ("a_cte_defined_but_not_used_does_not_count", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM paid", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM orders"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY p.store_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY o.store_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.created_at >= '2026-09-01'", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.created_at >= '2026-09-01'"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT p.amount - o.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(d) FROM (SELECT o.amount - p.amount AS d FROM paid p JOIN open o ON o.user_id = p.user_id) x"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(o.amount) FROM paid p JOIN open o ON o.user_id = p.user_id"), + ("which_pass_through_derived_table_a_column_comes_from_is_repo", "sqlite", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY p.created_at DESC LIMIT 10", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.id FROM paid p JOIN open o ON o.user_id = p.user_id ORDER BY o.created_at DESC LIMIT 10"), + ("how_a_derived_column_is_computed_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, p.total - r.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT SUM(v) FROM net", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, r.total - p.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT SUM(v) FROM net"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total - r.total > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE r.total - p.total > 100"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE r.total > 100"), + ("a_reversed_comparison_between_two_derived_tables_is_still_re", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE r.total > p.total"), + ("an_extra_source_written_in_from_is_named_correctly", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM vip v JOIN refunded r ON r.user_id = v.id JOIN paid p ON p.user_id = r.user_id WHERE p.total > r.total"), + ("unused_ctes_take_part_in_pairing", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN paid p ON p.user_id = u.id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN refunded r ON r.user_id = u.id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(r.total - p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("how_an_output_column_is_computed_is_compared", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) - SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(r.total) - SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id"), + ("renamed_and_reordered_derived_tables_still_tell_their_column", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH ref_sum AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), pay_sum AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(r.total) FROM ref_sum r JOIN pay_sum p ON p.user_id = r.user_id"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT p.user_id FROM paid p JOIN refunded r ON r.user_id = p.user_id GROUP BY p.user_id HAVING MAX(p.total) > 10", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT p.user_id FROM paid p JOIN refunded r ON r.user_id = p.user_id GROUP BY p.user_id HAVING MAX(r.total) > 10"), + ("which_derived_table_a_column_comes_from_is_reported", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT p.user_id FROM paid p JOIN refunded r ON r.user_id = p.user_id ORDER BY p.total DESC LIMIT 10", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT p.user_id FROM paid p JOIN refunded r ON r.user_id = p.user_id ORDER BY r.total DESC LIMIT 10"), + ("same_named_derived_tables_that_differ_only_in_conditions_say", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' AND is_deleted = 0 GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' AND is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > 100"), + ("two_same_shape_derived_tables_are_paired_by_their_conditions", "sqlite", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), open AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.total > o.total", + "WITH unpaid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'open' AND is_deleted = 0 GROUP BY user_id), settled AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' AND is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM unpaid u JOIN settled s ON s.user_id = u.user_id WHERE u.total < s.total"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.user_id = o.referrer_id", + "WITH paid AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'paid'), open AS (SELECT user_id, referrer_id, amount FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON p.referrer_id = o.user_id"), + ("the_same_item_on_another_layer_is_counted_once_per_layer", "sqlite", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id WHERE o.status = 'paid'", + "WITH pu AS (SELECT user_id, COUNT(*) AS n FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT SUM(o.amount) FROM orders o JOIN pu ON pu.user_id = o.user_id"), + ("filtering_before_or_after_a_window_or_limit_is_reported", "sqlite", + "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t WHERE paid_at >= '2026-09-01'", + "WITH t AS (SELECT * FROM orders WHERE paid_at >= '2026-09-01' ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t"), + ("filtering_before_or_after_a_window_or_limit_is_reported", "sqlite", + "WITH t AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t WHERE status = 'paid'", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid' ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM t"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.referrer_id = a.user_id"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(a.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(b.amount) FROM t a JOIN t b ON b.user_id = a.referrer_id"), + ("derived_tables_reading_the_same_table_keep_apart", "sqlite", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM t p JOIN orders c ON c.parent_id = p.id", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(c.amount) FROM t p JOIN orders c ON c.parent_id = p.id"), + ("time_columns_inside_derived_tables", "sqlite", + "WITH t AS (SELECT amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT amount, DATE(paid_at, '+8 hours') AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT store_id, user_id, SUM(amount) AS total FROM orders GROUP BY store_id, user_id) SELECT store_id FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) GROUP BY store_id HAVING MAX(total) > 100", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT store_id, user_id, SUM(amount) AS total FROM orders GROUP BY store_id, user_id) SELECT t.store_id FROM vip JOIN t ON t.user_id = vip.id GROUP BY t.store_id HAVING MAX(t.total) > 100"), + ("how_a_derived_column_is_computed_is_reported", "sqlite", + "WITH t AS (SELECT strftime('%Y-%m', paid_at) AS m, amount FROM orders) SELECT m, SUM(amount) FROM t GROUP BY m", + "WITH t AS (SELECT strftime('%Y-%m', created_at) AS m, amount FROM orders) SELECT m, SUM(amount) FROM t GROUP BY m"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT user_id, DATE(paid_at) AS day, amount FROM orders) SELECT SUM(amount) FROM t WHERE day >= '2026-09-01' AND user_id IN (SELECT id FROM users WHERE vip = 1)", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, DATE(paid_at) AS day, amount FROM orders) SELECT SUM(t.amount) FROM vip JOIN t ON t.user_id = vip.id WHERE t.day >= '2026-09-01'"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT AVG(total) FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1)", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT AVG(t.total) FROM vip JOIN t ON t.user_id = vip.id"), + ("a_changed_column_inside_a_renamed_derived_table_is_said_once", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), x AS (SELECT user_id, SUM(paid_amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM vip JOIN x ON x.user_id = vip.id WHERE total > 100"), + ("derived_columns_renamed_on_one_side_are_paired", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100", + "WITH x AS (SELECT user_id, SUM(amount - discount) AS net_total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM x WHERE net_total > 100"), + ("a_shifted_derived_number_does_not_split_one_condition_in_two", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100 AND user_id IN (SELECT id FROM users WHERE vip = 1)", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM vip JOIN t ON t.user_id = vip.id WHERE t.total > 100"), + ("a_paired_derived_join_with_another_name_says_both_names", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id", + "WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN totals ON totals.user_id = u.referrer_id"), + ("an_extra_derived_table_on_one_side_is_not_paired_with_anothe", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id JOIN t ON t.user_id = u.referrer_id"), + ("a_shifted_derived_join_is_paired_with_its_counterpart", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id WHERE t.total > 100", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id JOIN t ON t.user_id = u.id WHERE t.total > 100"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT user_id FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) ORDER BY total DESC LIMIT 10", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT t.user_id FROM vip JOIN t ON t.user_id = vip.id ORDER BY t.total DESC LIMIT 10"), + ("a_shifted_derived_number_is_no_difference_in_any_aspect", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) SELECT n, COUNT(*) FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) GROUP BY n", + "WITH vip AS (SELECT id FROM users WHERE vip = 1), t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) SELECT t.n, COUNT(*) FROM vip JOIN t ON t.user_id = vip.id GROUP BY t.n"), + ("how_a_derived_column_is_computed_is_reported", "sqlite", + "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT * FROM t", + "WITH t AS (SELECT user_id, amount * 0.8 AS net FROM orders) SELECT * FROM t"), + ("how_a_derived_column_is_computed_is_reported", "sqlite", + "WITH t AS (SELECT user_id, amount * 0.9 AS net FROM orders) SELECT SUM(net) FROM t", + "WITH t AS (SELECT user_id, amount * 0.8 AS net FROM orders) SELECT SUM(net) FROM t"), + ("a_time_column_computed_in_a_cte_is_seen_through", "sqlite", + "WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(created_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'"), + ("which_rows_are_taken_is_compared", "sqlite", + "WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM top", + "WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 100) SELECT SUM(amount) FROM top"), + ("limits_inside_derived_tables_are_paired_by_structure", "sqlite", + "WITH top AS (SELECT user_id, amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM top", + "WITH vip AS (SELECT id FROM users WHERE vip = 1 LIMIT 5), top AS (SELECT user_id, amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(top.amount) FROM top JOIN vip ON vip.id = top.user_id"), + ("a_real_difference_next_to_renamed_derived_tables_is_reported", "sqlite", + "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) SELECT COUNT(*) FROM user_totals WHERE total > 100", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t WHERE total > 100"), +] + +#: 过去复验者自己造的 SQL 对里,在这几组数据上结果不同的(SQLite,进执行比对) +VALIDATOR_DIFF: list[tuple[str, str, str]] = [ + ("cases.py:A29a", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(DISTINCT user_id) FROM orders WHERE status = 'paid'"), + ("cases.py:A29b", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status = 'paid') t", + "SELECT COUNT(user_id) FROM orders WHERE status = 'paid'"), + ("real1.py:W08 distinct in subq vs none", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders) t"), + ("exec_check3.py:E4", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id, store_id FROM orders) t", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders) t"), + ("batch4.py:N1 nested 3-level DISTINCT in middle", + "SELECT COUNT(*) FROM (SELECT user_id FROM (SELECT DISTINCT user_id, store_id FROM orders) a) b", + "SELECT COUNT(*) FROM (SELECT user_id FROM (SELECT user_id, store_id FROM orders) a) b"), + ("exec_check.py:F2", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders EXCEPT SELECT user_id FROM refunds) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM refunds EXCEPT SELECT user_id FROM orders) t"), + ("real1.py:W09 grain", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders GROUP BY user_id) t", + "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id"), + ("exec_check3.py:F3", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders INTERSECT SELECT user_id FROM refunds) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders EXCEPT SELECT user_id FROM refunds) t"), + ("exec_check.py:F1", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION SELECT user_id FROM refunds) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) t"), + ("cases.py:mysql", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r LEFT JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p ON p.user_id = r.user_id"), + ("cases.py:postgres", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'", + "SELECT COUNT(*) FROM (SELECT user_id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders WHERE status = 'paid') t WHERE t.rn = 1"), + ("batch3.py:K2 extra time col on one side; shared paid_at uppe", + "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT COUNT(*) FROM orders WHERE created_at >= '2026-01-01' AND paid_at >= '2026-09-01' AND paid_at < '2026-11-01'"), + ("p123c.py:Z08 real diff in 3-table chain same order (wording)", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM orders o JOIN users u ON u.id = o.user_id JOIN refunds r ON r.user_id = u.id"), + ("p123c.py:Z07 real diff in 3-table chain (wording)", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id JOIN refunds r ON r.user_id = u.id"), + ("p123d.py:Z10 real diff + permutation (tables)", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id JOIN users u ON u.id = o.user_id", + "SELECT COUNT(*) FROM users u JOIN refunds r ON r.user_id = u.id JOIN orders o ON o.id = r.order_id"), + ("real1.py:W03 join swap + LEFT vs INNER", + "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r JOIN orders o ON o.id = r.order_id"), + ("cases.py:postgres", + "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("batch2.py:C4 amount above own-user avg vs below (correlated ", + "SELECT COUNT(*) FROM orders o WHERE o.amount > (SELECT AVG(x.amount) FROM orders x WHERE x.user_id = o.user_id)", + "SELECT COUNT(*) FROM orders o WHERE (SELECT AVG(x.amount) FROM orders x WHERE x.user_id = o.user_id) > o.amount"), + ("real1.py:W01 alias + value", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid'", + "SELECT COUNT(*) FROM orders x WHERE x.status = 'done'"), + ("real1.py:W10 extra cond in outer vs IN subq", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND o.user_id IN (SELECT p.user_id FROM orders p WHERE p.status = 'paid')", + "SELECT COUNT(*) FROM orders o WHERE o.user_id IN (SELECT p.user_id FROM orders p WHERE p.status = 'paid')"), + ("exec_check3.py:SJ2", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.user_id = o1.user_id AND o2.created_at > o1.created_at WHERE o1.amount > 100", + "SELECT COUNT(*) FROM orders o1 JOIN orders o2 ON o2.user_id = o1.user_id AND o2.created_at > o1.created_at WHERE o2.amount > 100"), + ("real2.py:X02 table self-join role reversed, WHERE", + "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE a.vip = 1", + "SELECT COUNT(*) FROM users a JOIN users b ON a.id = b.referrer_id WHERE a.vip = 1"), + ("real1.py:W04 self-join direction reversed", + "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE a.vip = 1", + "SELECT COUNT(*) FROM users a JOIN users b ON b.id = a.referrer_id WHERE b.vip = 1"), + ("batch4.py:V1 COUNT(DISTINCT user_id) vs DISTINCT COUNT(user_", + "SELECT COUNT(DISTINCT user_id) FROM orders", + "SELECT DISTINCT COUNT(user_id) FROM orders"), + ("compare.py:count-distinct-vs-sub", + "SELECT COUNT(DISTINCT user_id) FROM orders WHERE status='paid'", + "SELECT COUNT(*) FROM (SELECT DISTINCT user_id FROM orders WHERE status='paid') t"), + ("batch2.py:E5 COUNT(t.user_id) over DISTINCT subquery vs COUN", + "SELECT COUNT(t.user_id) FROM (SELECT DISTINCT user_id FROM orders) t", + "SELECT COUNT(user_id) FROM orders"), + ("batch6.py:Q3s GROUP BY alias shadowing a column (sqlite)", + "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY created_at", + "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY DATE(created_at)"), + ("exec_check2.py:SQ3 sqlite", + "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY created_at ORDER BY 1", + "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY DATE(created_at) ORDER BY 1"), + ("fp4.py:outer-date-vs-cast", + "SELECT DATE(paid_at) AS d, COUNT(*) FROM orders GROUP BY DATE(paid_at)", + "SELECT CAST(paid_at AS DATE) AS d, COUNT(*) FROM orders GROUP BY CAST(paid_at AS DATE)"), + ("regress2.py:group-date-forms2", + "SELECT DATE(paid_at) AS d, SUM(amount) FROM orders GROUP BY DATE(paid_at)", + "SELECT CAST(paid_at AS DATE) AS d, SUM(amount) FROM orders GROUP BY CAST(paid_at AS DATE)"), + ("regress2.py:group-date-forms", + "SELECT DATE(paid_at), SUM(amount) FROM orders GROUP BY DATE(paid_at)", + "SELECT CAST(paid_at AS DATE) AS d, SUM(amount) FROM orders GROUP BY 1"), + ("batch2.py:E6 outer SELECT DISTINCT vs no DISTINCT", + "SELECT DISTINCT user_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid'"), + ("eq2.py:T01 distinct col order", + "SELECT DISTINCT user_id, store_id FROM orders", + "SELECT DISTINCT store_id, user_id FROM orders"), + ("exec_check4.py:F3 (sets)", + "SELECT GROUP_CONCAT(user_id) FROM (SELECT user_id FROM orders INTERSECT SELECT user_id FROM refunds) t", + "SELECT GROUP_CONCAT(user_id) FROM (SELECT user_id FROM orders EXCEPT SELECT user_id FROM refunds) t"), + ("real1.py:W14 self-join direction reversed renamed", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.id = a.parent_id", + "SELECT SUM(y.amount) FROM orders x JOIN orders y ON y.id = x.parent_id"), + ("batch1.py:B4 real-table self-join control (should report)", + "SELECT SUM(a.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id", + "SELECT SUM(b.amount) FROM orders a JOIN orders b ON b.user_id = a.referrer_id"), + ("batch4.py:N3 LIMIT two levels down, filter at top vs inside ", + "SELECT SUM(amount) FROM (SELECT * FROM (SELECT * FROM orders ORDER BY amount DESC LIMIT 10) a) b WHERE status = 'paid'", + "SELECT SUM(amount) FROM (SELECT * FROM (SELECT * FROM orders WHERE status = 'paid' ORDER BY amount DESC LIMIT 10) a) b"), + ("cases.py:postgres", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at) AS rn FROM orders) t WHERE rn = 1", + "SELECT SUM(amount) FROM (SELECT user_id, amount, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at DESC) AS rn FROM orders) t WHERE rn = 1"), + ("fp2.py:cast-date-where", + "SELECT SUM(amount) FROM orders WHERE CAST(paid_at AS DATE) >= '2026-09-01'", + "SELECT SUM(amount) FROM orders WHERE DATE(paid_at) >= '2026-09-01'"), + ("batch3.py:L2 table swapped AND SUM moved from order amount t", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(r.amount) FROM orders_archive o JOIN refunds r ON r.order_id = o.id"), + ("exec_check.py:L1", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE o.amount > 100", + "SELECT SUM(o.amount) FROM orders_archive o JOIN refunds r ON r.order_id = o.id WHERE r.amount > 100"), + ("cases.py:postgres", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE o.status = 'paid'"), + ("exec_check2.py:C2 (sum of qualifying orders)", + "SELECT SUM(o.amount), GROUP_CONCAT(o.id) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND p.created_at < o.created_at)", + "SELECT SUM(o.amount), GROUP_CONCAT(o.id) FROM orders o WHERE EXISTS (SELECT 1 FROM orders p WHERE p.user_id = o.user_id AND o.created_at < p.created_at)"), + ("exec_check3.py:SJ1", + "SELECT SUM(o1.amount) FROM orders o1 JOIN orders o2 ON o2.user_id = o1.user_id AND o2.created_at > o1.created_at", + "SELECT SUM(o2.amount) FROM orders o1 JOIN orders o2 ON o2.user_id = o1.user_id AND o2.created_at > o1.created_at"), + ("cases.py:postgres", + "SELECT SUM(r.total) / SUM(p.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id", + "SELECT SUM(p.total) / SUM(r.total) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id"), + ("batch4.py:U1 UNION branch signs swapped (control)", + "SELECT SUM(x) FROM (SELECT amount AS x FROM orders UNION ALL SELECT -amount FROM refunds) t", + "SELECT SUM(x) FROM (SELECT -amount AS x FROM orders UNION ALL SELECT amount FROM refunds) t"), + ("fp6.py:time-derived-renamed", + "SELECT SUM(x.amount) FROM (SELECT amount, DATE(paid_at) AS day FROM orders) x WHERE x.day BETWEEN '2026-09-01' AND '2026-09-30'", + "WITH y AS (SELECT amount, CAST(paid_at AS DATE) AS d FROM orders) SELECT SUM(amount) FROM y WHERE d BETWEEN '2026-09-01' AND '2026-09-30'"), + ("cases.py:mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x", + "SELECT SUM(x.total) FROM (SELECT p.user_id, r.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x"), + ("batch3.py:J1 ORDER BY DESC NULLS LAST vs NULLS FIRST LIMIT 1", + "SELECT id FROM orders ORDER BY amount DESC NULLS LAST LIMIT 10", + "SELECT id FROM orders ORDER BY amount DESC NULLS FIRST LIMIT 10"), + ("exec_check.py:J1", + "SELECT id FROM orders ORDER BY amount DESC NULLS LAST LIMIT 2", + "SELECT id FROM orders ORDER BY amount DESC NULLS FIRST LIMIT 2"), + ("batch3.py:H3 window frame 6 vs 2 preceding (control)", + "SELECT id, SUM(amount) OVER (ORDER BY created_at ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS s FROM orders", + "SELECT id, SUM(amount) OVER (ORDER BY created_at ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS s FROM orders"), + ("batch3.py:H1 named WINDOW partition user_id vs store_id", + "SELECT id, SUM(amount) OVER w AS s FROM orders WINDOW w AS (PARTITION BY user_id)", + "SELECT id, SUM(amount) OVER w AS s FROM orders WINDOW w AS (PARTITION BY store_id)"), + ("exec_check.py:H1", + "SELECT id, SUM(amount) OVER w AS s FROM orders WINDOW w AS (PARTITION BY user_id) ORDER BY id", + "SELECT id, SUM(amount) OVER w AS s FROM orders WINDOW w AS (PARTITION BY store_id) ORDER BY id"), + ("exec_check3.py:C3", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.parent_id = o.id) AS n FROM orders o ORDER BY o.id", + "SELECT o.id, (SELECT COUNT(*) FROM orders c WHERE c.id = o.parent_id) AS n FROM orders o ORDER BY o.id"), + ("eq1.py:E05 column order", + "SELECT store_id, user_id, SUM(amount) FROM orders GROUP BY store_id, user_id", + "SELECT SUM(amount), user_id, store_id FROM orders GROUP BY user_id, store_id"), + ("real1.py:W07 union vs union all wrapped", + "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT * FROM (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) t"), + ("batch2.py:F4 outer UNION vs UNION ALL", + "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds"), + ("compare.py:union-root-extra-branch-cond", + "SELECT user_id FROM orders WHERE status='paid' UNION SELECT referrer_id FROM orders WHERE status='paid'", + "SELECT user_id FROM orders WHERE status='paid' UNION SELECT referrer_id FROM orders"), + ("fp1.py:select-order", + "SELECT user_id, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY user_id", + "SELECT COUNT(*) AS cnt, o.user_id, SUM(o.amount) AS total FROM orders o GROUP BY o.user_id"), + ("eq2.py:T02 distinct vs group by", + "SELECT user_id, store_id FROM orders GROUP BY user_id, store_id", + "SELECT DISTINCT store_id, user_id FROM orders"), + ("compare.py:multi-cte-same-cond", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status='paid'), b AS (SELECT * FROM orders WHERE is_deleted = 0 AND status='open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT * FROM orders WHERE is_deleted = 0 AND status='paid'), b AS (SELECT * FROM orders WHERE status='open') SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id"), + ("cases.py:mysql", + "WITH a AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id), b AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'", + "WITH a AS (SELECT user_id, MAX(refunded_at) AS ts FROM refunds GROUP BY user_id), b AS (SELECT user_id, MAX(paid_at) AS ts FROM orders GROUP BY user_id) SELECT COUNT(*) FROM a JOIN b ON b.user_id = a.user_id WHERE a.ts >= '2026-09-01'"), + ("cases.py:sqlite", + "WITH a AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), b AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(a.total) FROM a JOIN b ON b.user_id = a.user_id", + "WITH a AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), b AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(a.total) FROM a JOIN b ON b.user_id = a.user_id"), + ("cases.py:mysql", + "WITH done AS (SELECT * FROM refunds WHERE state = 'done') SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id", + "WITH done AS (SELECT * FROM refunds) SELECT SUM(o.amount) FROM orders o LEFT JOIN done d ON d.order_id = o.id WHERE d.state = 'done'"), + ("real2.py:X05 condition moved into CTE + extra in outer", + "WITH p AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT SUM(amount) FROM p WHERE status = 'paid'", + "WITH p AS (SELECT * FROM orders) SELECT SUM(amount) FROM p WHERE status = 'paid'"), + ("real1.py:W02 CTE renamed + agg func", + "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(p.amount) FROM p", + "WITH q AS (SELECT * FROM orders WHERE status = 'paid') SELECT AVG(q.amount) FROM q"), + ("batch4.py:O1 LEFT JOIN keep side swapped (control)", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p LEFT JOIN open o ON o.user_id = p.user_id", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM open o LEFT JOIN paid p ON p.user_id = o.user_id"), + ("batch4.py:O2 LEFT JOIN, WHERE on keep side vs nullable side", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p LEFT JOIN open o ON o.user_id = p.user_id WHERE p.amount > 100", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p LEFT JOIN open o ON o.user_id = p.user_id WHERE o.amount > 100"), + ("real1.py:W12 sealed renamed CTEs, used the other one", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(p.amount) FROM paid p JOIN open o ON o.user_id = p.user_id", + "WITH a1 AS (SELECT * FROM orders WHERE status = 'open'), b1 AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(x.amount) FROM b1 y JOIN a1 x ON x.user_id = y.user_id"), + ("exec_check3.py:A5", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT p.store_id, COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY p.store_id ORDER BY 1", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid'), open AS (SELECT * FROM orders WHERE status = 'open') SELECT o.store_id, COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id GROUP BY o.store_id ORDER BY 1"), + ("batch4.py:T2 DATE(created_at) AS d in paid/open CTEs, p.d vs", + "WITH paid AS (SELECT user_id, DATE(created_at) AS d FROM orders WHERE status = 'paid'), open AS (SELECT user_id, DATE(created_at) AS d FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE p.d >= '2026-09-01'", + "WITH paid AS (SELECT user_id, DATE(created_at) AS d FROM orders WHERE status = 'paid'), open AS (SELECT user_id, DATE(created_at) AS d FROM orders WHERE status = 'open') SELECT COUNT(*) FROM paid p JOIN open o ON o.user_id = p.user_id WHERE o.d >= '2026-09-01'"), + ("cases.py:postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, p.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT COUNT(*) FROM net WHERE v > 100", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) , net AS (SELECT p.user_id, r.total AS v FROM paid p JOIN refunded r ON r.user_id = p.user_id) SELECT COUNT(*) FROM net WHERE v > 100"), + ("p123.py:1k renamed WHERE", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > 100", + "WITH a1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), b1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM a1 x JOIN b1 y ON y.user_id = x.user_id WHERE y.total > 100"), + ("cases.py:postgres", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM paid p LEFT JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM refunded r LEFT JOIN paid p ON p.user_id = r.user_id"), + ("p123.py:1l renamed diff", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total - r.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH a1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), b1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(y.total - x.total) FROM a1 x JOIN b1 y ON y.user_id = x.user_id"), + ("p123.py:1j renamed SUM", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH a1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), b1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT SUM(y.total) FROM a1 x JOIN b1 y ON y.user_id = x.user_id"), + ("p123b.py:Z04 diff survives join swap", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), vip AS (SELECT id FROM users WHERE vip = 1) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), vip AS (SELECT id FROM users WHERE vip = 1) SELECT SUM(r.total) FROM refunded r JOIN paid p ON p.user_id = r.user_id"), + ("p123b.py:Z05 diff survives names swapped", + "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), vip AS (SELECT id FROM users WHERE vip = 1) SELECT SUM(p.total) FROM paid p JOIN refunded r ON r.user_id = p.user_id", + "WITH refunded AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), paid AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), v AS (SELECT id FROM users WHERE vip = 1) SELECT SUM(x.total) FROM paid x JOIN refunded y ON y.user_id = x.user_id"), + ("cases.py:mysql", + "WITH r AS (SELECT user_id, amount, RANK() OVER (ORDER BY amount DESC) AS k FROM orders) SELECT SUM(amount) FROM r WHERE k <= 10", + "WITH r AS (SELECT user_id, amount, ROW_NUMBER() OVER (ORDER BY amount DESC) AS k FROM orders) SELECT SUM(amount) FROM r WHERE k <= 10"), + ("real1.py:W13 same name CTE different meaning", + "WITH t AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(t.amount) FROM t", + "WITH t AS (SELECT * FROM orders WHERE status = 'open') SELECT SUM(t.amount) FROM t"), + ("real1.py:W06 time col via renamed CTE", + "WITH t AS (SELECT DATE(paid_at) AS d, amount FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH x AS (SELECT DATE(created_at) AS d, amount FROM orders) SELECT SUM(amount) FROM x WHERE d >= '2026-09-01'"), + ("exec_check4.py:E3", + "WITH t AS (SELECT DISTINCT user_id, amount FROM orders) SELECT SUM(amount) FROM t", + "SELECT SUM(amount) FROM orders"), + ("batch3.py:H2 named WINDOW in CTE + outer filter", + "WITH t AS (SELECT id, amount, ROW_NUMBER() OVER w AS rn FROM orders WINDOW w AS (PARTITION BY user_id ORDER BY created_at)) SELECT SUM(amount) FROM t WHERE rn = 1", + "WITH t AS (SELECT id, amount, ROW_NUMBER() OVER w AS rn FROM orders WINDOW w AS (PARTITION BY user_id ORDER BY created_at DESC)) SELECT SUM(amount) FROM t WHERE rn = 1"), + ("cases.py:mysql", + "WITH t AS (SELECT user_id, CASE WHEN status = 'paid' THEN amount ELSE 0 END AS paid_amt FROM orders) SELECT SUM(paid_amt) FROM t", + "WITH t AS (SELECT user_id, CASE WHEN status IN ('paid', 'shipped') THEN amount ELSE 0 END AS paid_amt FROM orders) SELECT SUM(paid_amt) FROM t"), + ("real2.py:X03 extra outer cond also in CTE layer", + "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT COUNT(*) FROM t JOIN orders o ON o.user_id = t.user_id WHERE o.status = 'paid'", + "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders WHERE status = 'paid' GROUP BY user_id) SELECT COUNT(*) FROM t JOIN orders o ON o.user_id = t.user_id"), + ("p123b.py:Z06 problem2 variant: extra CTE after t, used via I", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), vip AS (SELECT id FROM users WHERE vip = 1) SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id WHERE u.id IN (SELECT id FROM vip)"), + ("real2.py:X01 CTE self-join role reversed", + "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT SUM(a.total) FROM t a JOIN t b ON b.user_id = a.referrer_id", + "WITH t AS (SELECT user_id, referrer_id, SUM(amount) AS total FROM orders GROUP BY user_id, referrer_id) SELECT SUM(a.total) FROM t a JOIN t b ON a.user_id = b.referrer_id"), + ("cases.py:postgres", + "WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), t2 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100", + "WITH t1 AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), t2 AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT COUNT(*) FROM t1 JOIN t2 ON t2.user_id = t1.user_id WHERE t1.total > 100"), + ("batch5.py:S1 filter on LIMIT-CTE column vs on another CTE's ", + "WITH top AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10), allo AS (SELECT * FROM orders) SELECT COUNT(*) FROM top t JOIN allo a ON a.user_id = t.user_id WHERE t.amount > 100", + "WITH top AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 10), allo AS (SELECT * FROM orders) SELECT COUNT(*) FROM top t JOIN allo a ON a.user_id = t.user_id WHERE a.amount > 100"), + ("exec_check.py:S1", + "WITH top AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 2), allo AS (SELECT * FROM orders) SELECT COUNT(*) FROM top t JOIN allo a ON a.user_id = t.user_id WHERE t.amount > 100", + "WITH top AS (SELECT * FROM orders ORDER BY amount DESC LIMIT 2), allo AS (SELECT * FROM orders) SELECT COUNT(*) FROM top t JOIN allo a ON a.user_id = t.user_id WHERE a.amount > 100"), + ("exec_check.py:E7", + "WITH u AS (SELECT DISTINCT user_id FROM refunds) SELECT SUM(o.amount) FROM orders o JOIN u ON u.user_id = o.user_id", + "WITH u AS (SELECT user_id FROM refunds) SELECT SUM(o.amount) FROM orders o JOIN u ON u.user_id = o.user_id"), +] + + +# ---------------------------------------------------------------------------- 第 2 轮复验的反例 +# 第 2 轮复验(重做之后的第一版)报的 blocker / major 和几位复验者留在 /private/tmp 的脚本里的对,改写到这个库的 +# 表上(列名换成这里有的),SQLite 能执行的进执行比对;只在 PostgreSQL / MySQL 才有的写法进 REGRESSIONS_R2 只查结论。 + +#: 意思一样、但要按语义推理才看得出的(条件挪进相关子查询、冗余条件、写法不同的等价条件……):规范化不保证认得出, +#: 认不出时只许给兜底说明,不许给具体说法(复验专挑这类对查「说出不存在的差异」) +_SAME_HARD = [ + # 条件挪进相关子查询(EXISTS 里引用外层的列):B 是有这个条件的,不能说「B 没有」 + ("r2-exists-moved-time", "SELECT COUNT(*) FROM orders o WHERE o.paid_at >= '2026-09-01' AND EXISTS " + "(SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id " + "AND o.paid_at >= '2026-09-01')"), + ("r2-exists-moved-filter", "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS " + "(SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND o.status = 'paid')"), + ("r2-scalar-moved-filter", "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND o.amount > " + "(SELECT AVG(r.amount) FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE o.amount > (SELECT AVG(r.amount) FROM refunds r WHERE r.order_id = o.id " + "AND o.status = 'paid')"), + # refunds 没有 status 列:子查询里的 status 是外层 orders 的 + ("r2-in-moved-outer-column", "SELECT COUNT(*) FROM orders WHERE status = 'paid' AND id IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE id IN (SELECT order_id FROM refunds WHERE status = 'paid')"), + # 一段时间拆成两段、IN 和 OR、多写一个下界:留下的行一样 + ("r2-time-split-range", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT COUNT(*) FROM orders WHERE (paid_at >= '2026-09-01' AND paid_at < '2026-09-16') OR " + "(paid_at >= '2026-09-16' AND paid_at < '2026-10-01')"), + ("r2-time-in-vs-or", "SELECT COUNT(*) FROM orders WHERE paid_at IN ('2026-09-01', '2026-09-30')", + "SELECT COUNT(*) FROM orders WHERE paid_at = '2026-09-01' OR paid_at = '2026-09-30'"), + ("r2-time-redundant-lower", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01' AND paid_at > '2026-08-01'"), + ("r2-in-vs-or", "SELECT COUNT(*) FROM orders WHERE status IN ('paid', 'done')", + "SELECT COUNT(*) FROM orders WHERE status = 'paid' OR status = 'done'"), + ("r2-redundant-not-null", "SELECT SUM(tax) FROM orders WHERE tax > 1 AND tax IS NOT NULL", + "SELECT SUM(tax) FROM orders WHERE tax > 1"), + ("r2-redundant-not-null-join", "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id " + "WHERE o.user_id IS NOT NULL", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id"), + ("r2-redundant-not-null-in", "SELECT COUNT(*) FROM orders WHERE store_id IN (SELECT id FROM stores " + "WHERE region = 'east') AND store_id IS NOT NULL", + "SELECT COUNT(*) FROM orders WHERE store_id IN (SELECT id FROM stores WHERE region = 'east')"), + ("r2-count-col-vs-not-null-filter", "SELECT COUNT(tax) FROM orders", "SELECT COUNT(*) FROM orders WHERE tax IS NOT NULL"), + ("r2-count-distinct-not-null", "SELECT COUNT(DISTINCT user_id) FROM orders WHERE user_id IS NOT NULL", + "SELECT COUNT(DISTINCT user_id) FROM orders"), + ("r2-coalesce-gt", "SELECT COUNT(*) FROM orders WHERE COALESCE(amount, 0) > 10", + "SELECT COUNT(*) FROM orders WHERE amount > 10"), + ("r2-left-coalesce-reject", "SELECT o.id FROM orders o LEFT JOIN users u ON u.id = o.user_id " + "WHERE COALESCE(u.grade, 0) > 1", + "SELECT o.id FROM orders o JOIN users u ON u.id = o.user_id WHERE COALESCE(u.grade, 0) > 1"), + ("r2-left-count-nullable-side", "SELECT COUNT(r.id) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(r.id) FROM orders o JOIN refunds r ON r.order_id = o.id"), + ("r2-having-count-gt-1-vs-ge-2", "SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1", + "SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) >= 2"), + ("r2-having-noop", "SELECT store_id, SUM(amount) FROM orders GROUP BY store_id HAVING COUNT(*) > 0", + "SELECT store_id, SUM(amount) FROM orders GROUP BY store_id"), + ("r2-scalar-max-commute", "SELECT max(amount, tax) FROM orders", "SELECT max(tax, amount) FROM orders"), + ("r2-case-vs-coalesce", "SELECT SUM(CASE WHEN discount IS NULL THEN 0 ELSE discount END) FROM orders", + "SELECT SUM(COALESCE(discount, 0)) FROM orders"), +] + +#: 意思一样、规范化认得出的改写(第 2 轮新增的几步) +_SAME_R2 = [ + ("r2-one-eq-one", "SELECT COUNT(*) FROM orders WHERE 1 = 1 AND status = 'paid'", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'"), + ("r2-not-is-not-null", "SELECT COUNT(*) FROM orders WHERE NOT (paid_at IS NOT NULL)", + "SELECT COUNT(*) FROM orders WHERE paid_at IS NULL"), + ("r2-not-like", "SELECT COUNT(*) FROM orders WHERE status NOT LIKE 'p%'", + "SELECT COUNT(*) FROM orders WHERE NOT status LIKE 'p%'"), + ("r2-iif-vs-case", "SELECT SUM(IIF(status = 'paid', amount, 0)) FROM orders", + "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE 0 END) FROM orders"), + ("r2-case-else-null", "SELECT SUM(CASE WHEN status = 'paid' THEN amount END) FROM orders", + "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE NULL END) FROM orders"), + ("r2-min-distinct", "SELECT MIN(DISTINCT amount), MAX(DISTINCT tax) FROM orders", + "SELECT MIN(amount), MAX(tax) FROM orders"), + ("r2-exists-limit-1", "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id LIMIT 1)", + "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)"), + ("r2-in-subquery-distinct", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT DISTINCT user_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM refunds)"), + ("r2-using-vs-on-select", "SELECT user_id FROM orders o LEFT JOIN refunds r USING (user_id)", + "SELECT o.user_id FROM orders o LEFT JOIN refunds r ON o.user_id = r.user_id"), + ("r2-multi-use-cte-correlated", "WITH p AS (SELECT * FROM orders WHERE status = 'paid') SELECT COUNT(*) FROM p " + "WHERE EXISTS (SELECT 1 FROM p AS p2 WHERE p2.parent_id = p.id)", + "SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM orders c " + "WHERE c.parent_id = o.id AND c.status = 'paid')"), + ("r2-outer-where-into-having", "WITH t AS (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id " + "HAVING SUM(amount) > 100) SELECT COUNT(*) FROM t", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS s FROM orders GROUP BY user_id) t WHERE t.s > 100"), +] + +#: 不许判等价的(第 2 轮复验报的 blocker 改写到这个库上;在这几组数据上大多结果不同) +_DIFF_R2 = [ + # 数字常量不折叠:SQLite 按浮点数算,0.1 + 0.2 ≠ 0.3;三项以上的加乘不换结合 + ("r2-float-fold", "SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2", "SELECT COUNT(*) FROM orders WHERE amount = 0.3"), + ("r2-float-fold-chain", "SELECT COUNT(*) FROM orders WHERE amount + 0.1 + 0.2 = 0.3", + "SELECT COUNT(*) FROM orders WHERE amount + 0.3 = 0.3"), + ("r2-float-fold-mul", "SELECT id, amount * 0.1 * 3 FROM orders", "SELECT id, amount * 0.3 FROM orders"), + ("r2-float-fold-compare", "SELECT COUNT(*) FROM orders WHERE amount < 3 * 0.1", + "SELECT COUNT(*) FROM orders WHERE amount < 0.3"), + ("r2-float-fold-007", "SELECT COUNT(*) FROM orders WHERE amount >= 100 * 0.07", + "SELECT COUNT(*) FROM orders WHERE amount >= 7.00"), + # 取两次负会把文本转成数 + ("r2-double-negation", "SELECT id, -(-name) FROM users", "SELECT id, name FROM users"), + ("r2-double-negation-compare", "SELECT COUNT(*) FROM users WHERE -(-name) = 100", + "SELECT COUNT(*) FROM users WHERE name = 100"), + # 列名表:CTE 自己写的列名决定了外面的名字指哪一列 + ("r2-cte-column-list-swap", "WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM p WHERE total > 100", + "WITH p(total, uid) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM p WHERE total > 100"), + ("r2-cte-column-list-swap-2", "WITH t(user_id, amount) AS (SELECT amount, user_id FROM orders) SELECT SUM(amount) FROM t", + "WITH t(amount, user_id) AS (SELECT amount, user_id FROM orders) SELECT SUM(amount) FROM t"), + # SQLite 不写 RECURSIVE 也是递归:A 引用的是自己,B 引用的是实表 nums + ("r2-recursive-without-keyword", "WITH nums(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM nums WHERE n < 5) " + "SELECT COUNT(*) FROM nums", + "WITH seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM nums WHERE n < 5) SELECT COUNT(*) FROM seq"), + # ORDER BY 有并列再取行、拼串、窗口函数:结果看行出来的先后,调换来源、DISTINCT 改 GROUP BY 都会变 + ("r2-distinct-group-limit-ties", "SELECT DISTINCT user_id, status FROM orders ORDER BY user_id LIMIT 4", + "SELECT user_id, status FROM orders GROUP BY user_id, status ORDER BY user_id LIMIT 4"), + ("r2-group-key-order-limit-ties", "SELECT user_id, store_id, COUNT(*) FROM orders GROUP BY user_id, store_id " + "ORDER BY COUNT(*) DESC LIMIT 2", + "SELECT user_id, store_id, COUNT(*) FROM orders GROUP BY store_id, user_id ORDER BY COUNT(*) DESC LIMIT 2"), + ("r2-cross-join-swap-limit-ties", "SELECT o.id, u.id FROM orders o CROSS JOIN users u WHERE u.id = o.user_id " + "ORDER BY o.amount LIMIT 3", + "SELECT o.id, u.id FROM users u CROSS JOIN orders o WHERE u.id = o.user_id ORDER BY o.amount LIMIT 3"), + ("r2-group-concat-join-order", "SELECT u.city, GROUP_CONCAT(o.id) FROM users u CROSS JOIN orders o " + "WHERE o.user_id = u.id GROUP BY u.city", + "SELECT u.city, GROUP_CONCAT(o.id) FROM orders o CROSS JOIN users u WHERE o.user_id = u.id GROUP BY u.city"), + ("r2-group-concat-cross-vs-comma", "SELECT u.city, GROUP_CONCAT(o.id) FROM users u CROSS JOIN orders o " + "WHERE o.user_id = u.id GROUP BY u.city", + "SELECT u.city, GROUP_CONCAT(o.id) FROM users u, orders o WHERE o.user_id = u.id GROUP BY u.city"), + ("r2-row-number-join-order", "SELECT o.id, ROW_NUMBER() OVER () AS rn FROM users u CROSS JOIN orders o " + "WHERE o.user_id = u.id", + "SELECT o.id, ROW_NUMBER() OVER () AS rn FROM orders o CROSS JOIN users u WHERE o.user_id = u.id"), + ("r2-row-number-ties-join-order", "SELECT o.id, ROW_NUMBER() OVER (ORDER BY o.status) AS rn FROM users u " + "JOIN orders o ON o.user_id = u.id", + "SELECT o.id, ROW_NUMBER() OVER (ORDER BY o.status) AS rn FROM orders o JOIN users u ON o.user_id = u.id"), + ("r2-group-concat-subquery-order", "SELECT GROUP_CONCAT(id) FROM (SELECT id FROM orders ORDER BY amount DESC, id) t", + "SELECT GROUP_CONCAT(id) FROM (SELECT id FROM orders) t"), + # SQLite 里 a IN (b) 是 a = +b:右边那一列不带类型亲和 + ("r2-in-single-column", "SELECT COUNT(*) FROM orders WHERE status IN (store_id)", + "SELECT COUNT(*) FROM orders WHERE status = store_id"), + ("r2-not-in-single-column", "SELECT COUNT(*) FROM orders WHERE status NOT IN (store_id)", + "SELECT COUNT(*) FROM orders WHERE status <> store_id"), + # 解析时丢了写法的:类型名(STRING 是数值亲和)、0x10 和 x'10' + ("r2-cast-string", "SELECT store_id, CAST(store_id AS STRING) FROM orders", + "SELECT store_id, CAST(store_id AS TEXT) FROM orders"), + ("r2-cast-bytea", "SELECT id, CAST(status AS BLOB) FROM orders", "SELECT id, CAST(status AS BYTEA) FROM orders"), + ("r2-hex-literal", "SELECT COUNT(*) FROM orders WHERE id = 0x10", "SELECT COUNT(*) FROM orders WHERE id = x'10'"), + # SELECT * 的列先后 + ("r2-star-cte-column-order", "WITH c AS (SELECT user_id, amount FROM orders) SELECT * FROM c JOIN users " + "ON users.id = c.user_id", + "WITH c AS (SELECT amount, user_id FROM orders) SELECT * FROM c JOIN users ON users.id = c.user_id"), + ("r2-star-join-reorder", "SELECT * FROM orders o LEFT JOIN refunds r ON r.order_id = o.id JOIN users u ON u.id = o.user_id", + "SELECT * FROM orders o JOIN users u ON u.id = o.user_id LEFT JOIN refunds r ON r.order_id = o.id"), + # EXISTS 里:ORDER BY 有聚合就是聚合查询(总出一行);DISTINCT 在 OFFSET 之前算(SQLite 恰好不管这个 DISTINCT) + ("r2-exists-order-by-aggregate", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r " + "WHERE r.order_id = o.id ORDER BY COUNT(*))", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)"), + ("r2-exists-distinct-offset", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT DISTINCT r.state FROM refunds r " + "WHERE r.amount > 0 LIMIT 1 OFFSET 1)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT r.state FROM refunds r WHERE r.amount > 0 LIMIT 1 OFFSET 1)"), + # HAVING 不往 WHERE 挪:不分大小写的排序规则下同一组里的值各不一样 + ("r2-having-key-vs-where", "SELECT city, COUNT(*) FROM users GROUP BY city HAVING city = 'SH'", + "SELECT city, COUNT(*) FROM users WHERE city = 'SH' GROUP BY city"), + # 两边对调、子查询配错位置(复验说成自相矛盾的话的几对) + ("r2-except-sides-swapped", "SELECT user_id FROM orders WHERE paid_at >= '2026-09-01' EXCEPT SELECT user_id FROM refunds", + "SELECT user_id FROM refunds EXCEPT SELECT user_id FROM orders WHERE created_at >= '2026-09-01'"), + ("r2-in-not-in-swapped", "WITH p AS (SELECT user_id FROM orders WHERE status = 'paid' AND user_id IS NOT NULL), " + "v AS (SELECT user_id FROM orders WHERE status = 'void' AND user_id IS NOT NULL) SELECT COUNT(*) FROM users " + "WHERE id IN (SELECT user_id FROM p) AND id NOT IN (SELECT user_id FROM v)", + "WITH p AS (SELECT user_id FROM orders WHERE status = 'paid' AND user_id IS NOT NULL), " + "v AS (SELECT user_id FROM orders WHERE status = 'void' AND user_id IS NOT NULL) SELECT COUNT(*) FROM users " + "WHERE id IN (SELECT user_id FROM v) AND id NOT IN (SELECT user_id FROM p)"), + ("r2-not-in-filter", "WITH p AS (SELECT user_id FROM orders WHERE status = 'paid'), v AS (SELECT user_id FROM orders " + "WHERE status = 'void' AND user_id IS NOT NULL) SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM p) " + "AND id NOT IN (SELECT user_id FROM v)", + "WITH p AS (SELECT user_id FROM orders WHERE status = 'paid'), v AS (SELECT user_id FROM orders " + "WHERE status = 'open' AND user_id IS NOT NULL) SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM p) " + "AND id NOT IN (SELECT user_id FROM v)"), + ("r2-exists-not-exists-swapped", "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r " + "WHERE r.order_id = o.id AND r.state = 'done') AND NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id " + "AND r.state = 'pending')", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND r.state = 'pending') " + "AND NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND r.state = 'done')"), + ("r2-in-left-column-swapped", "SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM orders WHERE status = 'paid') " + "AND referrer_id IN (SELECT user_id FROM orders WHERE status = 'void')", + "SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM orders WHERE status = 'void') " + "AND referrer_id IN (SELECT user_id FROM orders WHERE status = 'paid')"), + # 换了表:各自的列是各自表的(复验报的 detail 归错了表) + ("r2-table-swap-time", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM users WHERE created_at >= '2026-09-01'"), + ("r2-table-swap-join", "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id WHERE u.city = 'SH'"), + ("r2-table-swap-time-2", "SELECT SUM(amount) FROM refunds WHERE refunded_at >= '2026-09-01'", + "SELECT SUM(amount) FROM payments WHERE paid_at >= '2026-09-01'"), +] + +#: 只在 PostgreSQL / MySQL 才有的写法(不执行,只查不判等价):第 2 轮复验在真实的库上跑出结果不同的 +REGRESSIONS_R2: list[tuple[str, str, str, str]] = [ + ("having_binary_is_not_where", "mysql", + "SELECT city, COUNT(*) FROM users GROUP BY city HAVING BINARY city = 'SH'", + "SELECT city, COUNT(*) FROM users WHERE BINARY city = 'SH' GROUP BY city"), + ("having_like_binary_is_not_where", "mysql", + "SELECT city, COUNT(*) FROM users GROUP BY city HAVING city LIKE BINARY 'S%'", + "SELECT city, COUNT(*) FROM users WHERE city LIKE BINARY 'S%' GROUP BY city"), + ("having_collate_is_not_where", "mysql", + "SELECT city, COUNT(*) FROM users GROUP BY city HAVING city COLLATE utf8mb4_bin = 'SH'", + "SELECT city, COUNT(*) FROM users WHERE city COLLATE utf8mb4_bin = 'SH' GROUP BY city"), + ("distinct_city_join_order_ci", "mysql", + "SELECT DISTINCT u.city FROM users u JOIN orders o ON o.user_id = u.id", + "SELECT DISTINCT u.city FROM orders o JOIN users u ON o.user_id = u.id"), + ("group_city_join_order_ci", "mysql", + "SELECT u.city, COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id GROUP BY u.city", + "SELECT u.city, COUNT(*) FROM orders o JOIN users u ON o.user_id = u.id GROUP BY u.city"), + ("order_by_aggregate_in_exists", "postgres", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id ORDER BY COUNT(*))", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)"), + ("order_by_aggregate_in_in", "postgres", + "SELECT COUNT(*) FROM users WHERE vip IN (SELECT 1 FROM refunds WHERE amount > 1000 ORDER BY COUNT(*))", + "SELECT COUNT(*) FROM users WHERE vip IN (SELECT 1 FROM refunds WHERE amount > 1000)"), + ("cte_column_list_swap", "postgres", + "WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) SELECT COUNT(*) FROM p WHERE total > 100", + "WITH p(total, uid) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) SELECT COUNT(*) FROM p WHERE total > 100"), + ("cte_column_list_swap", "mysql", + "WITH t(user_id, amount) AS (SELECT amount, user_id FROM orders) SELECT SUM(amount) FROM t", + "WITH t(amount, user_id) AS (SELECT amount, user_id FROM orders) SELECT SUM(amount) FROM t"), + ("derived_column_list_swap", "postgres", + "SELECT SUM(t.x) FROM (SELECT amount, tax FROM orders) AS t(x, y)", + "SELECT SUM(t.x) FROM (SELECT amount, tax FROM orders) AS t(y, x)"), + ("derived_column_list_swap", "mysql", + "SELECT t.b FROM (SELECT user_id, amount FROM orders) AS t(a, b) WHERE t.a > 3", + "SELECT t.b FROM (SELECT user_id, amount FROM orders) AS t(b, a) WHERE t.a > 3"), + ("derived_column_list_renames", "postgres", + "SELECT amount, user_id FROM (SELECT amount, user_id FROM orders) AS t(user_id, amount)", + "SELECT amount, user_id FROM orders"), + ("table_alias_column_list", "postgres", + "SELECT COUNT(*) FROM orders AS o(user_id, id) WHERE o.user_id > 5", + "SELECT COUNT(*) FROM orders AS o WHERE o.user_id > 5"), + ("union_all_in_cte_alias_order", "postgres", + "WITH u AS (SELECT user_id AS a, amount AS b FROM orders UNION ALL SELECT amount AS b, user_id AS a FROM refunds) " + "SELECT SUM(b), SUM(a) FROM u", + "WITH u AS (SELECT amount AS b, user_id AS a FROM refunds UNION ALL SELECT user_id AS a, amount AS b FROM orders) " + "SELECT SUM(b), SUM(a) FROM u"), + ("union_all_in_cte_alias_order", "mysql", + "WITH u AS (SELECT user_id AS a, amount AS b FROM orders UNION ALL SELECT amount AS b, user_id AS a FROM refunds) " + "SELECT SUM(b), SUM(a) FROM u", + "WITH u AS (SELECT amount AS b, user_id AS a FROM refunds UNION ALL SELECT user_id AS a, amount AS b FROM orders) " + "SELECT SUM(b), SUM(a) FROM u"), + ("between_symmetric", "postgres", + "SELECT COUNT(*) FROM orders WHERE amount BETWEEN SYMMETRIC 100 AND 10", + "SELECT COUNT(*) FROM orders WHERE amount BETWEEN 100 AND 10"), + ("group_key_order_with_limit", "postgres", + "SELECT user_id, store_id, COUNT(*) FROM orders GROUP BY user_id, store_id ORDER BY COUNT(*) DESC LIMIT 2", + "SELECT user_id, store_id, COUNT(*) FROM orders GROUP BY store_id, user_id ORDER BY COUNT(*) DESC LIMIT 2"), + ("exists_distinct_offset", "postgres", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT DISTINCT r.state FROM refunds r WHERE r.amount > 0 LIMIT 1 OFFSET 1)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT r.state FROM refunds r WHERE r.amount > 0 LIMIT 1 OFFSET 1)"), + ("distinct_on_keeps_a_different_row", "postgres", + "SELECT DISTINCT ON (user_id) user_id, status FROM orders ORDER BY user_id, created_at DESC NULLS LAST, id", + "SELECT DISTINCT ON (user_id) user_id, status FROM orders ORDER BY user_id, paid_at DESC NULLS LAST, id"), + ("float_folding", "postgres", + "SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2", "SELECT COUNT(*) FROM orders WHERE amount = 0.3"), + ("double_negation", "mysql", "SELECT id, -(-name) FROM users", "SELECT id, name FROM users"), + # 本轮自己在私起的 PostgreSQL 18、MySQL 8.0 上跑出来的:调换 UNION ALL 各支,浮点数加的先后变了 + # (1902.2 对 1902.2000000000003);MySQL 把外层只涉及分组键的条件推进子查询的 WHERE,和 HAVING 结果不同 + ("float_sum_union_branch_order", "postgres", + "SELECT SUM(x) FROM (SELECT amount AS x FROM orders UNION ALL SELECT amount AS x FROM refunds) t", + "SELECT SUM(x) FROM (SELECT amount AS x FROM refunds UNION ALL SELECT amount AS x FROM orders) t"), + ("float_sum_union_branch_order", "mysql", + "SELECT SUM(x) FROM (SELECT amount AS x FROM orders UNION ALL SELECT amount AS x FROM refunds) t", + "SELECT SUM(x) FROM (SELECT amount AS x FROM refunds UNION ALL SELECT amount AS x FROM orders) t"), + ("outer_where_on_group_key_vs_having", "mysql", + "SELECT COUNT(*) FROM (SELECT city, COUNT(*) AS n FROM users GROUP BY city) t WHERE BINARY t.city = 'SH'", + "SELECT COUNT(*) FROM (SELECT city, COUNT(*) AS n FROM users GROUP BY city HAVING BINARY city = 'SH') t"), +] + +#: 意思一样、规范化认得出的(PostgreSQL / MySQL 的写法) +DIALECT_SAME_R2: list[tuple[str, str, str, str]] = [ + ("fetch_first_is_limit", "postgres", "SELECT id FROM orders ORDER BY id FETCH FIRST 4 ROWS ONLY", + "SELECT id FROM orders ORDER BY id LIMIT 4"), + ("offset_fetch_is_limit_offset", "postgres", "SELECT id FROM orders ORDER BY id OFFSET 2 ROWS FETCH NEXT 4 ROWS ONLY", + "SELECT id FROM orders ORDER BY id LIMIT 4 OFFSET 2"), + ("mysql_column_names_ignore_case", "mysql", "SELECT SUM(amount) FROM orders WHERE is_deleted = 0", + "SELECT SUM(AMOUNT) FROM orders WHERE IS_DELETED = 0"), + ("is_not_null_forms", "postgres", "SELECT COUNT(*) FROM orders WHERE NOT (paid_at IS NOT NULL)", + "SELECT COUNT(*) FROM orders WHERE paid_at IS NULL"), + ("not_like_forms", "postgres", "SELECT COUNT(*) FROM orders WHERE status NOT LIKE 'p%'", + "SELECT COUNT(*) FROM orders WHERE NOT status LIKE 'p%'"), + ("if_vs_case", "mysql", "SELECT SUM(IF(status = 'paid', amount, 0)) FROM orders", + "SELECT SUM(CASE WHEN status = 'paid' THEN amount ELSE 0 END) FROM orders"), +] + +#: 意思一样、规范化故意不认的(认了就要假设「相等就是一样」、或者 PostgreSQL / MySQL 里浮点数求和的先后不变,见模块 +#: 说明):只给兜底说明,不给具体说法 +KNOWN_UNRECOGNIZED: list[tuple[str, str, str, str]] = [ + # PostgreSQL / MySQL 里对浮点数求和的一段调换关联先后:加的先后变了,结果的最后几位可能不同 + ("the_same_derived_column_written_differently_is_no_difference", "mysql", + "SELECT SUM(x.total) FROM (SELECT p.user_id, p.total FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) p JOIN (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) r ON r.user_id = p.user_id) x", + "SELECT SUM(y.total) FROM (SELECT a.user_id, a.total FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) b JOIN (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) a ON a.user_id = b.user_id) y"), + ("derived_tables_reading_the_same_table_written_differently_ar", "postgres", + "SELECT SUM(p.amount) FROM (SELECT * FROM orders WHERE status = 'paid') p JOIN (SELECT * FROM orders WHERE status = 'open') o ON o.user_id = p.user_id", + "SELECT SUM(x.amount) FROM (SELECT * FROM orders WHERE status = 'open') y JOIN (SELECT * FROM orders WHERE status = 'paid') x ON x.user_id = y.user_id"), + ("having_on_group_key_vs_where", "sqlite", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE user_id > 100 GROUP BY user_id) SELECT SUM(total) FROM t", + "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) SELECT SUM(total) FROM t WHERE user_id > 100"), + ("union_dedup_branch_order", "sqlite", + "SELECT id, amount FROM orders UNION SELECT id, amount FROM orders_archive", + "SELECT id, amount FROM orders_archive UNION SELECT id, amount FROM orders"), + ("window_partition_key_order", "sqlite", + "SELECT id, SUM(amount) OVER (PARTITION BY user_id, store_id) AS s FROM orders", + "SELECT id, SUM(amount) OVER (PARTITION BY store_id, user_id) AS s FROM orders"), + ("distinct_vs_group_mysql_strings", "mysql", + "SELECT DISTINCT city FROM users", "SELECT city FROM users GROUP BY city"), +] + + +PAIRS: list[Pair] = [Pair(f"same:{i}", a, b, "same") for i, a, b in _SAME] + \ + [Pair(f"diff:{i}", a, b, "diff") for i, a, b in _DIFF] + \ + [Pair(f"validator:{n}:{k}", a, b, "diff") for k, (n, a, b) in enumerate(VALIDATOR_DIFF)] + \ + [Pair(f"same:{i}", a, b, "same") for i, a, b in _SAME_R2] + \ + [Pair(f"hard:{i}", a, b, "same") for i, a, b in _SAME_HARD] + \ + [Pair(f"diff:{i}", a, b, "diff") for i, a, b in _DIFF_R2] + +#: 按编号找语料里的一对(测试里引用第 2 轮的反例用) +PAIRS_BY_ID: dict[str, Pair] = {p.id: p for p in PAIRS} diff --git a/backend/tests/test_sql_equivalence.py b/backend/tests/test_sql_equivalence.py new file mode 100644 index 00000000..dd6e38a7 --- /dev/null +++ b/backend/tests/test_sql_equivalence.py @@ -0,0 +1,103 @@ +"""SQL 差异说明的「等价」结论拿真实执行守门(data/sql_semantics.py 的 explain_sql_diff)。 + +explain_sql_diff 判两条 SQL 等价,当且仅当两边经过只保持语义的变换之后规范形式一字不差。规范化的每一步都必须 +保持语义,可光看代码保证不了(sqlglot 自带的优化器里就有不保持语义的改写),所以这里在虚构的业务库上执行: + +- 判了等价的两条,在每组数据上结果都必须一样(结果集按行排序比较); +- 在某组数据上结果不一样的两条,绝不能判等价; +- 语料里标 diff 的(语义不同,或者只在某个方言、某种排序规则下才等价),一律不能判等价; +- 标 same 的(已知等价),没认出来时只许给兜底说明,不许给具体说法(具体说法说的是两边确实不同的地方); +- 等价改写(编号不以 hard: 开头的 same)记下认出来的比例:目标是大多数都认得出,但不强求全部。 + +库和语料见 tests/fixtures/sql_pairs.py:5 组按固定种子生成的数据加一组极端数据、一组专挑写法差异的数据(0.1 + 0.2、 +像数字的文本、大小写不同的值),带空值、重复、边界值、已删除的行、退款;语料里有手写的等价改写和语义不同的对,也有 +复验者以前造的、在这几组数据上结果不同的对,以及第 2 轮复验报的反例。每一对都在不给表结构和给表结构(数据源的 +schema_cache)两种情况下各判一次,两种都要守住上面的规矩。 +""" +from __future__ import annotations + +import pytest + +from app.data.sql_semantics import explain_sql_diff +from tests.fixtures import sql_pairs + +SCHEMA = sql_pairs.schema_cache() + + +@pytest.fixture(scope="module") +def databases(): + conns = sql_pairs.build_databases() + yield conns + for c in conns: + c.close() + + +def _results(databases, sql: str) -> list[list[tuple]]: + return [sql_pairs.run(c, sql) for c in databases] + + +def _check_shape(diff, a: str, b: str) -> None: + """SqlDiff 自己的承诺:等价才没有条目;不等价一定有条目,兜底时一定有 aspect=other 的那条,并给出不同的行。""" + if diff.equivalent: + assert diff.items == [] and not diff.fallback and not diff.text_diff, (a, b) + return + assert diff.items, (a, b) + if diff.fallback: + assert any(d.aspect == "other" for d in diff.items), (a, b, diff.items) + assert diff.text_diff, (a, b) + assert len(diff.text_diff) <= 40 + + +@pytest.mark.parametrize("pair", sql_pairs.PAIRS, ids=[p.id for p in sql_pairs.PAIRS]) +def test_equivalence_is_guarded_by_execution(databases, pair): + ra, rb = _results(databases, pair.a), _results(databases, pair.b) + differs = [i for i, (x, y) in enumerate(zip(ra, rb)) if x != y] + if pair.expect == "same": + assert not differs, f"语料标错了:{pair.id} 在第 {differs} 组数据上结果不同" + for schema in (None, SCHEMA): + diff = explain_sql_diff(pair.a, pair.b, "sqlite", schema=schema) + _check_shape(diff, pair.a, pair.b) + if differs: + assert not diff.equivalent, f"{pair.id}:第 {differs} 组数据上结果不同,却判了等价(schema={bool(schema)})" + if pair.expect == "diff": + assert not diff.equivalent, f"{pair.id} 不该判等价(schema={bool(schema)})" + if pair.expect == "same" and not diff.equivalent: + specific = [d.text for d in diff.items if d.aspect != "other"] + assert not specific, f"{pair.id} 意思一样,却说了具体的不同(schema={bool(schema)}):{specific}" + # 换个先后问,结论一样 + assert explain_sql_diff(pair.b, pair.a, "sqlite", schema=schema).equivalent == diff.equivalent, pair.id + + +def test_most_known_equivalent_rewrites_are_recognized(): + """已知等价的改写(别名、CTE 改名、关联先后、条件左右、子查询和 CTE 互换……)里认出来的比例。不强求全部认出来: + 认不出的给的是兜底说明,不是错误的结论。要按语义推理才看得出的(hard:)另记,不算进比例。""" + same = [p for p in sql_pairs.PAIRS if p.expect == "same" and not p.id.startswith("hard:")] + hits = [p.id for p in same if explain_sql_diff(p.a, p.b, "sqlite", schema=SCHEMA).equivalent] + bare = [p.id for p in same if explain_sql_diff(p.a, p.b, "sqlite").equivalent] + ratio, bare_ratio = len(hits) / len(same), len(bare) / len(same) + print(f"\n已知等价的改写 {len(same)} 对里认出 {len(hits)} 对({ratio:.0%}),不给表结构时 {len(bare)} 对({bare_ratio:.0%})") + print("没认出来的:", sorted(set(p.id for p in same) - set(hits))) + hard = [p for p in sql_pairs.PAIRS if p.id.startswith("hard:")] + hard_hits = [p.id for p in hard if explain_sql_diff(p.a, p.b, "sqlite", schema=SCHEMA).equivalent] + print(f"要按语义推理的 {len(hard)} 对里认出 {len(hard_hits)} 对(其余只给兜底说明)") + assert ratio >= 0.9 and bare_ratio >= 0.9 + + +def test_the_corpus_is_large_and_varied_enough(): + assert len(sql_pairs.PAIRS) >= 150 and len(sql_pairs.SEEDS) >= 5 + text = " ".join(p.a + " " + p.b for p in sql_pairs.PAIRS).upper() + for construct in ("WITH ", "EXISTS", " IN (SELECT", "UNION", "OVER (", "DISTINCT", "LEFT JOIN", "HAVING", + "LIMIT", "ORDER BY", "RIGHT JOIN", "FULL JOIN", "NOT IN", "CROSS JOIN", "EXCEPT", "GROUP_CONCAT", + "USING (", "IIF("): + assert construct in text, construct + + +def test_the_tricky_dataset_exposes_the_round_two_counterexamples(databases): + """第 2 轮复验的反例在专挑写法差异的那组数据上确实结果不同(不然执行比对守不住它们)。""" + tricky = databases[-1] + pairs = {p.id: p for p in sql_pairs.PAIRS} + for pid in ("diff:r2-float-fold", "diff:r2-float-fold-chain", "diff:r2-double-negation", "diff:r2-in-single-column", + "diff:r2-cast-string", "diff:r2-hex-literal", "diff:r2-recursive-without-keyword", + "diff:r2-cte-column-list-swap-2"): + pair = pairs[pid] + assert sql_pairs.run(tricky, pair.a) != sql_pairs.run(tricky, pair.b), pid diff --git a/backend/tests/test_sql_semantics.py b/backend/tests/test_sql_semantics.py index 1e62c3ab..61c9e967 100644 --- a/backend/tests/test_sql_semantics.py +++ b/backend/tests/test_sql_semantics.py @@ -1,14 +1,12 @@ -"""SQL 语义(data/sql_semantics.py):两条 SQL 算的是不是一回事,不是的话差在哪。 +"""SQL 语义(data/sql_semantics.py):两条 SQL 是不是等价,不是的话差在哪。 - normalize_sql:字面量换成占位符后的规范文本 + 字面量。只有字面量不同的两条 SQL 规范文本相同(定时运行每期的日期 - 不同,不能算模板变了);关键字大小写、空白、注释也不算不同;解析不了的退回逐词规范化,照样认得出只改了字面量 -- facets:表、连接(类型 / 关联进来的表 / ON 条件)、WHERE 合取式、时间条件(列 / 运算符 / 取值)、分组、聚合 - (函数 / 参数 / 去重)、HAVING、取哪几行(各段的 LIMIT / OFFSET / 排序)。别名换成真实表名(条件里套着的子查询、 - HAVING 里的别名也是),CTE、子查询按位置编号、不按起的名字比,条件里的子查询选出来的列名不算;比较的两边排好 - 顺序;解析不了带 parse_error,不抛 -- diff_facets:每处不同一句中文:「A 按支付时间 paid_at 筛时间,B 按下单时间 created_at」「A 排除了 - is_deleted = 1 的行,B 没有」「A 对 order_id 去重计数,B 没去重」……写法不同、口径相同的不报,一边多出一个排在 - 前面的 CTE、编号错开了也不报 + 不同,不能算模板变了);关键字大小写、空白、注释也不算不同;结构变了的一定不同;解析不了的退回逐词规范化 +- facets:把一条查询拆成表、连接、WHERE 合取式、时间条件、分组、聚合、HAVING、取哪几行(给调用方取结构用) +- explain_sql_diff:两边规范化后的形式一字不差才算等价(规范化的每一步都只做保持语义的变换);不等价时只把有把握 + 的不同写成具体的中文句子(时间列、筛选条件、聚合与去重、关联、分组、HAVING、取行、UNION),说不准的给兜底说明, + 并给出规范化后不同的行。等价的结论另有执行比对守门:见 test_sql_equivalence.py +- diff_facets:照旧收两份 facets,交给 explain_sql_diff 比原文,返回它的条目(含兜底条目) 夹具全是虚构的表:订单 orders、退款 refunds、门店 stores、用户 users、付款 payments,以及一张中文表「订单」。 """ @@ -19,24 +17,27 @@ import json import pytest from app.data.sql_semantics import ( + FacetDiff, Facets, + SqlDiff, catalog_labels, diff_facets, + explain_sql_diff, facets, normalize_sql, ) +from tests.fixtures import sql_pairs DIALECTS = ("sqlite", "mysql", "postgres") +LABELS = ("原流程", "盲算") +FALLBACK = "两条 SQL 的写法不同,未能归纳成具体差异,请对照原文" +FALLBACK_REST = "另有未能归纳的差异,请对照原文" def texts(a: str, b: str, dialect: str = "sqlite", **kwargs) -> list[str]: return [d.text for d in diff_facets(facets(a, dialect), facets(b, dialect), **kwargs)] -def aspects(a: str, b: str, dialect: str = "sqlite") -> list[str]: - return [d.aspect for d in diff_facets(facets(a, dialect), facets(b, dialect))] - - # -------------------------------------------------------------------------- # normalize_sql # -------------------------------------------------------------------------- @@ -273,414 +274,436 @@ def test_facets_are_json_ready(): f = facets("SELECT COUNT(DISTINCT order_id) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY region") data = json.loads(json.dumps(f.as_dict(), ensure_ascii=False)) assert data["aggregates"][0]["distinct"] is True and data["time"][0]["op"] == ">=" + assert not [k for k in data if k.startswith("_")], "拿来重新分析的原文不进 as_dict" + # -------------------------------------------------------------------------- -# diff_facets:每个方面一对 +# normalize_sql:「只改了参数」的判断不受差异说明改动影响 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b", [ + ("SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-10-01' AND paid_at < '2026-11-01'"), + ("WITH p AS (SELECT * FROM orders WHERE status = 'paid' AND amount > 100) SELECT store_id, SUM(amount) FROM p " + "GROUP BY store_id HAVING SUM(amount) > 1000 ORDER BY 2 DESC LIMIT 5", + "WITH p AS (SELECT * FROM orders WHERE status = 'done' AND amount > 50) SELECT store_id, SUM(amount) FROM p " + "GROUP BY store_id HAVING SUM(amount) > 10 ORDER BY 2 DESC LIMIT 20"), + ("SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM orders WHERE status IN ('paid', 'done'))", + "SELECT COUNT(*) FROM users u WHERE u.id IN (SELECT user_id FROM orders WHERE status IN ('open', 'void'))"), + ("SELECT o.id, CASE WHEN o.amount > 100 THEN 'big' ELSE 'small' END FROM orders o LEFT JOIN refunds r " + "ON r.order_id = o.id AND r.amount > 0 WHERE o.created_at BETWEEN '2026-09-01' AND '2026-09-30'", + "SELECT o.id, CASE WHEN o.amount > 200 THEN 'huge' ELSE 'tiny' END FROM orders o LEFT JOIN refunds r " + "ON r.order_id = o.id AND r.amount > 5 WHERE o.created_at BETWEEN '2026-10-01' AND '2026-10-31'"), +], ids=["dates", "cte-having-limit", "in-list", "case-join-between"]) +def test_only_parameters_changed_is_still_the_same_template(dialect, a, b): + """定时运行每期只换日期、阈值:规范文本相同(同一个模板),字面量各是各的;同时 explain_sql_diff 照样说两条 + 不等价(取值不同就是不同的口径)。""" + na, nb = normalize_sql(a, dialect), normalize_sql(b, dialect) + assert na.parse_error is None and na.same_shape(nb) and na.literals != nb.literals + assert not explain_sql_diff(a, b, dialect).equivalent + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("b", [ + "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' GROUP BY store_id", + "SELECT SUM(amount) FROM orders WHERE paid_at > '2026-09-01' GROUP BY store_id", + "SELECT AVG(amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY store_id", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND is_deleted = 0 GROUP BY store_id", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY user_id", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE paid_at >= '2026-09-01' GROUP BY store_id", + "SELECT SUM(DISTINCT amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY store_id", + "SELECT SUM(amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE paid_at >= '2026-09-01' " + "GROUP BY store_id", + "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY store_id ORDER BY 1", + "SELECT SUM(amount) FROM orders WHERE paid_at IN ('2026-09-01') GROUP BY store_id", + "WITH t AS (SELECT * FROM orders) SELECT SUM(amount) FROM t WHERE paid_at >= '2026-09-01' GROUP BY store_id", +]) +def test_a_structural_change_is_never_the_same_template(dialect, b): + a = "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' GROUP BY store_id" + assert not normalize_sql(a, dialect).same_shape(normalize_sql(b, dialect)), b + + +# -------------------------------------------------------------------------- +# explain_sql_diff:等价怎么判 +# -------------------------------------------------------------------------- + + +def check_shape(diff: SqlDiff) -> None: + """SqlDiff 自己的承诺:等价才没有条目;不等价一定有条目,兜底时最后一条是 aspect=other,并给出不同的行。""" + assert isinstance(diff, SqlDiff) + if diff.equivalent: + assert diff.items == [] and not diff.fallback and diff.text_diff == () + return + assert diff.items and diff.text_diff and len(diff.text_diff) <= 40 + assert all(line[:2] in ("- ", "+ ") for line in diff.text_diff) + if diff.fallback: + assert diff.items[-1].aspect == "other" and diff.items[-1].text in (FALLBACK, FALLBACK_REST) + else: + assert all(d.aspect != "other" for d in diff.items) + # 不自相矛盾:同一句话不说两遍,配成一对的两边写法不会一样 + assert len({d.text for d in diff.items}) == len(diff.items) + assert all(d.a != d.b for d in diff.items if isinstance(d.a, str) and isinstance(d.b, str)) + + +@pytest.mark.parametrize("dialect,a,b", [ + ("sqlite", "SELECT SUM(o.amount) FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted = 0", + "select sum(amount) from orders where 0 = is_deleted and '2026-09-01' <= paid_at"), + ("sqlite", "WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT v.order_id) FROM v", + "SELECT COUNT(DISTINCT order_id) FROM orders WHERE is_deleted = 0"), + ("sqlite", "SELECT SUM(o.amount) FROM orders o, stores s WHERE o.store_id = s.id AND s.city = '虚构市'", + "SELECT SUM(orders.amount) FROM orders INNER JOIN stores ON stores.id = orders.store_id " + "WHERE stores.city = '虚构市'"), + ("sqlite", "SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10 OFFSET 0", + "SELECT o.amount FROM orders o ORDER BY o.amount DESC LIMIT 10"), + ("postgres", "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM user_totals WHERE total > 100", + "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) AS agg " + "WHERE agg.total > 100"), + ("mysql", "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) r " + "ON r.order_id = o.id WHERE r.n > 1", + "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) " + "`退款` ON o.id = `退款`.order_id WHERE `退款`.n > 1"), + ("postgres", "SELECT SUM(amount) FROM orders WHERE user_id IN " + "(WITH t AS (SELECT id FROM users WHERE grade >= 3) SELECT id FROM t)", + "SELECT SUM(amount) FROM orders WHERE user_id IN " + "(WITH g AS (SELECT id FROM users WHERE grade >= 3) SELECT g.id FROM g)"), + ("postgres", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r RIGHT JOIN orders o ON o.id = r.order_id"), + ("mysql", "SELECT SUM(`amount`) FROM `orders` WHERE `status` = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("postgres", 'SELECT SUM("amount") FROM "orders" WHERE "status" = \'paid\'', + "SELECT SUM(amount) FROM orders WHERE status = 'paid'"), + ("postgres", "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT * FROM (SELECT user_id FROM orders UNION SELECT user_id FROM refunds) t"), + # UNION ALL 调换各支:行一样多、一样的值,各方言都认得出(SQLite 只限整条语句最外层、不排序的,见下面的反例) + ("sqlite", "SELECT user_id FROM orders WHERE amount > 1 UNION ALL SELECT user_id FROM refunds", + "SELECT user_id FROM refunds UNION ALL SELECT user_id FROM orders WHERE amount > 1"), + ("postgres", "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds ORDER BY 1", + "SELECT user_id FROM refunds UNION ALL SELECT user_id FROM orders ORDER BY 1"), + ("mysql", "SELECT COUNT(*) FROM (SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds) t", + "SELECT COUNT(*) FROM (SELECT user_id FROM refunds UNION ALL SELECT user_id FROM orders) t"), +], ids=["aliases-and-flips", "filter-in-cte", "comma-join", "offset-0", "cte-vs-subquery", + "mysql-chinese-alias", "cte-inside-in", "left-vs-right", "mysql-backticks", "pg-quotes", "pg-union-wrapped", + "sqlite-union-all-branches", "pg-union-all-order-by-position", "mysql-union-all-in-subquery"]) +def test_rewrites_that_keep_the_meaning_are_equivalent(dialect, a, b): + diff = explain_sql_diff(a, b, dialect) + check_shape(diff) + assert diff.equivalent and diff.level in ("optimizer", "light", "parsed") + assert texts(a, b, dialect) == [] + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.DIALECT_SAME, ids=[f"{n}-{d}-{i}" for i, (n, d, _, _) + in enumerate(sql_pairs.DIALECT_SAME)]) +def test_rewrites_from_earlier_rounds_are_recognized(name, dialect, a, b): + """旧测试里「写法不同、意思一样」的对(各方言,SQLite 以外的不能执行,只在这里查):都认得出是等价的。""" + diff = explain_sql_diff(a, b, dialect) + check_shape(diff) + assert diff.equivalent, name + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.REGRESSIONS, ids=[f"{n}-{d}-{i}" for i, (n, d, _, _) + in enumerate(sql_pairs.REGRESSIONS)]) +def test_differences_reported_in_earlier_rounds_are_never_called_equivalent(name, dialect, a, b): + """过去五轮复验、旧测试报过的每一处不同:一律不判等价,条目不为空、不自相矛盾;换个先后问也一样。第 5 轮回退的 + 「只在 IN / EXISTS / 标量子查询里用到的 CTE 里的不同」也在里面。""" + for x, y in ((a, b), (b, a)): + diff = explain_sql_diff(x, y, dialect, labels=LABELS) + check_shape(diff) + assert not diff.equivalent, name + + +@pytest.mark.parametrize("dialect,a,b,why", [ + ("sqlite", "SELECT o.id, r.flag FROM orders o LEFT JOIN (SELECT order_id, 1 AS flag FROM refunds) r " + "ON r.order_id = o.id", + "SELECT o.id, 1 AS flag FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "merge_subqueries 会把外连接可空一侧的常量列并进外层:没关联上的行本该是空值"), + ("sqlite", "SELECT id FROM orders WHERE status = 'paid' OR amount > (SELECT amount FROM refunds WHERE id = 999)", + "SELECT orders.id FROM orders CROSS JOIN (SELECT amount FROM refunds WHERE id = 999) AS q " + "WHERE orders.status = 'paid' OR orders.amount > q.amount", + "unnest_subqueries 把 OR 里的标量子查询改成 CROSS JOIN:子查询没有行时整条都没了"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= CAST('2026-09-01' AS DATE)", + "canonicalize 给日期比较加 CAST;SQLite 里 CAST(… AS DATE) 取的是数字"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE CAST(paid_at AS DATE) >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE DATE(paid_at) >= '2026-09-01'", + "sqlglot 生成 SQLite 时把 CAST(x AS DATE) 写成 DATE(x):比规范形式不能比生成的文本"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE +amount = '10'", "SELECT COUNT(*) FROM orders WHERE amount = '10'", + "一元加号在 SQLite 里去掉了类型亲和,解析时却被丢掉了"), + ("postgres", "SELECT COUNT(*) FROM orders WHERE status > 'b' AND status > 'O'", + "SELECT COUNT(*) FROM orders WHERE status > 'b'", + "字符串比较的强弱取决于排序规则:simplify 的这条化简不做"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE region > 10 AND region > 9", + "SELECT COUNT(*) FROM orders WHERE region > 10", "TEXT 列和数字比是按文本比:不合并同一列上的比较"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE amount + 1 = 101", "SELECT COUNT(*) FROM orders WHERE amount = 100", + "等式两边移项对 TEXT 列不成立"), + ("sqlite", "SELECT user_id, 1 AS total FROM orders GROUP BY user_id ORDER BY total DESC, user_id LIMIT 3", + "SELECT user_id, 1 AS total FROM orders GROUP BY user_id ORDER BY 1 DESC, user_id LIMIT 3", + "ORDER BY 别名换成常量 1 就成了按第一列排"), + ("sqlite", "SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT * FROM (SELECT user_id FROM orders UNION SELECT user_id FROM refunds) t", + "SQLite 从子查询读 UNION 时按第一支的类型亲和转换别的支的值"), + ("sqlite", "SELECT COUNT(*) FROM orders o WHERE o.id NOT IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "NOT IN 遇到空值整个不成立,NOT EXISTS 不会"), + ("sqlite", "SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.id IS NULL OR r.amount > 5", + "SELECT COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id WHERE r.id IS NULL OR r.amount > 5", + "IS NULL 留得下没关联上的行:这次外连接不是内连接"), + # 合并的各支调换先后:下面这些都会改变结果(复验压测时撞到的是第一种,SQLite 里留下的是 2 还是 2.0) + ("sqlite", "SELECT tax FROM orders UNION SELECT referrer_id FROM orders", + "SELECT referrer_id FROM orders UNION SELECT tax FROM orders", + "UNION 去重时留先出现的那个值:SQLite 里 REAL 列的 2.0 和 INTEGER 列的 2 算重复,留下哪个取决于哪支在前"), + ("mysql", "SELECT city FROM stores UNION SELECT city FROM users", + "SELECT city FROM users UNION SELECT city FROM stores", + "MySQL 默认的排序规则不分大小写,'SH' 和 'sh' 去重时留先出现的那个"), + ("postgres", "SELECT amount FROM orders INTERSECT SELECT amount FROM refunds", + "SELECT amount FROM refunds INTERSECT SELECT amount FROM orders", + "INTERSECT 留左边那支的值:numeric 的 2 和 2.00 相等,写出来却不一样"), + ("postgres", "SELECT id FROM orders UNION ALL SELECT id FROM refunds LIMIT 3", + "SELECT id FROM refunds UNION ALL SELECT id FROM orders LIMIT 3", + "没有 ORDER BY 的 LIMIT 取先出来的几行:各支调换先后,取到的就是另一张表的行"), + ("postgres", "SELECT id, user_id FROM orders UNION ALL SELECT user_id, id FROM refunds ORDER BY id", + "SELECT user_id, id FROM refunds UNION ALL SELECT id, user_id FROM orders ORDER BY id", + "合并结果的列名取第一支的:调换之后 ORDER BY id 排的是另一列"), + ("sqlite", "SELECT name FROM users UNION ALL SELECT city FROM stores ORDER BY 1", + "SELECT city FROM stores UNION ALL SELECT name FROM users ORDER BY 1", + "SQLite 给合并的结果排序时用最左一支那列声明的排序规则(COLLATE NOCASE 的列在前就不分大小写)"), + ("sqlite", "SELECT COUNT(*) FROM (SELECT name AS v FROM users UNION ALL SELECT city AS v FROM stores) t " + "WHERE v = 'sh'", + "SELECT COUNT(*) FROM (SELECT city AS v FROM stores UNION ALL SELECT name AS v FROM users) t WHERE v = 'sh'", + "SQLite 从子查询读合并的结果再比较时,列的排序规则也取最左一支的"), + ("sqlite", "SELECT COUNT(*) FROM users u JOIN stores s ON u.name COLLATE NOCASE = s.city COLLATE BINARY", + "SELECT COUNT(*) FROM users u JOIN stores s ON s.city COLLATE BINARY = u.name COLLATE NOCASE", + "比较的两边都写了 COLLATE 时 SQLite 用左边那个:两边不能对调"), + # 别名和名字:qualify 没有表结构时把 SELECT 列表里和前面别名同名的名字当成别名展开,库里却不认 + ("sqlite", "SELECT amount * 2 AS x, x + 1 AS y FROM orders", + "SELECT amount * 2 AS x, amount * 2 + 1 AS y FROM orders", + "SELECT 列表里不能引用前面列的别名:x 不是表的列,前一条在库里报错"), + ("sqlite", "SELECT user_id AS amount, amount AS user_id FROM orders ORDER BY amount", + "SELECT user_id AS amount, amount AS user_id FROM orders ORDER BY user_id", + "ORDER BY 里单独一个名字先认别名:两条排的是不同的列(第二列的 amount 是表的列)"), + # 取到哪几行、哪一行的值 SQL 本来就没规定的:库里看扫描的先后,调换关联先后就变了(压测撞到的是第一种) + ("sqlite", "SELECT o.created_at, COUNT(*) FROM orders o JOIN refunds r ON r.order_id = o.id GROUP BY o.store_id", + "SELECT o.created_at, COUNT(*) FROM refunds r JOIN orders o ON r.order_id = o.id GROUP BY o.store_id", + "聚合查询里既不分组也不聚合的列,SQLite 取组里任意一行的值"), + ("sqlite", "SELECT o.id, r.id FROM orders o JOIN refunds r ON r.order_id = o.id LIMIT 5", + "SELECT o.id, r.id FROM refunds r JOIN orders o ON r.order_id = o.id LIMIT 5", + "没有 ORDER BY 的 LIMIT 取先扫到的几行"), + ("sqlite", "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT DISTINCT user_id FROM refunds LIMIT 2)", + "SELECT COUNT(*) FROM orders WHERE user_id IN (SELECT user_id FROM refunds GROUP BY user_id LIMIT 2)", + "没有 ORDER BY 的 LIMIT:DISTINCT 按扫到的先后出行,GROUP BY 按分组键排好了出,取到的两个不同"), + ("sqlite", "SELECT r.id, o.id FROM refunds r RIGHT JOIN orders o ON r.order_id = o.id LIMIT 5", + "SELECT r.id, o.id FROM orders o LEFT JOIN refunds r ON r.order_id = o.id LIMIT 5", + "没有 ORDER BY 的 LIMIT:RIGHT JOIN 改写成 LEFT JOIN,扫描的先后就变了"), +], ids=["nullable-merge", "unnest-or", "cast-date", "cast-vs-date", "unary-plus", "collation", "text-number", + "equality-move", "order-by-constant", "sqlite-union-affinity", "not-in-null", "outer-join-is-null", + "union-dedup-storage-class", "union-dedup-collation", "intersect-numeric-scale", "union-all-limit", + "union-all-order-by-name", "sqlite-union-all-order-collation", "sqlite-nested-union-all", "collate-sides", + "select-list-alias-ref", "order-by-swapped-aliases", "loose-column-join-order", "limit-join-order", + "limit-distinct-vs-group", "limit-right-vs-left"]) +def test_rewrites_that_change_the_meaning_are_not_equivalent(dialect, a, b, why): + """这些写法看着能改写成同一个样子,其实结果可能不同(why):sqlglot 优化器里有的改写不保持语义,规范化不用它们, + 或者守住了用法。""" + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, dialect, schema=schema) + check_shape(diff) + assert not diff.equivalent, why + + +def test_the_schema_expands_select_star_and_qualifies_columns(): + """给了表结构(数据源的 schema_cache)才认得出 SELECT * 和列全写出来是一回事、几张表时不写表名的列是哪张表的。""" + schema = sql_pairs.schema_cache() + star = "SELECT * FROM stores" + listed = "SELECT id, name, city, region, owner_id FROM stores" + assert explain_sql_diff(star, listed, "sqlite", schema=schema).equivalent + assert not explain_sql_diff(star, listed, "sqlite").equivalent + a = "SELECT SUM(amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE city = 'SH'" + b = "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE s.city = 'SH'" + assert explain_sql_diff(a, b, "sqlite", schema=schema).equivalent + + +def test_an_alias_used_in_group_by_needs_the_schema_to_be_told_from_a_column(): + """WHERE、GROUP BY、HAVING 里的名字库里先认表的列、没有这一列才认别名:GROUP BY r 里的 r 是不是 orders 的列, + 给了表结构才知道。给了就认得出和按 region 分组是一回事;没给不猜(DATE(created_at) AS created_at 再 GROUP BY + created_at 按的是原来的时间列,当成别名就成了按日期分组)。""" + a = "SELECT o.region, SUM(o.amount) FROM orders o GROUP BY o.region HAVING SUM(o.amount) > 100" + b = "SELECT region AS r, SUM(amount) AS s FROM orders GROUP BY r HAVING s > 100" + assert explain_sql_diff(a, b, "sqlite", schema=sql_pairs.schema_cache()).equivalent + assert not explain_sql_diff(a, b, "sqlite").equivalent + shadow = "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY created_at" + by_day = "SELECT DATE(created_at) AS created_at, COUNT(*) FROM orders GROUP BY DATE(created_at)" + for schema in (None, sql_pairs.schema_cache()): + assert not explain_sql_diff(shadow, by_day, "sqlite", schema=schema).equivalent + + +def test_equivalence_comes_with_the_level_it_was_decided_at(): + diff = explain_sql_diff("SELECT COUNT(1) FROM orders", "select count(*) from ORDERS", "sqlite") + assert diff.equivalent and diff.level == "optimizer" + data = json.loads(json.dumps(diff.as_dict(), ensure_ascii=False)) + assert data == {"equivalent": True, "items": [], "fallback": False, "text_diff": [], "level": "optimizer"} + + +# -------------------------------------------------------------------------- +# explain_sql_diff:有把握的具体说法 # -------------------------------------------------------------------------- BASE = "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'" def test_time_column_differs_with_catalog_names(): - a = "SELECT SUM(amount) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'" + a = BASE b = "SELECT SUM(amount) FROM orders WHERE created_at >= '2026-09-01' AND created_at < '2026-10-01'" - [diff] = diff_facets(facets(a), facets(b), names={"orders.paid_at": "支付时间", "created_at": "下单时间"}) - assert diff.aspect == "time" - assert diff.text == "A 按支付时间 paid_at 筛时间,B 按下单时间 created_at" - assert (diff.a, diff.b) == ("paid_at", "created_at") - assert diff.detail == {"a": [{"table": "orders", "column": "paid_at"}], + diff = explain_sql_diff(a, b, names={"orders.paid_at": "支付时间", "created_at": "下单时间"}) + check_shape(diff) + [item] = diff.items + assert item.aspect == "time" and item.text == "A 按支付时间 paid_at 筛时间,B 按下单时间 created_at" + assert (item.a, item.b) == ("paid_at", "created_at") + assert item.detail == {"a": [{"table": "orders", "column": "paid_at"}], "b": [{"table": "orders", "column": "created_at"}]} assert texts(a, b) == ["A 按 paid_at 筛时间,B 按 created_at"], "没有中文名就只写列名" -def test_time_filter_on_one_side_only(): - assert texts(BASE, "SELECT SUM(amount) FROM orders") == ["A 按 paid_at 筛时间,B 没有按时间筛选"] - assert texts("SELECT SUM(amount) FROM orders", BASE) == ["A 没有按时间筛选,B 按 paid_at 筛时间"] - - -def test_same_time_column_different_range(): - other = "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'" - [diff] = diff_facets(facets(BASE), facets(other)) - assert diff.aspect == "time_range" - assert diff.text == ("A 的时间范围是 paid_at >= '2026-09-01' AND paid_at < '2026-10-01'," - "B 是 paid_at BETWEEN '2026-09-01' AND '2026-09-30'") - - -def test_only_literals_differ_is_still_a_different_range(): - """规范文本相同(同一个模板),但比口径时日期不同就是不同的范围:两个函数回答的是两个问题。""" - later = BASE.replace("2026-09-01", "2026-09-08").replace("2026-10-01", "2026-10-08") - assert normalize_sql(BASE).same_shape(normalize_sql(later)) - assert aspects(BASE, later) == ["time_range"] - - -def test_an_exclusion_on_one_side(): - assert texts("SELECT SUM(amount) FROM orders WHERE is_deleted <> 1", "SELECT SUM(amount) FROM orders") == \ - ["A 排除了 is_deleted = 1 的行,B 没有"] - assert texts("SELECT SUM(amount) FROM orders", "SELECT SUM(amount) FROM orders WHERE refund_id IS NOT NULL") == \ - ["B 排除了 refund_id 为空的行,A 没有"] - - -def test_a_kept_condition_on_one_side(): - assert texts("SELECT SUM(amount) FROM orders", "SELECT SUM(amount) FROM orders WHERE status = 'paid'") == \ - ["B 只保留 status = 'paid' 的行,A 没有这个条件"] - - -def test_two_conditions_on_the_same_column_are_paired(): - [diff] = diff_facets(facets("SELECT SUM(amount) FROM orders WHERE status = 'paid'"), - facets("SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')")) - assert (diff.aspect, diff.text) == ("filter", "A 按 status = 'paid' 筛选,B 按 status IN ('paid', 'shipped') 筛选") - assert (diff.a, diff.b) == ("status = 'paid'", "status IN ('paid', 'shipped')") - - -def test_distinct_count_versus_plain_count(): - assert texts("SELECT COUNT(DISTINCT order_id) FROM orders", "SELECT COUNT(order_id) FROM orders") == \ - ["A 对 order_id 去重计数,B 没去重"] - assert texts("SELECT COUNT(order_id) FROM orders", "SELECT COUNT(DISTINCT order_id) FROM orders") == \ - ["A 对 order_id 计数时没去重,B 去重计数"] - - -@pytest.mark.parametrize("a,b,text", [ - ("SELECT SUM(amount) FROM orders", "SELECT AVG(amount) FROM orders", "A 对 amount 求和,B 求平均"), - ("SELECT SUM(paid_amount) FROM orders", "SELECT SUM(amount) FROM orders", "A 对 paid_amount 求和,B 对 amount 求和"), +@pytest.mark.parametrize("a,b,expected", [ + # 时间 + (BASE, "SELECT SUM(amount) FROM orders", [("time", "A 按 paid_at 筛时间,B 没有按时间筛选")]), + ("SELECT SUM(amount) FROM orders", BASE, [("time", "A 没有按时间筛选,B 按 paid_at 筛时间")]), + (BASE, "SELECT SUM(amount) FROM orders WHERE paid_at BETWEEN '2026-09-01' AND '2026-09-30'", + [("time_range", "A 的时间范围是 paid_at >= '2026-09-01' AND paid_at < '2026-10-01'," + "B 是 paid_at >= '2026-09-01' AND paid_at <= '2026-09-30'")]), + ("WITH t AS (SELECT user_id, amount, DATE(paid_at) AS d FROM orders) SELECT SUM(amount) FROM t WHERE d >= '2026-09-01'", + "WITH t AS (SELECT user_id, amount, DATE(created_at) AS d FROM orders) SELECT SUM(amount) FROM t " + "WHERE d >= '2026-09-01'", + [("time", "A 按 paid_at 筛时间,B 按 created_at")]), + # 筛选 + ("SELECT SUM(amount) FROM orders WHERE is_deleted <> 1", "SELECT SUM(amount) FROM orders", + [("filter", "A 排除了 is_deleted = 1 的行,B 没有")]), + ("SELECT SUM(amount) FROM orders", "SELECT SUM(amount) FROM orders WHERE refund_id IS NOT NULL", + [("filter", "B 排除了 refund_id 为空的行,A 没有")]), + ("SELECT SUM(amount) FROM orders", "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + [("filter", "B 只保留 status = 'paid' 的行,A 没有这个条件")]), + ("SELECT SUM(amount) FROM orders WHERE status = 'paid'", + "SELECT SUM(amount) FROM orders WHERE status IN ('paid', 'shipped')", + [("filter", "A 按 status = 'paid' 筛选,B 按 status IN ('paid', 'shipped') 筛选")]), + ("WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM paid", + "WITH paid AS (SELECT * FROM orders WHERE status = 'paid') SELECT SUM(amount) FROM orders", + [("filter", "A 只保留 status = 'paid' 的行,B 没有这个条件")]), + # 聚合、去重 + ("SELECT COUNT(DISTINCT order_id) FROM orders", "SELECT COUNT(order_id) FROM orders", + [("aggregate", "A 对 order_id 去重计数,B 没去重")]), + ("SELECT COUNT(order_id) FROM orders", "SELECT COUNT(DISTINCT order_id) FROM orders", + [("aggregate", "A 对 order_id 计数时没去重,B 去重计数")]), + ("SELECT SUM(amount) FROM orders", "SELECT AVG(amount) FROM orders", [("aggregate", "A 对 amount 求和,B 求平均")]), + ("SELECT SUM(paid_amount) FROM orders", "SELECT SUM(amount) FROM orders", + [("aggregate", "A 对 paid_amount 求和,B 对 amount 求和")]), ("SELECT COUNT(*) FROM orders", "SELECT COUNT(DISTINCT order_id) FROM orders", - "A 数的是行数,B 对 order_id 去重计数"), -]) -def test_other_aggregate_differences(a, b, text): - assert texts(a, b) == [text] - - -def test_join_differences(): - left = "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id" - inner = "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id" - other_key = "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.user_id = o.user_id" - plain = "SELECT SUM(amount) FROM orders" - assert texts(left, inner) == ["A 用 LEFT JOIN 关联 refunds 表,B 用 INNER JOIN"] - assert texts(left, other_key) == ["A 关联 refunds 表的条件是 orders.id = refunds.order_id," - "B 是 orders.user_id = refunds.user_id"] - assert texts(left, plain) == ["A 关联了 refunds 表,B 没有"] - - -def test_a_different_source_table_is_said_once(): - """表换了一张:说一次「查的是哪张表」就够了,同一个聚合、同一个条件不再因为表名不同各报一遍。""" - a = "SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'" - b = "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 0 AND paid_at >= '2026-09-01'" - [diff] = diff_facets(facets(a), facets(b)) - assert (diff.aspect, diff.text) == ("tables", "A 查的是 orders 表,B 查的是 orders_archive 表") - assert aspects(a, b.replace("is_deleted = 0", "is_deleted = 1")) == ["filter", "tables"], "换表之外的不同照报" - - -def test_two_columns_that_read_the_same_are_told_apart(): - """各自只写列名时两边读起来一样,其实是两张表的列:改写成带表名的写法,不说「A 对 amount 求和,B 对 amount 求和」。""" - a = "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id" - b = "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id" - assert texts(a, b) == ["A 对 orders.amount 求和,B 对 refunds.amount 求和"] - - -def test_group_by_difference(): - assert texts("SELECT region, SUM(amount) FROM orders GROUP BY region", - "SELECT region, city, SUM(amount) FROM orders GROUP BY region, city") == \ - ["A 按 region 分组,B 按 region、city 分组"] - assert texts("SELECT SUM(amount) FROM orders", "SELECT region, SUM(amount) FROM orders GROUP BY region") == \ - ["A 没有分组,B 按 region 分组"] - - -def test_having_difference(): - assert texts("SELECT region, SUM(amount) FROM orders GROUP BY region HAVING SUM(amount) > 100", - "SELECT region, SUM(amount) FROM orders GROUP BY region") == \ - ["A 分组后只保留 SUM(amount) > 100 的组,B 没有这个条件"] - - -def test_limit_difference(): - assert texts("SELECT amount FROM orders LIMIT 10", "SELECT amount FROM orders") == ["A 只取前 10 行,B 不限行数"] - assert texts("SELECT amount FROM orders LIMIT 10", "SELECT amount FROM orders LIMIT 100") == \ - ["A 只取前 10 行,B 只取前 100 行"] - - -def test_an_unparsable_side_is_reported_once_without_item_by_item_comparison(): - [diff] = diff_facets(facets("SELECT FROM WHERE"), facets(BASE)) - assert diff.aspect == "parse" and diff.text.startswith("A:SQL 无法解析") and "没有逐项比较" in diff.text - assert diff.b is None - - -@pytest.mark.parametrize("a,b", [ - # 别名、比较两边的顺序、大小写 - ("SELECT SUM(o.amount) FROM orders o WHERE o.paid_at >= '2026-09-01' AND o.is_deleted = 0", - "select sum(amount) from orders where 0 = is_deleted and '2026-09-01' <= paid_at"), - # 有效记录条件写在 CTE 里、还是写在外层 - ("WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT v.order_id) FROM v", - "SELECT COUNT(DISTINCT order_id) FROM orders WHERE is_deleted = 0"), - # 逗号连接和 INNER JOIN - ("SELECT SUM(o.amount) FROM orders o, stores s WHERE o.store_id = s.id AND s.city = '虚构市'", - "SELECT SUM(orders.amount) FROM orders INNER JOIN stores ON stores.id = orders.store_id " - "WHERE stores.city = '虚构市'"), - # 条件里套着的子查询用了别名:NOT EXISTS、IN (SELECT …) - ("SELECT o.id FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", - "SELECT orders.id FROM orders WHERE NOT EXISTS (SELECT 1 FROM refunds WHERE refunds.order_id = orders.id)"), - ("SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)", - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)"), - # HAVING 里用 SELECT 的别名 - ("SELECT o.region, SUM(o.amount) FROM orders o GROUP BY o.region HAVING SUM(o.amount) > 100", - "SELECT region AS r, SUM(amount) AS s FROM orders GROUP BY r HAVING s > 100"), - # OFFSET 0 等于没写;取前几行时 ORDER BY 1 和写出列名是同一个排序 - ("SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10 OFFSET 0", - "SELECT o.amount FROM orders o ORDER BY o.amount DESC LIMIT 10"), -]) -def test_the_same_meaning_written_differently_is_no_difference(a, b): - assert texts(a, b) == [] - - -def test_a_real_difference_inside_a_subquery_is_still_reported(): - a = "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id FROM users u WHERE u.vip = 1)" - b = "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.referrer_id FROM users u WHERE u.vip = 1)" - assert texts(a, b) == ["A 按 user_id IN (SELECT id FROM users WHERE vip = 1) 筛选," - "B 按 user_id IN (SELECT referrer_id FROM users WHERE vip = 1) 筛选"] - - -@pytest.mark.parametrize("dialect,a,b", [ - # 条件里的子查询给选出来的列起了名字:IN、标量子查询只看取值,列名不影响口径 - ("sqlite", - "SELECT SUM(o.amount) FROM orders o WHERE o.amount > " - "(SELECT AVG(o2.amount) AS avg_amount FROM orders o2 WHERE o2.is_deleted = 0)", - "SELECT SUM(amount) FROM orders WHERE amount > (SELECT AVG(amount) FROM orders WHERE is_deleted = 0)"), - ("sqlite", - "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT u.id AS uid FROM users u WHERE u.vip = 1)", - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM users WHERE vip = 1)"), - # 条件里的子查询从 FROM 里的子查询取数,两边给它起的别名不同 - ("sqlite", - "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT s.user_id FROM " - "(SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) s WHERE s.total > 100)", - "SELECT SUM(o.amount) FROM orders o WHERE o.user_id IN (SELECT agg.user_id FROM " - "(SELECT user_id, SUM(amount) AS total FROM payments GROUP BY user_id) agg WHERE agg.total > 100)"), - # CTE 起的名字不同,外层按 CTE 算出来的列筛选 - ("sqlite", - "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " - "SELECT COUNT(*) FROM user_totals WHERE total > 100", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE t.total > 100"), - # 写成 CTE 还是写成 FROM 里的子查询 - ("postgres", - "WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM user_totals WHERE total > 100", - "SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) AS agg " - "WHERE agg.total > 100"), - # LATERAL 子查询的别名不同:别名挂在 LATERAL 上,不在子查询上 - ("postgres", - "SELECT SUM(o.amount) FROM orders o, LATERAL (SELECT COUNT(*) AS n FROM refunds r WHERE r.order_id = o.id) l " - "WHERE l.n > 0", - "SELECT SUM(o.amount) FROM orders o CROSS JOIN LATERAL (SELECT COUNT(*) AS n FROM refunds r " - "WHERE r.order_id = o.id) AS x WHERE x.n > 0"), - # 关联进来的是子查询,两边的别名不同(一边是带反引号的中文别名) - ("mysql", - "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) r " - "ON r.order_id = o.id WHERE r.n > 1", - "SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds GROUP BY order_id) " - "`退款` ON o.id = `退款`.order_id WHERE `退款`.n > 1"), - # 条件里的子查询用到 CTE,两边给 CTE 起的名字不同 - ("sqlite", - "WITH vip AS (SELECT id FROM users WHERE grade >= 3) " - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT id FROM vip)", - "WITH good_users AS (SELECT id FROM users WHERE grade >= 3) " - "SELECT SUM(amount) FROM orders WHERE user_id IN (SELECT g.id FROM good_users g)"), - # CTE 定义在条件里的子查询里:定义连同子查询整个写进外层的条件,定义上的名字也不算 - ("postgres", - "SELECT SUM(amount) FROM orders WHERE user_id IN " - "(WITH t AS (SELECT id FROM users WHERE grade >= 3) SELECT id FROM t)", - "SELECT SUM(amount) FROM orders WHERE user_id IN " - "(WITH g AS (SELECT id FROM users WHERE grade >= 3) SELECT g.id FROM g)"), - # 两个 CTE 定义的先后、起的名字都不同,按用到的先后它们是同一对 - ("sqlite", - "WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id), " - "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " - "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total", - "WITH r AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " - "p AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE status = 'paid' GROUP BY user_id) " - "SELECT COUNT(*) FROM p JOIN r ON p.user_id = r.user_id WHERE p.total > r.total"), - # CTE 定义的先后和用到的先后不一致,另一边又把它们写成了 FROM 里的子查询:按用到的先后编号才对得上 - ("sqlite", - "WITH c AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "d AS (SELECT user_id, COUNT(*) AS n FROM refunds GROUP BY user_id), " - "a AS (SELECT * FROM c WHERE total > 100), b AS (SELECT * FROM d WHERE n > 1) " - "SELECT COUNT(*) FROM b JOIN a ON a.user_id = b.user_id", - "SELECT COUNT(*) FROM (SELECT * FROM (SELECT user_id, COUNT(*) AS n FROM refunds GROUP BY user_id) x " - "WHERE n > 1) b JOIN (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) y " - "WHERE total > 100) a ON a.user_id = b.user_id"), - # 同上,两个 CTE 还算出了同名的列、外层拿它们互相比:不写编号的写法分不出是哪个的 total,编号得对得上 - ("sqlite", - "WITH c AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "d AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id), " - "a AS (SELECT * FROM c), b AS (SELECT * FROM d) " - "SELECT COUNT(*) FROM b JOIN a ON a.user_id = b.user_id WHERE a.total > b.total", - "SELECT COUNT(*) FROM (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) x) b " - "JOIN (SELECT * FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) y) a " - "ON a.user_id = b.user_id WHERE a.total > b.total"), - # 先用一个 CTE 筛一遍再汇总,还是直接在汇总的 CTE 里筛:里层多一段,不挪动外层那一段的编号 - ("sqlite", - "WITH base AS (SELECT * FROM orders WHERE is_deleted = 0), " - "per_user AS (SELECT user_id, SUM(amount) AS total FROM base GROUP BY user_id) " - "SELECT COUNT(*) FROM per_user WHERE total > 100", - "WITH totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " - "SELECT COUNT(*) FROM totals WHERE total > 100"), -]) -def test_names_given_to_derived_tables_are_no_difference(dialect, a, b): - """CTE 名、子查询的别名、条件里的子查询选出来的列名都是各自起的(双盲复算的盲算更是):只因起的名字不同, - 不算换了口径。""" - assert texts(a, b, dialect) == [] - - -def test_derived_tables_are_compared_by_position_and_shown_by_name(): - """派生来源(CTE、FROM 里的子查询)比较用的写法按它在语句里的位置编号,不写名字;给人看的写法照原样写名字。""" - a = facets("WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM user_totals WHERE total > 100") - b = facets("SELECT COUNT(*) FROM (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) agg " - "WHERE agg.total > 100") - assert a.where[0].key == b.where[0].key - assert "user_totals" not in a.where[0].key and "agg" not in b.where[0].key - assert (a.where[0].text, a.where[0].full) == ("total > 100", "user_totals.total > 100") - assert (b.where[0].text, b.where[0].full) == ("total > 100", "agg.total > 100") - joined = facets("SELECT SUM(o.amount) FROM orders o JOIN (SELECT order_id, COUNT(*) AS n FROM refunds " - "GROUP BY order_id) r ON r.order_id = o.id") - [join] = joined.joins - assert join.table.text == "r" and join.on.text == "orders.id = refunds.order_id" - - -SHIFTED_A = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 100 AND user_id IN (SELECT id FROM users WHERE vip = 1)") -SHIFTED_B = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM vip JOIN t ON t.user_id = vip.id WHERE t.total > 100") - - -def test_a_shifted_derived_number_does_not_split_one_condition_in_two(): - """编号只看位置:一边多出一个排在前面的 CTE(筛 VIP 用户),后面那个 CTE 的编号就错开了。按编号比不上的再按 - 不写编号的写法一对一地配,同一个条件不会被说成两边各有一个;真正的不同(VIP 用 IN 筛还是关联进来)照报。""" - found = diff_facets(facets(SHIFTED_A), facets(SHIFTED_B), labels=("原流程", "盲算")) - filters = [d.text for d in found if d.aspect == "filter"] - assert filters == ["原流程只保留 user_id IN (SELECT id FROM users WHERE vip = 1) 的行,盲算没有这个条件"] - # 错开之后的条件真不同,也照样配成一对说 - other = SHIFTED_B.replace("t.total > 100", "t.total > 200") - filters = [d.text for d in diff_facets(facets(SHIFTED_A), facets(other), labels=("原流程", "盲算")) - if d.aspect == "filter"] - assert filters[0] == "原流程按 total > 100 筛选,盲算按 total > 200 筛选" and len(filters) == 2 - - -@pytest.mark.parametrize("aspect,a,b", [ - # 时间条件、分组键、聚合、HAVING、取前几行时的排序都是按 CTE 算出来的列 - ("time", - "WITH t AS (SELECT user_id, DATE(paid_at) AS day, amount FROM orders) " - "SELECT SUM(amount) FROM t WHERE day >= '2026-09-01' AND user_id IN (SELECT id FROM users WHERE vip = 1)", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, DATE(paid_at) AS day, amount FROM orders) " - "SELECT SUM(t.amount) FROM vip JOIN t ON t.user_id = vip.id WHERE t.day >= '2026-09-01'"), - ("group_by", - "WITH t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) " - "SELECT n, COUNT(*) FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) GROUP BY n", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total, COUNT(*) AS n FROM orders GROUP BY user_id) " - "SELECT t.n, COUNT(*) FROM vip JOIN t ON t.user_id = vip.id GROUP BY t.n"), - ("aggregate", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT AVG(total) FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1)", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT AVG(t.total) FROM vip JOIN t ON t.user_id = vip.id"), - ("having", - "WITH t AS (SELECT store_id, user_id, SUM(amount) AS total FROM orders GROUP BY store_id, user_id) " - "SELECT store_id FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) GROUP BY store_id " - "HAVING MAX(total) > 100", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT store_id, user_id, SUM(amount) AS total FROM orders GROUP BY store_id, user_id) " - "SELECT t.store_id FROM vip JOIN t ON t.user_id = vip.id GROUP BY t.store_id HAVING MAX(t.total) > 100"), - ("limit", - "WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT user_id FROM t WHERE user_id IN (SELECT id FROM users WHERE vip = 1) ORDER BY total DESC LIMIT 10", - "WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT t.user_id FROM vip JOIN t ON t.user_id = vip.id ORDER BY t.total DESC LIMIT 10"), -]) -def test_a_shifted_derived_number_is_no_difference_in_any_aspect(aspect, a, b): - """编号错开之后,按 CTE 算出来的列写的时间条件、分组键、聚合、HAVING、排序也不算不同:只剩 VIP 那处真正的 - 不同。""" - assert not [x for x in aspects(a, b) if x.startswith(aspect)] - - -def test_a_shifted_number_inside_a_condition_subquery_is_no_difference(): - """条件里的子查询用到的 CTE 编号错开了:子查询整个写进外层条件的那一份,按不写编号的写法也写「@」,照样配得上。""" - a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u WHERE u.vip = 1 AND u.id IN (SELECT user_id FROM t WHERE total > 100)") - b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM vip WHERE vip.id IN (SELECT user_id FROM t WHERE total > 100)") - assert texts(a, b) == [] - - -def test_a_shifted_derived_join_is_paired_with_its_counterpart(): - """关联进来的 CTE 编号错开了(盲算多关联了一个排在前面的 CTE):先把完全一样的关联配掉,再按关联的是哪张表配, - 不会把原流程的 t 和盲算的 vip 当成同一处关联、说成「关联条件不同」。""" - a = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u JOIN t ON t.user_id = u.id WHERE t.total > 100") - b = ("WITH vip AS (SELECT id FROM users WHERE vip = 1), " - "t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM users u JOIN vip ON vip.id = u.id JOIN t ON t.user_id = u.id WHERE t.total > 100") - assert texts(a, b, labels=("原流程", "盲算")) == ["盲算只保留 vip = 1 的行,原流程没有这个条件", - "盲算关联了 vip 表,原流程没有"] - - -def test_swapped_join_order_of_two_derived_tables_is_no_difference(): - """两个 CTE 关联的先后对调,编号也跟着对调:拿它们的列互相比的条件,两边先按不写编号的写法排好两边的先后, - 才配得上(只按编号排,同一个比较会排成两个方向)。""" - ctes = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "u AS (SELECT user_id, COUNT(*) AS cnt FROM refunds GROUP BY user_id) ") - assert texts(ctes + "SELECT COUNT(*) FROM t JOIN u ON u.user_id = t.user_id WHERE t.total > u.cnt", - ctes + "SELECT COUNT(*) FROM u JOIN t ON t.user_id = u.user_id WHERE t.total > u.cnt") == [] - - -def test_a_reversed_comparison_between_two_derived_tables_is_still_reported(): - """两个 CTE 算出了同名的列:编号对得上时照样分得清是哪个 CTE 的,比较的方向反了要报出来(不写编号的写法只在 - 按编号比不上时才拿来配,不会把它吞掉)。""" - a = ("WITH paid AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id), " - "refunded AS (SELECT user_id, SUM(amount) AS total FROM refunds GROUP BY user_id) " - "SELECT COUNT(*) FROM paid p JOIN refunded r ON r.user_id = p.user_id WHERE p.total > r.total") - b = a.replace("WHERE p.total > r.total", "WHERE r.total > p.total") - assert texts(a, b) == ["A 只保留 paid.total > refunded.total 的行,B 没有这个条件", - "B 只保留 paid.total < refunded.total 的行,A 没有这个条件"] - - -def test_a_real_difference_next_to_renamed_derived_tables_is_reported_once(): - """派生来源起的名字不同、里面又真有不同时,只报真正不同的那一处:CTE 里少一个条件,外层按 CTE 算出来的列 - 筛选的条件不跟着报(不然两边各报一句「只保留 total > 100 的行」,自相矛盾);外层的条件本身不同,两边配成 - 一对说。""" - a = ("WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 " - "GROUP BY user_id) SELECT COUNT(*) FROM user_totals WHERE total > 100") - inner = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 100") - assert texts(a, inner, labels=("原流程", "盲算")) == ["原流程只保留 is_deleted = 0 的行,盲算没有这个条件"] - outer = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " - "SELECT COUNT(*) FROM t WHERE total > 200") - assert texts(a, outer, labels=("原流程", "盲算")) == ["原流程按 total > 100 筛选,盲算按 total > 200 筛选"] - - -@pytest.mark.parametrize("a,b,text", [ - ("WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 10) SELECT SUM(amount) FROM top", - "WITH top AS (SELECT amount FROM orders ORDER BY amount DESC LIMIT 100) SELECT SUM(amount) FROM top", - "A 在 top 里只取前 10 行,B 只取前 100 行"), + [("aggregate", "A 数的是行数,B 对 order_id 去重计数")]), + ("SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(r.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + [("aggregate", "A 对 orders.amount 求和,B 对 refunds.amount 求和")]), + ("SELECT DISTINCT region FROM orders", "SELECT region FROM orders", [("group_by", "A 对 region 去重,B 没有去重")]), + ("SELECT user_id, SUM(amount) FROM orders GROUP BY user_id", "SELECT SUM(amount), user_id FROM orders GROUP BY user_id", + [("expression", "两边选出来的列一样,先后不同:A 是 user_id、SUM(amount),B 是 SUM(amount)、user_id")]), + # 分组、HAVING、取行 + ("SELECT region, SUM(amount) FROM orders GROUP BY region", + "SELECT region, city, SUM(amount) FROM orders GROUP BY region, city", + [("group_by", "A 按 region 分组,B 按 city、region 分组")]), + ("SELECT SUM(amount) FROM orders", "SELECT region, SUM(amount) FROM orders GROUP BY region", + [("group_by", "A 没有分组,B 按 region 分组")]), + ("SELECT region, SUM(amount) FROM orders GROUP BY region HAVING SUM(amount) > 100", + "SELECT region, SUM(amount) FROM orders GROUP BY region", + [("having", "A 分组后只保留 SUM(amount) > 100 的组,B 没有这个条件")]), + ("SELECT amount FROM orders LIMIT 10", "SELECT amount FROM orders", [("limit", "A 只取前 10 行,B 不限行数")]), + ("SELECT amount FROM orders LIMIT 10", "SELECT amount FROM orders LIMIT 100", + [("limit", "A 只取前 10 行,B 只取前 100 行")]), ("SELECT amount FROM orders LIMIT 10 OFFSET 0", "SELECT amount FROM orders LIMIT 10 OFFSET 10", - "A 不跳过行,B 跳过前 10 行"), + [("limit", "A 不跳过行,B 跳过前 10 行")]), ("SELECT amount FROM orders ORDER BY 1 DESC LIMIT 10", "SELECT amount FROM orders ORDER BY 1 ASC LIMIT 10", - "A 取行前按 amount 降序排,B 按 amount 升序排"), - ("SELECT amount FROM orders ORDER BY amount DESC LIMIT 10", "SELECT amount FROM orders LIMIT 10", - "A 取行前按 amount 降序排,B 没有排序"), -]) -def test_which_rows_are_taken_is_compared(a, b, text): - """取哪几行:每一段的行数、跳过的行数,以及取前几行时的排序。排序只在有 LIMIT / OFFSET 的段里比: - 没有取前几行,排序不影响算出来的数。""" - assert texts(a, b) == [text] - assert texts("SELECT SUM(amount) FROM orders ORDER BY 1 DESC", "SELECT SUM(amount) FROM orders") == [] + [("limit", "A 取行前按 amount 降序排,B 按 amount 升序排")]), + ("SELECT id, amount FROM orders ORDER BY amount DESC", "SELECT id, amount FROM orders", + [("order", "A 按 amount 降序排,B 没有排序")]), + # 连接、表 + ("SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id", + [("join", "A 用 LEFT JOIN 关联 refunds 表,B 用 INNER JOIN")]), + ("SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.user_id = o.user_id", + [("join", "A 关联 refunds 表的条件是 orders.id = refunds.order_id,B 是 orders.user_id = refunds.user_id")]), + ("SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", "SELECT SUM(amount) FROM orders", + [("join", "A 用 LEFT JOIN 关联 refunds 表,B 没有")]), + ("SELECT SUM(o.amount) FROM orders o WHERE o.status = 'paid'", + "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE o.status = 'paid'", + [("join", "B 关联了 stores 表(关联的条件是 orders.store_id = stores.id),A 没有")]), + ("SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(*) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id", + [("join", "A 以 orders 为主表外连接 refunds,B 以 refunds 为主表外连接 orders")]), + ("SELECT SUM(amount) FROM orders WHERE is_deleted = 0 AND paid_at >= '2026-09-01'", + "SELECT SUM(amount) FROM orders_archive WHERE is_deleted = 1 AND paid_at >= '2026-09-01'", + [("tables", "A 查的是 orders 表,B 查的是 orders_archive 表"), + ("filter", "A 按 is_deleted = 0 筛选,B 按 is_deleted = 1 筛选")]), + # UNION + ("SELECT user_id FROM orders UNION SELECT user_id FROM refunds", + "SELECT user_id FROM orders UNION ALL SELECT user_id FROM refunds", + [("set_op", "A 用 UNION 合并(去掉重复的行),B 用 UNION ALL(重复的行都留着)")]), + ("SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders WHERE status = 'paid'", + "SELECT user_id FROM orders WHERE status = 'paid' UNION SELECT referrer_id FROM orders", + [("filter", "A 在合并的一支里只保留 status = 'paid' 的行,B 在合并的一支里没有这个条件")]), +], ids=lambda v: None) +def test_specific_differences_are_said_in_one_sentence_each(a, b, expected): + diff = explain_sql_diff(a, b, "sqlite") + check_shape(diff) + assert not diff.fallback and [(d.aspect, d.text) for d in diff.items] == expected + + +def test_a_difference_inside_a_cte_used_only_in_a_condition_subquery_is_reported(): + """第 5 轮回退的例子:只在 IN 子查询里用到的 CTE 里的条件不同,以前整个漏报(返回空列表)。现在 CTE 并进了 IN + 子查询,往里比,说得出是哪个条件,位置带上 IN 左边的列(两个 IN 子查询时才分得清)。EXISTS、标量子查询一样。""" + head = "WITH p AS (SELECT * FROM orders WHERE status = '{}') SELECT COUNT(*) FROM users u WHERE " + for tail, place in (("u.id IN (SELECT user_id FROM p)", "在 id IN 的子查询里"), + ("EXISTS (SELECT 1 FROM p WHERE p.user_id = u.id)", "在 EXISTS 子查询里"), + ("(SELECT COUNT(*) FROM p WHERE p.user_id = u.id) > 0", "在条件的子查询里")): + a, b = head.format("paid") + tail, head.format("open") + tail + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, "sqlite", schema=schema, labels=LABELS) + check_shape(diff) + assert [(d.aspect, d.text) for d in diff.items] == [ + ("filter", f"原流程{place}按 status = 'paid' 筛选,盲算{place}按 status = 'open' 筛选")] + assert diff_facets(facets(a), facets(b), labels=LABELS) == diff.items + + +def test_a_difference_inside_an_aggregating_cte_names_the_cte_on_each_side(): + a = ("WITH user_totals AS (SELECT user_id, SUM(amount) AS total FROM orders WHERE is_deleted = 0 GROUP BY user_id) " + "SELECT COUNT(*) FROM user_totals WHERE total > 100") + b = ("WITH t AS (SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id) " + "SELECT COUNT(*) FROM t WHERE total > 100") + diff = explain_sql_diff(a, b, labels=LABELS) + check_shape(diff) + assert [d.text for d in diff.items] == ["原流程在 user_totals 里只保留 is_deleted = 0 的行,盲算在 t 里没有这个条件"] + + +def test_an_outer_join_turned_inner_by_where_says_what_was_written(): + """原文写的是 LEFT / RIGHT JOIN、WHERE 里的条件把没关联上的行筛掉了:按内连接比,说法照实写「写的是 LEFT JOIN, + 实际是 INNER JOIN」,不说成「用 INNER JOIN」。从 RIGHT JOIN 改写来的外连接说以哪一侧为主表,不说 LEFT JOIN。""" + a = "SELECT SUM(o.amount) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE o.status = 'paid'" + b = "SELECT SUM(o.amount) FROM refunds r LEFT JOIN orders o ON o.id = r.order_id WHERE o.status = 'paid'" + assert texts(a, b, "postgres") == [ + "A 用 LEFT JOIN 关联 refunds 表,B 写的是 LEFT JOIN,但 WHERE 里的条件把关联不上 orders 的行筛掉了,实际是 INNER JOIN"] + a = "SELECT COUNT(*) FROM orders o RIGHT JOIN refunds r ON r.order_id = o.id" + b = a + " WHERE o.status = 'paid'" + assert texts(a, b, "postgres") == [ + "A 以 refunds 为主表外连接 orders 表,B 写的是 RIGHT JOIN,但 WHERE 里的条件把关联不上 orders 的行筛掉了," + "实际是 INNER JOIN", "B 只保留 orders.status = 'paid' 的行,A 没有这个条件"] + # 本来就是内连接、和被 WHERE 变成内连接的 LEFT JOIN 是同一个意思 + assert explain_sql_diff("SELECT COUNT(*) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id WHERE r.amount > 10", + "SELECT COUNT(*) FROM refunds r JOIN orders o ON o.id = r.order_id WHERE r.amount > 10" + ).equivalent @pytest.mark.parametrize("dialect", DIALECTS) def test_each_dialect_gives_the_same_three_findings(dialect): - q = {"sqlite": '"', "postgres": '"', "mysql": "`"}[dialect] + q = "`" if dialect == "mysql" else '"' a = (f"SELECT COUNT(DISTINCT o.order_id) AS {q}订单数{q} FROM orders o " "WHERE o.paid_at >= '2026-09-01' AND o.is_deleted <> 1") b = f"SELECT COUNT(o.order_id) AS {q}订单数{q} FROM orders o WHERE o.created_at >= '2026-09-01'" @@ -691,8 +714,141 @@ def test_each_dialect_gives_the_same_three_findings(dialect): def test_side_labels_in_chinese_have_no_stray_spaces(): a = "SELECT COUNT(DISTINCT order_id) FROM orders WHERE paid_at >= '2026-09-01'" b = "SELECT COUNT(order_id) FROM orders WHERE created_at >= '2026-09-01'" - assert texts(a, b, labels=("原流程", "盲算"), names={"paid_at": "支付时间"}) == [ - "原流程按支付时间 paid_at 筛时间,盲算按 created_at", "原流程对 order_id 去重计数,盲算没去重"] + assert texts(a, b, labels=LABELS) == ["原流程按 paid_at 筛时间,盲算按 created_at", "原流程对 order_id 去重计数,盲算没去重"] + + +def test_filters_on_real_columns_carry_the_columns_for_the_catalog(): + """筛选条件只涉及的实表列记在 detail 里:调用方据此对照数据目录里的有效行过滤。""" + [item] = explain_sql_diff("SELECT SUM(o.amount) FROM orders o WHERE o.is_deleted = 0", + "SELECT SUM(amount) FROM orders").items + assert item.aspect == "filter" and item.detail == {"columns": [{"table": "orders", "column": "is_deleted"}]} + + +def test_a_displayed_condition_means_what_the_sql_means(): + """给人看的条件按方言生成;方言生成的写法读回来不是同一个意思时(SQLite 把 CAST(x AS DATE) 生成 DATE(x))改用 + 通用写法,不拿 DATE(paid_at) 冒充原文。几个条件连起来时 OR 带上括号。""" + [item] = explain_sql_diff("SELECT COUNT(*) FROM orders WHERE CAST(paid_at AS DATE) >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE CAST(paid_at AS DATE) >= '2026-10-01'", "sqlite").items + assert item.aspect == "time_range" and "CAST(paid_at AS DATE)" in item.text and "DATE(paid_at)" not in item.text + a = "SELECT SUM(amount) FROM orders" + b = "SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id AND (s.city = 'SH' OR s.city = 'BJ')" + [item] = explain_sql_diff(a, b).items + assert item.text == ("B 关联了 stores 表(关联的条件是 orders.store_id = stores.id AND " + "(stores.city = 'BJ' OR stores.city = 'SH')),A 没有") + + +# -------------------------------------------------------------------------- +# explain_sql_diff:兜底说明 +# -------------------------------------------------------------------------- + + +def test_nothing_specific_gives_one_fallback_item_and_the_lines_that_differ(): + a = "SELECT COUNT(*) FROM orders o WHERE o.id NOT IN (SELECT order_id FROM refunds)" + b = "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)" + diff = explain_sql_diff(a, b, "sqlite") + check_shape(diff) + assert diff.fallback and diff.items == [FacetDiff("other", None, None, FALLBACK)] + assert any(line.startswith("- ") for line in diff.text_diff) + assert any(line.startswith("+ ") for line in diff.text_diff) + assert diff_facets(facets(a), facets(b)) == diff.items + + +def test_specific_items_that_cover_only_part_end_with_the_rest_note(): + """一边多关联了 refunds(会把订单行按退款数复制),另一边只在 EXISTS 里用它:关联那一句说得准,EXISTS 那处说不准, + 补一句「另有未能归纳的差异」。""" + a = "SELECT SUM(o.amount) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)" + b = "SELECT SUM(o.amount) FROM orders o JOIN refunds r ON r.order_id = o.id" + diff = explain_sql_diff(a, b, labels=LABELS) + check_shape(diff) + assert diff.fallback and [d.text for d in diff.items] == [ + "盲算关联了 refunds 表(关联的条件是 orders.id = refunds.order_id),原流程没有关联它(只在子查询里用到了 refunds)", + FALLBACK_REST] + + +def test_columns_inside_a_cte_are_not_compared_because_canonical_forms_prune_them(): + """规范化会去掉 CTE 里没用到的列、重排剩下的:CTE 里选出来的列不同不说具体的(原文里两边的 CTE 都选了三列, + 说「A 在 t 里选出 2 列」就是错的),交给兜底说明。""" + a = ("SELECT COUNT(*) FROM (SELECT id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at, id) AS rn " + "FROM orders) t WHERE t.rn = 1 AND t.status = 'paid'") + b = ("SELECT COUNT(*) FROM (SELECT id, status, ROW_NUMBER() OVER (PARTITION BY user_id ORDER BY paid_at, id) AS rn " + "FROM orders WHERE status = 'paid') t WHERE t.rn = 1") + diff = explain_sql_diff(a, b) + check_shape(diff) + assert diff.fallback and not [d for d in diff.items if "选出" in d.text] + + +def test_differences_that_cannot_be_checked_get_only_the_fallback(): + """两边算法不同、又判断不了是不是真的不同的(ROUND 包了一层、CASE 和 COALESCE、两种写法的时间常量),不说具体的: + 「A 算的是 X,B 算的是 Y」读起来就是两边不同,说错了比不说更糟。""" + for a, b in (("SELECT SUM(amount) FROM orders", "SELECT ROUND(SUM(amount), 2) FROM orders"), + ("SELECT SUM(COALESCE(discount, 0)) FROM orders", + "SELECT SUM(CASE WHEN discount IS NULL THEN 0 ELSE discount END) FROM orders"), + ("SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at >= DATE('2026-09-01')")): + diff = explain_sql_diff(a, b, "sqlite") + check_shape(diff) + assert diff.fallback and diff.items == [FacetDiff("other", None, None, FALLBACK)], (a, b, diff.items) + + +def test_text_diff_lists_at_most_forty_lines(): + cols_a = ", ".join(f"SUM(amount * {i}) AS c{i}" for i in range(60)) + cols_b = ", ".join(f"SUM(amount * {i + 100}) AS c{i}" for i in range(60)) + diff = explain_sql_diff(f"SELECT {cols_a} FROM orders", f"SELECT {cols_b} FROM orders") + assert not diff.equivalent and len(diff.text_diff) == 40 + + +@pytest.mark.parametrize("a,b", [ + ("SELECT FROM WHERE", BASE), + (BASE, "SELECT 1; SELECT 2"), + ("", BASE), + (None, BASE), + (BASE, "UPDATE orders SET amount = 0"), +]) +def test_an_unparsable_side_gives_one_parse_item_and_never_raises(a, b): + diff = explain_sql_diff(a, b, "sqlite") + assert not diff.equivalent and len(diff.items) == 1 and diff.items[0].aspect == "parse" + assert "没有逐项比较" in diff.items[0].text + json.dumps(diff.as_dict(), ensure_ascii=False) + + +# -------------------------------------------------------------------------- +# diff_facets:照旧收 facets,交给 explain_sql_diff +# -------------------------------------------------------------------------- + + +def test_diff_facets_returns_the_items_of_explain_sql_diff(): + pairs = [(BASE, "SELECT SUM(amount) FROM orders"), + ("SELECT COUNT(DISTINCT order_id) FROM orders", "SELECT COUNT(order_id) FROM orders"), + ("SELECT COUNT(*) FROM orders o WHERE o.id NOT IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders o WHERE NOT EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)")] + for a, b in pairs: + for dialect in DIALECTS: + expected = explain_sql_diff(a, b, dialect, labels=LABELS, names={"paid_at": "支付时间"}).items + assert diff_facets(facets(a, dialect), facets(b, dialect), labels=LABELS, + names={"paid_at": "支付时间"}) == expected + + +def test_diff_facets_passes_the_schema_given_to_facets(): + schema = sql_pairs.schema_cache() + a = "SELECT SUM(amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE city = 'SH'" + b = "SELECT SUM(o.amount) FROM stores s JOIN orders o ON o.store_id = s.id WHERE s.city = 'SH'" + assert diff_facets(facets(a, "sqlite", schema=schema), facets(b, "sqlite", schema=schema)) == [] + assert diff_facets(facets(a, "sqlite"), facets(b, "sqlite")) != [] + + +def test_an_unparsable_side_is_reported_once_without_item_by_item_comparison(): + [diff] = diff_facets(facets("SELECT FROM WHERE"), facets(BASE)) + assert diff.aspect == "parse" and diff.text.startswith("A:SQL 无法解析") and "没有逐项比较" in diff.text + assert diff.b is None + + +def test_facets_without_the_original_sql_cannot_be_called_equivalent(): + """不是 facets 算出来的 Facets(没记原文)比不了原文:facets 记的结构不全,两份一样也不说明 SQL 一样,只有同一个 + 对象才算没有不同,否则给兜底说明,不猜。""" + f = Facets(dialect="sqlite") + assert diff_facets(f, f) == [] + for other in (Facets(dialect="sqlite"), Facets(dialect="sqlite", limit="10")): + assert diff_facets(f, other) == [FacetDiff("other", None, None, FALLBACK)] def test_catalog_labels_come_from_visible_catalog_items(): @@ -710,5 +866,236 @@ def test_catalog_labels_come_from_visible_catalog_items(): def test_a_facets_object_compares_with_itself_cleanly(): - f = facets(BASE) + f = facets("WITH v AS (SELECT * FROM orders WHERE is_deleted = 0) SELECT COUNT(DISTINCT order_id) FROM v") assert isinstance(f, Facets) and diff_facets(f, f) == [] + + +# -------------------------------------------------------------------------- +# 第 2 轮复验的反例:等价的结论、具体说法的可靠性 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.REGRESSIONS_R2, + ids=[f"{n}-{d}" for n, d, _, _ in sql_pairs.REGRESSIONS_R2]) +def test_round_two_counterexamples_are_never_equivalent(name, dialect, a, b): + """第 2 轮复验在 PostgreSQL / MySQL 上跑出结果不同的对:常量折叠(Decimal 和浮点数)、取两次负、HAVING 挪进 WHERE + (不分大小写的排序规则下同组的值各不一样)、ORDER BY 里有聚合的 EXISTS / IN、列名表、合并的各支调换、BETWEEN + SYMMETRIC、有并列的取行、DISTINCT ON。不管给不给表结构、谁先谁后,一律不判等价。""" + for schema in (None, sql_pairs.schema_cache()): + for x, y in ((a, b), (b, a)): + diff = explain_sql_diff(x, y, dialect, schema=schema) + check_shape(diff) + assert not diff.equivalent, name + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.DIALECT_SAME_R2, + ids=[f"{n}-{d}" for n, d, _, _ in sql_pairs.DIALECT_SAME_R2]) +def test_round_two_rewrites_are_recognized(name, dialect, a, b): + """FETCH FIRST 和 LIMIT、MySQL 列名的大小写、各方言 IS NOT NULL / NOT LIKE 的不同解析形状、MySQL 的 IF 和 CASE: + 认得出是一回事(以前把 FETCH 读成「不限行数」)。""" + diff = explain_sql_diff(a, b, dialect) + check_shape(diff) + assert diff.equivalent, (name, diff.items) + + +@pytest.mark.parametrize("name,dialect,a,b", sql_pairs.KNOWN_UNRECOGNIZED, + ids=[f"{n}-{d}" for n, d, _, _ in sql_pairs.KNOWN_UNRECOGNIZED]) +def test_equivalences_left_unrecognized_on_purpose_get_only_the_fallback(name, dialect, a, b): + """意思一样、规范化故意不认的(要假设「相等就是一样」才认得出):只给兜底说明,不说具体的不同。""" + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, dialect, schema=schema) + check_shape(diff) + assert diff.equivalent or all(d.aspect == "other" for d in diff.items), (name, diff.items) + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b", [ + ("SELECT COUNT(*) FROM orders o WHERE o.paid_at >= '2026-09-01' AND EXISTS " + "(SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id " + "AND o.paid_at >= '2026-09-01')"), + ("SELECT COUNT(*) FROM orders o WHERE o.status = 'paid' AND EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id)", + "SELECT COUNT(*) FROM orders o WHERE EXISTS (SELECT 1 FROM refunds r WHERE r.order_id = o.id AND o.status = 'paid')"), + ("SELECT COUNT(*) FROM orders WHERE status = 'paid' AND id IN (SELECT order_id FROM refunds)", + "SELECT COUNT(*) FROM orders WHERE id IN (SELECT order_id FROM refunds WHERE status = 'paid')"), + ("SELECT COUNT(*) FROM users u WHERE u.city = 'BJ' AND EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id)", + "SELECT COUNT(*) FROM users u WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id AND u.city = 'BJ')"), +], ids=["exists-time", "exists-status", "in-outer-column", "exists-city"]) +def test_a_condition_moved_into_a_correlated_subquery_is_not_called_missing(dialect, a, b): + """条件挪进了相关子查询(引用的还是外层那一行):两条意思一样,B 是有这个条件的。以前说「B 没有按时间筛选」 + 「B 没有这个条件」,现在说不清就只给兜底说明。""" + for schema in (None, sql_pairs.schema_cache()): + for x, y in ((a, b), (b, a)): + diff = explain_sql_diff(x, y, dialect, schema=schema) + check_shape(diff) + assert all(d.aspect == "other" for d in diff.items), diff.items + + +@pytest.mark.parametrize("dialect", DIALECTS) +def test_time_filters_written_another_way_are_never_called_no_time_filter(dialect): + """时间条件换了写法(拆成两段、IN 和 OR、多一个下界、LIKE 和区间):另一边是按时间筛了的,不能说「没有按时间筛选」, + 也不能说成两个范围不同。""" + base = "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01'" + for other in ("SELECT COUNT(*) FROM orders WHERE (paid_at >= '2026-09-01' AND paid_at < '2026-09-16') OR " + "(paid_at >= '2026-09-16' AND paid_at < '2026-10-01')", + "SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01' AND paid_at < '2026-10-01' " + "AND paid_at > '2026-08-01'", + "SELECT COUNT(*) FROM orders WHERE paid_at LIKE '2026-09%'"): + diff = explain_sql_diff(base, other, dialect) + check_shape(diff) + assert not any("没有按时间筛选" in d.text or d.aspect == "time_range" for d in diff.items), diff.items + diff = explain_sql_diff("SELECT COUNT(*) FROM orders WHERE paid_at IN ('2026-09-01', '2026-09-30')", + "SELECT COUNT(*) FROM orders WHERE paid_at = '2026-09-01' OR paid_at = '2026-09-30'", dialect) + assert all(d.aspect == "other" for d in diff.items) + + +def test_except_with_its_sides_swapped_is_not_one_time_difference(): + """EXCEPT 两边对调,外加一边换了时间列:以前只说时间列不同、还说全了(会写进数据目录)。EXCEPT 按先后逐支比,对不上 + 就只给兜底说明;子查询、CTE、IN 子查询里套着的也一样。""" + a = "SELECT user_id FROM orders WHERE paid_at >= '2026-09-01' EXCEPT SELECT user_id FROM refunds" + b = "SELECT user_id FROM refunds EXCEPT SELECT user_id FROM orders WHERE created_at >= '2026-09-01'" + for x, y in ((a, b), (f"SELECT COUNT(*) FROM ({a}) t", f"SELECT COUNT(*) FROM ({b}) t"), + (f"WITH e AS ({a}) SELECT COUNT(*) FROM e", f"WITH e AS ({b}) SELECT COUNT(*) FROM e"), + (f"SELECT COUNT(*) FROM users WHERE id IN ({a})", f"SELECT COUNT(*) FROM users WHERE id IN ({b})")): + for dialect in ("sqlite", "postgres"): + diff = explain_sql_diff(x, y, dialect) + check_shape(diff) + assert diff.fallback and all(d.aspect == "other" for d in diff.items), (x, diff.items) + + +@pytest.mark.parametrize("a,b", [ + (sql_pairs.PAIRS_BY_ID["diff:r2-in-not-in-swapped"].a, sql_pairs.PAIRS_BY_ID["diff:r2-in-not-in-swapped"].b), + (sql_pairs.PAIRS_BY_ID["diff:r2-exists-not-exists-swapped"].a, + sql_pairs.PAIRS_BY_ID["diff:r2-exists-not-exists-swapped"].b), + (sql_pairs.PAIRS_BY_ID["diff:r2-in-left-column-swapped"].a, sql_pairs.PAIRS_BY_ID["diff:r2-in-left-column-swapped"].b), +], ids=["in-not-in", "exists-not-exists", "left-column"]) +def test_subqueries_swapped_between_in_and_not_in_are_not_explained_both_ways(a, b): + """IN 和 NOT IN(EXISTS 和 NOT EXISTS、id IN 和 referrer_id IN)两个子查询对调:以前配错了对,说出一对互相矛盾的话、 + 还把 NOT IN 说成「IN 子查询」。子查询要外面的写法(含 NOT、IN 左边的列)一模一样才配对,配不上就只给兜底。""" + for schema in (None, sql_pairs.schema_cache()): + diff = explain_sql_diff(a, b, "sqlite", schema=schema, labels=LABELS) + check_shape(diff) + assert all(d.aspect == "other" for d in diff.items), diff.items + + +def test_a_difference_inside_a_not_in_subquery_says_not_in(): + """NOT IN 子查询里的条件不同,说的是「NOT IN 的子查询」(以前说成「IN 子查询」,读的人会以为是保留这些用户)。子查询 + 里不带表名的列,没给表结构时分不清是子查询里的表的还是外层的(users 也可能有 status),不说具体的;给了表结构、 + 或者写了表名,就说得出。""" + a = ("SELECT COUNT(*) FROM users WHERE id NOT IN (SELECT user_id FROM orders WHERE status = 'void' " + "AND user_id IS NOT NULL)") + b = a.replace("'void'", "'open'") + expected = [("filter", "原流程在 id NOT IN 的子查询里按 status = 'void' 筛选,盲算在 id NOT IN 的子查询里按 " + "status = 'open' 筛选")] + diff = explain_sql_diff(a, b, "sqlite", labels=LABELS, schema=sql_pairs.schema_cache()) + check_shape(diff) + assert [(d.aspect, d.text) for d in diff.items] == expected + assert explain_sql_diff(a, b, "sqlite", labels=LABELS).items == [FacetDiff("other", None, None, FALLBACK)] + qa, qb = a.replace("FROM orders WHERE status", "FROM orders o WHERE o.status"), \ + b.replace("FROM orders WHERE status", "FROM orders o WHERE o.status") + assert [(d.aspect, d.text) for d in explain_sql_diff(qa, qb, "sqlite", labels=LABELS).items] == expected + diff = explain_sql_diff(a.replace("NOT IN", "IN"), b.replace("NOT IN", "IN"), "sqlite", labels=LABELS, + schema=sql_pairs.schema_cache()) + assert "id IN 的子查询" in diff.items[0].text and "NOT IN" not in diff.items[0].text + + +def test_a_swapped_table_keeps_each_sides_own_columns(): + """换了一张表:盲算的 created_at 是 users 的(注册时间),以前把 B 的表改名成 A 的表再比,中文名和 detail 都归到了 + orders。现在两边各用各的表说。""" + names = {"orders.created_at": "下单时间", "users.created_at": "注册时间", "orders.paid_at": "支付时间"} + diff = explain_sql_diff("SELECT COUNT(*) FROM orders WHERE paid_at >= '2026-09-01'", + "SELECT COUNT(*) FROM users WHERE created_at >= '2026-09-01'", "sqlite", names=names, + labels=LABELS) + check_shape(diff) + assert [(d.aspect, d.text) for d in diff.items] == [ + ("tables", "原流程查的是 orders 表,盲算查的是 users 表"), + ("time", "原流程按支付时间 paid_at 筛时间,盲算按注册时间 created_at")] + assert diff.items[1].detail == {"a": [{"table": "orders", "column": "paid_at"}], + "b": [{"table": "users", "column": "created_at"}]} + diff = explain_sql_diff("SELECT SUM(o.amount) FROM orders o JOIN stores s ON s.id = o.store_id WHERE s.city = 'SH'", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id WHERE u.city = 'SH'", + "sqlite") + check_shape(diff) + assert [d.text for d in diff.items] == [ + "A 查的是 stores 表,B 查的是 users 表", + "A 关联的条件是 orders.store_id = stores.id,B 是 orders.user_id = users.id"] + + +def test_ordering_that_decides_which_row_distinct_on_keeps_is_not_just_ordering(): + """DISTINCT ON 留哪一行由 ORDER BY 决定:排序不同就是留下的行不同,不能说成「只是排序不同」(aspect=order)。""" + a = "SELECT DISTINCT ON (user_id) user_id, status FROM orders ORDER BY user_id, created_at DESC NULLS LAST, id" + b = "SELECT DISTINCT ON (user_id) user_id, status FROM orders ORDER BY user_id, paid_at DESC NULLS LAST, id" + diff = explain_sql_diff(a, b, "postgres") + check_shape(diff) + assert not diff.equivalent and all(d.aspect != "order" for d in diff.items) + + +def test_a_table_name_differing_only_in_case_is_not_called_another_table_in_mysql(): + """MySQL 的表名在 Linux 上分大小写、在 macOS / Windows 上不分:BILLS 和 bills 不判等价,也不说「查的是另一张表」。""" + diff = explain_sql_diff("SELECT store_id, SUM(amount) AS s FROM orders WHERE is_deleted = 0 GROUP BY store_id", + "select STORE_ID, sum( AMOUNT ) as s from ORDERS where IS_DELETED = 0 group by store_id", + "mysql") + check_shape(diff) + assert all(d.aspect == "other" for d in diff.items) + + +def test_redundant_or_constant_conditions_are_not_claimed(): + """多写一个本来就成立的条件(1 = 1、已经被 > 筛掉的空值、关联键上的 IS NOT NULL、分组后 COUNT(*) > 0):两条意思一样, + 不说「A 只保留 … 的行,B 没有这个条件」。1 = 1 直接认成等价。""" + assert explain_sql_diff("SELECT COUNT(*) FROM orders WHERE 1 = 1 AND status = 'paid'", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'", "sqlite").equivalent + for dialect in DIALECTS: + for a, b in (("SELECT SUM(tax) FROM orders WHERE tax > 1 AND tax IS NOT NULL", "SELECT SUM(tax) FROM orders WHERE tax > 1"), + ("SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id WHERE o.user_id IS NOT NULL", + "SELECT SUM(o.amount) FROM orders o JOIN users u ON u.id = o.user_id"), + ("SELECT store_id, SUM(amount) FROM orders GROUP BY store_id HAVING COUNT(*) > 0", + "SELECT store_id, SUM(amount) FROM orders GROUP BY store_id"), + ("SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 1", + "SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) >= 2"), + ("SELECT COUNT(tax) FROM orders", "SELECT COUNT(*) FROM orders WHERE tax IS NOT NULL"), + ("SELECT COUNT(r.id) FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + "SELECT COUNT(r.id) FROM orders o JOIN refunds r ON r.order_id = o.id")): + diff = explain_sql_diff(a, b, dialect) + check_shape(diff) + assert all(d.aspect == "other" for d in diff.items), (dialect, a, diff.items) + + +def test_single_item_in_list_becomes_equality_only_for_a_constant(): + """IN 列表只有一项:是常量才写成 =;是一列的话 SQLite 里 a IN (b) 是 a = +b(右边不带类型亲和),和 a = b 不同。""" + assert explain_sql_diff("SELECT COUNT(*) FROM orders WHERE status IN ('paid')", + "SELECT COUNT(*) FROM orders WHERE status = 'paid'", "sqlite").equivalent + assert not explain_sql_diff("SELECT id FROM t WHERE a IN (b)", "SELECT id FROM t WHERE a = b", "sqlite").equivalent + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b", [ + ("SELECT id FROM orders ORDER BY amount DESC LIMIT 5 OFFSET 10", "SELECT id FROM orders ORDER BY amount DESC LIMIT 20 OFFSET 0"), + ("WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders WHERE amount > 10 GROUP BY user_id) " + "SELECT COUNT(*) FROM p WHERE total > 100", + "WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders WHERE amount > 50 GROUP BY user_id) " + "SELECT COUNT(*) FROM p WHERE total > 999"), + ("SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2 AND status IN ('paid', 'done')", + "SELECT COUNT(*) FROM orders WHERE amount = 0.5 + 0.25 AND status IN ('open', 'void')"), + ("SELECT COUNT(*) FROM users WHERE id NOT IN (SELECT user_id FROM orders WHERE created_at >= '2026-09-01')", + "SELECT COUNT(*) FROM users WHERE id NOT IN (SELECT user_id FROM orders WHERE created_at >= '2026-10-01')"), +], ids=["limit-offset", "cte-column-list", "arith-and-in-list", "not-in"]) +def test_only_literals_changed_stays_the_same_template_after_the_rewrite(dialect, a, b): + """normalize_sql「只改了参数」的判断不受这次改动影响:只改字面量的照旧同一个模板(等价判定照样说不等价)。""" + na, nb = normalize_sql(a, dialect), normalize_sql(b, dialect) + assert na.parse_error is None and na.same_shape(nb) and na.literals != nb.literals + assert not explain_sql_diff(a, b, dialect).equivalent + + +@pytest.mark.parametrize("dialect", DIALECTS) +@pytest.mark.parametrize("a,b", [ + ("WITH p(uid, total) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) SELECT COUNT(*) FROM p WHERE total > 1", + "WITH p(total, uid) AS (SELECT user_id, SUM(amount) FROM orders GROUP BY user_id) SELECT COUNT(*) FROM p WHERE total > 1"), + ("SELECT COUNT(*) FROM users WHERE id IN (SELECT user_id FROM orders)", + "SELECT COUNT(*) FROM users WHERE id NOT IN (SELECT user_id FROM orders)"), + ("SELECT COUNT(*) FROM orders WHERE amount = 0.1 + 0.2", "SELECT COUNT(*) FROM orders WHERE amount = 0.3"), + ("SELECT u.city, COUNT(*) FROM users u JOIN orders o ON o.user_id = u.id GROUP BY u.city", + "SELECT u.city, COUNT(*) FROM users u LEFT JOIN orders o ON o.user_id = u.id GROUP BY u.city"), +], ids=["cte-column-list-order", "in-vs-not-in", "folded-constant", "join-kind"]) +def test_a_structural_change_is_still_a_different_template(dialect, a, b): + """结构变了的(列名表的先后、IN 和 NOT IN、常量写成一个数、连接类型)照旧不是同一个模板。""" + assert not normalize_sql(a, dialect).same_shape(normalize_sql(b, dialect))