diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index 13f99203..a71624b1 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -149,8 +149,8 @@ WHERE / GROUP BY 里用列别名的),而且只说四类: **兜底。** 规范形式不一致、一条具体说法都没有时,给一条 aspect="other" 的「两条 SQL 的写法不同,未能归纳成具体差异, 请对照原文」;具体说法只覆盖了一部分时追加「另有未能归纳的差异,请对照原文」;fallback 为真。text_diff 给出规范化后 -不同的行(difflib 比两边 sqlglot pretty 的输出,最多 40 行);有一边没有规范形式(保真守卫拦下的)时给原文的逐行 -对照。 +不同的行(difflib 比两边 sqlglot pretty 的输出,WHERE、ON、HAVING 的合取式每个条件一行,最多 40 行,见 _text_diff); +有一边没有规范形式(保真守卫拦下的)时给原文的逐行对照。 仍做不到的(只给兜底说明,或者认不出等价): - 认不出的等价改写:要靠约束才成立的(主键不为空时的 COUNT(主键) 和 COUNT(*)、按主键关联又没用到的 LEFT JOIN、带 @@ -5923,8 +5923,8 @@ class SqlDiff: equivalent:两边经过只保持语义的变换之后,规范形式一字不差(level 是哪一级的规范形式,见 _canonical_forms)。 items:具体差异,等价时为空;不等价时一定不为空——归纳不出具体差异时是一条 aspect="other" 的兜底说明。 fallback:用了兜底说明(一条具体说法都没有,或者具体说法只覆盖了一部分)。 - text_diff:规范化之后两条 SQL 不同的行(difflib 比两边 sqlglot pretty 的输出,「- 」开头是 A 的、「+ 」开头是 - B 的,最多 40 行),给界面高亮;等价时为空。""" + text_diff:规范化之后两条 SQL 不同的行(difflib 比两边 sqlglot pretty 的输出,WHERE、ON、HAVING 的合取式每个条件 + 一行,「- 」开头是 A 的、「+ 」开头是 B 的,最多 40 行,见 _text_diff),给界面高亮;等价时为空。""" equivalent: bool items: list[FacetDiff] = field(default_factory=list) @@ -5942,6 +5942,10 @@ _FALLBACK = "两条 SQL 的写法不同,未能归纳成具体差异,请对 _FALLBACK_REST = "另有未能归纳的差异,请对照原文" #: text_diff 最多列几行 _MAX_DIFF_LINES = 40 +#: 逐行比较时不看的、条件前面的 AND(见 _text_diff) +_LEADING_AND = re.compile(r"^(\s*)AND ") +#: 各方言「每个条件一行」的生成器类,按 sqlglot 的生成器类缓存(见 _line_generator) +_LINE_GENERATORS: dict[type, type] = {} _CJK_PUNCT = frozenset(",。;:、!?「」『』()《》【】…—") #: 聚合函数的说法 @@ -7761,21 +7765,99 @@ def _by_bound(conds: list[Any]) -> list[Any]: def _text_diff(a: _Form | None, b: _Form | None, read: str | None) -> tuple[str, ...]: """规范化之后两条 SQL 不同的行:difflib 比两边 sqlglot pretty 的输出,「- 」是 A 的、「+ 」是 B 的。只有一边有这一级 - 的规范形式时不比(只列一边的行读起来像另一边是空的),返回空的,调用方改给原文的逐行对照(见 _raw_diff)。""" - import difflib + 的规范形式时不比(只列一边的行读起来像另一边是空的),返回空的,调用方改给原文的逐行对照(见 _raw_diff)。 - def lines(form: _Form) -> list[str]: - try: - return form.tree.sql(dialect=read, pretty=True, comments=False).splitlines() - except Exception: # noqa: BLE001 - return form.text.splitlines() + 两边按同一套排版写(_diff_lines):WHERE、ON、HAVING 的合取式每个条件一行。sqlglot 的 pretty 按宽度决定换不换行, + 两个条件放得下就排在一行、三个就拆开,同一个条件在两边落在不同的行上,就被当成「不同的行」(2026-10 验收:两边完全 + 相同的日期区间被标成了差异,审核人会以为统计期也不一样)。比的时候不看条件前面的 AND:一边多出来的条件排在最前面 + 时,另一边同一个条件只差在前面有没有 AND。列出来的仍是各自原样的行。只动排版,判不判等价、差异说明都不看这里。""" + import difflib if a is None or b is None: return () - out = [line for line in difflib.ndiff(lines(a), lines(b)) if line[:2] in ("- ", "+ ")] + la, lb = _diff_lines(a, read), _diff_lines(b, read) + out: list[str] = [] + i = j = 0 + # ndiff 按顺序走完两边:「 」两边各进一行,「- 」「+ 」各进自己那边一行,「? 」是行内提示、不占行 + for line in difflib.ndiff([_LEADING_AND.sub(r"\1", x) for x in la], [_LEADING_AND.sub(r"\1", x) for x in lb]): + tag = line[:2] + if tag == " ": + i, j = i + 1, j + 1 + elif tag == "- ": + out.append(f"- {la[i]}") + i += 1 + elif tag == "+ ": + out.append(f"+ {lb[j]}") + j += 1 return tuple(out[:_MAX_DIFF_LINES]) +def _diff_lines(form: _Form, read: str | None) -> list[str]: + """逐行对照用的写法:sqlglot pretty,WHERE、HAVING、ON 的合取式每个条件一行(见 _line_generator)。生成不了就退回 + 普通的 pretty,再不行用规范形式的原文。""" + try: + text = _line_generator(read).generate(form.tree) + except Exception: # noqa: BLE001 - 只是给人看的排版,出错不能连累差异说明 + try: + text = form.tree.sql(dialect=read, pretty=True, comments=False) + except Exception: # noqa: BLE001 + text = form.text + return [line.rstrip() for line in text.splitlines()] + + +def _condition_root(node: Any) -> str | None: + """node 是 WHERE、HAVING 的整个条件("clause")、JOIN 的 ON 条件("on"),还是别处的(None)。""" + from sqlglot import exp + + parent = node.parent + if isinstance(parent, (exp.Where, exp.Having)) and node.arg_key == "this": + return "clause" + if isinstance(parent, exp.Join) and node.arg_key == "on": + return "on" + return None + + +def _line_generator(read: str | None) -> Any: + """这种方言的 sqlglot 生成器(pretty、不带注释),只改一处:WHERE、HAVING、ON 的整个条件按 AND 拆开,每个条件一行, + 第二个起前面写 AND(各层子查询里的也一样)。别的照 sqlglot 原样:条件本身太长时照样折行,两边同一个条件折得一样。 + + - 条件本身是 OR(XOR)连起来的一律加括号:和别的条件并列时本来就要加,单独一个时不加的话,同一个条件两边写法不同。 + 条件外面多余的括号、AND 套 AND 的括号去掉(_conjuncts),不改意思。 + - ON 照 WHERE 的样子单独一行、条件缩进在下面:写在 ON 后面的话,第一个条件和后面的差在「ON」「AND」上,换个先后 + 就对不上;缩进比 WHERE 的条件深一级,ON 里和 WHERE 里的同一个条件不会被当成同一行。 + """ + from sqlglot import exp + from sqlglot.dialects.dialect import Dialect + + dialect = Dialect.get_or_raise(read) + base = dialect.generator_class + cls = _LINE_GENERATORS.get(base) + if cls is None: + class _ConditionLines(base): # type: ignore[misc, valid-type] + def sql(self, expression: Any, key: str | None = None, comment: bool = True) -> str: + root = _condition_root(expression) if key is None and isinstance(expression, exp.Expr) else None + conds = _conjuncts(expression) if root else [] + if not conds: + return super().sql(expression, key, comment) + lines = [self._one_condition(c) for c in conds] + text = "\n".join([lines[0], *(f"AND {line}" for line in lines[1:])]) + # ON:join_sql 拼成「ON <条件>」并给条件的各行加一级缩进,开头换行就是 ON 单独一行、条件在下面 + return "\n" + self.indent(text) if root == "on" else text + + def _one_condition(self, node: Any) -> str: + # 只有一个条件时它就是整个条件,再走 self.sql 又会被认成整个条件、绕不出来:这一层直接用 sqlglot 原来的 + # 写法(里面各层子查询的条件照样经过上面的 sql) + text = super().sql(node) + if not isinstance(node, exp.Connector): + return text + if "\n" not in text: + return f"({text})" + return f"({self.seg(self.indent(text), sep='')}{self.seg(')', sep='')}" + + cls = _LINE_GENERATORS[base] = _ConditionLines + return cls(dialect=dialect, pretty=True, comments=False) + + def _raw_diff(a: str, b: str) -> tuple[str, ...]: """原文的逐行对照(没有规范形式可比时用:解析时会读错、丢掉写法的,见 _lost_in_parsing):difflib 比两边原文的行 (去掉行尾空白),「- 」是 A 的、「+ 」是 B 的;比不出不同的行(只差在行内空白)就整条各列一行。""" diff --git a/backend/tests/test_dual_check_fixture.py b/backend/tests/test_dual_check_fixture.py index 0ee341c9..51f2ea8e 100644 --- a/backend/tests/test_dual_check_fixture.py +++ b/backend/tests/test_dual_check_fixture.py @@ -1,7 +1,7 @@ """双盲复算审批卡的前端夹具(frontend/src/dev/fixtures/dual_check.json)要和服务端的真实输出对得上。 审批卡上「对照两条 SQL」的高亮、「只差一个方面会记入数据目录」的提示,都是照 explain_sql_diff 的输出画的。它的 -text_diff 是规范化之后的行(sqlglot pretty:列名带表名、条件并成一行),和原文的行对不上;夹具要是照原文手写, +text_diff 是规范化之后的行(sqlglot pretty:列名带表名、每个条件一行),和原文的行对不上;夹具要是照原文手写, check-verify 在夹具上能过,真实数据上却一处都不高亮(2026-10 复核时发现)。所以这里按夹具里的两条 SQL 重新算一遍 差异说明,逐个指标、逐个变体和夹具里写的比。sql_semantics 改了说法或规范形式,这里先变红,按下面的办法重新生成。 diff --git a/backend/tests/test_sql_text_diff.py b/backend/tests/test_sql_text_diff.py new file mode 100644 index 00000000..263c2505 --- /dev/null +++ b/backend/tests/test_sql_text_diff.py @@ -0,0 +1,101 @@ +"""explain_sql_diff 的 text_diff(「规范化后不同的行」):两边相同的条件不因为换行的位置不同被标成差异。 + +2026-10 验收:双盲复算审批卡上,两边完全相同的日期区间被标成了差异—— + + orders.paid_date >= '2026-09-28' AND orders.paid_date <= '2026-10-04' + - orders.paid_date >= '2026-09-28' + - AND orders.paid_date <= '2026-10-04' + - AND orders.status <> 'refunded' +sqlglot 的 pretty 按宽度决定换不换行:两个条件放得下就排在一行,三个就拆成三行,同一个条件在两边落在不同的行上。原文 +SQL 都是单行时原文栏一行都不标,这一栏是唯一逐行指出差在哪的地方,审核人会以为统计期也不一样。 + +现在逐行比之前两边按同一套排版写:WHERE、ON、HAVING 的合取式每个条件一行(嵌在子查询里的也一样),条件本身是 OR +连起来的加上括号;比的时候不看条件前面的 AND(一边多出来的条件排在最前面时,另一边同一个条件只差在有没有 AND 上)。 +只动排版:判不判等价、差异说明(items)不变,sql_semantics、sql_equivalence 原有的测试照样全过。 + +表名、列名、日期、取值都是虚构的。 +""" +from __future__ import annotations + +from app.data.sql_semantics import explain_sql_diff + +LABELS = ("原流程", "盲算") +#: 包裹表、仓库表的结构(虚构):给了表结构才认列,同验收时双盲复算节点的调用 +SCHEMA = {"tables": { + "parcels": {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "fee", "type": "REAL"}, + {"name": "sent_on", "type": "TEXT"}, {"name": "state", "type": "TEXT"}, + {"name": "hub", "type": "TEXT"}, {"name": "depot_id", "type": "INTEGER"}]}, + "depots": {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "zone", "type": "TEXT"}, + {"name": "kind", "type": "TEXT"}]}, +}} +WEEK = "sent_on BETWEEN '2031-03-03' AND '2031-03-09'" + + +def diff(a: str, b: str): + return explain_sql_diff(a, b, "sqlite", schema=SCHEMA, labels=LABELS) + + +def test_the_same_date_range_is_not_marked_when_one_side_has_one_more_condition(): + """验收时的形状:原流程比盲算多一个排除作废的条件,两边的日期区间一样——只标多出来的那一个条件。""" + a = f"SELECT SUM(fee) AS week_fee FROM parcels WHERE {WEEK} AND state <> 'void'" + b = f"SELECT SUM(fee) AS total_fee FROM parcels WHERE {WEEK}" + got = diff(a, b) + assert got.text_diff == ("- AND parcels.state <> 'void'",) + # 只动排版:判定和差异说明照旧 + assert not got.equivalent and not got.fallback and got.level == "optimizer" + assert [(i.aspect, i.text) for i in got.items] == [("filter", "原流程排除了 state = 'void' 的行和 state 为空的行,盲算没有")] + # 反过来比也一样,只是换到「+ 」那一边 + assert diff(b, a).text_diff == ("+ AND parcels.state <> 'void'",) + + +def test_an_extra_condition_sorted_first_does_not_drag_the_shared_ones_along(): + """多出来的条件排在最前面:另一边同一个日期条件只差在前面有没有 AND,不算不同。""" + a = f"SELECT SUM(fee) FROM parcels WHERE {WEEK} AND hub = 'north'" + b = f"SELECT SUM(fee) FROM parcels WHERE {WEEK}" + assert diff(a, b).text_diff == ("- parcels.hub = 'north'",) + assert diff(b, a).text_diff == ("+ parcels.hub = 'north'",) + + +def test_two_conditions_on_each_side_mark_only_the_one_that_differs(): + """同一列上的区间只差结束日期(这种只给兜底说明,审核人全靠逐行对照):开始日期那一行两边一样,不标。""" + a = f"SELECT SUM(fee) FROM parcels WHERE {WEEK}" + b = "SELECT SUM(fee) FROM parcels WHERE sent_on BETWEEN '2031-03-03' AND '2031-03-10'" + got = diff(a, b) + assert got.fallback + assert got.text_diff == ("- AND parcels.sent_on <= '2031-03-09'", "+ AND parcels.sent_on <= '2031-03-10'") + + +def test_on_conditions_get_one_line_each_too(): + """LEFT JOIN 的 ON:ON 单独一行,条件各占一行,多出来的那个条件单独标出来。""" + a = "SELECT COUNT(*) FROM parcels p LEFT JOIN depots d ON d.id = p.depot_id AND d.kind = 'cold'" + b = "SELECT COUNT(*) FROM parcels p LEFT JOIN depots d ON d.id = p.depot_id" + assert diff(a, b).text_diff == ("- AND depots.kind = 'cold'",) + # 关联条件本身不同:只标那一行,ON 那一行两边一样 + c = "SELECT COUNT(*) FROM parcels p LEFT JOIN depots d ON d.zone = p.hub" + assert diff(b, c).text_diff == ("- depots.id = parcels.depot_id", "+ depots.zone = parcels.hub") + + +def test_having_conditions_get_one_line_each_too(): + a = "SELECT hub, SUM(fee) FROM parcels GROUP BY hub HAVING SUM(fee) > 10 AND COUNT(*) > 2" + b = "SELECT hub, SUM(fee) FROM parcels GROUP BY hub HAVING SUM(fee) > 10" + assert diff(a, b).text_diff == ("- AND COUNT(*) > 2",) + + +def test_conditions_inside_a_subquery_get_one_line_each_too(): + a = ("SELECT COUNT(*) FROM parcels p WHERE p.state <> 'void' " + "AND EXISTS (SELECT 1 FROM depots d WHERE d.id = p.depot_id AND d.zone = 'east')") + b = ("SELECT COUNT(*) FROM parcels p WHERE p.state <> 'void' " + "AND EXISTS (SELECT 1 FROM depots d WHERE d.id = p.depot_id)") + [line] = diff(a, b).text_diff + assert line.startswith("- AND ") and line.endswith(".zone = 'east'") + + +def test_an_or_condition_reads_the_same_alone_and_among_others(): + """OR 连起来的条件和别的条件并列时要加括号,单独一个时以前不加:同一个条件两边写法不同,就被标成了差异。现在条件 + 本身是 OR 的一律加括号,两边写法一样;加括号不改意思。""" + either = "(hub = 'north' OR hub = 'south')" + a = f"SELECT COUNT(*) FROM parcels WHERE {either} AND state <> 'void'" + b = f"SELECT COUNT(*) FROM parcels WHERE {either}" + assert diff(a, b).text_diff == ("- parcels.state <> 'void'",) + c = "SELECT COUNT(*) FROM parcels WHERE (hub = 'north' OR hub = 'west') AND state <> 'void'" + assert diff(a, c).text_diff == ("- AND (parcels.hub = 'north' OR parcels.hub = 'south')", + "+ AND (parcels.hub = 'north' OR parcels.hub = 'west')") diff --git a/frontend/src/dev/fixtures/dual_check.json b/frontend/src/dev/fixtures/dual_check.json index f39233a5..dd63120c 100644 --- a/frontend/src/dev/fixtures/dual_check.json +++ b/frontend/src/dev/fixtures/dual_check.json @@ -247,8 +247,10 @@ ], "fallback": false, "text_diff": [ - "- orders.paid_at >= '2026-09-01' AND orders.paid_at < '2026-09-08'", - "+ orders.created_at >= '2026-09-01' AND orders.created_at < '2026-09-08'" + "- orders.paid_at >= '2026-09-01'", + "+ orders.created_at >= '2026-09-01'", + "- AND orders.paid_at < '2026-09-08'", + "+ AND orders.created_at < '2026-09-08'" ] } }, @@ -314,7 +316,8 @@ "- 1", "+ \"orders@1\".id", "- WHERE", - "- \"orders@1\".id = refunds.order_id AND \"orders@1\".status <> 'test'" + "- \"orders@1\".id = refunds.order_id", + "- AND \"orders@1\".status <> 'test'" ] } } @@ -580,8 +583,10 @@ ], "fallback": false, "text_diff": [ - "- orders.created_at >= '2026-09-01' AND orders.created_at < '2026-09-08'", - "+ orders.paid_at >= '2026-09-01' AND orders.paid_at < '2026-09-08'" + "- orders.created_at >= '2026-09-01'", + "+ orders.paid_at >= '2026-09-01'", + "- AND orders.created_at < '2026-09-08'", + "+ AND orders.paid_at < '2026-09-08'" ] }, "catalog": { @@ -621,7 +626,8 @@ "fallback": false, "text_diff": [ "- WHERE", - "- orders.paid_at >= '2026-09-01' AND orders.paid_at < '2026-09-08'" + "- orders.paid_at >= '2026-09-01'", + "- AND orders.paid_at < '2026-09-08'" ] }, "catalog": { @@ -718,7 +724,8 @@ "fallback": false, "text_diff": [ "- WHERE", - "- orders.channel = 'web' AND orders.status <> 'refunded'" + "- orders.channel = 'web'", + "- AND orders.status <> 'refunded'" ] }, "catalog": { @@ -747,11 +754,12 @@ ], "fallback": false, "text_diff": [ - "- FROM orders AS orders", - "- LEFT JOIN refunds AS refunds", "+ FROM orders AS orders, refunds AS refunds", "+ WHERE", - "- ON orders.id = refunds.order_id", + "- FROM orders AS orders", + "- LEFT JOIN refunds AS refunds", + "- ON", + "- orders.id = refunds.order_id", "+ orders.id = refunds.order_id" ] }, @@ -814,8 +822,8 @@ ], "fallback": true, "text_diff": [ - "- orders.paid_at >= '2026-09-01' AND orders.paid_at < '2026-09-08'", - "+ orders.paid_at >= '2026-09-01' AND orders.paid_at < '2026-09-09'" + "- AND orders.paid_at < '2026-09-08'", + "+ AND orders.paid_at < '2026-09-09'" ] }, "catalog": { @@ -948,8 +956,7 @@ ], "fallback": false, "text_diff": [ - "- orders.id = refunds.order_id AND orders.status <> 'cancelled'", - "+ orders.id = refunds.order_id" + "- AND orders.status <> 'cancelled'" ] }, "catalog": { diff --git a/scripts/check-verify.mjs b/scripts/check-verify.mjs index 667a7b93..8209bd07 100644 --- a/scripts/check-verify.mjs +++ b/scripts/check-verify.mjs @@ -297,7 +297,7 @@ await section('审批卡:并排对照与高亮(服务端真实的差异输 const blind = gmv.locator('[data-dual-sql="blind"]') check('两条 SQL 都在,各标是哪一边', (await text(orig)).includes('paid_at') && (await text(blind)).includes('created_at') && (await text(gmv.locator('[data-dual-pair]'))).includes('原流程的 SQL') && (await text(gmv.locator('[data-dual-pair]'))).includes('盲算的 SQL')) - // 夹具的 text_diff 是 explain_sql_diff 的真实输出:规范化后列名带表名、两个条件并成一行,和原文的任何一行都对不上 + // 夹具的 text_diff 是 explain_sql_diff 的真实输出:规范化后列名带表名、每个条件一行,和原文的任何一行都对不上 const real = sidesOf(fx.card.metrics[0].diff) const rawLines = new Set(fx.card.metrics[0].original.inputs[0].sql.split('\n').map((l) => l.trim())) check('(夹具)text_diff 是规范化后的行,和原文逐行对不上', real.del.length > 0 && !real.del.some((l) => rawLines.has(l)), real.del.join(' | ')) @@ -462,6 +462,18 @@ await section('审批卡:没有预告时,「记入数据目录」只说必 if (!real.add.length) { check('……一边没有不同的行时照实说', (await text(gmv.locator('[data-dual-norm="blind"]'))).includes('没有不同的行')) } + // 规范化后每个条件一行(2026-10 验收:两个条件排在一行、三个拆成三行,两边相同的日期条件因为换行的位置不同被 + // 标成了差异)。只差结束日期时,两边一样的开始日期不标,只标结束日期那一行 + if (name === 'fallback') { + const marked = [...m.normDel, ...m.normAdd] + check('……两边相同的条件(开始日期)不标,只标不同的那一行', m.normDel.length === 1 && m.normAdd.length === 1 + && !marked.some((l) => l.includes(">= '2026-09-01'")) + && marked.every((l) => l.includes("orders.paid_at < '2026-09-")), marked.join(' ‖ ')) + } + if (name === 'two_filters') { + check('……一边多出的两个条件各占一行', m.normDel.includes("orders.channel = 'web'") + && m.normDel.includes("AND orders.status <> 'refunded'"), m.normDel.join(' | ')) + } } check(`${what}:页面没有报错`, errors.length === 0, errors.join(';')) }