diff --git a/backend/app/analysis/comparable.py b/backend/app/analysis/comparable.py index 76041bb6..bc157432 100644 --- a/backend/app/analysis/comparable.py +++ b/backend/app/analysis/comparable.py @@ -3,11 +3,12 @@ 五项检查，每项 pass / warn / fail / unknown（unknown 只写「核对不了」，不判不可比）： -- 同口径：两份 SQL 一字不差 → pass；规范化（data/sql_semantics.normalize_sql）后只差字面量：把时间条件的界（data/coverage - 认出的时间谓词的界，整个表达式）和像日期的字面量换成同一个占位再比，一样就是只差日期 → pass，不一样就是比的两段不是 - 两期 → fail「两边除了日期，筛选的还不一样」；规范形式不同交给口径账本的 explain_sql_diff，等价 → pass，只差时间范围、排序 - → pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里的值）。两份来自不同的数据源 fail。同一份快照里按期次分组 - 的天然同口径。 +- 同口径：两份 SQL 一字不差 → pass；否则两边都去掉最外层的 ORDER BY（没有 LIMIT 时排序不改结果），把时间条件的界（data/ + coverage 认出的时间谓词的界，整个表达式：字面量、按运行当天现算的都算）和像日期的字面量换成同一个占位，再规范化 + （data/sql_semantics.normalize_sql）比：一样就是只差日期 → pass；只差字面量就是比的两段不是两期 → fail「两边除了日期， + 筛选的还不一样」；还不一样交给口径账本的 explain_sql_diff（同样去掉最外层的 ORDER BY，现算的界换成按先后编号的日期字面量： + 它只认字面量的时间范围），等价 → pass，只差时间范围、排序 → pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里 + 的值）。两份来自不同的数据源 fail。同一份快照里按期次分组的天然同口径。 - 窗口长度：用 data/coverage 的 filters 取业务日期列（目录的 business_date；没写就取唯一那一列时间谓词）上的区间，界是字面 量的直接解析，按运行当天现算的在库里算（Book 的 bound，记进 probes）；算两边天数。整月对整月、整季对整季算一样长。 不相等：度量是流量 fail，比率、存量、没标的 warn。按期次分组的：两期粒度相同，一样长。 @@ -94,14 +95,26 @@ class Outcome: # -------------------------------------------------------------------------- +def _unordered(sql: str, read: str | None) -> Any: + """解析成语法树，去掉最外层的 ORDER BY（同一层有 LIMIT / OFFSET 时排序决定取哪几行，不去）。解析不了为 None。""" + from app.data import coverage + + tree = coverage.parse_sql(sql, read) + if tree is not None and tree.args.get("order") is not None and not tree.args.get("limit") \ + and not tree.args.get("offset"): + tree.set("order", None) + return tree + + def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None: - """时间条件的界、像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为 None。""" + """去掉最外层的 ORDER BY、时间条件的界和像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为 + None。""" from sqlglot import exp from app.data import coverage from app.data.sql_semantics import normalize_sql - tree = coverage.parse_sql(sql, read) + tree = _unordered(sql, read) if tree is None: return None for flt in coverage.filters(tree): @@ -117,6 +130,28 @@ def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None: return normalized.text, normalized.literals +def _for_diff(sql: str, read: str | None) -> str: + """交给 explain_sql_diff 的写法：去掉最外层的 ORDER BY；按运行当天现算的界换成按先后编号的日期字面量（它只认字面量的 + 时间范围，现算的界写法不一样会落进兜底的「写法不同」）。解析不了的原样交。""" + from sqlglot import exp + + from app.data import coverage + + tree = _unordered(sql, read) + if tree is None: + return sql + n = 0 + for flt in coverage.filters(tree): + for _, node, _ in flt.conds: + if node.parent is not None and not coverage.literal_bound(node)[0]: + n += 1 + node.replace(exp.Literal.string(f"1900-01-{n:02d}")) + try: + return tree.sql(dialect=read) + except Exception: # noqa: BLE001 - 写不回去就交原文 + return sql + + def same_query(base: Snap, target: Snap) -> Check: from app.data.sql_semantics import explain_sql_diff, normalize_sql @@ -133,17 +168,17 @@ def same_query(base: Snap, target: Snap) -> Check: if na.parse_error or nb.parse_error: check.reasons = ["SQL 解析不了"] return check - if na.text == nb.text: - ua, ub = _undated(a, read), _undated(b, read) - if ua is None or ub is None: - return check - if ua == ub: - check.status = "pass" - else: - check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"] + ua, ub = _undated(a, read), _undated(b, read) + if ua is None or ub is None: + return check + if ua == ub: + check.status = "pass" + return check + if ua[0] == ub[0]: + check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"] return check try: - diff = explain_sql_diff(a, b, read) + diff = explain_sql_diff(_for_diff(a, read), _for_diff(b, read), read) except Exception: # noqa: BLE001 - 比不了就当核对不了 return check aspects = [d.aspect for d in diff.items] diff --git a/backend/tests/test_analysis_comparable.py b/backend/tests/test_analysis_comparable.py index caf8d433..6dcb8bc3 100644 --- a/backend/tests/test_analysis_comparable.py +++ b/backend/tests/test_analysis_comparable.py @@ -204,3 +204,55 @@ async def test_one_snapshot_of_sqlite_weeks_uses_the_real_days_of_each_week(): plain = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"}) unknown = await analyze("comparable", snaps=[plain], query="Q1", period="week", base="2026-W41", target="2026-W42") assert unknown["phrase"] == "两期可比，窗口长度、整期、星期构成核对不了" + + +def _same(a, b): + from app.analysis.comparable import same_query + from app.analysis.inputs import Snap + + return same_query(Snap("Q1", {"sql": a, "source": "shop"}, "sqlite"), + Snap("Q2", {"sql": b, "source": "shop"}, "sqlite")) + + +def test_the_same_query_ignores_the_outer_order_and_how_run_day_bounds_are_written(): + by_region = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '{a}' AND '{b}' " + "GROUP BY region{tail}") + # 只有一边带 ORDER BY：排序不改结果 + check = _same(by_region.format(a="2026-10-05", b="2026-10-11", tail=""), + by_region.format(a="2026-10-12", b="2026-10-18", tail=" ORDER BY gmv DESC")) + assert check.status == "pass" and check.reasons == [] + # 按运行当天现算的界写法不一样（上周对本周到今天） + window = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= {a} AND order_date < {b}" + assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"), + window.format(a="date('now', 'weekday 1', '-7 days')", b="date('now')")).status == "pass" + assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"), + "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', 'weekday 1', '-7 days')" + ).status == "pass" + + +def test_what_still_makes_two_queries_different(): + base = "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' GROUP BY region" + # 带 LIMIT 时排序决定取哪几段 + top = _same(base + " ORDER BY gmv DESC LIMIT 2", base.replace("2026-10-05", "2026-10-12") + " LIMIT 2") + assert top.status == "fail" + # 除了日期还差别的筛选、按别的时间列、多一个条件 + other = _same(base.replace("WHERE", "WHERE region = '华东' AND"), + base.replace("WHERE", "WHERE region = '华南' AND").replace("2026-10-05", "2026-10-12")) + assert (other.status, other.reasons) == ("fail", ["两边除了日期，筛选的还不一样"]) + paid = _same(base, base.replace("order_date", "paid_at").replace("2026-10-05", "2026-10-12")) + assert paid.status == "fail" and paid.reasons == ["两份查询的口径结构不同（差在时间列）"] + extra = _same(base, base.replace("GROUP BY", "AND status = 'paid' GROUP BY").replace("2026-10-05", "2026-10-12")) + assert extra.status == "fail" and "筛选条件" in extra.reasons[0] + + +async def test_decomposing_two_snapshots_that_differ_only_in_order_is_not_refused(): + a = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-05' AND '2026-10-11' " + "GROUP BY region") + b = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-12' AND '2026-10-18' " + "GROUP BY region ORDER BY gmv DESC") + base = snap("Q1", a, ["region", "gmv"], [["华东", 820], ["华南", 700], ["华北", 600]], types={"region": "text"}) + target = snap("Q2", b, ["region", "gmv"], [["华东", 700], ["华南", 666], ["华北", 560]], types={"region": "text"}) + result = await analyze("decompose", "additive", snaps=[base, target], probe=fixed_prober(RANGE), base_query="Q1", + target_query="Q2", by=["region"], measure="gmv", label="销售额") + assert result["status"] == "ok" and result["verdict"]["code"] == "driver" + assert not [c for c in result["caveats"] if c["code"] == "comparability"]