From 460af221f9c5d2a2bcf5a913ba2e42a785f582da Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 21 Oct 2026 15:51:30 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=8C=E5=8F=A3=E5=BE=84=E5=85=88=E5=8E=BB?= =?UTF-8?q?=E6=8E=89=E6=9C=80=E5=A4=96=E5=B1=82=E6=8E=92=E5=BA=8F=E3=80=81?= =?UTF-8?q?=E6=8C=89=E5=8E=BB=E6=8E=89=E6=97=A5=E6=9C=9F=E7=9A=84=E5=86=99?= =?UTF-8?q?=E6=B3=95=E6=AF=94=EF=BC=88=E5=A4=8D=E6=A0=B8=20R7=EF=BC=89?= =?UTF-8?q?=EF=BC=9Asame=5Fquery=20=E4=B8=8D=E5=86=8D=E5=8F=AA=E5=9C=A8?= =?UTF-8?q?=E8=A7=84=E8=8C=83=E5=BD=A2=E5=BC=8F=E5=AE=8C=E5=85=A8=E7=9B=B8?= =?UTF-8?q?=E5=90=8C=E6=97=B6=E6=89=8D=E6=8A=8A=E6=97=A5=E6=9C=9F=E6=8D=A2?= =?UTF-8?q?=E6=88=90=E5=8D=A0=E4=BD=8D=E2=80=94=E2=80=94=E4=B8=A4=E8=BE=B9?= =?UTF-8?q?=E9=83=BD=E5=8E=BB=E6=8E=89=E6=9C=80=E5=A4=96=E5=B1=82=E7=9A=84?= =?UTF-8?q?=20ORDER=20BY=EF=BC=88=E5=90=8C=E4=B8=80=E5=B1=82=E6=9C=89=20LI?= =?UTF-8?q?MIT=20/=20OFFSET=20=E6=97=B6=E4=B8=8D=E5=8E=BB=EF=BC=9A?= =?UTF-8?q?=E6=8E=92=E5=BA=8F=E5=86=B3=E5=AE=9A=E5=8F=96=E5=93=AA=E5=87=A0?= =?UTF-8?q?=E8=A1=8C=EF=BC=89=EF=BC=8C=E6=97=B6=E9=97=B4=E6=9D=A1=E4=BB=B6?= =?UTF-8?q?=E7=9A=84=E7=95=8C=EF=BC=88=E5=AD=97=E9=9D=A2=E9=87=8F=E3=80=81?= =?UTF-8?q?=E6=8C=89=E8=BF=90=E8=A1=8C=E5=BD=93=E5=A4=A9=E7=8E=B0=E7=AE=97?= =?UTF-8?q?=E7=9A=84=E9=83=BD=E7=AE=97=EF=BC=89=E5=92=8C=E5=83=8F=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E7=9A=84=E5=AD=97=E9=9D=A2=E9=87=8F=E6=8D=A2=E6=88=90?= =?UTF-8?q?=E5=90=8C=E4=B8=80=E4=B8=AA=E5=8D=A0=E4=BD=8D=E5=90=8E=E8=A7=84?= =?UTF-8?q?=E8=8C=83=E5=8C=96=E5=86=8D=E6=AF=94=EF=BC=9A=E4=B8=80=E6=A0=B7?= =?UTF-8?q?=E5=B0=B1=E9=80=9A=E8=BF=87=EF=BC=8C=E5=8F=AA=E5=B7=AE=E5=AD=97?= =?UTF-8?q?=E9=9D=A2=E9=87=8F=E5=88=A4=E3=80=8C=E4=B8=A4=E8=BE=B9=E9=99=A4?= =?UTF-8?q?=E4=BA=86=E6=97=A5=E6=9C=9F=EF=BC=8C=E7=AD=9B=E9=80=89=E7=9A=84?= =?UTF-8?q?=E8=BF=98=E4=B8=8D=E4=B8=80=E6=A0=B7=E3=80=8D=EF=BC=8C=E8=BF=98?= =?UTF-8?q?=E4=B8=8D=E4=B8=80=E6=A0=B7=E6=89=8D=E4=BA=A4=E7=BB=99=20explai?= =?UTF-8?q?n=5Fsql=5Fdiff=EF=BC=9B=E4=BA=A4=E7=BB=99=E5=AE=83=E7=9A=84?= =?UTF-8?q?=E4=B9=9F=E5=8E=BB=E6=8E=89=E6=9C=80=E5=A4=96=E5=B1=82=E6=8E=92?= =?UTF-8?q?=E5=BA=8F=E3=80=81=E7=8E=B0=E7=AE=97=E7=9A=84=E7=95=8C=E6=8D=A2?= =?UTF-8?q?=E6=88=90=E6=8C=89=E5=85=88=E5=90=8E=E7=BC=96=E5=8F=B7=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E6=9C=9F=E5=AD=97=E9=9D=A2=E9=87=8F=EF=BC=88=E5=AE=83?= =?UTF-8?q?=E5=8F=AA=E8=AE=A4=E5=AD=97=E9=9D=A2=E9=87=8F=E7=9A=84=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E8=8C=83=E5=9B=B4=EF=BC=89=EF=BC=8C=E5=8F=AA=E5=B7=AE?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E8=8C=83=E5=9B=B4=E3=80=81=E6=8E=92=E5=BA=8F?= =?UTF-8?q?=E7=9A=84=E7=85=A7=E6=97=A7=E7=AE=97=E5=90=8C=E5=8F=A3=E5=BE=84?= =?UTF-8?q?=E3=80=82=E5=8F=AA=E6=9C=89=E4=B8=80=E8=BE=B9=E5=B8=A6=20ORDER?= =?UTF-8?q?=20BY=E3=80=81=E4=B8=8A=E5=91=A8=E5=AF=B9=E6=9C=AC=E5=91=A8?= =?UTF-8?q?=E5=88=B0=E4=BB=8A=E5=A4=A9=E8=BF=99=E7=B1=BB=E7=8E=B0=E7=AE=97?= =?UTF-8?q?=E7=95=8C=E5=86=99=E6=B3=95=E4=B8=8D=E5=90=8C=E7=9A=84=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E8=90=BD=E8=BF=9B=E5=85=9C=E5=BA=95=E5=88=A4=E6=88=90?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E4=B8=8D=E5=90=8C=EF=BC=8C=E4=B8=A4=E4=BB=BD?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E7=9A=84=E5=8F=98=E5=8C=96=E6=8B=86=E8=A7=A3?= =?UTF-8?q?=E4=B9=9F=E4=B8=8D=E5=86=8D=E5=9B=A0=E6=AD=A4=E4=B8=8D=E6=8B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/analysis/comparable.py | 67 +++++++++++++++++------ backend/tests/test_analysis_comparable.py | 52 ++++++++++++++++++ 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/backend/app/analysis/comparable.py b/backend/app/analysis/comparable.py index 76041bb6..bc157432 100644 --- a/backend/app/analysis/comparable.py +++ b/backend/app/analysis/comparable.py @@ -3,11 +3,12 @@ 五项检查，每项 pass / warn / fail / unknown（unknown 只写「核对不了」，不判不可比）： -- 同口径：两份 SQL 一字不差 → pass；规范化（data/sql_semantics.normalize_sql）后只差字面量：把时间条件的界（data/coverage - 认出的时间谓词的界，整个表达式）和像日期的字面量换成同一个占位再比，一样就是只差日期 → pass，不一样就是比的两段不是 - 两期 → fail「两边除了日期，筛选的还不一样」；规范形式不同交给口径账本的 explain_sql_diff，等价 → pass，只差时间范围、排序 - → pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里的值）。两份来自不同的数据源 fail。同一份快照里按期次分组 - 的天然同口径。 +- 同口径：两份 SQL 一字不差 → pass；否则两边都去掉最外层的 ORDER BY（没有 LIMIT 时排序不改结果），把时间条件的界（data/ + coverage 认出的时间谓词的界，整个表达式：字面量、按运行当天现算的都算）和像日期的字面量换成同一个占位，再规范化 + （data/sql_semantics.normalize_sql）比：一样就是只差日期 → pass；只差字面量就是比的两段不是两期 → fail「两边除了日期， + 筛选的还不一样」；还不一样交给口径账本的 explain_sql_diff（同样去掉最外层的 ORDER BY，现算的界换成按先后编号的日期字面量： + 它只认字面量的时间范围），等价 → pass，只差时间范围、排序 → pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里 + 的值）。两份来自不同的数据源 fail。同一份快照里按期次分组的天然同口径。 - 窗口长度：用 data/coverage 的 filters 取业务日期列（目录的 business_date；没写就取唯一那一列时间谓词）上的区间，界是字面 量的直接解析，按运行当天现算的在库里算（Book 的 bound，记进 probes）；算两边天数。整月对整月、整季对整季算一样长。 不相等：度量是流量 fail，比率、存量、没标的 warn。按期次分组的：两期粒度相同，一样长。 @@ -94,14 +95,26 @@ class Outcome: # -------------------------------------------------------------------------- +def _unordered(sql: str, read: str | None) -> Any: + """解析成语法树，去掉最外层的 ORDER BY（同一层有 LIMIT / OFFSET 时排序决定取哪几行，不去）。解析不了为 None。""" + from app.data import coverage + + tree = coverage.parse_sql(sql, read) + if tree is not None and tree.args.get("order") is not None and not tree.args.get("limit") \ + and not tree.args.get("offset"): + tree.set("order", None) + return tree + + def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None: - """时间条件的界、像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为 None。""" + """去掉最外层的 ORDER BY、时间条件的界和像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为 + None。""" from sqlglot import exp from app.data import coverage from app.data.sql_semantics import normalize_sql - tree = coverage.parse_sql(sql, read) + tree = _unordered(sql, read) if tree is None: return None for flt in coverage.filters(tree): @@ -117,6 +130,28 @@ def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None: return normalized.text, normalized.literals +def _for_diff(sql: str, read: str | None) -> str: + """交给 explain_sql_diff 的写法：去掉最外层的 ORDER BY；按运行当天现算的界换成按先后编号的日期字面量（它只认字面量的 + 时间范围，现算的界写法不一样会落进兜底的「写法不同」）。解析不了的原样交。""" + from sqlglot import exp + + from app.data import coverage + + tree = _unordered(sql, read) + if tree is None: + return sql + n = 0 + for flt in coverage.filters(tree): + for _, node, _ in flt.conds: + if node.parent is not None and not coverage.literal_bound(node)[0]: + n += 1 + node.replace(exp.Literal.string(f"1900-01-{n:02d}")) + try: + return tree.sql(dialect=read) + except Exception: # noqa: BLE001 - 写不回去就交原文 + return sql + + def same_query(base: Snap, target: Snap) -> Check: from app.data.sql_semantics import explain_sql_diff, normalize_sql @@ -133,17 +168,17 @@ def same_query(base: Snap, target: Snap) -> Check: if na.parse_error or nb.parse_error: check.reasons = ["SQL 解析不了"] return check - if na.text == nb.text: - ua, ub = _undated(a, read), _undated(b, read) - if ua is None or ub is None: - return check - if ua == ub: - check.status = "pass" - else: - check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"] + ua, ub = _undated(a, read), _undated(b, read) + if ua is None or ub is None: + return check + if ua == ub: + check.status = "pass" + return check + if ua[0] == ub[0]: + check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"] return check try: - diff = explain_sql_diff(a, b, read) + diff = explain_sql_diff(_for_diff(a, read), _for_diff(b, read), read) except Exception: # noqa: BLE001 - 比不了就当核对不了 return check aspects = [d.aspect for d in diff.items] diff --git a/backend/tests/test_analysis_comparable.py b/backend/tests/test_analysis_comparable.py index caf8d433..6dcb8bc3 100644 --- a/backend/tests/test_analysis_comparable.py +++ b/backend/tests/test_analysis_comparable.py @@ -204,3 +204,55 @@ async def test_one_snapshot_of_sqlite_weeks_uses_the_real_days_of_each_week(): plain = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"}) unknown = await analyze("comparable", snaps=[plain], query="Q1", period="week", base="2026-W41", target="2026-W42") assert unknown["phrase"] == "两期可比，窗口长度、整期、星期构成核对不了" + + +def _same(a, b): + from app.analysis.comparable import same_query + from app.analysis.inputs import Snap + + return same_query(Snap("Q1", {"sql": a, "source": "shop"}, "sqlite"), + Snap("Q2", {"sql": b, "source": "shop"}, "sqlite")) + + +def test_the_same_query_ignores_the_outer_order_and_how_run_day_bounds_are_written(): + by_region = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '{a}' AND '{b}' " + "GROUP BY region{tail}") + # 只有一边带 ORDER BY：排序不改结果 + check = _same(by_region.format(a="2026-10-05", b="2026-10-11", tail=""), + by_region.format(a="2026-10-12", b="2026-10-18", tail=" ORDER BY gmv DESC")) + assert check.status == "pass" and check.reasons == [] + # 按运行当天现算的界写法不一样（上周对本周到今天） + window = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= {a} AND order_date < {b}" + assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"), + window.format(a="date('now', 'weekday 1', '-7 days')", b="date('now')")).status == "pass" + assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"), + "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', 'weekday 1', '-7 days')" + ).status == "pass" + + +def test_what_still_makes_two_queries_different(): + base = "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' GROUP BY region" + # 带 LIMIT 时排序决定取哪几段 + top = _same(base + " ORDER BY gmv DESC LIMIT 2", base.replace("2026-10-05", "2026-10-12") + " LIMIT 2") + assert top.status == "fail" + # 除了日期还差别的筛选、按别的时间列、多一个条件 + other = _same(base.replace("WHERE", "WHERE region = '华东' AND"), + base.replace("WHERE", "WHERE region = '华南' AND").replace("2026-10-05", "2026-10-12")) + assert (other.status, other.reasons) == ("fail", ["两边除了日期，筛选的还不一样"]) + paid = _same(base, base.replace("order_date", "paid_at").replace("2026-10-05", "2026-10-12")) + assert paid.status == "fail" and paid.reasons == ["两份查询的口径结构不同（差在时间列）"] + extra = _same(base, base.replace("GROUP BY", "AND status = 'paid' GROUP BY").replace("2026-10-05", "2026-10-12")) + assert extra.status == "fail" and "筛选条件" in extra.reasons[0] + + +async def test_decomposing_two_snapshots_that_differ_only_in_order_is_not_refused(): + a = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-05' AND '2026-10-11' " + "GROUP BY region") + b = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-12' AND '2026-10-18' " + "GROUP BY region ORDER BY gmv DESC") + base = snap("Q1", a, ["region", "gmv"], [["华东", 820], ["华南", 700], ["华北", 600]], types={"region": "text"}) + target = snap("Q2", b, ["region", "gmv"], [["华东", 700], ["华南", 666], ["华北", 560]], types={"region": "text"}) + result = await analyze("decompose", "additive", snaps=[base, target], probe=fixed_prober(RANGE), base_query="Q1", + target_query="Q2", by=["region"], measure="gmv", label="销售额") + assert result["status"] == "ok" and result["verdict"]["code"] == "driver" + assert not [c for c in result["caveats"] if c["code"] == "comparability"]