mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
同口径先去掉最外层排序、按去掉日期的写法比（复核 R7）：same_query 不再只在规范形式完全相同时才把日期换成占位——两边都去掉最外层的 ORDER BY（同一层有 LIMIT / OFFSET 时不去：排序决定取哪几行），时间条件的界（字面量、按运行当天现算的都算）和像日期的字面量换成同一个占位后规范化再比：一样就通过，只差字面量判「两边除了日期，筛选的还不一样」，还不一样才交给 explain_sql_diff；交给它的也去掉最外层排序、现算的界换成按先后编号的日期字面量（它只认字面量的时间范围），只差时间范围、排序的照旧算同口径。只有一边带 ORDER BY、上周对本周到今天这类现算界写法不同的不再落进兜底判成结构不同，两份快照的变化拆解也不再因此不拆
This commit is contained in:
ThearchyHelios
@@ -3,11 +3,12 @@
|
||||
|
||||
五项检查，每项 pass / warn / fail / unknown（unknown 只写「核对不了」，不判不可比）：
|
||||
|
||||
- 同口径：两份 SQL 一字不差 → pass；规范化（data/sql_semantics.normalize_sql）后只差字面量：把时间条件的界（data/coverage
|
||||
认出的时间谓词的界，整个表达式）和像日期的字面量换成同一个占位再比，一样就是只差日期 → pass，不一样就是比的两段不是
|
||||
两期 → fail「两边除了日期，筛选的还不一样」；规范形式不同交给口径账本的 explain_sql_diff，等价 → pass，只差时间范围、排序
|
||||
→ pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里的值）。两份来自不同的数据源 fail。同一份快照里按期次分组
|
||||
的天然同口径。
|
||||
- 同口径：两份 SQL 一字不差 → pass；否则两边都去掉最外层的 ORDER BY（没有 LIMIT 时排序不改结果），把时间条件的界（data/
|
||||
coverage 认出的时间谓词的界，整个表达式：字面量、按运行当天现算的都算）和像日期的字面量换成同一个占位，再规范化
|
||||
（data/sql_semantics.normalize_sql）比：一样就是只差日期 → pass；只差字面量就是比的两段不是两期 → fail「两边除了日期，
|
||||
筛选的还不一样」；还不一样交给口径账本的 explain_sql_diff（同样去掉最外层的 ORDER BY，现算的界换成按先后编号的日期字面量：
|
||||
它只认字面量的时间范围），等价 → pass，只差时间范围、排序 → pass，否则 fail，写出差在哪些方面（只写方面，不写 SQL 原文里
|
||||
的值）。两份来自不同的数据源 fail。同一份快照里按期次分组的天然同口径。
|
||||
- 窗口长度：用 data/coverage 的 filters 取业务日期列（目录的 business_date；没写就取唯一那一列时间谓词）上的区间，界是字面
|
||||
量的直接解析，按运行当天现算的在库里算（Book 的 bound，记进 probes）；算两边天数。整月对整月、整季对整季算一样长。
|
||||
不相等：度量是流量 fail，比率、存量、没标的 warn。按期次分组的：两期粒度相同，一样长。
|
||||
@@ -94,14 +95,26 @@ class Outcome:
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _unordered(sql: str, read: str | None) -> Any:
|
||||
"""解析成语法树，去掉最外层的 ORDER BY（同一层有 LIMIT / OFFSET 时排序决定取哪几行，不去）。解析不了为 None。"""
|
||||
from app.data import coverage
|
||||
|
||||
tree = coverage.parse_sql(sql, read)
|
||||
if tree is not None and tree.args.get("order") is not None and not tree.args.get("limit") \
|
||||
and not tree.args.get("offset"):
|
||||
tree.set("order", None)
|
||||
return tree
|
||||
|
||||
|
||||
def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None:
|
||||
"""时间条件的界、像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为 None。"""
|
||||
"""去掉最外层的 ORDER BY、时间条件的界和像日期的字面量换成同一个占位之后的规范形式（文本 + 剩下的字面量）。解析不了为
|
||||
None。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data import coverage
|
||||
from app.data.sql_semantics import normalize_sql
|
||||
|
||||
tree = coverage.parse_sql(sql, read)
|
||||
tree = _unordered(sql, read)
|
||||
if tree is None:
|
||||
return None
|
||||
for flt in coverage.filters(tree):
|
||||
@@ -117,6 +130,28 @@ def _undated(sql: str, read: str | None) -> tuple[str, tuple[str, ...]] | None:
|
||||
return normalized.text, normalized.literals
|
||||
|
||||
|
||||
def _for_diff(sql: str, read: str | None) -> str:
|
||||
"""交给 explain_sql_diff 的写法：去掉最外层的 ORDER BY；按运行当天现算的界换成按先后编号的日期字面量（它只认字面量的
|
||||
时间范围，现算的界写法不一样会落进兜底的「写法不同」）。解析不了的原样交。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data import coverage
|
||||
|
||||
tree = _unordered(sql, read)
|
||||
if tree is None:
|
||||
return sql
|
||||
n = 0
|
||||
for flt in coverage.filters(tree):
|
||||
for _, node, _ in flt.conds:
|
||||
if node.parent is not None and not coverage.literal_bound(node)[0]:
|
||||
n += 1
|
||||
node.replace(exp.Literal.string(f"1900-01-{n:02d}"))
|
||||
try:
|
||||
return tree.sql(dialect=read)
|
||||
except Exception: # noqa: BLE001 - 写不回去就交原文
|
||||
return sql
|
||||
|
||||
|
||||
def same_query(base: Snap, target: Snap) -> Check:
|
||||
from app.data.sql_semantics import explain_sql_diff, normalize_sql
|
||||
|
||||
@@ -133,17 +168,17 @@ def same_query(base: Snap, target: Snap) -> Check:
|
||||
if na.parse_error or nb.parse_error:
|
||||
check.reasons = ["SQL 解析不了"]
|
||||
return check
|
||||
if na.text == nb.text:
|
||||
ua, ub = _undated(a, read), _undated(b, read)
|
||||
if ua is None or ub is None:
|
||||
return check
|
||||
if ua == ub:
|
||||
check.status = "pass"
|
||||
else:
|
||||
check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"]
|
||||
ua, ub = _undated(a, read), _undated(b, read)
|
||||
if ua is None or ub is None:
|
||||
return check
|
||||
if ua == ub:
|
||||
check.status = "pass"
|
||||
return check
|
||||
if ua[0] == ub[0]:
|
||||
check.status, check.reasons = "fail", ["两边除了日期，筛选的还不一样"]
|
||||
return check
|
||||
try:
|
||||
diff = explain_sql_diff(a, b, read)
|
||||
diff = explain_sql_diff(_for_diff(a, read), _for_diff(b, read), read)
|
||||
except Exception: # noqa: BLE001 - 比不了就当核对不了
|
||||
return check
|
||||
aspects = [d.aspect for d in diff.items]
|
||||
|
||||
@@ -204,3 +204,55 @@ async def test_one_snapshot_of_sqlite_weeks_uses_the_real_days_of_each_week():
|
||||
plain = snap("Q1", sql, ["week", "orders"], rows, types={"week": "text", "orders": "number"})
|
||||
unknown = await analyze("comparable", snaps=[plain], query="Q1", period="week", base="2026-W41", target="2026-W42")
|
||||
assert unknown["phrase"] == "两期可比，窗口长度、整期、星期构成核对不了"
|
||||
|
||||
|
||||
def _same(a, b):
|
||||
from app.analysis.comparable import same_query
|
||||
from app.analysis.inputs import Snap
|
||||
|
||||
return same_query(Snap("Q1", {"sql": a, "source": "shop"}, "sqlite"),
|
||||
Snap("Q2", {"sql": b, "source": "shop"}, "sqlite"))
|
||||
|
||||
|
||||
def test_the_same_query_ignores_the_outer_order_and_how_run_day_bounds_are_written():
|
||||
by_region = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '{a}' AND '{b}' "
|
||||
"GROUP BY region{tail}")
|
||||
# 只有一边带 ORDER BY：排序不改结果
|
||||
check = _same(by_region.format(a="2026-10-05", b="2026-10-11", tail=""),
|
||||
by_region.format(a="2026-10-12", b="2026-10-18", tail=" ORDER BY gmv DESC"))
|
||||
assert check.status == "pass" and check.reasons == []
|
||||
# 按运行当天现算的界写法不一样（上周对本周到今天）
|
||||
window = "SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= {a} AND order_date < {b}"
|
||||
assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"),
|
||||
window.format(a="date('now', 'weekday 1', '-7 days')", b="date('now')")).status == "pass"
|
||||
assert _same(window.format(a="date('now', 'weekday 1', '-14 days')", b="date('now', 'weekday 1', '-7 days')"),
|
||||
"SELECT SUM(amount) AS gmv FROM orders WHERE order_date >= date('now', 'weekday 1', '-7 days')"
|
||||
).status == "pass"
|
||||
|
||||
|
||||
def test_what_still_makes_two_queries_different():
|
||||
base = "SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date >= '2026-10-05' GROUP BY region"
|
||||
# 带 LIMIT 时排序决定取哪几段
|
||||
top = _same(base + " ORDER BY gmv DESC LIMIT 2", base.replace("2026-10-05", "2026-10-12") + " LIMIT 2")
|
||||
assert top.status == "fail"
|
||||
# 除了日期还差别的筛选、按别的时间列、多一个条件
|
||||
other = _same(base.replace("WHERE", "WHERE region = '华东' AND"),
|
||||
base.replace("WHERE", "WHERE region = '华南' AND").replace("2026-10-05", "2026-10-12"))
|
||||
assert (other.status, other.reasons) == ("fail", ["两边除了日期，筛选的还不一样"])
|
||||
paid = _same(base, base.replace("order_date", "paid_at").replace("2026-10-05", "2026-10-12"))
|
||||
assert paid.status == "fail" and paid.reasons == ["两份查询的口径结构不同（差在时间列）"]
|
||||
extra = _same(base, base.replace("GROUP BY", "AND status = 'paid' GROUP BY").replace("2026-10-05", "2026-10-12"))
|
||||
assert extra.status == "fail" and "筛选条件" in extra.reasons[0]
|
||||
|
||||
|
||||
async def test_decomposing_two_snapshots_that_differ_only_in_order_is_not_refused():
|
||||
a = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-05' AND '2026-10-11' "
|
||||
"GROUP BY region")
|
||||
b = ("SELECT region, SUM(amount) AS gmv FROM orders WHERE order_date BETWEEN '2026-10-12' AND '2026-10-18' "
|
||||
"GROUP BY region ORDER BY gmv DESC")
|
||||
base = snap("Q1", a, ["region", "gmv"], [["华东", 820], ["华南", 700], ["华北", 600]], types={"region": "text"})
|
||||
target = snap("Q2", b, ["region", "gmv"], [["华东", 700], ["华南", 666], ["华北", 560]], types={"region": "text"})
|
||||
result = await analyze("decompose", "additive", snaps=[base, target], probe=fixed_prober(RANGE), base_query="Q1",
|
||||
target_query="Q2", by=["region"], measure="gmv", label="销售额")
|
||||
assert result["status"] == "ok" and result["verdict"]["code"] == "driver"
|
||||
assert not [c for c in result["caveats"] if c["code"] == "comparability"]
|
||||
|
||||
Reference in New Issue
Block a user