diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index 26962b35..bceb8f22 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -7037,6 +7037,8 @@ class _Explainer: pb, jb = self._shape(B) if pa == pb and ja == jb: return True, {}, set(), set() + if (said := self._left_on_times(A, B, pa, ja, pb, jb)) is not None: + return said if not ja and not jb: only_a = [r for r in A.rels if r.alias not in pb] only_b = [r for r in B.rels if r.alias not in pa] @@ -7066,6 +7068,80 @@ class _Explainer: x = flipped[0] return self._left_inner(A, B, x, 0) if kinds_a[x] == "left" else self._left_inner(B, A, x, 1) + def _left_on_times(self, A: _Block, B: _Block, pa: Any, ja: Any, pb: Any, jb: Any + ) -> tuple[bool, dict[str, str], set[str], set[str]] | None: + """来源两边一样，只差在一个 LEFT JOIN 的 ON 里的时间条件（日期条件写在 LEFT JOIN 的 ON 里）：说「A 用 LEFT JOIN + 关联 X 表时只关联 … 的行，B 没有这个条件」「A 用 LEFT JOIN 关联 X 表时对 paid_at 的条件是 …，B 是 …」「A … 时按 + paid_at 筛时间，B 按 created_at」。说不准的返回 None（照旧整条兜底）。 + + 说得准要（每一条都是为了不说出不存在的不同）：开头那组来源一样，后面的连接只有这一个的 ON 不同（别名、表、LEFT 都 + 一样）；两边 ON 里没配上的条件每一个都是时间条件（timed）、只涉及这张可空一侧的表的列；这张表在 ON 之外只出现在 + 结果里（SELECT 列表、分组、排序），没在 WHERE、HAVING、别的连接里用到（用到了就是反连接、或者关联条件的事）； + SELECT 列表里用到了它的列（不然 ON 里的不同只影响行会不会重复，说不准）；结果看得见行；这张表在这一段只出现一次。 + 两边都有、同一列上的，按取值逐段算过留下的取值确实不同（_Range.compare，同 _rewrites）；只有一边有的，不是恒为真的 + （_always）。aspect 是 join：说的是关联条件，dual_check_facts 不会把它当成业务日期写进数据目录。 + + 只改说明，不碰等价判定：走到这里时两边已经判了不等价（见 explain_sql_diff）。""" + from sqlglot import exp + + if pa != pb or len(ja) != len(jb) or not self.visible: + return None + differing = [i for i, (x, y) in enumerate(zip(ja, jb)) if x != y] + if len(differing) != 1: + return None + i = differing[0] + if ja[i][:3] != jb[i][:3] or ja[i][2] != "left": + return None + alias = ja[i][0] + join_a, join_b = A.joins[i], B.joins[i] + rel = A.rel(alias) + if rel is None or not rel.table or A.names().count(rel.name) != 1 or B.names().count(rel.name) != 1: + return None + on_a, on_b = _conjuncts(join_a.args["on"]), _conjuncts(join_b.args["on"]) + only_a, only_b = self._unmatched(on_a, on_b, {}) + keys: set[tuple[str, str]] = set() + for block, conds in ((A, only_a), (B, only_b)): + for cond in conds: + cols = self.base_columns(cond, block) + found = self.timed(cond, block) + if found is None or not cols or any(r.alias != alias for r, _ in cols): + return None + keys.add(found) + for block, join in ((A, join_a), (B, join_b)): + if self._used_outside(block, alias, join, outputs=False): + return None + if not any(col.table == alias for e in block.exprs for col in e.find_all(exp.Column)): + return None + what = self.what(rel) + if only_a and only_b: + if len(keys) == 1: + key = next(iter(keys)) + same_a = [c for c in on_a if self.refs(c, A) == {key}] + same_b = [c for c in on_b if self.refs(c, B) == {key}] + if self.range.compare(same_a, same_b, never_null=False) is not False: + return None + shown_a, shown_b = self.show_all(_by_bound(same_a), A), self.show_all(_by_bound(same_b), B) + text = _say(self.la, "用 LEFT JOIN 关联", what, "时对", self.named(*key), "的条件是", shown_a, "，", + self.lb, "是", shown_b) + else: + cols_a = {self.timed(c, A) for c in only_a} + cols_b = {self.timed(c, B) for c in only_b} + if len(cols_a) != 1 or len(cols_b) != 1: + return None + shown_a, shown_b = self.show_all(only_a, A), self.show_all(only_b, B) + text = _say(self.la, "用 LEFT JOIN 关联", what, "时按", self.named(*next(iter(cols_a))), "筛时间，", + self.lb, "按", self.named(*next(iter(cols_b)))) + self.add("join", shown_a, shown_b, text) + return True, {}, set(), set() + side, block, conds = (0, A, only_a) if only_a else (1, B, only_b) + if self._always(conds, block) is not False: + return None + shown = self.show_all(_by_bound(conds), block) + me, them = (self.la, self.lb) if side == 0 else (self.lb, self.la) + self.add("join", shown if side == 0 else None, shown if side == 1 else None, + _say(me, "用 LEFT JOIN 关联", what, "时只关联", shown, "的行，", them, "没有这个条件")) + return True, {}, set(), set() + def _swapped(self, A: _Block, B: _Block, x: _Rel, y: _Rel) -> tuple[bool, dict[str, str], set[str], set[str]] | None: """换了一张表：开头那组来源只差一张实表（各自只出现一次、名字不只差在大小写上，都没带库名）。换了表的那一边， 列照它自己的表说。""" diff --git a/backend/app/ledger/match.py b/backend/app/ledger/match.py index dbb774de..4b77fc05 100644 --- a/backend/app/ledger/match.py +++ b/backend/app/ledger/match.py @@ -21,8 +21,8 @@ MAX_CANDIDATES 份（按工作流、版本、节点排好的先后）。 和选择列表的项数要一样）；口径一侧按口径卡 cell(nodes.取数, 行, '列') 里写的列名，在取数 SQL 选择列表的输出名（别名， 没有别名的是列名，不分大小写，同名的不止一项时不投影）里找位置。投影后两边留下的几列按位置配对（和整条比时的列映射一样： 外层别名不计、列的先后计），两边列数一样才比；答案一侧留下两三列时另试调换先后。每份口径取数最多试 MAX_PROJECTIONS 种。 -证明了的候选记 projection {ours, dropped, caliber, caliber_dropped}；没证明的给第一种投影的比较（差在哪是投影之后的说法）。 -没有投影可试的（报告没引用这条查询、两边都不投影、 +证明了的候选记 projection {ours, dropped, caliber, caliber_dropped}；没证明的给第一种投影的比较（差在哪是投影之后的说法， +答案多选了列的在最前面写一句「这次多查了 N 列，不影响这个数」）。没有投影可试的（报告没引用这条查询、两边都不投影、 投影不了）照旧是整条比的结果，逐对一字不差。 **助手选用的（composed）。** 问数据的助手 use_caliber 选用口径时，取数节点是服务端照那一版拷进来的（ledger/compose.py）， @@ -93,6 +93,8 @@ PARSE_FAILED = "这次的 SQL 无法解析，没有比对" MERGED = "合并查询的格不和口径比对（它的数来自各个输入的查询）" COMPOSED_UNPROVEN = "这条查询出自助手选用的口径，但和口径的取数没能证明一致（可能被改过）" MORE_CANDIDATES = "另有 {n} 份相关口径没有比对（每条查询最多比 {max} 份）" +#: 按引用的列投影之后没证明等价时，差在哪的第一句：答案只是多选了几列（去掉它们不影响被引用的数，见 sql_semantics 的投影） +EXTRA_COLUMNS = "这次多查了 {n} 列，不影响这个数" #: 不比对的原因（GET 账本时照实说） NOT_FOUND = "运行记录不存在" @@ -395,7 +397,7 @@ def _projected(caliber: Caliber, fetch: Fetch, query: Query, wanted: tuple[int, 没有投影可试的为 None（照旧用整条比的结果）。""" from itertools import permutations - from app.data.sql_semantics import match_template, output_names, project_select, project_template + from app.data.sql_semantics import FacetDiff, match_template, output_names, project_select, project_template width = len(query.columns or []) whole = project_select(query.sql, tuple(range(width)), query.dialect, width=width) @@ -438,7 +440,11 @@ def _projected(caliber: Caliber, fetch: Fetch, query: Query, wanted: tuple[int, if first is None: return None a, c, m = first - return {**_candidate(caliber, fetch, m), "projection": _projection_view(query, a, c)} + cand = {**_candidate(caliber, fetch, m), "projection": _projection_view(query, a, c)} + if a.dropped and isinstance(cand.get("diff"), dict): + note = FacetDiff("projection", None, None, EXTRA_COLUMNS.format(n=a.dropped)).as_dict() + cand["diff"] = {**cand["diff"], "items": [note, *cand["diff"]["items"]]} + return cand def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: float | None = None, @@ -737,7 +743,7 @@ async def shutdown() -> None: await asyncio.gather(*tasks, return_exceptions=True) -__all__ = ["BACKFILL_DAYS", "CONCURRENCY", "EXPLORE_ORIGINS", "MAX_PROJECTIONS", "TIMEOUT_S", "Caliber", +__all__ = ["BACKFILL_DAYS", "CONCURRENCY", "EXPLORE_ORIGINS", "EXTRA_COLUMNS", "MAX_PROJECTIONS", "TIMEOUT_S", "Caliber", "Fetch", "Query", "backfill", "caliber_of", "compare_queries", "composed_nodes", "compute", "drain", "ineligible", "load_calibers", "match_run", "pending", "read_queries", "schedule", "shutdown", "start_backfill", "wanted"] diff --git a/backend/app/ledger/stats.py b/backend/app/ledger/stats.py index ae2f68e2..4c4ead14 100644 --- a/backend/app/ledger/stats.py +++ b/backend/app/ledger/stats.py @@ -10,7 +10,8 @@ - E1 等价命中：比对过的查询里 proven / composed / related / none / skipped 各几条（skipped 单独列，不混进 none），有候选口径 的查询里各自的比率；证明等价的绑定是在哪一级配上的，其中几份是按引用的列投影之后才证明的（proven_projected，ledger/match 的 _projected）；没证明的原因（不可比、参数绑不上、配上了但不等价）；不等价的差异类型分布（time / filter / aggregate / - join / other / parse，每份口径的差异每类只计一次）；逐个数的标签各几个、整题等价几次。 + join / other / parse，每份口径的差异每类只计一次；投影之后那句「这次多查了 N 列」不是差异，不计）；逐个数的标签各几个、 + 整题等价几次。 - E2 对账：对账运行（origin=reconcile，第 3 波起由 ledger/reconcile 发起）发起、完成、失败各几次，对过账的答案几个，其中参数 经人核对过的、对照的指标由人选的各几次；相关口径的差异说明里兜底说明的占比；对账结果上「说明有用 / 看不懂」的反馈 （第 4 波，ledger/feedback：对账运行上的封存后事件 ledger.feedback，每人只算最后一次）各几个、有用的比率。 @@ -43,6 +44,8 @@ REPEAT_CONVERSATIONS = 2 ASPECT_GROUP = {"time": "time", "time_range": "time", "filter": "filter", "aggregate": "aggregate", "group_by": "aggregate", "join": "join", "tables": "join", "parse": "parse"} DIFF_GROUPS = ("time", "filter", "aggregate", "join", "other", "parse") +#: 差在哪里不算差异的条目：投影之后的「这次多查了 N 列，不影响这个数」 +NOT_DIFFS = ("projection",) STATUSES = ("proven", "composed", "related", "none", "skipped") #: 聚类最多取几个问题（同治理接口的上限） QUESTION_LIMIT = 500 @@ -126,7 +129,7 @@ def e1_of(contents: list[dict[str, Any]]) -> dict[str, Any]: else: unproven["参数配上了，但和口径不等价"] += 1 seen = {ASPECT_GROUP.get(str(item.get("aspect")), "other") - for item in (cand.get("diff") or {}).get("items") or []} + for item in (cand.get("diff") or {}).get("items") or [] if item.get("aspect") not in NOT_DIFFS} for group in seen: diffs[group] += 1 for report in (content.get("reports") or {}).values(): diff --git a/backend/tests/test_ledger_projection.py b/backend/tests/test_ledger_projection.py index b8b18351..4a5cb25c 100644 --- a/backend/tests/test_ledger_projection.py +++ b/backend/tests/test_ledger_projection.py @@ -6,6 +6,7 @@ - 不错给信任：多一个分组列、DISTINCT、按别的列排序再取前几行、HAVING 用到多出来的聚合的，一律仍然证明不了，结果和 不投影时一字不差； - 没投影的（报告没引用、两边都不用投影、投影不了）逐对和以前一字不差； +- 没证明的，差在哪按投影之后说：先说「这次多查了 N 列，不影响这个数」，再照常说条件上的不同； - 跑一次真的探索运行：报告引用销售额那一格，整题等价；E1 照实数出几份是投影之后才证明的。 """ from __future__ import annotations @@ -132,13 +133,35 @@ def test_without_a_projection_every_pair_is_exactly_the_old_result(): assert compare(sql, columns, cited) == compare(sql, columns, None), sql -def test_e1_counts_the_proofs_that_needed_a_projection(): +# -------------------------------------------------------------------------- +# 说得清：没证明的按投影之后说 +# -------------------------------------------------------------------------- + + +def test_an_unproven_projection_says_the_extra_columns_first_then_the_real_difference(): + sql = ("SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数, MIN(paid_at) AS 起, MAX(paid_at) AS 止 FROM orders " + "WHERE paid_at >= '2026-09-01' AND paid_at < '2026-09-08'") + row = compare(sql, ["销售额", "单数", "起", "止"], {"销售额"}) + assert row["status"] == "related" + [cand] = row["candidates"] + items = cand["diff"]["items"] + assert items[0] == {"aspect": "projection", "a": None, "b": None, "text": "这次多查了 3 列，不影响这个数"} + assert items[1]["aspect"] == "filter" and "status = 'paid'" in items[1]["text"] + assert cand["diff"]["fallback"] is False + assert cand["projection"]["dropped"] == 3 + # 不投影时整段退回兜底（真实验收里 E2 的兜底比例 45% 就是这么来的） + [old] = compare(sql, ["销售额", "单数", "起", "止"], None)["candidates"] + assert old["diff"]["fallback"] is True + + +def test_e1_does_not_count_the_extra_columns_note_as_a_difference_and_counts_projected_proofs(): proven = compare(f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW}", ["销售额", "单数"], {"销售额"}) related = compare("SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE paid_at >= '2026-09-01' " "AND paid_at < '2026-09-08'", ["销售额", "单数"], {"销售额"}) e1 = ledger_stats.e1_of([{"queries": [proven, related], "reports": {}, "whole": {}}]) assert e1["status"]["proven"] == 1 and e1["proven_projected"] == 1 + assert e1["diff_aspects"]["filter"] == 1 and e1["diff_aspects"]["other"] == 0 # -------------------------------------------------------------------------- @@ -194,6 +217,7 @@ async def test_reconcile_pairs_the_columns_left_after_the_projection(engine_up, [q] = body["queries"] [cand] = q["candidates"] assert cand["projection"]["ours"] == ["销售额"] and cand["projection"]["caliber"] == ["gmv"] + assert cand["diff"]["items"][0]["text"] == "这次多查了 2 列，不影响这个数" res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json={ "query": q["artifact"], "workflow_id": wf, "version": 1, "confirm": True, "inputs": {"start": "2026-09-01", "end": "2026-09-08"}}) diff --git a/backend/tests/test_sql_left_on_times.py b/backend/tests/test_sql_left_on_times.py new file mode 100644 index 00000000..93b5617c --- /dev/null +++ b/backend/tests/test_sql_left_on_times.py @@ -0,0 +1,82 @@ +"""日期条件写在 LEFT JOIN 的 ON 里时的差异说明（data/sql_semantics.py 的 _Explainer._left_on_times）。 + +以前两边来源一样、只差在一个 LEFT JOIN 的 ON 里的日期条件时，整段退回「两条 SQL 的写法不同，未能归纳成具体差异」。现在 +说得准的照实说（aspect 是 join：说的是关联条件，不会被当成业务日期写进数据目录），说不准的照实兜底。只改说明：判等价的 +结论一个不变；说出来的不同在虚构库上确实让结果不一样。 +""" +from __future__ import annotations + +import pytest + +from app.data.sql_semantics import explain_sql_diff +from tests.fixtures import sql_pairs + +SCHEMA = sql_pairs.schema_cache() +BASE = "SELECT u.city, SUM(o.amount) AS s FROM users u LEFT JOIN orders o ON o.user_id = u.id{on} GROUP BY u.city" + +SAID = [ + # 一边多一个日期条件 + (BASE.format(on=" AND o.paid_at >= '2026-09-10'"), BASE.format(on=""), + "这次用 LEFT JOIN 关联 orders 表时只关联 orders.paid_at >= '2026-09-10' 的行，口径没有这个条件"), + (BASE.format(on=""), BASE.format(on=" AND o.created_at < '2026-09-20'"), + "口径用 LEFT JOIN 关联 orders 表时只关联 orders.created_at < '2026-09-20' 的行，这次没有这个条件"), + # 按不同的时间列筛 + (BASE.format(on=" AND o.paid_at >= '2026-09-10'"), BASE.format(on=" AND o.created_at >= '2026-09-10'"), + "这次用 LEFT JOIN 关联 orders 表时按 paid_at 筛时间，口径按 created_at"), + # 同一列、范围不同 + (BASE.format(on=" AND o.paid_at >= '2026-09-10' AND o.paid_at < '2026-10-01'"), + BASE.format(on=" AND o.paid_at >= '2026-09-01' AND o.paid_at < '2026-10-01'"), + "这次用 LEFT JOIN 关联 orders 表时对 paid_at 的条件是 orders.paid_at >= '2026-09-10' AND orders.paid_at < " + "'2026-10-01'，口径是 orders.paid_at >= '2026-09-01' AND orders.paid_at < '2026-10-01'"), +] + + +@pytest.fixture(scope="module") +def databases(): + conns = sql_pairs.build_databases() + yield conns + for c in conns: + c.close() + + +@pytest.mark.parametrize("a, b, text", SAID, ids=["extra-on-ours", "extra-on-caliber", "other-column", "range"]) +@pytest.mark.parametrize("schema", [None, SCHEMA], ids=["bare", "schema"]) +def test_date_conditions_in_a_left_join_on_are_said(a, b, text, schema, databases): + diff = explain_sql_diff(a, b, "sqlite", schema=schema, labels=("这次", "口径")) + assert diff.equivalent is False and diff.fallback is False + assert [(i.aspect, i.text) for i in diff.items] == [("join", text)] + # 说出来的不同是真的：在虚构库的某组数据上两条结果确实不一样 + assert any(sql_pairs.run(c, a) != sql_pairs.run(c, b) for c in databases) + + +@pytest.mark.parametrize("a, b", [ + # 套着日期函数，按取值认不准两边是不是一样：照实兜底 + (BASE.format(on=" AND DATE(o.paid_at) >= '2026-09-10' AND DATE(o.paid_at) <= '2026-09-30'"), + BASE.format(on=" AND o.paid_at >= '2026-09-10' AND o.paid_at < '2026-10-01'")), + # 可空一侧在 WHERE 里用到了（反连接）：说的就不是时间窗口了 + ("SELECT u.city, COUNT(*) AS n FROM users u LEFT JOIN orders o ON o.user_id = u.id AND o.paid_at >= '2026-09-10' " + "WHERE o.id IS NULL GROUP BY u.city", + "SELECT u.city, COUNT(*) AS n FROM users u LEFT JOIN orders o ON o.user_id = u.id WHERE o.id IS NULL " + "GROUP BY u.city"), + # 结果里没用到可空一侧的列：ON 里的不同只影响行会不会重复，说不准 + ("SELECT u.city, COUNT(*) AS n FROM users u LEFT JOIN orders o ON o.user_id = u.id AND o.paid_at >= '2026-09-10' " + "GROUP BY u.city", + "SELECT u.city, COUNT(*) AS n FROM users u LEFT JOIN orders o ON o.user_id = u.id GROUP BY u.city"), + # ON 里多出来的不是时间条件 + (BASE.format(on=" AND o.status = 'paid'"), BASE.format(on="")), +], ids=["date-function", "anti-join", "not-in-output", "not-time"]) +def test_what_cannot_be_said_precisely_still_gets_the_fallback(a, b): + diff = explain_sql_diff(a, b, "sqlite", labels=("这次", "口径")) + assert diff.equivalent is False and diff.fallback is True + assert not [i for i in diff.items if i.aspect == "join"] + + +@pytest.mark.parametrize("a, b, same", [ + (BASE.format(on=" AND o.paid_at >= '2026-09-10'"), BASE.format(on=" AND o.paid_at >= '2026-09-10'"), True), + ("SELECT u.city, SUM(o.amount) AS s FROM users u LEFT JOIN orders o ON o.paid_at >= '2026-09-10' AND " + "o.user_id = u.id GROUP BY u.city", BASE.format(on=" AND o.paid_at >= '2026-09-10'"), True), + *[(a, b, False) for a, b, _ in SAID], +]) +def test_only_the_explanation_changed_not_the_verdict(a, b, same): + for schema in (None, SCHEMA): + assert explain_sql_diff(a, b, "sqlite", schema=schema).equivalent is same