diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index ffb7ae58..26962b35 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -8727,7 +8727,366 @@ def _column_map(sql: str, rendered: str, read: str | None, info: _Schema, return tuple({"index": i, "ours": a, "caliber": b} for i, (a, b) in enumerate(zip(ours, theirs))) +# ========================================================================== +# 按引用的列投影(next-round 口径账本:整题等价按答案引用的那几列比) +# ========================================================================== +# +# 真实的助手写 SQL 总会多选几列(订单数、日期常量、MIN / MAX、COALESCE……),报告只引用其中一列;口径的取数也只选它的 +# 指标要的那几列。拿整条 SQL 去比永远不等价,「上周的销售额是多少」就永远只标相关口径。这里把最外层 SELECT 列表里没被 +# 引用、去掉也不改变被引用那几列取值的选择项去掉(需要时也调换留下的几项的先后),得到一条改写后的 SQL,交给调用方再走 +# 现有的绑定和判等价(match_template → explain_sql_diff):投影本身不判等价、不给信任,判等价的那一步一点不放宽。 +# +# 只在确定去掉不影响时才投影,不然照实给不投影的原因(Projection.refused),调用方照旧整条比: +# - 整条语句就是一段 SELECT(可以带 WITH):UNION 这类合并、外面套着括号的不投影——各支的列按位置合并,去掉一支的一列 +# 就错位了; +# - SELECT 里只有 WITH、选择列表、FROM、JOIN、WHERE、GROUP BY、HAVING、QUALIFY、WINDOW、ORDER BY、LIMIT、OFFSET; +# DISTINCT / DISTINCT ON(按整行去重,去掉一列行数就可能变)和别的修饰(提示、INTO、加锁、采样、LATERAL VIEW……) +# 不投影; +# - 选择列表里没有 * / t.*(展开之后按位置对不准);给了查询结果的列数(width)时要和选择列表的项数一样:被引用的列按 +# 位置对上选择项; +# - ORDER BY / GROUP BY 里没有按位置引用的(ORDER BY 2:去掉、调换之后指到别的列); +# - 有 LIMIT / OFFSET 时只投影不分组的聚合(只出一行,取哪一行和选出来的列无关):别的「取前几行」有并列时取到哪几行 +# 可能跟着选择列表变; +# - 去掉的每一项:不含认不准会不会改变行数的函数(UNNEST / EXPLODE / generate_series 这类展开成多行的、sqlglot 不认识的 +# 函数);它的输出名(别名,没有别名的是列名)在整条语句别处(同一段的 WHERE、GROUP BY、HAVING、QUALIFY、ORDER BY、 +# 窗口、别的选择项、子查询)一次都没有被不带表名地写到——SQLite 的 WHERE、GROUP BY 都认列别名,和表列同名的也算, +# 宁可不投影;不是常量的话,同样的写法没有出现在 GROUP BY、HAVING、ORDER BY、QUALIFY、窗口里(「出现在 GROUP BY 里」 +# 「被 HAVING 用到」都在这一条); +# - 去掉之后这一段聚不聚合不变:没有 GROUP BY、HAVING 时去掉的不能是仅有的聚合(剩下的列就不止一行了);留下的列里没有 +# 既不在聚合里、也不是分组键的列(SQLite 取 MAX 那一行的值、MySQL 取任意一行:去掉别的聚合可能换了取值的那一行); +# - 调换先后时,留下的各项不按名字写到别的选择项的输出名(DuckDB 这类认「前面的列别名」)。 +# +# 改写在原文上做:按方言切词、按括号深度认出最外层 SELECT 列表每一项在原文里的起止,只把留下的几项按新的先后原样拼回去, +# 选择列表前后的原文一字不动(explain_sql_diff 的保真守卫照样看得到原文里的写法)。拼完重新解析,语法树要和「原来的树换成 +# 留下的几项」一模一样,不然不投影。口径模板({{ input.x }})按 _template_slots 的哨兵渲染之后认起止,再按位置换回模板 +# 的原文;MySQL 原文里有被当成两个减号的 --(_mysql_dashes 会改写的)不投影:改写后位置对不上原文。 + +#: 投影的那一段能有的子句 +_PROJECTABLE_ARGS = frozenset({"with_", "expressions", "from_", "joins", "where", "group", "having", "qualify", + "windows", "order", "limit", "offset"}) +#: 去掉的选择项里不许有的、可能把一行展开成多行的函数(sqlglot 的类名;表函数的基类 UDTF、不认识的函数另认) +_ROW_CHANGING = frozenset({"Explode", "Posexplode", "ExplodeOuter", "PosexplodeOuter", "Unnest", "Inline", "Stack", + "ExplodingGenerateSeries", "GenerateSeries"}) +#: 最外层 SELECT 列表到这些词就结束了 +_LIST_ENDS = _CLAUSE_ENDS | {"FROM", "INTO"} +_OPENS = frozenset({"L_PAREN", "L_BRACKET", "L_BRACE"}) +_CLOSES = frozenset({"R_PAREN", "R_BRACKET", "R_BRACE"}) + +#: 不投影的原因(只进比对的说明和测试,不判等价) +_PROJ_UNPARSED = "SQL 无法解析,不投影" +_PROJ_SETOP = "不是一段 SELECT(UNION 这类合并、外面套着括号),不投影" +_PROJ_DISTINCT = "有 DISTINCT,去掉一列行数就可能变,不投影" +_PROJ_CLAUSE = "SELECT 里有投影认不准的子句({clause}),不投影" +_PROJ_STAR = "选择列表里有 *,按位置对不准,不投影" +_PROJ_WIDTH = "查询结果的列数({width})和选择列表的项数({items})对不上,不投影" +_PROJ_KEEP = "要留下的列不对(空的、重复的、超出选择列表),不投影" +_PROJ_ORDINAL = "ORDER BY / GROUP BY 按位置引用了选择项,去掉、调换之后会指到别的列,不投影" +_PROJ_LIMIT = "有 LIMIT / OFFSET,取哪几行可能跟着选择列表变,不投影" +_PROJ_ROWS = "去掉的「{item}」里有可能改变行数的函数,不投影" +_PROJ_NAMED = "去掉的「{item}」在别处被按名字写到了,不投影" +_PROJ_USED = "去掉的「{item}」同样的写法出现在 GROUP BY、HAVING、ORDER BY、QUALIFY 或窗口里,不投影" +_PROJ_AGGREGATE = "去掉的是仅有的聚合,剩下的列就不止一行了,不投影" +_PROJ_LOOSE = "留下的列里有既不在聚合里、也不是分组键的列,去掉别的列可能换了取值的那一行,不投影" +_PROJ_REORDER = "留下的「{item}」按名字写到了别的选择项,不调换先后" +_PROJ_TEXT = "认不出选择列表在原文里的起止,不投影" +_PROJ_DASHES = "原文里有 MySQL 当成两个减号的 --,不投影" + + +@dataclass(frozen=True) +class Projection: + """project_select / project_template 的结果。 + + sql 是投影后的 SQL(模板投影之后仍是模板,{{ input.x }} 原样留着);kept 是留下的选择项在原来的选择列表里的位置 + (按投影后的先后);dropped 是去掉了几项;names 是留下的各项的输出名(别名、列名,别的写成它的 SQL,同 + _result_columns)。refused 不为空时不投影:sql 为 None,写为什么。要留下的就是全部、先后也没变时原样交回(dropped + 为 0)。""" + + sql: str | None + kept: tuple[int, ...] = () + dropped: int = 0 + names: tuple[str, ...] = () + refused: str | None = None + + +class _NotProjectable(Exception): + """不投影:args[0] 是原因。""" + + +def _output_name(item: Any, read: str | None) -> str: + from sqlglot import exp + + name = item.alias_or_name if isinstance(item, (exp.Alias, exp.Column)) else "" + return name or item.sql(dialect=read) + + +def _name_of(item: Any) -> str: + """选择项按名字被引用时的那个名字(别名、列名;小写)。表达式、常量没有。""" + from sqlglot import exp + + return (item.alias_or_name if isinstance(item, (exp.Alias, exp.Column)) else "").translate(_ASCII_LOWER) + + +def _ordinal(node: Any) -> bool: + """ORDER BY / GROUP BY 的一项是按位置引用(一个整数常量)。""" + from sqlglot import exp + + inner = _unparen(node.this if isinstance(node, exp.Ordered) else node) + return isinstance(inner, exp.Literal) and not inner.is_string + + +def _projection_problem(root: Any, keep: Sequence[int], read: str | None) -> str | None: + """按 keep 投影这一段会不会改变留下的那几列的取值(见本节开头):会、或者认不准时交回原因,确定不会时为 None。""" + from sqlglot import exp + + if not isinstance(root, exp.Select): + return _PROJ_SETOP + if root.args.get("distinct") is not None: + return _PROJ_DISTINCT + extra = sorted(k for k, v in root.args.items() if v not in (None, False, []) and k not in _PROJECTABLE_ARGS) + if extra: + return _PROJ_CLAUSE.format(clause=extra[0]) + items = list(root.expressions) + if any(_is_star(item) for item in items): + return _PROJ_STAR + if not keep or len(set(keep)) != len(keep) or any(not isinstance(k, int) or not 0 <= k < len(items) for k in keep): + return _PROJ_KEEP + group, order = root.args.get("group"), root.args.get("order") + for clause in (group, order): + if clause is not None and any(_ordinal(e) for e in clause.expressions): + return _PROJ_ORDINAL + if (root.args.get("limit") is not None or root.args.get("offset") is not None) and \ + not (group is None and _aggregating(root)): + return _PROJ_LIMIT + kept = set(keep) + removed = [i for i in range(len(items)) if i not in kept] + holders = [c for c in (group, root.args.get("having"), order, root.args.get("qualify")) if c is not None] + holders += [w for w in root.args.get("windows") or [] if isinstance(w, exp.Expr)] + holders += [w for k in keep for w in items[k].find_all(exp.Window)] + used = {_fingerprint(n) for holder in holders for n in holder.walk()} + for i in removed: + item = items[i] + shown = item.sql(dialect=read, comments=False)[:60] + if any(isinstance(n, (exp.UDTF, exp.Anonymous)) or type(n).__name__ in _ROW_CHANGING for n in item.walk()): + return _PROJ_ROWS.format(item=shown) + name = _name_of(item) + if name: + inside = {id(n) for n in item.walk()} + if any(isinstance(n, exp.Column) and not n.table and id(n) not in inside + and n.name.translate(_ASCII_LOWER) == name for n in root.walk()): + return _PROJ_NAMED.format(item=shown) + body = item.unalias() + if not _constant(body) and _fingerprint(body) in used: + return _PROJ_USED.format(item=shown) + projected = root.copy() + projected.set("expressions", [items[k].copy() for k in keep]) + if group is None and root.args.get("having") is None and _aggregating(root) != _aggregating(projected): + return _PROJ_AGGREGATE + if _loose_columns(projected): + return _PROJ_LOOSE + if list(keep) != sorted(keep): + names = {_name_of(item) for item in items} - {""} + for k in keep: + others = names - {_name_of(items[k])} + if any(isinstance(n, exp.Column) and not n.table and n.name.translate(_ASCII_LOWER) in others + for n in items[k].walk()): + return _PROJ_REORDER.format(item=items[k].sql(dialect=read, comments=False)[:60]) + return None + + +def _list_spans(text: str, read: str | None) -> list[tuple[int, int]] | None: + """最外层 SELECT 列表每一项在原文里的起止(含两头):按方言切词、按括号深度,深度 0 的 SELECT 之后、深度 0 的子句 + 开头的词(FROM、WHERE……)之前,深度 0 的逗号分开。深度 0 的 SELECT 不止一个、括号对不上、有空项的为 None。""" + try: + tokens = _tokens(text, read) + except Exception: # noqa: BLE001 - 切词都不行:认不出 + return None + depth = 0 + state = 0 # 0:还没到最外层的 SELECT;1:在选择列表里;2:选择列表之后 + spans: list[tuple[int, int]] = [] + start: int | None = None + end = 0 + for token in tokens: + kind = token.token_type.name + if state == 1 and depth == 0 and (kind == "COMMA" or kind in _LIST_ENDS): + if start is None: + return None + spans.append((start, end)) + start = None + if kind != "COMMA": + state = 2 + continue + if kind in _OPENS: + depth += 1 + elif kind in _CLOSES: + depth -= 1 + if depth < 0: + return None + elif depth == 0 and kind == "SELECT": + if state != 0: + return None + state = 1 + continue + if state == 1: + start = token.start if start is None else start + end = token.end + if state == 1: + if start is None: + return None + spans.append((start, end)) + return spans or None + + +def _cut(text: str, spans: Sequence[tuple[int, int]], keep: Sequence[int]) -> str: + """把选择列表换成留下的几项(按 keep 的先后,原文照抄、逗号隔开):选择列表前后的原文一字不动。""" + head, tail = text[:spans[0][0]], text[spans[-1][1] + 1:] + return head + ", ".join(text[spans[k][0]:spans[k][1] + 1] for k in keep) + tail + + +def _plan_projection(text: str, keep: Sequence[int], read: str | None, width: int | None + ) -> tuple[Any, list[tuple[int, int]] | None, Any]: + """(原文的语法树, 选择列表各项的起止, 投影后应有的语法树)。原样(keep 就是全部、先后不变)时后两样为 None。不投影抛 + _NotProjectable。""" + if read == "mysql" and _mysql_dashes(text) != text: + raise _NotProjectable(_PROJ_DASHES) + root, _ = _parse(text, read) + if root is None: + raise _NotProjectable(_PROJ_UNPARSED) + from sqlglot import exp + + items = list(root.expressions) if isinstance(root, exp.Select) else [] + if width is not None and isinstance(root, exp.Select) and len(items) != width: + raise _NotProjectable(_PROJ_WIDTH.format(width=width, items=len(items))) + if isinstance(root, exp.Select) and list(keep) == list(range(len(items))): + return root, None, None + why = _projection_problem(root, keep, read) + if why: + raise _NotProjectable(why) + spans = _list_spans(text, read) + if spans is None or len(spans) != len(items): + raise _NotProjectable(_PROJ_TEXT) + expected = root.copy() + expected.set("expressions", [items[k].copy() for k in keep]) + return root, spans, expected + + +def _same_tree(text: str, read: str | None, expected: Any) -> bool: + root, _ = _parse(text, read) + return root is not None and _fingerprint(root) == _fingerprint(expected) + + +def project_select(sql: str, keep: Sequence[int], dialect: str | None = None, *, + width: int | None = None) -> Projection: + """按引用的列投影一条查询(见本节开头):最外层选择列表只留 keep 那几项(位置从 0 数,按 keep 的先后)。从不抛异常。 + + width 是查询结果的列数(查询快照里记的):给了就要和选择列表的项数一样,被引用的列才按位置对得上选择项。不投影时 + refused 写为什么,调用方照旧整条比。投影出来的 SQL 不判等价:交给 match_template / explain_sql_diff 判。""" + read = _read(dialect) + text = (sql or "").strip() + keep = tuple(keep) + try: + root, spans, expected = _plan_projection(text, keep, read, width) + items = list(root.expressions) + names = tuple(_output_name(items[k], read) for k in keep) + if spans is None: + return Projection(text, keep, 0, names) + out = _cut(text, spans, keep) + if not _same_tree(out, read, expected): + raise _NotProjectable(_PROJ_TEXT) + return Projection(out, keep, len(items) - len(keep), names) + except _NotProjectable as e: + return Projection(None, keep, refused=str(e)) + except Exception: # noqa: BLE001 - 投影本身的缺陷不能连累调用方:当作不投影,照旧整条比 + logger.warning("SQL 投影失败", exc_info=True) + return Projection(None, keep, refused=_PROJ_TEXT) + + +def _template_marks(template: str, spans: Sequence[tuple[int, int, TemplateSlot]]) -> list[tuple[int, int, int, int]]: + """模板里每一处槽位在哨兵渲染的 SQL 里、在模板原文里的起止:[(哨兵起, 哨兵止, 模板起, 模板止)](止不含)。槽位之外 + 两边逐字相同(_template_slots 只换槽位)。""" + out: list[tuple[int, int, int, int]] = [] + shift = 0 + for start, end, slot in spans: + size = len(_STRING_SENTINEL.format(slot.index)) if slot.context == "quoted" else \ + len(str(_NUMBER_SENTINEL + slot.index)) if slot.context == "bare" else 0 + out.append((start - shift, start - shift + size, start, end)) + shift += (end - start) - size + return out + + +def _to_template(pos: int, marks: Sequence[tuple[int, int, int, int]], *, last: bool) -> int | None: + """哨兵 SQL 里的一个位置 → 模板原文里的位置。落在哨兵中间的认不出(选择项的起止只会在一个词的两头)。last:这是一项 + 的最后一个字(含),落在哨兵最后一个字上时对到槽位的最后一个字。""" + delta = 0 + for s0, s1, t0, t1 in marks: + if pos < s0: + break + if pos < s1: + if not last and pos == s0: + return t0 + if last and pos == s1 - 1: + return t1 - 1 + return None + delta = t1 - s1 + return pos + delta + + +def project_template(template_sql: str, keep: Sequence[int], dialect: str | None = None) -> Projection: + """按口径卡读到的列投影口径的取数模板({{ input.<字段> }} 原样留着,见本节开头)。规则同 project_select,在哨兵渲染 + 的 SQL 上判;起止按位置换回模板原文。从不抛异常。""" + read = _read(dialect) + template = (template_sql or "").strip() + keep = tuple(keep) + try: + slots, sentinel = _template_slots(template) + root, spans, expected = _plan_projection(sentinel, keep, read, None) + items = list(root.expressions) + names = tuple(_output_name(items[k], read) for k in keep) + if spans is None: + return Projection(template, keep, 0, names) + if not _same_tree(_cut(sentinel, spans, keep), read, expected): + raise _NotProjectable(_PROJ_TEXT) + marks = _template_marks(template, slots) + mapped = [(_to_template(s, marks, last=False), _to_template(e, marks, last=True)) for s, e in spans] + if any(s is None or e is None for s, e in mapped): + raise _NotProjectable(_PROJ_TEXT) + out = _cut(template, mapped, keep) # type: ignore[arg-type] + # 换回模板之后再按哨兵渲染一遍:结构(字面量只看位置,槽位的编号会因为去掉了几处而变)要和投影后的一样 + _, again = _template_slots(out) + root_again, _ = _parse(again, read) + lits_a: list[Any] = [] + lits_b: list[Any] = [] + if root_again is None or _erased(root_again, lits_a) != _erased(expected, lits_b) or len(lits_a) != len(lits_b): + raise _NotProjectable(_PROJ_TEXT) + return Projection(out, keep, len(items) - len(keep), names) + except _NotComparable as e: + return Projection(None, keep, refused=str(e)) + except _NotProjectable as e: + return Projection(None, keep, refused=str(e)) + except Exception: # noqa: BLE001 - 同 project_select + logger.warning("口径模板投影失败", exc_info=True) + return Projection(None, keep, refused=_PROJ_TEXT) + + +def output_names(sql: str, dialect: str | None = None, *, template: bool = False) -> list[str] | None: + """最外层选择列表各项的输出名(按位置;同 _result_columns,不展开 *:有 * 的为 None)。template 为真时 sql 是口径 + 模板,按哨兵渲染之后认。解析不了、不是一段 SELECT 的为 None。""" + from sqlglot import exp + + read = _read(dialect) + text = (sql or "").strip() + try: + if template: + _, text = _template_slots(text) + root, _ = _parse(_mysql_dashes(text) if read == "mysql" else text, read) + except _NotComparable: + return None + if not isinstance(root, exp.Select) or any(_is_star(i) for i in root.expressions): + return None + return [_output_name(i, read) for i in root.expressions] + + __all__ = ["BINDING_LEVELS", "MAX_SQL_CHARS", "PLACEHOLDER", "Aggregate", "ColumnRef", "Computed", "FacetDiff", "Facets", - "JoinFacet", "LimitFacet", "NormalizedSql", "Part", "SqlDiff", "TemplateBinding", "TemplateMatch", - "TemplateSlot", "TimePredicate", "bind_template", "catalog_labels", "diff_facets", "explain_sql_diff", - "facets", "match_template", "normalize_sql", "render_bound"] + "JoinFacet", "LimitFacet", "NormalizedSql", "Part", "Projection", "SqlDiff", "TemplateBinding", + "TemplateMatch", "TemplateSlot", "TimePredicate", "bind_template", "catalog_labels", "diff_facets", + "explain_sql_diff", "facets", "match_template", "normalize_sql", "output_names", "project_select", + "project_template", "render_bound"] diff --git a/backend/app/ledger/labels.py b/backend/app/ledger/labels.py index 08a23cac..5ded531b 100644 --- a/backend/app/ledger/labels.py +++ b/backend/app/ledger/labels.py @@ -162,6 +162,78 @@ def card_expressions(cards: list[Mapping[str, Any]]) -> list[tuple[str, str, str return out +def card_reads(cards: list[Mapping[str, Any]]) -> list[tuple[str, str, tuple[tuple[str, str], ...]]]: + """口径版本里每张源头口径卡的每个指标读了哪几格的列:(卡的节点 id, 指标 id, ((取数节点, 列), …)),按式子里出现的先后、 + 去重。只认 cell(nodes.X, 行, '列') 这种输入(同 card_expressions);有别的写法的指标不收。比对时口径一侧按它投影到 + 「这个指标引用的列」(ledger/match 的 _projected)。""" + import ast + + from app.engine.expressions import ExpressionError, cell_parts, leaf_refs, parse_expression + + out: list[tuple[str, str, tuple[tuple[str, str], ...]]] = [] + for card in cards: + for metric in card.get("metrics") or []: + try: + tree = parse_expression(str((metric or {}).get("expression") or ""))[0] + except ExpressionError: + continue + reads: list[tuple[str, str]] = [] + for leaf in leaf_refs(tree): + parts = cell_parts(leaf) + if parts is None or not parts[0].startswith("nodes."): + break + try: + column = ast.literal_eval(parts[2]) + except (ValueError, SyntaxError): + break + if not isinstance(column, str): + break + read = (parts[0][len("nodes."):], column) + if read not in reads: + reads.append(read) + else: + if reads: + out.append((str(card.get("node_id")), str(metric.get("id")), tuple(reads))) + return out + + +def referenced_columns(docs: list[Mapping[str, Any]], + load_card: Callable[[str], Mapping[str, Any] | None]) -> dict[str, set[str]]: + """报告里引用了哪条查询的哪几列:{查询快照: {列名}}。和 label_doc 标的是同一批数——单元格([[v:]]、[[table:]] 里的格) + 按证据目录找到的快照和列;指标([[m:]])按指标集里这个指标的输入(落在查询快照一格上的)。比对时按它投影:只比被 + 引用的那几列(ledger/match 的 _projected)。""" + from app.engine.evidence import iter_units + + out: dict[str, set[str]] = {} + cards: dict[str, Mapping[str, Any] | None] = {} + for doc in docs: + catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {} + for _, unit in iter_units(doc): + for seg in unit.get("segments") or []: + if not isinstance(seg, dict) or seg.get("kind") not in ("number", "value") or not seg.get("id"): + continue + cite = seg.get("cite") if isinstance(seg.get("cite"), dict) else {} + if cite.get("status") != "resolved": + continue + entry = catalog.get(cite.get("alias")) if isinstance(catalog.get(cite.get("alias")), dict) else {} + locator = cite.get("locator") if isinstance(cite.get("locator"), dict) else {} + artifact = str(entry.get("artifact") or "") + if cite.get("kind") == "cell" and artifact and isinstance(locator.get("column"), str): + out.setdefault(artifact, set()).add(locator["column"]) + elif cite.get("kind") == "metric" and artifact: + if artifact not in cards: + cards[artifact] = load_card(artifact) + card = cards[artifact] or {} + metric = next((m for m in card.get("metrics") or [] + if isinstance(m, dict) and m.get("id") == locator.get("metric")), None) + for item in (metric or {}).get("inputs") or []: + where = item.get("locator") if isinstance(item, dict) else None + if isinstance(item, dict) and item.get("artifact") and item.get("via") in _SNAPSHOT_VIAS \ + and isinstance(where, dict) and isinstance(where.get("column"), str): + out.setdefault(str(item["artifact"]), set()).add(where["column"]) + return out + + def metric_label(card: Mapping[str, Any] | None, metric_id: Any, queries: Mapping[str, Mapping[str, Any]], calibers: Mapping[str, Mapping[str, Any]]) -> dict[str, Any]: """指标的标签(见模块说明)。card 是封存的指标集工件(读不出为 None);calibers 是 {「工作流@版本」: {level, cards}}, @@ -359,5 +431,5 @@ def related_view(item: Mapping[str, Any], *, hide: Callable[[Any], bool], shown: __all__ = ["CERTIFIED_NOTE", "HOWS", "KINDS", "LEVEL_TEXT", "POST_SEAL_NOTE", "RETRO_NOTE", "binding_view", "caliber_key", - "card_expressions", "cell_label", "diff_view", "label_doc", "metric_label", "related_view", "view_label", - "whole_of"] + "card_expressions", "card_reads", "cell_label", "diff_view", "label_doc", "metric_label", "referenced_columns", + "related_view", "view_label", "whole_of"] diff --git a/backend/app/ledger/match.py b/backend/app/ledger/match.py index 85804248..dbb774de 100644 --- a/backend/app/ledger/match.py +++ b/backend/app/ledger/match.py @@ -11,6 +11,20 @@ MAX_CANDIDATES 份(按工作流、版本、节点排好的先后)。 **怎么比。** data/sql_semantics.match_template:先绑定参数(只是启发),再按绑定渲染口径的取数 SQL 交 explain_sql_diff 证明。 +**按引用的列投影**(_projected,algo 3)。助手写的 SQL 总会多选几列(订单数、日期常量、MIN / MAX……),口径的取数也只选 +它的指标要的那几列:整条比永远不等价。整条比不等价时,答案一侧按报告引用了这条查询的哪几列(labels.referenced_columns: +单元格的列、指标输入落在的列)投影,口径一侧按口径卡上每个指标读了这个取数节点的哪几列(labels.card_reads)投影 +(sql_semantics.project_select / project_template:去掉没被引用、去掉也不改变被引用那几列取值的选择项,认不准的一律 +不投影),投影后的两条照样交 match_template 绑定、判等价——不另开一条给信任的路,判等价那一步一点不放宽。 + +被引用的列两边怎么对上:答案一侧按列名在查询快照记的结果列(columns)里找位置,结果列按位置对上选择列表的各项(列数 +和选择列表的项数要一样);口径一侧按口径卡 cell(nodes.取数, 行, '列') 里写的列名,在取数 SQL 选择列表的输出名(别名, +没有别名的是列名,不分大小写,同名的不止一项时不投影)里找位置。投影后两边留下的几列按位置配对(和整条比时的列映射一样: +外层别名不计、列的先后计),两边列数一样才比;答案一侧留下两三列时另试调换先后。每份口径取数最多试 MAX_PROJECTIONS 种。 +证明了的候选记 projection {ours, dropped, caliber, caliber_dropped};没证明的给第一种投影的比较(差在哪是投影之后的说法)。 +没有投影可试的(报告没引用这条查询、两边都不投影、 +投影不了)照旧是整条比的结果,逐对一字不差。 + **助手选用的(composed)。** 问数据的助手 use_caliber 选用口径时,取数节点是服务端照那一版拷进来的(ledger/compose.py), 口径卡的 caliber_from 带着改名映射 {原 id: 新 id}。这几个节点交回的查询照样要证明:证明了的那一份记 how=composed,查询 状态记 composed——数确实来自口径,但「这份口径答的就是这个问题」是助手凭问法判断的,不能说成「等价已证明」(拍板 L1: @@ -40,6 +54,7 @@ import logging import time import weakref from concurrent.futures import ThreadPoolExecutor +from collections.abc import Collection, Mapping from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any @@ -62,6 +77,8 @@ CONCURRENCY = 2 GRACE_S = 2.0 #: 每条查询最多比几份口径 MAX_CANDIDATES = 8 +#: 按引用的列投影之后,每份口径取数最多再试几种比法(答案一侧留下的列的先后 × 口径一侧的列组,见 _projected) +MAX_PROJECTIONS = 12 #: 启动时补比最近几天、最多几个 BACKFILL_DAYS = 7 BACKFILL_LIMIT = 200 @@ -124,7 +141,8 @@ class Fetch: @dataclass(frozen=True) class Caliber: """一个工作流最新的已发布、未退役版本(只收里面有查库节点的)。fields 是入口节点的字段原样(取缺省值用,不进工件), - cards 是源头口径卡的节点 id,expressions 是这些卡上每个指标换成记号后的式子(labels.card_expressions)。""" + cards 是源头口径卡的节点 id,expressions 是这些卡上每个指标换成记号后的式子(labels.card_expressions),reads 是每个 + 指标读了哪几个取数节点的哪几列(labels.card_reads:口径一侧按它投影,不进工件)。""" workflow_id: str version: int @@ -134,6 +152,7 @@ class Caliber: fields: tuple[dict[str, Any], ...] = () cards: tuple[str, ...] = () expressions: tuple[tuple[str, str, str], ...] = () + reads: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...] = () @property def key(self) -> str: @@ -201,7 +220,8 @@ def caliber_of(workflow_id: str, row: Any) -> Caliber | None: return Caliber(workflow_id=workflow_id, version=int(row.version), level=str(getattr(row, "level", None) or ""), graph_hash=getattr(row, "graph_hash", None), fetches=tuple(fetches), fields=tuple(fields), cards=tuple(c["node_id"] for c in cards), - expressions=tuple(ledger_labels.card_expressions(cards))) + expressions=tuple(ledger_labels.card_expressions(cards)), + reads=tuple(ledger_labels.card_reads(cards))) async def load_calibers(session: Any) -> list[Caliber]: @@ -331,12 +351,104 @@ def composed_nodes(graph: Any) -> dict[str, tuple[str, str]]: return out +def _wanted(query: Query, referenced: Mapping[str, Collection[str]] | None) -> tuple[int, ...] | None: + """报告引用了这条查询的哪几列:按列名在查询快照记的结果列里找位置(按位置先后,同名的几列都算)。没引用、快照没记 + 结果列的为 None(不投影)。""" + names = (referenced or {}).get(query.artifact) + if not names or query.columns is None: + return None + return tuple(i for i, c in enumerate(query.columns) if c in names) or None + + +def _caliber_sets(caliber: Caliber, fetch: Fetch, names: list[str]) -> list[tuple[int, ...]]: + """口径一侧按哪几组列投影:口径卡上每个指标读了这个取数节点的哪几列(labels.card_reads),按列名(不分大小写)在取数 + SQL 的输出名里找位置、按位置排好;最后加上所有指标读到的列合在一起的那一组。找不到、同名的不止一项的那一组不要。""" + lowered = [n.lower() for n in names] + out: list[tuple[int, ...]] = [] + union: set[int] = set() + for _, _, reads in caliber.reads: + columns = [column for node, column in reads if node == fetch.node_id] + if not columns: + continue + found = [[i for i, n in enumerate(lowered) if n == c.lower()] for c in columns] + if any(len(f) != 1 for f in found): + continue + group = tuple(sorted({f[0] for f in found})) + union |= set(group) + if group not in out: + out.append(group) + whole = tuple(sorted(union)) + if whole and whole not in out: + out.append(whole) + return out + + +def _projection_view(query: Query, ours: Any, theirs: Any) -> dict[str, Any]: + """候选上记的「按引用的列比的」:答案留下的列(快照里的列名)、去掉了几列,口径留下的列、去掉了几列。""" + return {"ours": [str((query.columns or [])[i]) for i in ours.kept], "dropped": ours.dropped, + "caliber": list(theirs.names), "caliber_dropped": theirs.dropped} + + +def _projected(caliber: Caliber, fetch: Fetch, query: Query, wanted: tuple[int, ...], + deadline: float | None) -> dict[str, Any] | None: + """整条比不等价时按引用的列投影再比(见模块说明):证明了的候选(带 projection);都没证明时第一种投影的那次比较; + 没有投影可试的为 None(照旧用整条比的结果)。""" + from itertools import permutations + + from app.data.sql_semantics import match_template, output_names, project_select, project_template + + width = len(query.columns or []) + whole = project_select(query.sql, tuple(range(width)), query.dialect, width=width) + if whole.sql is None: + return None + ours = [] + if len(wanted) < width: + cut = project_select(query.sql, wanted, query.dialect, width=width) + if cut.sql is not None: + ours.append(cut) + ours = ours or [whole] + for base in list(ours): + if 2 <= len(base.kept) <= 3: + for order in permutations(base.kept): + if order != base.kept and (p := project_select(query.sql, order, query.dialect, width=width)).sql: + ours.append(p) + names = output_names(fetch.sql, query.dialect, template=True) + if names is None: + return None + theirs = [p for group in _caliber_sets(caliber, fetch, names) if len(group) < len(names) + for p in [project_template(fetch.sql, group, query.dialect)] if p.sql is not None] + theirs.append(project_template(fetch.sql, tuple(range(len(names))), query.dialect)) + first: tuple[Any, Any, Any] | None = None + tries = 0 + for a in ours: + for c in theirs: + if c.sql is None or len(a.kept) != len(c.kept) or (not a.dropped and not c.dropped + and a.kept == whole.kept): + continue # 两边都没投影:就是整条比过的那一次 + if tries >= MAX_PROJECTIONS or (deadline is not None and time.monotonic() > deadline): + break + tries += 1 + m = match_template(c.sql, list(caliber.fields), a.sql, query.dialect, schema=query.schema, + names=query.names, columns=[str((query.columns or [])[i]) for i in a.kept], + deadline=deadline) + if m.equivalent: + return {**_candidate(caliber, fetch, m), "projection": _projection_view(query, a, c)} + if first is None: + first = (a, c, m) + if first is None: + return None + a, c, m = first + return {**_candidate(caliber, fetch, m), "projection": _projection_view(query, a, c)} + + def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: float | None = None, - composed: dict[str, tuple[str, str]] | None = None) -> list[dict[str, Any]]: + composed: dict[str, tuple[str, str]] | None = None, + referenced: Mapping[str, Collection[str]] | None = None) -> list[dict[str, Any]]: """每条查询和预筛出来的口径取数逐个比(match_template),给出 [{artifact, node_id, source, tables, status, reason?, candidates}]。status:proven(和某一份等价)/ composed(助手选用的那一份,证明了)/ related(有同源同表的口径,都不 等价)/ none(没有相关的口径)/ skipped(没比完:超时、解析失败、快照读不出)。composed 是 composed_nodes 的结果, - 不给就没有助手选用的。纯计算,从不抛异常。评测逐题比 SQL 也调它。""" + 不给就没有助手选用的。referenced 是报告引用了哪条查询的哪几列(labels.referenced_columns:{查询快照: {列名}}): + 给了的,整条比不等价时按引用的列投影再比(_projected);不给就只整条比。纯计算,从不抛异常。评测逐题比 SQL 也调它。""" from app.data.sql_semantics import match_template, normalize_sql out: list[dict[str, Any]] = [] @@ -347,6 +459,7 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: out.append({**row, "status": "skipped", "reason": query.problem, "candidates": []}) continue chosen = (composed or {}).get(str(query.node_id or "")) + wanted = _wanted(query, referenced) pool = [(c, f) for c in calibers for f in c.fetches if f.source == query.source and f.tables & query.tables] if chosen is not None: # 助手选用的那一份排最前面:候选多的时候也一定比到它 @@ -370,6 +483,11 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: candidates.append(_skipped(caliber, fetch, "比对出错")) continue cand = _candidate(caliber, fetch, m) + if not m.equivalent and m.comparable and wanted is not None: + try: + cand = _projected(caliber, fetch, query, wanted, deadline) or cand + except Exception: # noqa: BLE001 - 投影再比是附加的一步:出错照旧用整条比的结果 + logger.exception("按引用的列投影再比出错(运行查询 %s,口径 %s)", query.artifact[:12], caliber.key) if chosen is not None and (caliber.key, fetch.node_id) == chosen: if cand["status"] == "proven": cand["how"] = "composed" @@ -396,20 +514,6 @@ def compute(sealed: Any, calibers: list[Caliber], kinds: dict[str, str | None], deadline = time.monotonic() + timeout queries = read_queries(sealed, kinds) - if not sealed.trusted: - results = [{"artifact": q.artifact, "node_id": q.node_id, "source": q.source, "tables": sorted(q.tables), - "status": "skipped", "reason": SEAL_BROKEN, "candidates": []} for q in queries] - else: - results = compare_queries(queries, calibers, deadline=deadline, composed=composed_nodes(sealed.graph)) - used = {c["caliber"] for q in results for c in q["candidates"]} - considered = [c for c in calibers if c.key in used] - by_key = {c.key: {"level": c.level, "fields": c.summary()["fields"], "expressions": list(c.expressions)} - for c in considered} - by_artifact = {q["artifact"]: q for q in results} - # 合并查询的快照不比(read_queries 跳过了它),引用了它的格照实写为什么没比对;不进工件的 queries,统计不数它 - for artifact in sealed.queries: - by_artifact.setdefault(artifact, {"artifact": artifact, "status": "skipped", "reason": MERGED, - "candidates": []}) sealed_cards = {artifact for _, artifact in sealed.ledger} def load_card(artifact: str) -> dict[str, Any] | None: @@ -421,12 +525,28 @@ def compute(sealed: Any, calibers: list[Caliber], kinds: dict[str, str | None], return None return card if isinstance(card, dict) else None + found = _reports(sealed) + readable = [report for report in found if report.doc is not None and report.hash_ok is True] + if not sealed.trusted: + results = [{"artifact": q.artifact, "node_id": q.node_id, "source": q.source, "tables": sorted(q.tables), + "status": "skipped", "reason": SEAL_BROKEN, "candidates": []} for q in queries] + else: + referenced = ledger_labels.referenced_columns([report.doc for report in readable], load_card) + results = compare_queries(queries, calibers, deadline=deadline, composed=composed_nodes(sealed.graph), + referenced=referenced) + used = {c["caliber"] for q in results for c in q["candidates"]} + considered = [c for c in calibers if c.key in used] + by_key = {c.key: {"level": c.level, "fields": c.summary()["fields"], "expressions": list(c.expressions)} + for c in considered} + by_artifact = {q["artifact"]: q for q in results} + # 合并查询的快照不比(read_queries 跳过了它),引用了它的格照实写为什么没比对;不进工件的 queries,统计不数它 + for artifact in sealed.queries: + by_artifact.setdefault(artifact, {"artifact": artifact, "status": "skipped", "reason": MERGED, + "candidates": []}) reports: dict[str, Any] = {} primary: str | None = None - found = _reports(sealed) - for report in found: - if report.doc is not None and report.hash_ok is True: - reports[report.node_id] = {"labels": ledger_labels.label_doc(report.doc, by_artifact, by_key, load_card)} + for report in readable: + reports[report.node_id] = {"labels": ledger_labels.label_doc(report.doc, by_artifact, by_key, load_card)} if found: primary = _choose(found, None, sealed).node_id whole = ledger_labels.whole_of((reports.get(primary) or {}).get("labels") or {}, by_key) if primary in reports \ @@ -617,6 +737,7 @@ async def shutdown() -> None: await asyncio.gather(*tasks, return_exceptions=True) -__all__ = ["BACKFILL_DAYS", "CONCURRENCY", "EXPLORE_ORIGINS", "TIMEOUT_S", "Caliber", "Fetch", "Query", "backfill", - "caliber_of", "compare_queries", "composed_nodes", "compute", "drain", "ineligible", "load_calibers", - "match_run", "pending", "read_queries", "schedule", "shutdown", "start_backfill", "wanted"] +__all__ = ["BACKFILL_DAYS", "CONCURRENCY", "EXPLORE_ORIGINS", "MAX_PROJECTIONS", "TIMEOUT_S", "Caliber", + "Fetch", "Query", "backfill", "caliber_of", "compare_queries", "composed_nodes", "compute", "drain", + "ineligible", "load_calibers", "match_run", "pending", "read_queries", "schedule", "shutdown", + "start_backfill", "wanted"] diff --git a/backend/app/ledger/reconcile.py b/backend/app/ledger/reconcile.py index dc997743..db2edfff 100644 --- a/backend/app/ledger/reconcile.py +++ b/backend/app/ledger/reconcile.py @@ -16,7 +16,8 @@ 的字段校验,ledger/compose.check_inputs 同一套),界面写「参数按这次的 SQL 推断,请核对」。 **列怎么配。** 只有一行结果时:口径卡上的指标式就是取数第一行某一列(cell(nodes.取数, 0, '列'))的,按那一列在口径取数 -结果里的位置配这次结果同一位置的列(等价时已实测列的先后计;不等价的两条 SQL 选择列表一样时位置照样对得上)。差异 +结果里的位置配这次结果同一位置的列(等价时已实测列的先后计;不等价的两条 SQL 选择列表一样时位置照样对得上)。比对时 +按引用的列投影过的(候选的 projection,ledger/match 的 _projected),按投影之后两边留下的列逐个配(这次多选的列不算)。差异 有兜底说明(fallback:两条 SQL 对不上结构)、列数不同、一个都配不上时不猜,要人选(pairs:{ours: 这次的列, metric: 口径卡 上的指标 id}),这个选择记进对账运行的 origin。 @@ -327,8 +328,9 @@ def _plain_cell(expression: str, fetch: str) -> str | None: def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str], caliber_columns: list[str] | None, metrics: list[dict[str, Any]], fetch: str, card: str, fallback: bool, chosen: Any, - referenced: list[tuple[int, str]]) -> Plan: - """配列、(多行时)定行和键:交回 Plan,配不了的抛 ReconcileRefused(见模块说明)。纯计算。""" + referenced: list[tuple[int, str]], projected: Any = None) -> Plan: + """配列、(多行时)定行和键:交回 Plan,配不了的抛 ReconcileRefused(见模块说明)。projected 是比对候选的 projection + (按引用的列投影过的:{ours: [这次留下的列], caliber: [口径留下的列]},按位置一一对应)。纯计算。""" def check_column(column: str) -> None: if column not in ours_columns: raise ReconcileRefused(422, PAIR_COLUMN.format(column=column)) @@ -360,7 +362,17 @@ def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str if fallback: raise ReconcileRefused(409, PAIRS_FALLBACK, NEEDS_PAIRS, extra=options) pairs = [] - if caliber_columns is not None and len(caliber_columns) == len(ours_columns): + kept = _projected_pairs(projected, ours_columns) + if kept is not None: + # 按投影之后留下的列配:这次多选的列不算,口径一侧只看投影留下的那几列(列名不分大小写) + for index, m in enumerate(metrics): + column = _plain_cell(str(m.get("expression") or ""), fetch) + ours = next((o for o, c in kept if column is not None and c.lower() == column.lower()), None) + if ours is None or ours.lower() in hidden: + continue + pairs.append(Pair(ours=ours, metric=str(m.get("id")), index=index, name=str(m.get("name") or m.get("id")), + unit=str(m.get("unit") or ""), decimals=m.get("decimals"))) + elif caliber_columns is not None and len(caliber_columns) == len(ours_columns): for index, m in enumerate(metrics): column = _plain_cell(str(m.get("expression") or ""), fetch) if column is None or column not in caliber_columns: @@ -375,6 +387,19 @@ def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str return Plan(pairs=pairs) +def _projected_pairs(projected: Any, ours_columns: list[str]) -> list[tuple[str, str]] | None: + """比对候选的 projection → [(这次的列, 口径的列)],按位置一一对应;不是投影过的、两边列数不一样、这次的列不在结果里的 + 为 None(照旧按整条的位置配)。""" + if not isinstance(projected, dict): + return None + ours, theirs = projected.get("ours"), projected.get("caliber") + if not isinstance(ours, list) or not isinstance(theirs, list) or not ours or len(ours) != len(theirs): + return None + if any(not isinstance(o, str) or o not in ours_columns or not isinstance(c, str) for o, c in zip(ours, theirs)): + return None + return list(zip(ours, theirs)) + + def _plan_rows(ours_columns: list[str], rows: list[Any], hidden: set[str], caliber_columns: list[str] | None, referenced: list[tuple[int, str]], check_column: Any) -> Plan: if caliber_columns is None: @@ -536,7 +561,7 @@ async def reconcile(run_id: str, req: Request, *, principal: Any) -> Run: caliber_columns=_caliber_columns(template, inputs, list(pinned.entry_fields), kind, schema), metrics=pinned.metrics, fetch=str(fetch["id"]), card=pinned.card_title, fallback=bool((cand.get("diff") or {}).get("fallback")), chosen=req.pairs, - referenced=_referenced(sealed, req.query)) + referenced=_referenced(sealed, req.query), projected=cand.get("projection")) graph = build(pinned, sql=snap["sql"], tool=f"{QUERY_PREFIX}{source}", plan=plan, inputs=inputs, fetch=str(fetch["id"])) diff --git a/backend/app/ledger/stats.py b/backend/app/ledger/stats.py index 64f37605..ae2f68e2 100644 --- a/backend/app/ledger/stats.py +++ b/backend/app/ledger/stats.py @@ -8,8 +8,9 @@ 这时不再按来源和计入统计筛——调用方点名的就是要算的)。 - E1 等价命中:比对过的查询里 proven / composed / related / none / skipped 各几条(skipped 单独列,不混进 none),有候选口径 - 的查询里各自的比率;证明等价的绑定是在哪一级配上的;没证明的原因(不可比、参数绑不上、配上了但不等价);不等价的差异类型 - 分布(time / filter / aggregate / join / other / parse,每份口径的差异每类只计一次);逐个数的标签各几个、整题等价几次。 + 的查询里各自的比率;证明等价的绑定是在哪一级配上的,其中几份是按引用的列投影之后才证明的(proven_projected,ledger/match + 的 _projected);没证明的原因(不可比、参数绑不上、配上了但不等价);不等价的差异类型分布(time / filter / aggregate / + join / other / parse,每份口径的差异每类只计一次);逐个数的标签各几个、整题等价几次。 - E2 对账:对账运行(origin=reconcile,第 3 波起由 ledger/reconcile 发起)发起、完成、失败各几次,对过账的答案几个,其中参数 经人核对过的、对照的指标由人选的各几次;相关口径的差异说明里兜底说明的占比;对账结果上「说明有用 / 看不懂」的反馈 (第 4 波,ledger/feedback:对账运行上的封存后事件 ledger.feedback,每人只算最后一次)各几个、有用的比率。 @@ -98,6 +99,7 @@ def e1_of(contents: list[dict[str, Any]]) -> dict[str, Any]: status = Counter({s: 0 for s in STATUSES}) with_candidates = Counter({s: 0 for s in STATUSES}) via: Counter[str] = Counter() + projected = 0 unproven: Counter[str] = Counter() diffs = Counter({g: 0 for g in DIFF_GROUPS}) labels = Counter({k: 0 for k in ledger_labels.KINDS}) @@ -113,6 +115,7 @@ def e1_of(contents: list[dict[str, Any]]) -> dict[str, Any]: for cand in query.get("candidates") or []: if cand.get("status") == "proven": via[str(cand.get("via") or "parsed")] += 1 + projected += isinstance(cand.get("projection"), dict) continue if cand.get("status") == "skipped": continue @@ -136,7 +139,8 @@ def e1_of(contents: list[dict[str, Any]]) -> dict[str, Any]: "runs": len(contents), "queries": queries, "status": dict(status), "with_candidates": candidates, "rates": {s: _ratio(with_candidates[s], candidates) for s in STATUSES if s != "none"}, - "proven_via": dict(via), "unproven_reasons": dict(unproven.most_common(20)), "diff_aspects": dict(diffs), + "proven_via": dict(via), "proven_projected": projected, "unproven_reasons": dict(unproven.most_common(20)), + "diff_aspects": dict(diffs), "labels": dict(labels), "whole_equivalent": whole, } diff --git a/backend/app/ledger/store.py b/backend/app/ledger/store.py index 6ccb7f39..abe1717c 100644 --- a/backend/app/ledger/store.py +++ b/backend/app/ledger/store.py @@ -31,7 +31,8 @@ FORMAT = "agentlab.caliber_match/1" #: 比对算法的版本:绑定、证明、标签的规则改了就加一,界面和统计据此分得清是哪一版算法比的。 #: 2(第 3 波):认出助手选用的口径(查询 composed、标签 how=composed),指标的标签记它取自哪几条查询(queries,认证队列 #: 认「被报告引用的查询」用) -ALGO = "2" +#: 3:整条比不等价时按报告引用的列投影再比(ledger/match 的 _projected),候选记 projection;没投影的照旧 +ALGO = "3" #: 触发方式(见模块说明)。certified(第 3 波):从问数据固化出的口径发布之后,对来源运行补比的(ledger/certify) TRIGGERS = ("sealed", "backfill", "on_demand", "certified") diff --git a/backend/app/ledger/view.py b/backend/app/ledger/view.py index ce88b960..6ea951d5 100644 --- a/backend/app/ledger/view.py +++ b/backend/app/ledger/view.py @@ -88,6 +88,9 @@ def _query_view(query: dict[str, Any], hider: _Hide, names: dict[str, str]) -> d item["how"] = cand["how"] if cand.get("columns") is not None: item["columns"] = cand["columns"] + if isinstance(cand.get("projection"), dict): + # 按引用的列投影之后比的(ledger/match 的 _projected):两边各留下了哪几列、去掉了几列(列名,不带值) + item["projection"] = cand["projection"] cands.append(item) out["candidates"] = cands return out diff --git a/backend/tests/test_ledger_compose.py b/backend/tests/test_ledger_compose.py index 0c3405b0..7dce61e4 100644 --- a/backend/tests/test_ledger_compose.py +++ b/backend/tests/test_ledger_compose.py @@ -253,7 +253,7 @@ async def test_an_answer_built_with_use_caliber_is_labelled_composed(client, mon await ledger_match.drain() recorded = await store.latest(row.id) content = recorded.content - assert content["algo"] == store.ALGO == "2" + assert content["algo"] == store.ALGO == "3" [query] = content["queries"] assert query["status"] == "composed" and query["node_id"] == "c1_fetch" [cand] = query["candidates"] diff --git a/backend/tests/test_ledger_projection.py b/backend/tests/test_ledger_projection.py new file mode 100644 index 00000000..b8b18351 --- /dev/null +++ b/backend/tests/test_ledger_projection.py @@ -0,0 +1,203 @@ +"""认口径时按引用的列投影(app/ledger/match.py 的 _projected,algo 3)。 + +真实助手的 SQL 总会多选几列(订单数、日期常量、MIN / MAX……),整条比永远不等价,「上周的销售额是多少」从不自动出具。 +整条比不等价时,答案一侧按报告引用的列、口径一侧按口径卡上指标读的列投影,再交给现有的绑定和判等价。这里守住: +- 多选一列、多几个聚合、日期常量列的,按引用的那一列证明等价;口径一侧多选了列的同样认得出;留下两列先后不同的另试调换; +- 不错给信任:多一个分组列、DISTINCT、按别的列排序再取前几行、HAVING 用到多出来的聚合的,一律仍然证明不了,结果和 + 不投影时一字不差; +- 没投影的(报告没引用、两边都不用投影、投影不了)逐对和以前一字不差; +- 跑一次真的探索运行:报告引用销售额那一格,整题等价;E1 照实数出几份是投影之后才证明的。 +""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from app.engine.runner import run_manager +from app.ledger import match as ledger_match +from app.ledger import stats as ledger_stats +from app.ledger import store +from tests import ledger_samples as S + +WINDOW = "status = 'paid' AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'" + + +def caliber(sql: str = S.TEMPLATE, metrics: list[dict] | None = None) -> ledger_match.Caliber: + graph = S.caliber_graph(sql) + if metrics is not None: + graph["nodes"][2]["data"]["config"]["metrics"] = metrics + found = ledger_match.caliber_of("wf", SimpleNamespace(graph=graph, version=1, level="governed", graph_hash="h")) + assert found is not None + return found + + +def query(sql: str, columns: list[str], artifact: str = "q1") -> ledger_match.Query: + return ledger_match.Query(artifact, "q", S.CURRENT["source"], sql, columns, frozenset({"orders"}), "sqlite") + + +def compare(sql: str, columns: list[str], cited: set[str] | None, cal: ledger_match.Caliber | None = None) -> dict: + [row] = ledger_match.compare_queries([query(sql, columns)], [cal or caliber()], + referenced={"q1": cited} if cited is not None else None) + return row + + +# -------------------------------------------------------------------------- +# 认得出 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sql, columns", [ + (f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW}", ["销售额", "单数"]), + (f"SELECT COUNT(*) AS 单数, MIN(paid_at) AS 起, MAX(paid_at) AS 止, SUM(amount) AS 销售额, " + f"AVG(amount) AS 客单价 FROM orders WHERE {WINDOW}", ["单数", "起", "止", "销售额", "客单价"]), + (f"SELECT '2026-09-01' AS 周一, '2026-09-07' AS 周日, SUM(amount) AS 销售额 FROM orders WHERE {WINDOW}", + ["周一", "周日", "销售额"]), +], ids=["extra-count", "extra-aggregates", "date-constants"]) +def test_extra_columns_are_projected_away_and_the_cited_one_is_proven(sql, columns): + row = compare(sql, columns, {"销售额"}) + assert row["status"] == "proven", row + [cand] = row["candidates"] + assert cand["status"] == "proven" and cand["binding"] == {"start": "2026-09-01", "end": "2026-09-08"} + assert cand["columns"] == [{"index": 0, "ours": "销售额", "caliber": "gmv"}] + assert cand["projection"] == {"ours": ["销售额"], "dropped": len(columns) - 1, "caliber": ["gmv"], + "caliber_dropped": 0} + # 不投影(报告没说引用了哪一列)时整条比,照旧证明不了 + assert compare(sql, columns, None)["status"] == "related" + + +def test_the_caliber_side_is_projected_to_the_columns_its_metrics_read(): + two = S.TEMPLATE.replace("SUM(amount) AS gmv", "SUM(amount) AS gmv, COUNT(*) AS n") + cal = caliber(two, [{"id": "gmv", "name": "销售额", "expression": "cell(nodes.fetch, 0, 'gmv')"}, + {"id": "n", "name": "单数", "expression": "cell(nodes.fetch, 0, 'n')"}]) + row = compare(f"SELECT SUM(amount) AS 销售额 FROM orders WHERE {WINDOW}", ["销售额"], {"销售额"}, cal) + assert row["status"] == "proven" + [cand] = row["candidates"] + assert cand["columns"] == [{"index": 0, "ours": "销售额", "caliber": "gmv"}] + assert cand["projection"] == {"ours": ["销售额"], "dropped": 0, "caliber": ["gmv"], "caliber_dropped": 1} + # 两列都引用、先后和口径的不同:调换之后按位置对上 + row = compare(f"SELECT COUNT(*) AS 单数, SUM(amount) AS 销售额, MAX(amount) AS 最大 FROM orders WHERE {WINDOW}", + ["单数", "销售额", "最大"], {"单数", "销售额"}, cal) + assert row["status"] == "proven" + [cand] = row["candidates"] + assert [(c["ours"], c["caliber"]) for c in cand["columns"]] == [("销售额", "gmv"), ("单数", "n")] + + +# -------------------------------------------------------------------------- +# 不错给信任 +# -------------------------------------------------------------------------- + + +GROUPED = S.TEMPLATE.replace("SUM(amount) AS gmv", "DISTINCT amount AS gmv") + + +@pytest.mark.parametrize("sql, columns, cited, cal_sql", [ + # 多一个分组列:答案是每个地区一行,口径只有一行 + (f"SELECT region, SUM(amount) AS 销售额 FROM orders WHERE {WINDOW} GROUP BY region", ["region", "销售额"], + {"销售额"}, S.TEMPLATE), + # DISTINCT 按整行去重:去掉 region 之后行数就变了(口径按金额去重,投影了就会错认) + (f"SELECT DISTINCT region, amount AS 金额 FROM orders WHERE {WINDOW}", ["region", "金额"], {"金额"}, GROUPED), + # 按别的列排序再取前几行 + (f"SELECT SUM(amount) AS 销售额, region FROM orders WHERE {WINDOW} GROUP BY region ORDER BY region LIMIT 1", + ["销售额", "region"], {"销售额"}, S.TEMPLATE), + (f"SELECT amount AS 金额, paid_at FROM orders WHERE {WINDOW} ORDER BY paid_at DESC LIMIT 1", ["金额", "paid_at"], + {"金额"}, S.TEMPLATE.replace("SUM(amount) AS gmv", "amount AS gmv") + " ORDER BY amount DESC LIMIT 1"), + # HAVING 用到多出来的聚合(按别名、按同样的写法) + (f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW} HAVING 单数 > 0", ["销售额", "单数"], + {"销售额"}, S.TEMPLATE), + (f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW} HAVING COUNT(*) > 0", + ["销售额", "单数"], {"销售额"}, S.TEMPLATE), +], ids=["extra-group-by", "distinct", "group-order-limit", "order-other-column-limit", "having-alias", + "having-same-aggregate"]) +def test_projection_never_proves_what_the_whole_sql_does_not_say(sql, columns, cited, cal_sql): + expression = "cell(nodes.fetch, 0, 'gmv')" + cal = caliber(cal_sql, [{"id": "gmv", "name": "销售额", "expression": expression}]) + row = compare(sql, columns, cited, cal) + assert row["status"] != "proven", row + assert all("projection" not in c for c in row["candidates"]), "这几种都不该投影" + # 不投影的:和没给引用列时一字不差 + assert row == compare(sql, columns, None, cal) + + +def test_without_a_projection_every_pair_is_exactly_the_old_result(): + """报告引用了全部的列、口径也不用投影的,整条比证明了的,投影不了的:逐对和不给引用列时一字不差。""" + cases = [ + (S.EQUIVALENT, ["销售额"], {"销售额"}), # 整条比就证明了 + (S.WITH_REFUNDS, ["gmv"], {"gmv"}), # 两边都只有这一列:没有可投影的 + (f"SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE {WINDOW}", ["gmv", "n"], {"gmv", "n"}), + (f"SELECT DISTINCT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE {WINDOW}", ["gmv", "n"], {"gmv"}), + (f"SELECT SUM(amount) AS gmv, COUNT(*) AS n FROM orders WHERE {WINDOW}", ["gmv"], {"gmv"}), # 列数对不上 + ] + for sql, columns, cited in cases: + assert compare(sql, columns, cited) == compare(sql, columns, None), sql + + +def test_e1_counts_the_proofs_that_needed_a_projection(): + proven = compare(f"SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE {WINDOW}", + ["销售额", "单数"], {"销售额"}) + related = compare("SELECT SUM(amount) AS 销售额, COUNT(*) AS 单数 FROM orders WHERE paid_at >= '2026-09-01' " + "AND paid_at < '2026-09-08'", ["销售额", "单数"], {"销售额"}) + e1 = ledger_stats.e1_of([{"queries": [proven, related], "reports": {}, "whole": {}}]) + assert e1["status"]["proven"] == 1 and e1["proven_projected"] == 1 + + +# -------------------------------------------------------------------------- +# 真的跑一遍 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def engine_up(tmp_path): + await run_manager.setup() + await S.fresh_source(tmp_path / "shop.db") + yield + await ledger_match.drain() + await run_manager.shutdown() + + +async def test_last_weeks_sales_with_extra_columns_is_whole_equivalent(engine_up, monkeypatch): + wf = await S.publish() + sql = ("SELECT '2026-09-01' AS 周一, SUM(amount) AS 销售额, COUNT(*) AS 单数, MAX(paid_at) AS 最后一单 " + f"FROM orders WHERE {WINDOW}") + row = await S.explore(monkeypatch, sql, text="上周销售额 [[v:Q1.r0.销售额]],共 [[v:Q1.r0.单数]] 单。") + recorded = await store.latest(row.id) + content = recorded.content + assert content["algo"] == "3" + [q] = content["queries"] + assert q["status"] == "related", "销售额、单数两列都被引用,口径只有销售额:投影之后列数对不上,照旧不等价" + + row = await S.explore(monkeypatch, sql, text="上周销售额 [[v:Q1.r0.销售额]]。") + content = (await store.latest(row.id)).content + [q] = content["queries"] + assert q["status"] == "proven" + [cand] = q["candidates"] + assert cand["projection"]["dropped"] == 3 and cand["columns"] == [{"index": 0, "ours": "销售额", "caliber": "gmv"}] + [labels] = [r["labels"] for r in content["reports"].values()] + [label] = labels.values() + assert label["kind"] == "caliber" and label["column"] == "gmv" and label["caliber"] == f"{wf}@1" + assert content["whole"]["equivalent"] is True and content["whole"]["binding"] == {"start": "2026-09-01", + "end": "2026-09-08"} + + +async def test_reconcile_pairs_the_columns_left_after_the_projection(engine_up, monkeypatch): + """含退款、又多选了几列的答案:对账时按投影之后两边留下的列配(这次的销售额 → 口径卡的 gmv),不用人选。""" + from httpx import ASGITransport, AsyncClient + + from app.main import app + + wf = await S.publish() + sql = ("SELECT COUNT(*) AS 单数, SUM(amount) AS 销售额, MAX(paid_at) AS 最后一单 FROM orders " + "WHERE paid_at >= '2026-09-01' AND paid_at < '2026-09-08'") + row = await S.explore(monkeypatch, sql, text="上周销售额 [[v:Q1.r0.销售额]]。") + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + body = (await client.get(f"/api/runs/{row.id}/ledger")).json() + [q] = body["queries"] + [cand] = q["candidates"] + assert cand["projection"]["ours"] == ["销售额"] and cand["projection"]["caliber"] == ["gmv"] + res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json={ + "query": q["artifact"], "workflow_id": wf, "version": 1, "confirm": True, + "inputs": {"start": "2026-09-01", "end": "2026-09-08"}}) + assert res.status_code == 201, res.text + done = await S.wait(res.json()["run_id"]) + assert done.status == "succeeded", done.error + assert done.origin["pairs"] == [{"ours": "销售额", "metric": "gmv"}] diff --git a/backend/tests/test_sql_projection.py b/backend/tests/test_sql_projection.py new file mode 100644 index 00000000..70bbe654 --- /dev/null +++ b/backend/tests/test_sql_projection.py @@ -0,0 +1,195 @@ +"""按引用的列投影(data/sql_semantics.py 的 project_select / project_template / output_names,next-round 口径账本:整题等价 +按答案引用的那几列比)。 + +投影本身不判等价,只把「没被引用、去掉也不改变被引用那几列取值」的选择项去掉,交给现有的绑定和判等价。这里守两件事: +- 该去的去得掉:多选一列、多几个聚合、日期常量列、带 WITH 的、只出一行的聚合带 LIMIT;口径模板里的 {{ input.x }} 原样留着; +- 去了会变的一律不去(宁可不投影):DISTINCT / DISTINCT ON、出现在 GROUP BY 里、被 ORDER BY / HAVING / QUALIFY / 窗口 + 用到(按别名、按同样的写法、按位置)、和 LIMIT / OFFSET 一起定取哪几行、集合运算、*、可能改变行数的函数、去掉仅有的聚合、 + 留下按任意一行取值的列。 + +另有执行守门:语料(tests/fixtures/sql_pairs.py)里每一条 SQL,按每一列(和「只去掉一列」)投影,投影得了的在虚构库的每组 +数据上,投影后的结果都得等于原结果的那几列。 +""" +from __future__ import annotations + +import pytest + +from app.data.sql_semantics import output_names, project_select, project_template +from tests.fixtures import sql_pairs + +WEEKLY = ("SELECT SUM(amount) AS gmv FROM orders WHERE status = 'paid' " + "AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}'") + + +# -------------------------------------------------------------------------- +# 该去的去得掉 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sql, keep, want", [ + ("SELECT SUM(amount) AS sales, COUNT(*) AS n FROM orders WHERE status = 'paid'", [0], + "SELECT SUM(amount) AS sales FROM orders WHERE status = 'paid'"), + ("SELECT COUNT(*) AS n, MIN(paid_at) AS first_day, MAX(paid_at) AS last_day, SUM(amount) AS sales " + "FROM orders WHERE paid_at >= '2026-10-05'", [3], + "SELECT SUM(amount) AS sales FROM orders WHERE paid_at >= '2026-10-05'"), + ("SELECT '2026-10-05' AS week_start, '2026-10-11' AS week_end, COALESCE(SUM(amount), 0) AS sales FROM orders", [2], + "SELECT COALESCE(SUM(amount), 0) AS sales FROM orders"), + ("WITH paid AS (SELECT amount FROM orders WHERE status = 'paid') SELECT SUM(amount) AS s, COUNT(*) AS n FROM paid", + [0], "WITH paid AS (SELECT amount FROM orders WHERE status = 'paid') SELECT SUM(amount) AS s FROM paid"), + # 不分组的聚合只出一行:LIMIT 取哪一行和选出来的列无关 + ("SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders LIMIT 100", [0], "SELECT SUM(amount) AS s FROM orders LIMIT 100"), + # 分组的一段去掉别的聚合(分组键留着) + ("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region", [0, 1], + "SELECT region, SUM(amount) AS s FROM orders GROUP BY region"), + # 调换先后 + ("SELECT COUNT(*) AS n, SUM(amount) AS s FROM orders", [1, 0], "SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders"), +], ids=["extra-count", "extra-aggregates", "date-constants", "cte", "single-row-limit", "grouped", "reorder"]) +def test_items_that_do_not_change_the_referenced_columns_are_dropped(sql, keep, want): + p = project_select(sql, keep, "sqlite") + assert p.refused is None and p.sql == want + assert p.kept == tuple(keep) and p.dropped == len(output_names(sql, "sqlite")) - len(keep) + + +def test_the_text_around_the_select_list_is_kept_as_written(): + sql = "SELECT\n SUM(amount) AS sales, -- 金额\n COUNT(*) AS n\nFROM orders o\nWHERE o.status = 'paid' -- 只算已支付" + p = project_select(sql, [0], "sqlite") + assert p.sql == "SELECT\n SUM(amount) AS sales\nFROM orders o\nWHERE o.status = 'paid' -- 只算已支付" + assert p.names == ("sales",) + + +def test_keeping_everything_in_order_gives_the_sql_back_unchanged(): + sql = "SELECT SUM(amount) AS s FROM orders" + p = project_select(sql, [0], "sqlite", width=1) + assert p.sql == sql and p.dropped == 0 and p.refused is None + + +def test_template_projection_keeps_the_slots(): + template = ("SELECT SUM(amount) AS gmv, COUNT(*) AS n, '{{ input.start }}' AS since FROM orders " + "WHERE status = 'paid' AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}'") + assert output_names(template, "sqlite", template=True) == ["gmv", "n", "since"] + p = project_template(template, [0], "sqlite") + assert p.refused is None and p.sql == WEEKLY and p.dropped == 2 and p.names == ("gmv",) + p = project_template(template, [1], "sqlite") + assert p.sql == WEEKLY.replace("SUM(amount) AS gmv", "COUNT(*) AS n") + + +# -------------------------------------------------------------------------- +# 去了会变的一律不去 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sql, keep, dialect", [ + ("SELECT DISTINCT region, amount FROM orders", [1], "sqlite"), + ("SELECT DISTINCT ON (region) region, amount FROM orders ORDER BY region, amount", [1], "postgres"), + # 多一个分组列:它在 GROUP BY 里 + ("SELECT region, SUM(amount) AS s FROM orders GROUP BY region", [1], "sqlite"), + ("SELECT region AS r, SUM(amount) AS s FROM orders GROUP BY 1", [1], "sqlite"), + # 按别的列排序再取前几行 + ("SELECT amount, paid_at FROM orders ORDER BY paid_at DESC LIMIT 5", [0], "sqlite"), + ("SELECT amount, id FROM orders ORDER BY amount LIMIT 3 OFFSET 1", [0], "sqlite"), + # HAVING 用到多出来的聚合:按别名、按同样的写法 + ("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region HAVING n > 2", [0, 1], "sqlite"), + ("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region HAVING COUNT(*) > 2", [0, 1], + "sqlite"), + # ORDER BY 用到、按位置引用 + ("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region ORDER BY n DESC", [0, 1], "sqlite"), + ("SELECT region, SUM(amount) AS s, COUNT(*) AS n FROM orders GROUP BY region ORDER BY 3 DESC", [0, 1], "sqlite"), + # 窗口、QUALIFY 用到 + ("SELECT id, amount, SUM(amount) OVER (ORDER BY id) AS run FROM orders", [0, 2], "sqlite"), + ("SELECT region, amount, ROW_NUMBER() OVER (PARTITION BY region ORDER BY amount) AS rn FROM orders QUALIFY rn = 1", + [0, 1], "duckdb"), + # SQLite 的 WHERE 认列别名 + ("SELECT amount, DATE(paid_at) AS d FROM orders WHERE d >= '2026-10-05'", [0], "sqlite"), + # 集合运算、* + ("SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders UNION ALL SELECT 1, 2", [0], "sqlite"), + ("SELECT *, amount * 2 AS twice FROM orders", [1], "sqlite"), + # 可能把一行展开成多行的函数、不认识的函数 + ("SELECT id, unnest(tags) AS tag FROM orders", [0], "postgres"), + ("SELECT id, generate_series(1, 3) AS k FROM orders", [0], "postgres"), + ("SELECT id, json_array_elements(payload) AS e FROM orders", [0], "postgres"), + # 去掉仅有的聚合:剩下的不再只是一行 + ("SELECT COUNT(*) AS n, '2026' AS y FROM orders", [1], "sqlite"), + # 留下按任意一行取值的列(SQLite 取 MAX 那一行) + ("SELECT region, MAX(amount) AS m, COUNT(*) AS n FROM orders", [0, 1], "sqlite"), + # MySQL 里紧跟着字的 -- 是两个减号:改写之后位置对不上原文 + ("SELECT SUM(amount)--1 AS s, COUNT(*) AS n FROM orders", [0], "mysql"), +], ids=["distinct", "distinct-on", "group-by-column", "group-by-ordinal", "order-limit", "order-limit-offset", + "having-alias", "having-same-aggregate", "order-by-alias", "order-by-ordinal", "window", "qualify", + "sqlite-where-alias", "union", "star", "unnest", "generate-series", "unknown-function", "only-aggregate", + "loose-column", "mysql-dashes"]) +def test_items_whose_removal_could_change_the_rest_are_never_dropped(sql, keep, dialect): + p = project_select(sql, keep, dialect) + assert p.sql is None and p.refused, (sql, p) + + +def test_the_result_width_must_match_the_select_list(): + p = project_select("SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders", [0], "sqlite", width=3) + assert p.sql is None and "列数" in (p.refused or "") + + +@pytest.mark.parametrize("keep", [[], [0, 0], [5]]) +def test_bad_keep_lists_are_refused(keep): + assert project_select("SELECT SUM(amount) AS s, COUNT(*) AS n FROM orders", keep, "sqlite").sql is None + + +def test_unreorderable_items_keep_their_order(): + # DuckDB 认前面的列别名:调换之后 b 就找不到 a 了 + p = project_select("SELECT amount + 1 AS a, a * 2 AS b FROM orders", [1, 0], "duckdb") + assert p.sql is None and p.refused + + +# -------------------------------------------------------------------------- +# 执行守门:投影得了的,结果就是原结果的那几列 +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def databases(): + conns = sql_pairs.build_databases() + yield conns + for c in conns: + c.close() + + +def _keeps(width: int) -> list[tuple[int, ...]]: + """每一列单独留下,再加上「只去掉一列」的几种(列多的只试前四列)。""" + singles = [(i,) for i in range(min(width, 4))] + drops = [tuple(j for j in range(width) if j != i) for i in range(min(width, 4))] if width > 2 else [] + return singles + drops + + +def _bag(rows: list[tuple]) -> list[tuple]: + """同 sql_pairs.run:行按值排序(先后不算结果的一部分),浮点数按 6 位小数取整。""" + rows = [tuple(sql_pairs._norm(v) for v in row) for row in rows] + return sorted(rows, key=lambda r: tuple((v is None, str(type(v)), v if v is not None else 0) for v in r)) + + +def _corpus() -> list[str]: + seen: dict[str, None] = {} + for pair in sql_pairs.PAIRS: + seen.setdefault(pair.a) + seen.setdefault(pair.b) + return list(seen) + + +def test_projection_keeps_the_values_of_the_kept_columns(databases): + """语料里每一条 SQL:投影得了的,在每组数据上投影后的结果(行的先后不算)都等于原结果只取留下的那几列。同时数一数 + 投影了多少种,免得规则收得太紧、守门形同虚设。""" + projected = 0 + for sql in _corpus(): + width = len(output_names(sql, "sqlite") or []) + if width < 2: + continue + for keep in _keeps(width): + p = project_select(sql, keep, "sqlite", width=width) + if p.sql is None: + continue + projected += 1 + for conn in databases: + try: + full = conn.execute(sql).fetchall() + except Exception: # noqa: BLE001 - 语料里有只在别的方言里成立的写法:原句跑不了的不比 + break + want = _bag([tuple(row[k] for k in keep) for row in full]) + assert sql_pairs.run(conn, p.sql) == want, (sql, keep, p.sql) + assert projected >= 100, projected