mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-19 07:42:00 +00:00
ad1092900c
复核发现两处(C-relation-coverage-inflated、C-profile-note-rewording-bumps-catalog): 一、覆盖率虚高。key_literals 把父列是数字、子列取到的不是整数写法的文字('WALKIN'、'GUEST0')从覆盖率的 分母里拿掉了:散客代号占九成的一列只剩 1 个整数写法的键对上,就测成覆盖率 100%、升为「已核实」,模型的 db_schema、自查、冻结快照都会照这条错的关系写 join。基线上同一份数据是 proposed、覆盖率 3.1%。 - 整数主键里永远没有这些文字,它们照样是子表的键值,算作没对上、留在分母里: 覆盖率 = 对上的 / (拼进 IN 的 + 按类型就对不上的)。全都不是整数写法时不发父表的核对查询,覆盖率 0%、 「可能不是这条关系」,不再整条判「无法核对」。 - key_literals 改回 KeyLiterals(literals, unmatchable, unsafe):不能安全拼进 SQL 的值单独计数,不再用 「取到的 − 拼进的 − 不同类型的」倒推('0012' 和 '12' 写成同一个字面量时会被错说成「类型不支持核对」)。 - 被指向列是日期时间的关系整条跳过,并且在发子表取值查询之前就跳过、不占这张表的查询预算;说明改成 「各库的时间写法和精度不同,不核对这种关系的覆盖率」——子表一侧也是时间时,以前写的「两端类型不同」不照实。 - 备注里「另有 N 个键值和被指向列的类型不同,未计入」改成「其中 N 个不是整数写法,被指向列是数字,算作没对上」。 - 报告里覆盖率不足的那句「抽样覆盖率 N,保持推断」也改成「覆盖率 N」,和备注一种说法。 二、升级后第一次重新剖析,数据没变也给每张有剖析结论的表升一个版本。剖析备注改了四处说法(抽样覆盖率→ 覆盖率、子表抽样→子表取了、(只看前→(按存储顺序只看前、按前 N 行统计→按存储顺序取前 N 行统计), catalog.same_profile_note 没认成同一句,已发布工作流记下的目录版本对不上,之后每次正式运行都报 catalog.drift。 - same_profile_note 比较前先把这四处固定句式的旧写法换成现在的写法(_LEGACY_PROFILE_WORDING,句式带着前后文 匹配,不会误换人写的原话);算相同时保留目录里的旧备注,旧说法等结论真变了、重写备注时才换掉。 - 起草时核对有效行过滤写在模型依据后面的那句「剖析核对:按前 N 行统计」同样只认那一句的旧写法。 - MySQL / MariaDB 上新加的排序规则说明、ENUM 列定义里没出现的取值、无效日期个数是新信息,这些备注照常 升一次版本,要写进发版说明。 测试:test_catalog_profile 加文字键指向整数主键的端到端用例(去掉修复回跑:测成 verified、覆盖率 100%); test_profile_dialect_units 加覆盖率分母、时间键不发查询、unsafe 计数的单测;test_catalog_semantics 加跨版本 用例(把剖析备注改回旧说法写进目录、同样的数据重新剖析,版本号和目录都不变;去掉规整回跑两条失败)。
179 lines
9.5 KiB
Python
179 lines
9.5 KiB
Python
"""剖析 SQL 的方言金样(下一轮 03 细案 C2):没有真库时挡住方言写法的回退。
|
||
|
||
三层,各挡各的:
|
||
|
||
1. **金样**:每种方言(mysql、mariadb、postgresql、oracle、sqlite)× 每类查询(统计信息、有上限的计数、取不同键值、
|
||
对键值、唯一性、取值分布、日期最小最大、时间窗口里的取值分布、按日行数、块采样),完整 SQL 钉在
|
||
tests/fixtures/dialect/<方言>.sql,人审过一次;以后拼法有任何改动都会出 diff。要更新金样:
|
||
``UPDATE_DIALECT_GOLDEN=1 pytest tests/test_profile_dialect_golden.py``,再逐行看 git diff。
|
||
2. **sqlglot 逐条解析**:按方言读一遍(mariadb 按 mysql 读),挡住语法写坏了(少括号、关键字拼错)。
|
||
3. **IN 两边类型族一致**:用 sqlglot 的 qualify + annotate_types,配上表结构里的类型,核对 IN 列表的字面量和被指向
|
||
列是同一类型族(PostgreSQL 上 varchar 父列配整数子列时,字面量必须带引号)。
|
||
|
||
挡不住的照实写:sqlglot 不知道真库的统计信息是不是过期、零日期驱动怎么返回、Oracle 小写对象是否存在、排序规则
|
||
怎么比——这些要靠 scripts/dialect-it.sh 在本机容器里跑真库(不进 CI)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
import sqlglot
|
||
from sqlglot import exp
|
||
from sqlglot.optimizer.annotate_types import annotate_types
|
||
from sqlglot.optimizer.qualify import qualify
|
||
|
||
from app.data import catalog, guard
|
||
from app.data.catalog_profile import IN_CHUNK, SqlDialect, key_literals
|
||
|
||
GOLDEN = Path(__file__).parent / "fixtures" / "dialect"
|
||
DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite")
|
||
#: sqlglot 的方言名
|
||
READ = {"mysql": "mysql", "mariadb": "mysql", "postgresql": "postgres", "oracle": "oracle", "sqlite": "sqlite"}
|
||
|
||
#: 夹具表结构:一张带 schema 的订单表、一张门店表(编码是文字),Oracle 的一张小写带引号的表和一个小写带引号的列
|
||
META = {
|
||
"orders": {"schema": "sales", "columns": [
|
||
{"name": "id", "type": "INTEGER"}, {"name": "store_code", "type": "INTEGER"},
|
||
{"name": "status", "type": "CHAR(2)"}, {"name": "paid_at", "type": "TIMESTAMP"},
|
||
{"name": "biz_day", "type": "VARCHAR(10)"}, {"name": "level", "type": "INTEGER"}],
|
||
"primary_key": ["id"]},
|
||
"stores": {"schema": None, "columns": [{"name": "code", "type": "VARCHAR(10)"}], "primary_key": ["code"]},
|
||
"visits": {"schema": None, "quote_name": True, "columns": [
|
||
{"name": "id", "type": "INTEGER"}, {"name": "level", "type": "INTEGER", "quote_name": True}],
|
||
"primary_key": ["id"]},
|
||
}
|
||
|
||
|
||
def _queries(kind: str) -> list[tuple[str, str | None]]:
|
||
"""一种方言的全部剖析、观察查询:(名字, SQL)。没有这种查询(SQLite 没有统计信息、MySQL 没有块采样)的 SQL 为 None。"""
|
||
d = SqlDialect(kind)
|
||
orders, stores, visits = d.table(META["orders"], "orders"), d.table(META["stores"], "stores"), \
|
||
d.table(META["visits"], "visits")
|
||
status, code = d.column(META["orders"], "status"), d.column(META["stores"], "code")
|
||
level = d.column(META["visits"], "level")
|
||
keys = key_literals([12, 7], "number", parent="text", child_type="INTEGER").literals
|
||
out: list[tuple[str, str | None]] = [
|
||
("stats", d.stats_sql(META["orders"], "orders")),
|
||
("stats_quoted", d.stats_sql(META["visits"], "visits")),
|
||
("bounded", d.bounded_count_sql(orders, 100_001)),
|
||
("distinct", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, scan_cap=None)),
|
||
("distinct_capped", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000,
|
||
scan_cap=100_000)),
|
||
("match", d.match_count_sql(stores, code, keys)),
|
||
("unique", d.unique_check_sql(visits, level)),
|
||
("value_counts", d.value_counts_sql(orders, status, 21, scan_cap=None)),
|
||
("value_counts_capped", d.value_counts_sql(orders, status, 21, scan_cap=100_000)),
|
||
("min_max", d.min_max_sql(orders, [(d.column(META["orders"], "paid_at"), False),
|
||
(d.column(META["orders"], "biz_day"), True)])),
|
||
("date_window", d.window_value_counts_sql(orders, status, d.column(META["orders"], "paid_at"),
|
||
"2026-08-27", 21)),
|
||
("day_volume", d.day_volume_sql(orders, d.column(META["orders"], "paid_at"), "2026-08-27")),
|
||
("sample", d.sample_value_counts_sql(orders, status, 1, 21)),
|
||
]
|
||
return out
|
||
|
||
|
||
def _render(kind: str) -> str:
|
||
lines = [f"-- 剖析与取值观察的 SQL 金样:{kind}(tests/test_profile_dialect_golden.py 生成,改动要人审)"]
|
||
for name, sql in _queries(kind):
|
||
lines.append(f"-- {name}")
|
||
lines.append(f"{sql};" if sql is not None else "-- (这种方言没有这类查询)")
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
@pytest.mark.parametrize("kind", DIALECTS)
|
||
def test_profile_sql_matches_the_golden_file(kind):
|
||
path = GOLDEN / f"{kind}.sql"
|
||
text = _render(kind)
|
||
if os.environ.get("UPDATE_DIALECT_GOLDEN") == "1":
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(text, encoding="utf-8")
|
||
assert path.read_text(encoding="utf-8") == text, f"{kind} 的剖析 SQL 和金样不一致:看 diff,确认无误后重新生成金样"
|
||
|
||
|
||
@pytest.mark.parametrize("kind", DIALECTS)
|
||
def test_every_profile_query_parses_and_passes_the_guard(kind):
|
||
for name, sql in _queries(kind):
|
||
if sql is None:
|
||
continue
|
||
parsed = sqlglot.parse(sql, read=READ[kind], error_level=sqlglot.ErrorLevel.RAISE)
|
||
assert len(parsed) == 1 and parsed[0] is not None, (kind, name)
|
||
assert guard.check(sql, readonly=True, dialect=kind) == sql, (kind, name)
|
||
|
||
|
||
@pytest.mark.parametrize("kind", ["postgresql", "oracle", "mysql"])
|
||
def test_observation_queries_use_the_right_day_and_sample_functions(kind):
|
||
"""按天截断和块采样各家写法不同,解析成语法树再看用的是哪个函数,不只比字符串。"""
|
||
sql = dict(_queries(kind))
|
||
day = sqlglot.parse_one(sql["day_volume"], read=READ[kind])
|
||
if kind == "oracle":
|
||
assert any(isinstance(n, (exp.DateTrunc, exp.TimestampTrunc, exp.Anonymous)) or n.key == "trunc"
|
||
for n in day.walk()), sql["day_volume"]
|
||
assert "TRUNC(paid_at)" in sql["day_volume"]
|
||
elif kind == "postgresql":
|
||
assert any(isinstance(n, exp.Cast) for n in day.walk())
|
||
else:
|
||
assert "DATE(paid_at)" in sql["day_volume"]
|
||
if kind == "mysql":
|
||
assert sql["sample"] is None
|
||
else:
|
||
sampled = sqlglot.parse_one(sql["sample"], read=READ[kind])
|
||
assert any(isinstance(n, exp.TableSample) for n in sampled.walk()), sql["sample"]
|
||
|
||
|
||
# ---------------------------------------------------------------- IN 两边的类型族
|
||
|
||
|
||
_FAMILY_OF = [(exp.DataType.TEXT_TYPES, "text"), (exp.DataType.NUMERIC_TYPES, "number"),
|
||
(exp.DataType.TEMPORAL_TYPES, "time")]
|
||
|
||
|
||
def _sg_family(t: exp.DataType | None) -> str | None:
|
||
if t is None:
|
||
return None
|
||
for group, name in _FAMILY_OF:
|
||
if t.this in group:
|
||
return name
|
||
return None
|
||
|
||
|
||
def _in_families(sql: str, kind: str, schema: dict[str, dict[str, str]]) -> list[tuple[str | None, set[str | None]]]:
|
||
"""sql 里每个 IN:(被比较的列的类型族, 字面量的类型族集合),按 annotate_types 配上表结构的类型推出来。"""
|
||
tree = qualify(sqlglot.parse_one(sql, read=READ[kind]), schema=schema, dialect=READ[kind],
|
||
validate_qualify_columns=False)
|
||
tree = annotate_types(tree, schema=schema, dialect=READ[kind])
|
||
return [(_sg_family(node.this.type), {_sg_family(x.type) for x in node.expressions})
|
||
for node in tree.find_all(exp.In)]
|
||
|
||
|
||
@pytest.mark.parametrize("kind", DIALECTS)
|
||
@pytest.mark.parametrize("parent_type, child_values, child_type", [
|
||
("VARCHAR(10)", [12, 7, 30], "INTEGER"), # 父列文字、子列数:字面量要带引号
|
||
("INTEGER", ["12", "0007", "A1"], "VARCHAR(10)"), # 父列数、子列文字:只收整数写法,写成数字
|
||
("VARCHAR(10)", ["A ", "BC"], "CHAR(2)"), # 定长子列:去掉补齐的空格
|
||
("INTEGER", [1, 2], "INTEGER"),
|
||
])
|
||
def test_in_list_literals_have_the_parent_columns_family(kind, parent_type, child_values, child_type):
|
||
d = SqlDialect(kind)
|
||
parent = {"name": "code", "type": parent_type}
|
||
literals = key_literals(child_values, None, parent=catalog.column_family(parent), child_type=child_type).literals
|
||
assert literals
|
||
sql = d.match_count_sql("stores", d.quote("code"), literals[:IN_CHUNK])
|
||
found = _in_families(sql, kind, {"stores": {"code": parent_type}})
|
||
assert found, sql
|
||
for column_family, literal_families in found:
|
||
assert literal_families == {column_family}, (kind, sql, column_family, literal_families)
|
||
|
||
|
||
def test_the_old_way_would_mismatch_on_postgres():
|
||
"""对照:以前按子表取回的值写字面量,varchar 父列配整数子列在 sqlglot 推出的类型上就两边不同——PostgreSQL 真库上
|
||
这是「operator does not exist: character varying = integer」。"""
|
||
from app.data.catalog_profile import sql_literal
|
||
|
||
old = [lit for v in (12, 7) if (lit := sql_literal(v, "number"))]
|
||
sql = SqlDialect("postgresql").match_count_sql("stores", "code", old)
|
||
[(column_family, literal_families)] = _in_families(sql, "postgresql", {"stores": {"code": "VARCHAR(10)"}})
|
||
assert column_family == "text" and literal_families == {"number"}
|