数据目录闭环第一步(后端 A):有效行过滤的校验和剖析核对

- sqlcheck.parse_valid_filter / valid_filter_problem:按数据源的方言用 sqlglot 把有效行过滤解析成一个条件,只能引用这张表的列(不带表名或带这张表的表名,不分大小写),不能有子查询、聚合、窗口函数、分号、注释、模板占位、参数占位、算术,只允许比较、IN(值的列表)、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE,不超过 500 字,至少用到一列;不认识的数据源类型不收
- sqlcheck.render_valid_filter:把条件按方言重新写出来,列换成表结构里的真名、按剖析查询的规矩加引号,拼进核对查询用
- catalog_profile.check_valid_filter:开了数据剖析的源经剖析的只读守卫(guard.check(readonly=True)、单条时限、查询次数上限)发一条查询,只取条件里的列、先取前 max(抽样键值数, 整表统计行数上限) 行,数出总行数和被排除的行数;排除 0% 或 100% 的不收,收下的写「剖析核对:排除 3.2% 的行(共 12,345 行)」,只数了前若干行的写明按前几行统计;没开剖析、数据源停用、条件里有遮罩的列、视图、取值拼不进查询、查询失败或超时照收,写明没有核对的原因
This commit is contained in:
ThearchyHelios
2026-10-14 08:30:14 +08:00
parent a87b935ebf
commit 447c66649c
3 changed files with 523 additions and 6 deletions
+124 -4
View File
@@ -1341,8 +1341,128 @@ async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[st
return report
__all__ = ["CODES_MAX", "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES",
# ==========================================================================
# 核对模型提议的有效行过滤:排除了多少行
# ==========================================================================
#: 有效行过滤核对的说法(写进这一项的 note;前面是模型给的依据)
FILTER_CHECKED = "剖析核对:排除 {share} 的行(共 {rows} 行)"
FILTER_SAMPLED = "剖析核对:按前 {rows} 行统计,排除 {share} 的行"
FILTER_SAMPLED_EDGE = "剖析核对:按前 {rows} 行统计,{what},请人工核对"
FILTER_UNCHECKED = "没有核对排除比例({why})"
FILTER_NOT_ENABLED = "数据源没有开数据剖析"
@dataclass(frozen=True)
class FilterCheck:
"""核对一条有效行过滤的结果。
keep:收不收(排除 0% 或 100% 的不收:前者什么都没筛掉,后者把所有行都筛掉了,多半写错了);note:写进这一项 note
的那句(剖析核对的排除比例,或者没有核对的原因);rows / excluded:数到的行数和其中被排除的行数(没核对是 None);
sampled:表太大或不知道多大,只数了前若干行;dropped:不收的原因(记日志用)。"""
keep: bool
note: str
rows: int | None = None
excluded: int | None = None
sampled: bool = False
dropped: str | None = None
def excluded_share(excluded: int, rows: int) -> str:
"""排除比例写成百分数:留一位小数(整数不带小数);不到 0.1% 的写「不足 0.1%」,不是全部的不四舍五入成 100%。"""
if rows <= 0:
return "0%"
pct = excluded * 100 / rows
if 0 < pct < 0.1:
return "不足 0.1%"
if 99.9 < pct < 100:
return "超过 99.9%"
return f"{pct:.1f}".rstrip("0").rstrip(".") + "%"
def _unchecked(why: str) -> FilterCheck:
return FilterCheck(True, FILTER_UNCHECKED.format(why=why))
def _unsafe_literal(value: str) -> bool:
"""拼不进核对查询的字符串取值:查询层用 SQLAlchemy 的 text() 执行,它会把字符串里的「:名字」当成绑定参数;
反斜杠在 MySQL 上读法不定、守卫会整条拒掉;控制字符不该出现(同 sql_literal 的规矩)。"""
return "\\" in value or bool(_CONTROL.search(value)) or bool(_BIND_LIKE.search(value))
async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | None, condition: str, *,
settings: ProfileSettings | None = None) -> FilterCheck:
"""开了数据剖析的源:数一数这条有效行过滤排除了多少行(模型起草时提议的那条,data/catalog_draft.py 调)。从不抛异常。
经剖析的只读守卫(_Runner:guard.check(readonly=True)、单条时限、查询次数上限)发一条查询:只取条件里用到的列,
先取前 max(sample_size, max_scan_rows) 行(大表不做整表扫描),再数总行数和不满足条件的行数(条件为 NULL 也算排除,
和 WHERE 一样)。数到的不满前那么多行,就是整张表。
- 排除 0% 或 100% 的不收(keep 为假);只数了前若干行的照收,写明「请人工核对」;
- 没开剖析、数据源停用、条件里有遮罩的列、视图、取值拼不进查询、查询失败或超时:照收,note 写没有核对的原因。
"""
from app.data import sqlcheck
options = getattr(source, "options", None)
settings = settings or profile_settings(options)
if not settings.enabled:
return _unchecked(FILTER_NOT_ENABLED)
if getattr(source, "enabled", True) is False:
return _unchecked("数据源已停用")
kind = getattr(source, "kind", None)
parsed = sqlcheck.parse_valid_filter(condition, meta, kind=kind, table=table)
if parsed.problem:
return FilterCheck(False, parsed.problem, dropped=parsed.problem)
masked = {c.lower() for c in data_engine.masked_columns(options)}
if hit := [c for c in parsed.columns if c.lower() in masked]:
return _unchecked(f"条件里的 {'、'.join(hit)} 在数据源设置中被遮罩,不取数")
if (meta or {}).get("is_view"):
return _unchecked("这张表是视图,剖析可能触发整个视图的计算")
from sqlglot import exp
if any(n.is_string and _unsafe_literal(str(n.this)) for n in parsed.expression.find_all(exp.Literal)):
return _unchecked("条件里的取值含有冒号或反斜杠,拼不进核对查询")
try:
dialect = SqlDialect(kind)
except ValueError as e:
return _unchecked(str(e))
cap = max(int(settings.sample_size), int(settings.max_scan_rows))
cond = sqlcheck.render_valid_filter(parsed, quoted=lambda name: dialect.quote(name) != name)
cols = ", ".join(dialect.quote(c) for c in parsed.columns)
inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap)
sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s"
run = _Runner(source, settings, dialect)
try:
res = await run(sql)
except _Skip as e:
return _unchecked(e.detail.removesuffix(",已跳过"))
except _Stop as e:
return _unchecked(_STOP_DETAIL[e.reason].format(max_queries=settings.max_queries,
max_total_s=_fmt(settings.max_total_s)).removesuffix(",未检查"))
except SnapshotTampered as e:
return _unchecked(str(e))
row = res.rows[0] if res.rows else [None, None]
rows, excluded = _as_int(row[0]), _as_int(row[1]) if len(row) > 1 else None
if not rows:
return _unchecked("表里还没有数据")
excluded = excluded or 0
sampled = rows >= cap
counted = f"{rows:,}"
if excluded in (0, rows):
what = "没有排除任何一行" if excluded == 0 else "所有行都被排除了"
if sampled:
return FilterCheck(True, FILTER_SAMPLED_EDGE.format(rows=counted, what=what), rows, excluded, True)
return FilterCheck(False, FILTER_CHECKED.format(share=excluded_share(excluded, rows), rows=counted), rows,
excluded, False, dropped=what)
share = excluded_share(excluded, rows)
note = (FILTER_SAMPLED if sampled else FILTER_CHECKED).format(share=share, rows=counted)
return FilterCheck(True, note, rows, excluded, sampled)
__all__ = ["CODES_MAX", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", "FilterCheck",
"IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES",
"PROFILE_OPTION", "PROFILE_PROBE_TABLES", "ProfileBusy", "ProfileDisabled", "ProfileReport",
"ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "ensure_enabled",
"estimate_size", "profile_catalog", "profile_settings", "profile_settings_problem", "profile_settings_rejection",
"sql_literal"]
"ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "check_valid_filter",
"ensure_enabled", "estimate_size", "excluded_share", "profile_catalog", "profile_settings",
"profile_settings_problem", "profile_settings_rejection", "sql_literal"]
+184 -2
View File
@@ -1131,5 +1131,187 @@ async def load_checkers_by_name(session: Any, names: Iterable[str]) -> dict[str,
return await load_checkers(session, rows)
__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "check_sql",
"dialect_of", "graph_checks", "graph_sources", "load_checkers", "load_checkers_by_name", "tool_query"]
# ==========================================================================
# 有效行过滤(valid_filter)的校验:模型起草的条件要能进目录、能拼进剖析的核对查询
# ==========================================================================
#: 有效行过滤最多几个字(同目录里文本项的上限)
VALID_FILTER_MAX = 500
#: 有效行过滤里能用的函数(其余函数、聚合、窗口一律不收)
VALID_FILTER_FUNCTIONS = ("UPPER", "LOWER", "TRIM", "COALESCE")
#: 模板占位:条件里不该有,有了也说不清最终拼出来的是什么(注释在解析之后按语法树认,字符串里的 -- 不算)
_FILTER_TEMPLATE = re.compile(r"\{\{|\{%")
_FILTER_PREFIX = "有效行过滤"
@dataclass(frozen=True)
class ValidFilter:
"""一段有效行过滤解析的结果。
expression:sqlglot 的条件(不合规时为 None);columns:条件里用到的列(表结构里的真名,按出现的先后去重);
names:小写列名 → 表结构里的真名;problem:不合规时给人看的一句话,合规是 None;dialect:解析用的 sqlglot 方言。
"""
expression: Any = None
columns: tuple[str, ...] = ()
names: Mapping[str, str] = field(default_factory=dict)
problem: str | None = None
dialect: str | None = None
def _bad_filter(problem: str, dialect: str | None = None) -> ValidFilter:
return ValidFilter(problem=f"{_FILTER_PREFIX}{problem}", dialect=dialect)
def _filter_node_problem(node: Any, *, predicate: bool) -> str | None:
"""一个不在允许范围里的节点为什么不收(没有问题返回 None 的调用方不会走到这里)。"""
from sqlglot import exp
if isinstance(node, exp.Query):
return "应为一个条件,不能是一条查询"
if any(isinstance(n, (exp.Query, exp.Subquery)) for n in node.walk()):
return "里不能有子查询"
if any(isinstance(n, exp.Window) for n in node.walk()):
return "里不能有窗口函数"
if any(isinstance(n, exp.AggFunc) for n in node.walk()):
return "里不能有聚合函数"
if isinstance(node, exp.Func):
name = str(node.this) if isinstance(node, exp.Anonymous) else node.sql_name()
return f"里只能用 {'、'.join(VALID_FILTER_FUNCTIONS)} 这几个函数,不能用 {name.upper()}"
if predicate:
return "应为一个条件(比较、IN、BETWEEN、LIKE、IS NULL,用 AND、OR、NOT 连接)"
text = node.sql()
return f"里有不支持的写法({text[:40]}{'…' if len(text) > 40 else ''})"
def _filter_value_problem(node: Any) -> str | None:
"""比较两边的一个值:列、字面量、负数、允许的几个函数(参数同样是值)。"""
from sqlglot import exp
node = _unparen(node)
if isinstance(node, (exp.Column, exp.Literal, exp.Null, exp.Boolean)):
return None
if isinstance(node, exp.Neg) and isinstance(_unparen(node.this), exp.Literal) and not _unparen(node.this).is_string:
return None
if isinstance(node, (exp.Upper, exp.Lower, exp.Trim, exp.Coalesce)):
for arg in node.iter_expressions():
if problem := _filter_value_problem(arg):
return problem
return None
return _filter_node_problem(node, predicate=False)
def _filter_predicate_problem(node: Any) -> str | None:
"""一个条件:比较、IN(只能是值的列表)、BETWEEN、LIKE、IS NULL,或者用 AND / OR / NOT 连起来的条件。"""
from sqlglot import exp
node = _unparen(node)
if isinstance(node, (exp.And, exp.Or)):
return _filter_predicate_problem(node.left) or _filter_predicate_problem(node.right)
if isinstance(node, exp.Not):
return _filter_predicate_problem(node.this)
if isinstance(node, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE)):
return _filter_value_problem(node.left) or _filter_value_problem(node.right)
if isinstance(node, exp.In):
if any(node.args.get(k) for k in ("query", "unnest", "field")):
return "里不能有子查询"
if not node.expressions:
return _filter_node_problem(node, predicate=True)
return _filter_value_problem(node.this) or next(
(p for p in (_filter_value_problem(e) for e in node.expressions) if p), None)
if isinstance(node, exp.Between):
return next((p for p in (_filter_value_problem(node.args.get(k)) for k in ("this", "low", "high")) if p), None)
if isinstance(node, exp.Like):
return _filter_value_problem(node.this) or _filter_value_problem(node.expression)
if isinstance(node, exp.Is):
if not isinstance(_unparen(node.expression), exp.Null):
return "里的 IS 只能写 IS NULL 或 IS NOT NULL"
return _filter_value_problem(node.this)
return _filter_node_problem(node, predicate=True)
def parse_valid_filter(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None,
table: str | None = None) -> ValidFilter:
"""按数据源的方言把有效行过滤解析成一个条件,并核对它只用了这张表的列。从不抛异常,不合规写在 problem 里。
规矩:一段文字、不超过 VALID_FILTER_MAX 字;不能有分号、模板占位、注释;解析出来恰好是一个条件,只由比较、IN(值的
列表)、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE 组成(不能有子查询、聚合、窗口、
别的函数、算术、参数占位);列不带表名或带这张表的表名(table),都在表结构里(不分大小写),至少用到一列。
"""
from app.data.names import name_key as _key
dialect = dialect_of(kind)
if not isinstance(condition, str) or not condition.strip():
return _bad_filter("应为一段 SQL 条件", dialect)
text = condition.strip()
if len(text) > VALID_FILTER_MAX:
return _bad_filter(f"超过 {VALID_FILTER_MAX} 字", dialect)
if ";" in text:
return _bad_filter("里不能有分号", dialect)
if _FILTER_TEMPLATE.search(text):
return _bad_filter("里不能有模板占位", dialect)
if dialect is None:
return _bad_filter(f"没法校验:数据源类型「{kind}」不支持", dialect)
try:
import sqlglot
statements = [s for s in sqlglot.parse(text, read=dialect) if s is not None]
except Exception: # noqa: BLE001 - 语法错、方言不支持的写法:一律不收
return _bad_filter("不是一段能解析的 SQL 条件", dialect)
if len(statements) != 1:
return _bad_filter("不是一段能解析的 SQL 条件", dialect)
expression = statements[0]
if any(node.comments for node in expression.walk()):
return _bad_filter("里不能有注释", dialect)
if problem := _filter_predicate_problem(expression):
return _bad_filter(problem, dialect)
from sqlglot import exp
lookup = {str(c.get("name")).lower(): str(c.get("name")) for c in (meta or {}).get("columns") or []
if isinstance(c, Mapping) and c.get("name") is not None}
own = {_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified")) if n}
used: dict[str, str] = {}
for column in expression.find_all(exp.Column, bfs=False): # 按书写的先后
if column.is_star:
return _bad_filter("里不能写 *", dialect)
qualifier = column.table
if column.args.get("db") or column.args.get("catalog") or (qualifier and _key(qualifier) not in own):
return _bad_filter(f"只能引用这张表的列,不能引用 {column.sql(dialect=dialect)}", dialect)
real = lookup.get(column.name.lower())
if real is None:
return _bad_filter(f"里的列「{column.name}」不在表结构里", dialect)
used.setdefault(column.name.lower(), real)
if not used:
return _bad_filter("没有用到这张表的任何列", dialect)
return ValidFilter(expression=expression, columns=tuple(dict.fromkeys(used.values())), names=used,
dialect=dialect)
def valid_filter_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None,
table: str | None = None) -> str | None:
"""有效行过滤哪里不合规(给人看的一句话);合规返回 None。规矩见 parse_valid_filter。"""
return parse_valid_filter(condition, meta, kind=kind, table=table).problem
def render_valid_filter(parsed: ValidFilter, *, quoted: Any) -> str:
"""把解析好的条件按数据源的方言重新写出来,列换成表结构里的真名、不带表名;quoted(真名) 为真的加引号(和剖析查询
其余部分的写法一致:SqlDialect.quote 要加引号的才加)。拼进剖析的核对查询用,不改目录里存的原文。"""
from sqlglot import exp
if parsed.expression is None:
raise ValueError(parsed.problem or "有效行过滤不合规")
def swap(node: Any) -> Any:
if isinstance(node, exp.Column):
real = parsed.names.get(node.name.lower(), node.name)
return exp.Column(this=exp.Identifier(this=real, quoted=bool(quoted(real))))
return node
return parsed.expression.copy().transform(swap).sql(dialect=parsed.dialect)
__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "VALID_FILTER_FUNCTIONS",
"VALID_FILTER_MAX", "ValidFilter", "check_sql", "dialect_of", "graph_checks", "graph_sources",
"load_checkers", "load_checkers_by_name", "parse_valid_filter", "render_valid_filter", "tool_query",
"valid_filter_problem"]
+215
View File
@@ -0,0 +1,215 @@
"""防错事实的提议:模型起草时提议有效行过滤(valid_filter)、去重规则(dedup),附一句依据。
- 提示词写清两项只在表结构、注释、码值能看出来时才写;结构化输出、纯文本两条路都认这两项;
- 有效行过滤按数据源的方言用 sqlglot 解析成一个条件:只能引用这张表的列,不能有子查询、聚合、窗口函数、分号,只允许
比较、IN、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE,不超过 500 字。不过就丢掉;
- 开了数据剖析、能连上的源:经剖析的只读守卫数出排除比例,排除 0% 或 100% 的不收,收下的在 note 里写剖析核对的结果;
没开剖析、连不上的照收,note 写没有核对的原因;
- 开了剖析的源,目录里记下的码值交给模型做背景(遮罩的列不给);
- 这两项照旧记模型起草 / 推断;SQL 检查只在人工确认、有确证时才把漏了有效行过滤报成警告,推断的只是提示。
用合成的景区业务库(tests/fixtures/catalog/scenic.py)和虚构的表结构,不调用真实模型。
"""
from __future__ import annotations
import logging
import sqlite3
from types import SimpleNamespace
import pytest
from app.data import catalog, catalog_profile, sqlcheck
from app.data import catalog_draft as D
from app.data.catalog import make_item
from app.data.catalog_profile import PROFILE_OPTION
from app.data.engine import MASK_COLUMNS_OPTION, engines
from app.data.introspect import introspect
from tests.fixtures.catalog.fake_model import FakeModel, answer, requested
# --------------------------------------------------------------------------
# 夹具
# --------------------------------------------------------------------------
def _col(name: str, type_: str = "INTEGER", comment: str | None = None) -> dict:
return {"name": name, "type": type_, "nullable": True, "comment": comment}
def _orders() -> dict:
return {"qualified": "orders", "schema": None, "is_view": False, "primary_key": ["id"], "foreign_keys": [],
"comment": None,
"columns": [_col("id"), _col("order_no", "TEXT"), _col("status", "TEXT", comment="C 作废,X 测试单"),
_col("amount", "REAL"), _col("updated_at", "TEXT")]}
def _plain(tables: dict, **kw) -> SimpleNamespace:
"""不连库的源(没有 options:没开数据剖析)。"""
return SimpleNamespace(id="facts-src", name="facts", kind=kw.pop("kind", "sqlite"), origin="manual",
description="", schema_cache={"tables": tables}, **kw)
_CACHE: dict[str, dict] = {}
async def _scenic_cache(path: str) -> dict:
if path not in _CACHE:
probe = SimpleNamespace(id="facts-scenic-probe", name="scenic", kind="sqlite", database=path, host=None,
port=None, username=None, password=None, options={}, readonly=True, description="",
schema_cache={}, origin="manual")
try:
_CACHE[path] = await introspect(probe)
finally:
await engines.invalidate(probe.id)
return _CACHE[path]
@pytest.fixture
async def scenic(scenic_db):
"""接上景区库的源(开了数据剖析)。用完作废连接池。"""
made: list[str] = []
async def make(*, options: dict | None = None, database: str | None = None, enabled: bool = True):
sid = f"facts-{len(made)}-{id(made)}"
made.append(sid)
opts = {PROFILE_OPTION: {"enabled": True}} if options is None else options
return SimpleNamespace(id=sid, name="scenic", kind="sqlite", database=database or scenic_db, host=None,
port=None, username=None, password=None, options=opts, readonly=True, description="",
schema_cache=await _scenic_cache(scenic_db), origin="manual", enabled=enabled)
yield make
for sid in made:
await engines.invalidate(sid)
def _count(path: str, where: str) -> tuple[int, int]:
"""(全表行数, 不满足条件的行数):直接用 sqlite3 数,核对剖析算出来的数。"""
db = sqlite3.connect(path)
try:
total = db.execute("SELECT COUNT(*) FROM visits").fetchone()[0]
kept = db.execute(f"SELECT COUNT(*) FROM visits WHERE {where}").fetchone()[0]
finally:
db.close()
return total, total - kept
# --------------------------------------------------------------------------
# 校验有效行过滤
# --------------------------------------------------------------------------
META = _orders()
@pytest.mark.parametrize("condition, columns", [
("status NOT IN ('C', 'X')", ["status"]),
("STATUS NOT IN ('C', 'X')", ["status"]), # 列名不分大小写,记表结构里的写法
("orders.status <> 'C'", ["status"]), # 带这张表的表名
("status IS NOT NULL AND amount > 0", ["status", "amount"]),
("NOT (status = 'C' OR status = 'X')", ["status"]),
("amount BETWEEN 0 AND 100000", ["amount"]),
("order_no NOT LIKE 'TEST%'", ["order_no"]),
("UPPER(TRIM(status)) <> 'C'", ["status"]),
("COALESCE(status, '') <> 'X' AND amount >= -1", ["status", "amount"]),
("status = 'C' OR updated_at IS NULL", ["status", "updated_at"]),
])
def test_valid_filters_that_pass(condition, columns):
parsed = sqlcheck.parse_valid_filter(condition, META, kind="sqlite", table="orders")
assert parsed.problem is None, parsed.problem
assert list(parsed.columns) == columns
assert sqlcheck.valid_filter_problem(condition, META, kind="mysql", table="orders") is None
@pytest.mark.parametrize("condition, said", [
("status = 'C'; DROP TABLE orders", "分号"),
("status IN (SELECT code FROM codes)", "子查询"),
("EXISTS (SELECT 1 FROM refunds)", "子查询"),
("COUNT(status) > 1", "聚合"),
("ROW_NUMBER() OVER (ORDER BY id) = 1", "窗口"),
("LENGTH(status) > 0", "LENGTH"),
("amount * 2 > 10", "不支持的写法"),
("payments.status = 1", "只能引用这张表的列"),
("nope = 1", "nope"),
("1 = 1", "没有用到这张表的任何列"),
("status", "一个条件"),
("status = ?", "不支持的写法"),
("status = 'C' -- 注释", "注释"),
("status = '{{ input.code }}'", "模板"),
("status = '" + "C" * 600 + "'", "500"),
("status = (", "解析"),
("status = 'C' UNION SELECT 1", "一个条件"),
])
def test_valid_filters_that_are_dropped(condition, said):
problem = sqlcheck.valid_filter_problem(condition, META, kind="sqlite", table="orders")
assert problem is not None and said in problem, problem
def test_a_source_kind_without_a_dialect_cannot_be_checked():
assert "不支持" in sqlcheck.valid_filter_problem("status = 'C'", META, kind="mssql", table="orders")
def test_the_condition_is_rendered_with_the_columns_quoted_like_the_source():
meta = {**META, "columns": [*META["columns"], _col("isTest")]}
parsed = sqlcheck.parse_valid_filter("istest = 0 AND STATUS <> 'C'", meta, kind="mysql", table="orders")
assert parsed.problem is None
rendered = sqlcheck.render_valid_filter(parsed, quoted=lambda name: name != name.lower())
assert rendered == "`isTest` = 0 AND status <> 'C'"
# --------------------------------------------------------------------------
# 剖析核对排除比例
# --------------------------------------------------------------------------
async def test_the_check_counts_excluded_rows_through_the_read_only_guard(scenic, scenic_db, monkeypatch):
seen: list[tuple[str, bool]] = []
real = catalog_profile.guard.check
def spy(sql, **kw):
seen.append((sql, kw.get("readonly")))
return real(sql, **kw)
monkeypatch.setattr(catalog_profile.guard, "check", spy)
source = await scenic()
meta = source.schema_cache["tables"]["visits"]
found = await catalog_profile.check_valid_filter(source, "visits", meta, "status = 1")
total, excluded = _count(scenic_db, "status = 1")
assert 0 < excluded < total
assert found.keep and (found.rows, found.excluded) == (total, excluded) and not found.sampled
share = catalog_profile.excluded_share(excluded, total)
assert found.note == f"剖析核对:排除 {share} 的行(共 {total:,} 行)"
assert len(seen) == 1 and seen[0][1] is True, "一条查询,过只读守卫"
assert "COUNT(*)" in seen[0][0] and "status = 1" in seen[0][0]
@pytest.mark.parametrize("condition", ["status IN (0, 1)", "status = 7"])
async def test_a_filter_that_excludes_nothing_or_everything_is_dropped(scenic, condition):
source = await scenic()
found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"],
condition)
assert not found.keep and found.dropped
async def test_a_big_table_is_checked_on_its_first_rows(scenic):
source = await scenic(options={PROFILE_OPTION: {"enabled": True, "max_scan_rows": 100, "sample_size": 10}})
found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"],
"status = 1")
assert found.sampled and found.rows == 100
assert found.note.startswith("剖析核对:按前 100 行统计,")
async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path):
meta_of = lambda s: s.schema_cache["tables"]["visits"] # noqa: E731
off = await scenic(options={})
found = await catalog_profile.check_valid_filter(off, "visits", meta_of(off), "status = 1")
assert found.keep and found.note == "没有核对排除比例(数据源没有开数据剖析)"
masked = await scenic(options={PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: "Status"})
found = await catalog_profile.check_valid_filter(masked, "visits", meta_of(masked), "status = 1")
assert found.keep and found.note == "没有核对排除比例(条件里的 status 在数据源设置中被遮罩,不取数)"
stopped = await scenic(enabled=False)
found = await catalog_profile.check_valid_filter(stopped, "visits", meta_of(stopped), "status = 1")
assert found.keep and found.note == "没有核对排除比例(数据源已停用)"
gone = await scenic(database=str(tmp_path / "missing" / "nope.db"))
found = await catalog_profile.check_valid_filter(gone, "visits", meta_of(gone), "status = 1")
assert found.keep and found.note.startswith("没有核对排除比例(查询失败:")