diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 6966b26d..1e9ec5bb 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -1341,8 +1341,128 @@ async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[st return report -__all__ = ["CODES_MAX", "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", +# ========================================================================== +# 核对模型提议的有效行过滤:排除了多少行 +# ========================================================================== + +#: 有效行过滤核对的说法(写进这一项的 note;前面是模型给的依据) +FILTER_CHECKED = "剖析核对:排除 {share} 的行(共 {rows} 行)" +FILTER_SAMPLED = "剖析核对:按前 {rows} 行统计,排除 {share} 的行" +FILTER_SAMPLED_EDGE = "剖析核对:按前 {rows} 行统计,{what},请人工核对" +FILTER_UNCHECKED = "没有核对排除比例({why})" +FILTER_NOT_ENABLED = "数据源没有开数据剖析" + + +@dataclass(frozen=True) +class FilterCheck: + """核对一条有效行过滤的结果。 + + keep:收不收(排除 0% 或 100% 的不收:前者什么都没筛掉,后者把所有行都筛掉了,多半写错了);note:写进这一项 note + 的那句(剖析核对的排除比例,或者没有核对的原因);rows / excluded:数到的行数和其中被排除的行数(没核对是 None); + sampled:表太大或不知道多大,只数了前若干行;dropped:不收的原因(记日志用)。""" + + keep: bool + note: str + rows: int | None = None + excluded: int | None = None + sampled: bool = False + dropped: str | None = None + + +def excluded_share(excluded: int, rows: int) -> str: + """排除比例写成百分数:留一位小数(整数不带小数);不到 0.1% 的写「不足 0.1%」,不是全部的不四舍五入成 100%。""" + if rows <= 0: + return "0%" + pct = excluded * 100 / rows + if 0 < pct < 0.1: + return "不足 0.1%" + if 99.9 < pct < 100: + return "超过 99.9%" + return f"{pct:.1f}".rstrip("0").rstrip(".") + "%" + + +def _unchecked(why: str) -> FilterCheck: + return FilterCheck(True, FILTER_UNCHECKED.format(why=why)) + + +def _unsafe_literal(value: str) -> bool: + """拼不进核对查询的字符串取值:查询层用 SQLAlchemy 的 text() 执行,它会把字符串里的「:名字」当成绑定参数; + 反斜杠在 MySQL 上读法不定、守卫会整条拒掉;控制字符不该出现(同 sql_literal 的规矩)。""" + return "\\" in value or bool(_CONTROL.search(value)) or bool(_BIND_LIKE.search(value)) + + +async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | None, condition: str, *, + settings: ProfileSettings | None = None) -> FilterCheck: + """开了数据剖析的源:数一数这条有效行过滤排除了多少行(模型起草时提议的那条,data/catalog_draft.py 调)。从不抛异常。 + + 经剖析的只读守卫(_Runner:guard.check(readonly=True)、单条时限、查询次数上限)发一条查询:只取条件里用到的列, + 先取前 max(sample_size, max_scan_rows) 行(大表不做整表扫描),再数总行数和不满足条件的行数(条件为 NULL 也算排除, + 和 WHERE 一样)。数到的不满前那么多行,就是整张表。 + + - 排除 0% 或 100% 的不收(keep 为假);只数了前若干行的照收,写明「请人工核对」; + - 没开剖析、数据源停用、条件里有遮罩的列、视图、取值拼不进查询、查询失败或超时:照收,note 写没有核对的原因。 + """ + from app.data import sqlcheck + + options = getattr(source, "options", None) + settings = settings or profile_settings(options) + if not settings.enabled: + return _unchecked(FILTER_NOT_ENABLED) + if getattr(source, "enabled", True) is False: + return _unchecked("数据源已停用") + kind = getattr(source, "kind", None) + parsed = sqlcheck.parse_valid_filter(condition, meta, kind=kind, table=table) + if parsed.problem: + return FilterCheck(False, parsed.problem, dropped=parsed.problem) + masked = {c.lower() for c in data_engine.masked_columns(options)} + if hit := [c for c in parsed.columns if c.lower() in masked]: + return _unchecked(f"条件里的 {'、'.join(hit)} 在数据源设置中被遮罩,不取数") + if (meta or {}).get("is_view"): + return _unchecked("这张表是视图,剖析可能触发整个视图的计算") + from sqlglot import exp + + if any(n.is_string and _unsafe_literal(str(n.this)) for n in parsed.expression.find_all(exp.Literal)): + return _unchecked("条件里的取值含有冒号或反斜杠,拼不进核对查询") + try: + dialect = SqlDialect(kind) + except ValueError as e: + return _unchecked(str(e)) + cap = max(int(settings.sample_size), int(settings.max_scan_rows)) + cond = sqlcheck.render_valid_filter(parsed, quoted=lambda name: dialect.quote(name) != name) + cols = ", ".join(dialect.quote(c) for c in parsed.columns) + inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap) + sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s" + run = _Runner(source, settings, dialect) + try: + res = await run(sql) + except _Skip as e: + return _unchecked(e.detail.removesuffix(",已跳过")) + except _Stop as e: + return _unchecked(_STOP_DETAIL[e.reason].format(max_queries=settings.max_queries, + max_total_s=_fmt(settings.max_total_s)).removesuffix(",未检查")) + except SnapshotTampered as e: + return _unchecked(str(e)) + row = res.rows[0] if res.rows else [None, None] + rows, excluded = _as_int(row[0]), _as_int(row[1]) if len(row) > 1 else None + if not rows: + return _unchecked("表里还没有数据") + excluded = excluded or 0 + sampled = rows >= cap + counted = f"{rows:,}" + if excluded in (0, rows): + what = "没有排除任何一行" if excluded == 0 else "所有行都被排除了" + if sampled: + return FilterCheck(True, FILTER_SAMPLED_EDGE.format(rows=counted, what=what), rows, excluded, True) + return FilterCheck(False, FILTER_CHECKED.format(share=excluded_share(excluded, rows), rows=counted), rows, + excluded, False, dropped=what) + share = excluded_share(excluded, rows) + note = (FILTER_SAMPLED if sampled else FILTER_CHECKED).format(share=share, rows=counted) + return FilterCheck(True, note, rows, excluded, sampled) + + +__all__ = ["CODES_MAX", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", "FilterCheck", + "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", "PROFILE_OPTION", "PROFILE_PROBE_TABLES", "ProfileBusy", "ProfileDisabled", "ProfileReport", - "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "ensure_enabled", - "estimate_size", "profile_catalog", "profile_settings", "profile_settings_problem", "profile_settings_rejection", - "sql_literal"] + "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "check_valid_filter", + "ensure_enabled", "estimate_size", "excluded_share", "profile_catalog", "profile_settings", + "profile_settings_problem", "profile_settings_rejection", "sql_literal"] diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index c233d430..8cd6e440 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -1131,5 +1131,187 @@ async def load_checkers_by_name(session: Any, names: Iterable[str]) -> dict[str, return await load_checkers(session, rows) -__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "check_sql", - "dialect_of", "graph_checks", "graph_sources", "load_checkers", "load_checkers_by_name", "tool_query"] +# ========================================================================== +# 有效行过滤(valid_filter)的校验:模型起草的条件要能进目录、能拼进剖析的核对查询 +# ========================================================================== + +#: 有效行过滤最多几个字(同目录里文本项的上限) +VALID_FILTER_MAX = 500 +#: 有效行过滤里能用的函数(其余函数、聚合、窗口一律不收) +VALID_FILTER_FUNCTIONS = ("UPPER", "LOWER", "TRIM", "COALESCE") +#: 模板占位:条件里不该有,有了也说不清最终拼出来的是什么(注释在解析之后按语法树认,字符串里的 -- 不算) +_FILTER_TEMPLATE = re.compile(r"\{\{|\{%") + +_FILTER_PREFIX = "有效行过滤" + + +@dataclass(frozen=True) +class ValidFilter: + """一段有效行过滤解析的结果。 + + expression:sqlglot 的条件(不合规时为 None);columns:条件里用到的列(表结构里的真名,按出现的先后去重); + names:小写列名 → 表结构里的真名;problem:不合规时给人看的一句话,合规是 None;dialect:解析用的 sqlglot 方言。 + """ + + expression: Any = None + columns: tuple[str, ...] = () + names: Mapping[str, str] = field(default_factory=dict) + problem: str | None = None + dialect: str | None = None + + +def _bad_filter(problem: str, dialect: str | None = None) -> ValidFilter: + return ValidFilter(problem=f"{_FILTER_PREFIX}{problem}", dialect=dialect) + + +def _filter_node_problem(node: Any, *, predicate: bool) -> str | None: + """一个不在允许范围里的节点为什么不收(没有问题返回 None 的调用方不会走到这里)。""" + from sqlglot import exp + + if isinstance(node, exp.Query): + return "应为一个条件,不能是一条查询" + if any(isinstance(n, (exp.Query, exp.Subquery)) for n in node.walk()): + return "里不能有子查询" + if any(isinstance(n, exp.Window) for n in node.walk()): + return "里不能有窗口函数" + if any(isinstance(n, exp.AggFunc) for n in node.walk()): + return "里不能有聚合函数" + if isinstance(node, exp.Func): + name = str(node.this) if isinstance(node, exp.Anonymous) else node.sql_name() + return f"里只能用 {'、'.join(VALID_FILTER_FUNCTIONS)} 这几个函数,不能用 {name.upper()}" + if predicate: + return "应为一个条件(比较、IN、BETWEEN、LIKE、IS NULL,用 AND、OR、NOT 连接)" + text = node.sql() + return f"里有不支持的写法({text[:40]}{'…' if len(text) > 40 else ''})" + + +def _filter_value_problem(node: Any) -> str | None: + """比较两边的一个值:列、字面量、负数、允许的几个函数(参数同样是值)。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.Column, exp.Literal, exp.Null, exp.Boolean)): + return None + if isinstance(node, exp.Neg) and isinstance(_unparen(node.this), exp.Literal) and not _unparen(node.this).is_string: + return None + if isinstance(node, (exp.Upper, exp.Lower, exp.Trim, exp.Coalesce)): + for arg in node.iter_expressions(): + if problem := _filter_value_problem(arg): + return problem + return None + return _filter_node_problem(node, predicate=False) + + +def _filter_predicate_problem(node: Any) -> str | None: + """一个条件:比较、IN(只能是值的列表)、BETWEEN、LIKE、IS NULL,或者用 AND / OR / NOT 连起来的条件。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.And, exp.Or)): + return _filter_predicate_problem(node.left) or _filter_predicate_problem(node.right) + if isinstance(node, exp.Not): + return _filter_predicate_problem(node.this) + if isinstance(node, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE)): + return _filter_value_problem(node.left) or _filter_value_problem(node.right) + if isinstance(node, exp.In): + if any(node.args.get(k) for k in ("query", "unnest", "field")): + return "里不能有子查询" + if not node.expressions: + return _filter_node_problem(node, predicate=True) + return _filter_value_problem(node.this) or next( + (p for p in (_filter_value_problem(e) for e in node.expressions) if p), None) + if isinstance(node, exp.Between): + return next((p for p in (_filter_value_problem(node.args.get(k)) for k in ("this", "low", "high")) if p), None) + if isinstance(node, exp.Like): + return _filter_value_problem(node.this) or _filter_value_problem(node.expression) + if isinstance(node, exp.Is): + if not isinstance(_unparen(node.expression), exp.Null): + return "里的 IS 只能写 IS NULL 或 IS NOT NULL" + return _filter_value_problem(node.this) + return _filter_node_problem(node, predicate=True) + + +def parse_valid_filter(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, + table: str | None = None) -> ValidFilter: + """按数据源的方言把有效行过滤解析成一个条件,并核对它只用了这张表的列。从不抛异常,不合规写在 problem 里。 + + 规矩:一段文字、不超过 VALID_FILTER_MAX 字;不能有分号、模板占位、注释;解析出来恰好是一个条件,只由比较、IN(值的 + 列表)、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE 组成(不能有子查询、聚合、窗口、 + 别的函数、算术、参数占位);列不带表名或带这张表的表名(table),都在表结构里(不分大小写),至少用到一列。 + """ + from app.data.names import name_key as _key + + dialect = dialect_of(kind) + if not isinstance(condition, str) or not condition.strip(): + return _bad_filter("应为一段 SQL 条件", dialect) + text = condition.strip() + if len(text) > VALID_FILTER_MAX: + return _bad_filter(f"超过 {VALID_FILTER_MAX} 字", dialect) + if ";" in text: + return _bad_filter("里不能有分号", dialect) + if _FILTER_TEMPLATE.search(text): + return _bad_filter("里不能有模板占位", dialect) + if dialect is None: + return _bad_filter(f"没法校验:数据源类型「{kind}」不支持", dialect) + try: + import sqlglot + + statements = [s for s in sqlglot.parse(text, read=dialect) if s is not None] + except Exception: # noqa: BLE001 - 语法错、方言不支持的写法:一律不收 + return _bad_filter("不是一段能解析的 SQL 条件", dialect) + if len(statements) != 1: + return _bad_filter("不是一段能解析的 SQL 条件", dialect) + expression = statements[0] + if any(node.comments for node in expression.walk()): + return _bad_filter("里不能有注释", dialect) + if problem := _filter_predicate_problem(expression): + return _bad_filter(problem, dialect) + from sqlglot import exp + + lookup = {str(c.get("name")).lower(): str(c.get("name")) for c in (meta or {}).get("columns") or [] + if isinstance(c, Mapping) and c.get("name") is not None} + own = {_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified")) if n} + used: dict[str, str] = {} + for column in expression.find_all(exp.Column, bfs=False): # 按书写的先后 + if column.is_star: + return _bad_filter("里不能写 *", dialect) + qualifier = column.table + if column.args.get("db") or column.args.get("catalog") or (qualifier and _key(qualifier) not in own): + return _bad_filter(f"只能引用这张表的列,不能引用 {column.sql(dialect=dialect)}", dialect) + real = lookup.get(column.name.lower()) + if real is None: + return _bad_filter(f"里的列「{column.name}」不在表结构里", dialect) + used.setdefault(column.name.lower(), real) + if not used: + return _bad_filter("没有用到这张表的任何列", dialect) + return ValidFilter(expression=expression, columns=tuple(dict.fromkeys(used.values())), names=used, + dialect=dialect) + + +def valid_filter_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, + table: str | None = None) -> str | None: + """有效行过滤哪里不合规(给人看的一句话);合规返回 None。规矩见 parse_valid_filter。""" + return parse_valid_filter(condition, meta, kind=kind, table=table).problem + + +def render_valid_filter(parsed: ValidFilter, *, quoted: Any) -> str: + """把解析好的条件按数据源的方言重新写出来,列换成表结构里的真名、不带表名;quoted(真名) 为真的加引号(和剖析查询 + 其余部分的写法一致:SqlDialect.quote 要加引号的才加)。拼进剖析的核对查询用,不改目录里存的原文。""" + from sqlglot import exp + + if parsed.expression is None: + raise ValueError(parsed.problem or "有效行过滤不合规") + + def swap(node: Any) -> Any: + if isinstance(node, exp.Column): + real = parsed.names.get(node.name.lower(), node.name) + return exp.Column(this=exp.Identifier(this=real, quoted=bool(quoted(real)))) + return node + + return parsed.expression.copy().transform(swap).sql(dialect=parsed.dialect) + + +__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "VALID_FILTER_FUNCTIONS", + "VALID_FILTER_MAX", "ValidFilter", "check_sql", "dialect_of", "graph_checks", "graph_sources", + "load_checkers", "load_checkers_by_name", "parse_valid_filter", "render_valid_filter", "tool_query", + "valid_filter_problem"] diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py new file mode 100644 index 00000000..dba6db48 --- /dev/null +++ b/backend/tests/test_catalog_draft_facts.py @@ -0,0 +1,215 @@ +"""防错事实的提议:模型起草时提议有效行过滤(valid_filter)、去重规则(dedup),附一句依据。 + +- 提示词写清两项只在表结构、注释、码值能看出来时才写;结构化输出、纯文本两条路都认这两项; +- 有效行过滤按数据源的方言用 sqlglot 解析成一个条件:只能引用这张表的列,不能有子查询、聚合、窗口函数、分号,只允许 + 比较、IN、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE,不超过 500 字。不过就丢掉; +- 开了数据剖析、能连上的源:经剖析的只读守卫数出排除比例,排除 0% 或 100% 的不收,收下的在 note 里写剖析核对的结果; + 没开剖析、连不上的照收,note 写没有核对的原因; +- 开了剖析的源,目录里记下的码值交给模型做背景(遮罩的列不给); +- 这两项照旧记模型起草 / 推断;SQL 检查只在人工确认、有确证时才把漏了有效行过滤报成警告,推断的只是提示。 + +用合成的景区业务库(tests/fixtures/catalog/scenic.py)和虚构的表结构,不调用真实模型。 +""" +from __future__ import annotations + +import logging +import sqlite3 +from types import SimpleNamespace + +import pytest + +from app.data import catalog, catalog_profile, sqlcheck +from app.data import catalog_draft as D +from app.data.catalog import make_item +from app.data.catalog_profile import PROFILE_OPTION +from app.data.engine import MASK_COLUMNS_OPTION, engines +from app.data.introspect import introspect +from tests.fixtures.catalog.fake_model import FakeModel, answer, requested + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _col(name: str, type_: str = "INTEGER", comment: str | None = None) -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": comment} + + +def _orders() -> dict: + return {"qualified": "orders", "schema": None, "is_view": False, "primary_key": ["id"], "foreign_keys": [], + "comment": None, + "columns": [_col("id"), _col("order_no", "TEXT"), _col("status", "TEXT", comment="C 作废,X 测试单"), + _col("amount", "REAL"), _col("updated_at", "TEXT")]} + + +def _plain(tables: dict, **kw) -> SimpleNamespace: + """不连库的源(没有 options:没开数据剖析)。""" + return SimpleNamespace(id="facts-src", name="facts", kind=kw.pop("kind", "sqlite"), origin="manual", + description="", schema_cache={"tables": tables}, **kw) + + +_CACHE: dict[str, dict] = {} + + +async def _scenic_cache(path: str) -> dict: + if path not in _CACHE: + probe = SimpleNamespace(id="facts-scenic-probe", name="scenic", kind="sqlite", database=path, host=None, + port=None, username=None, password=None, options={}, readonly=True, description="", + schema_cache={}, origin="manual") + try: + _CACHE[path] = await introspect(probe) + finally: + await engines.invalidate(probe.id) + return _CACHE[path] + + +@pytest.fixture +async def scenic(scenic_db): + """接上景区库的源(开了数据剖析)。用完作废连接池。""" + made: list[str] = [] + + async def make(*, options: dict | None = None, database: str | None = None, enabled: bool = True): + sid = f"facts-{len(made)}-{id(made)}" + made.append(sid) + opts = {PROFILE_OPTION: {"enabled": True}} if options is None else options + return SimpleNamespace(id=sid, name="scenic", kind="sqlite", database=database or scenic_db, host=None, + port=None, username=None, password=None, options=opts, readonly=True, description="", + schema_cache=await _scenic_cache(scenic_db), origin="manual", enabled=enabled) + + yield make + for sid in made: + await engines.invalidate(sid) + + +def _count(path: str, where: str) -> tuple[int, int]: + """(全表行数, 不满足条件的行数):直接用 sqlite3 数,核对剖析算出来的数。""" + db = sqlite3.connect(path) + try: + total = db.execute("SELECT COUNT(*) FROM visits").fetchone()[0] + kept = db.execute(f"SELECT COUNT(*) FROM visits WHERE {where}").fetchone()[0] + finally: + db.close() + return total, total - kept + + +# -------------------------------------------------------------------------- +# 校验有效行过滤 +# -------------------------------------------------------------------------- + +META = _orders() + + +@pytest.mark.parametrize("condition, columns", [ + ("status NOT IN ('C', 'X')", ["status"]), + ("STATUS NOT IN ('C', 'X')", ["status"]), # 列名不分大小写,记表结构里的写法 + ("orders.status <> 'C'", ["status"]), # 带这张表的表名 + ("status IS NOT NULL AND amount > 0", ["status", "amount"]), + ("NOT (status = 'C' OR status = 'X')", ["status"]), + ("amount BETWEEN 0 AND 100000", ["amount"]), + ("order_no NOT LIKE 'TEST%'", ["order_no"]), + ("UPPER(TRIM(status)) <> 'C'", ["status"]), + ("COALESCE(status, '') <> 'X' AND amount >= -1", ["status", "amount"]), + ("status = 'C' OR updated_at IS NULL", ["status", "updated_at"]), +]) +def test_valid_filters_that_pass(condition, columns): + parsed = sqlcheck.parse_valid_filter(condition, META, kind="sqlite", table="orders") + assert parsed.problem is None, parsed.problem + assert list(parsed.columns) == columns + assert sqlcheck.valid_filter_problem(condition, META, kind="mysql", table="orders") is None + + +@pytest.mark.parametrize("condition, said", [ + ("status = 'C'; DROP TABLE orders", "分号"), + ("status IN (SELECT code FROM codes)", "子查询"), + ("EXISTS (SELECT 1 FROM refunds)", "子查询"), + ("COUNT(status) > 1", "聚合"), + ("ROW_NUMBER() OVER (ORDER BY id) = 1", "窗口"), + ("LENGTH(status) > 0", "LENGTH"), + ("amount * 2 > 10", "不支持的写法"), + ("payments.status = 1", "只能引用这张表的列"), + ("nope = 1", "nope"), + ("1 = 1", "没有用到这张表的任何列"), + ("status", "一个条件"), + ("status = ?", "不支持的写法"), + ("status = 'C' -- 注释", "注释"), + ("status = '{{ input.code }}'", "模板"), + ("status = '" + "C" * 600 + "'", "500"), + ("status = (", "解析"), + ("status = 'C' UNION SELECT 1", "一个条件"), +]) +def test_valid_filters_that_are_dropped(condition, said): + problem = sqlcheck.valid_filter_problem(condition, META, kind="sqlite", table="orders") + assert problem is not None and said in problem, problem + + +def test_a_source_kind_without_a_dialect_cannot_be_checked(): + assert "不支持" in sqlcheck.valid_filter_problem("status = 'C'", META, kind="mssql", table="orders") + + +def test_the_condition_is_rendered_with_the_columns_quoted_like_the_source(): + meta = {**META, "columns": [*META["columns"], _col("isTest")]} + parsed = sqlcheck.parse_valid_filter("istest = 0 AND STATUS <> 'C'", meta, kind="mysql", table="orders") + assert parsed.problem is None + rendered = sqlcheck.render_valid_filter(parsed, quoted=lambda name: name != name.lower()) + assert rendered == "`isTest` = 0 AND status <> 'C'" + + +# -------------------------------------------------------------------------- +# 剖析核对排除比例 +# -------------------------------------------------------------------------- + + +async def test_the_check_counts_excluded_rows_through_the_read_only_guard(scenic, scenic_db, monkeypatch): + seen: list[tuple[str, bool]] = [] + real = catalog_profile.guard.check + + def spy(sql, **kw): + seen.append((sql, kw.get("readonly"))) + return real(sql, **kw) + + monkeypatch.setattr(catalog_profile.guard, "check", spy) + source = await scenic() + meta = source.schema_cache["tables"]["visits"] + found = await catalog_profile.check_valid_filter(source, "visits", meta, "status = 1") + total, excluded = _count(scenic_db, "status = 1") + assert 0 < excluded < total + assert found.keep and (found.rows, found.excluded) == (total, excluded) and not found.sampled + share = catalog_profile.excluded_share(excluded, total) + assert found.note == f"剖析核对:排除 {share} 的行(共 {total:,} 行)" + assert len(seen) == 1 and seen[0][1] is True, "一条查询,过只读守卫" + assert "COUNT(*)" in seen[0][0] and "status = 1" in seen[0][0] + + +@pytest.mark.parametrize("condition", ["status IN (0, 1)", "status = 7"]) +async def test_a_filter_that_excludes_nothing_or_everything_is_dropped(scenic, condition): + source = await scenic() + found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], + condition) + assert not found.keep and found.dropped + + +async def test_a_big_table_is_checked_on_its_first_rows(scenic): + source = await scenic(options={PROFILE_OPTION: {"enabled": True, "max_scan_rows": 100, "sample_size": 10}}) + found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], + "status = 1") + assert found.sampled and found.rows == 100 + assert found.note.startswith("剖析核对:按前 100 行统计,") + + +async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): + meta_of = lambda s: s.schema_cache["tables"]["visits"] # noqa: E731 + off = await scenic(options={}) + found = await catalog_profile.check_valid_filter(off, "visits", meta_of(off), "status = 1") + assert found.keep and found.note == "没有核对排除比例(数据源没有开数据剖析)" + + masked = await scenic(options={PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: "Status"}) + found = await catalog_profile.check_valid_filter(masked, "visits", meta_of(masked), "status = 1") + assert found.keep and found.note == "没有核对排除比例(条件里的 status 在数据源设置中被遮罩,不取数)" + + stopped = await scenic(enabled=False) + found = await catalog_profile.check_valid_filter(stopped, "visits", meta_of(stopped), "status = 1") + assert found.keep and found.note == "没有核对排除比例(数据源已停用)" + + gone = await scenic(database=str(tmp_path / "missing" / "nope.db")) + found = await catalog_profile.check_valid_filter(gone, "visits", meta_of(gone), "status = 1") + assert found.keep and found.note.startswith("没有核对排除比例(查询失败:")