From 7383877ee7b5c089c03ea6311f953dd7043bff92 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 09:59:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95=E9=97=AD?= =?UTF-8?q?=E7=8E=AF=E7=AC=AC=E4=B8=80=E6=AD=A5=EF=BC=88=E5=90=8E=E7=AB=AF?= =?UTF-8?q?=20A=EF=BC=89=E5=A4=8D=E6=A0=B8=E4=BF=AE=E6=AD=A3=EF=BC=9A?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E6=8F=90=E8=AE=AE=E7=9A=84=E6=9C=89=E6=95=88?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=9D=A1=E4=BB=B6=E5=AD=98=E6=95=B4=E7=90=86?= =?UTF-8?q?=E8=BF=87=E7=9A=84=E5=86=99=E6=B3=95=E2=80=94=E2=80=94=E5=8E=9F?= =?UTF-8?q?=E6=96=87=E7=85=A7=E6=97=A7=EF=BC=88NOT=20IN=E3=80=81IS=20NOT?= =?UTF-8?q?=20NULL=20=E4=B8=8D=E8=A2=AB=E6=94=B9=E5=86=99=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E5=8F=AA=E6=8A=8A=E5=88=97=E6=8D=A2=E6=88=90=E8=A1=A8?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E9=87=8C=E7=9A=84=E7=9C=9F=E5=90=8D=E3=80=81?= =?UTF-8?q?=E4=B8=8D=E5=B8=A6=E8=A1=A8=E5=90=8D=E3=80=81=E6=8C=89=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=BA=90=E7=9A=84=E6=96=B9=E8=A8=80=E8=A6=81=E5=8A=A0?= =?UTF-8?q?=E5=BC=95=E5=8F=B7=E7=9A=84=E5=8A=A0=E5=BC=95=E5=8F=B7=EF=BC=8C?= =?UTF-8?q?=E5=89=96=E6=9E=90=E6=A0=B8=E5=AF=B9=E7=9A=84=E5=B0=B1=E6=98=AF?= =?UTF-8?q?=E5=AD=98=E4=B8=8B=E7=9A=84=E8=BF=99=E4=B8=80=E6=9D=A1=E3=80=82?= =?UTF-8?q?=E4=BB=A5=E5=89=8D=E5=AD=98=E7=9A=84=E6=98=AF=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E7=9A=84=E5=8E=9F=E6=96=87=EF=BC=8C=E6=A0=B8=E5=AF=B9=E6=97=B6?= =?UTF-8?q?=E5=8D=B4=E6=8C=89=E7=9C=9F=E5=90=8D=E5=8A=A0=E4=BA=86=E5=BC=95?= =?UTF-8?q?=E5=8F=B7=EF=BC=9APostgreSQL=20=E4=B8=8A=E6=B7=B7=E5=90=88?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E5=86=99=E7=9A=84=E5=88=97=EF=BC=88"OrderSta?= =?UTF-8?q?tus"=EF=BC=89=E5=AD=98=E6=88=90=20orderstatus=20<>=20'X'?= =?UTF-8?q?=EF=BC=8C=E4=B8=80=E8=B7=91=E5=B0=B1=E6=8A=A5=E5=88=97=E4=B8=8D?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=EF=BC=8CSQL=20=E6=A3=80=E6=9F=A5=E8=BF=98?= =?UTF-8?q?=E4=BC=9A=E8=AE=A9=E6=A8=A1=E5=9E=8B=E7=85=A7=E6=8A=84=E3=80=82?= =?UTF-8?q?=E5=88=97=E5=90=8D=E5=85=88=E7=85=A7=E5=8E=9F=E6=A0=B7=E5=AF=B9?= =?UTF-8?q?=E8=A1=A8=E7=BB=93=E6=9E=84=EF=BC=8C=E5=8F=AA=E5=B7=AE=E5=A4=A7?= =?UTF-8?q?=E5=B0=8F=E5=86=99=E7=9A=84=E5=87=A0=E5=88=97=E5=86=99=E4=B8=8D?= =?UTF-8?q?=E6=B8=85=E6=98=AF=E5=93=AA=E4=B8=80=E5=88=97=E7=9A=84=E4=B8=8D?= =?UTF-8?q?=E6=94=B6=EF=BC=9B=E5=8A=A0=E4=BA=86=E5=BC=95=E5=8F=B7=E8=B6=85?= =?UTF-8?q?=E8=BF=87=20500=20=E5=AD=97=E7=9A=84=E4=B8=8D=E6=94=B6=EF=BC=88?= =?UTF-8?q?=E5=86=99=E8=BF=9B=E5=8E=BB=E4=BC=9A=E8=AE=A9=E6=95=B4=E5=BC=A0?= =?UTF-8?q?=E8=A1=A8=E5=86=99=E5=85=A5=E5=A4=B1=E8=B4=A5=EF=BC=89=EF=BC=9B?= =?UTF-8?q?=E4=B8=8D=E5=90=88=E8=A7=84=E7=9A=84=E8=AF=B4=E6=B3=95=E7=94=A8?= =?UTF-8?q?=E7=95=8C=E9=9D=A2=E4=B8=8A=E7=9A=84=E5=8F=AB=E6=B3=95=E3=80=8C?= =?UTF-8?q?=E6=9C=89=E6=95=88=E8=AE=B0=E5=BD=95=E6=9D=A1=E4=BB=B6=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog.py | 17 +++- backend/app/data/catalog_draft.py | 9 +- backend/app/data/catalog_profile.py | 3 +- backend/app/data/sqlcheck.py | 117 +++++++++++++++++++--- backend/tests/test_catalog_draft_facts.py | 68 +++++++++++++ 5 files changed, 190 insertions(+), 24 deletions(-) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index d6016713..0157e4bd 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1738,6 +1738,8 @@ def _clean_text(value: Any, limit: int = _TEXT_MAX) -> str | None: #: 模型给的依据(有效行过滤、去重规则的 reason)写进这一项的 note FACT_REASON_NOTE = "依据:{reason}" +#: 有效行过滤按数据源的写法整理(列名加了引号)以后超过目录里这一项的长度上限 +FACT_FILTER_TOO_LONG = "{label}按数据源的写法整理后超过 {limit} 字" def _model_fact(raw: Any, key: str) -> tuple[str, str | None] | None: @@ -1765,17 +1767,24 @@ def _model_facts(entry: Mapping[str, Any], meta: Mapping[str, Any], *, kind: str """模型提议的有效行过滤、去重规则(llm / proposed,依据写进 note)。 有效行过滤按数据源的方言解析成一个条件、只能用这张表的列(sqlcheck.parse_valid_filter),不过就丢掉、日志记一条, - 不算这张表失败;存的是模型写的原文(只去掉首尾空白,字符串里的空格不动)。去重规则是一句话,同文字项的规矩。""" + 不算这张表失败。存的是 sqlcheck.stored_valid_filter 整理过的写法:原文照旧,只把列换成表结构里的真名、不带 + 表名,按数据源的需要加引号——剖析核对的、SQL 检查让模型照着加的都是这一条。去重规则是一句话,同文字项的规矩。""" from app.data import sqlcheck out: dict[str, Any] = {} found = _model_fact(entry.get("valid_filter"), "condition") if found is not None: text, reason = found - problem = sqlcheck.valid_filter_problem(text, meta, kind=kind, table=table) + parsed = sqlcheck.parse_valid_filter(text, meta, kind=kind, table=table) + problem = parsed.problem if problem is None: - out["valid_filter"] = _fact_item(text, reason) - else: + stored = sqlcheck.stored_valid_filter(parsed, kind) + if _text_ok(stored): + out["valid_filter"] = _fact_item(stored, reason) + else: + # 加了引号以后超过上限:写进去会让整张表写入失败(validate_notes),不收 + problem = FACT_FILTER_TOO_LONG.format(label=TABLE_FIELD_LABEL["valid_filter"], limit=_TEXT_MAX) + if problem is not None: import logging logging.getLogger(__name__).info("起草数据目录:表 %s 的有效行过滤没有通过校验,已丢掉:%s(%s)", diff --git a/backend/app/data/catalog_draft.py b/backend/app/data/catalog_draft.py index 282a63c2..84ba96a0 100644 --- a/backend/app/data/catalog_draft.py +++ b/backend/app/data/catalog_draft.py @@ -79,10 +79,11 @@ 一段被缩小的上限分开成几块、被拆成两半,块数就多几块;一块问完(写没写出来都算)算结束一块。 **防错事实。** 表级的项里模型可以提议有效行过滤(valid_filter)和去重规则(dedup),各附一句依据(写进这一项的 note), -校验见 catalog._from_model(有效行过滤按方言解析,不过就丢掉)。开了数据剖析的源(catalog_profile.profile_settings): -目录里记下的码值交给模型做背景(遮罩的列不给,「已确认的目录项」里遮罩列的码值同样不给),表级那段写完后经剖析的只读 -守卫数一数有效行过滤排除了多少行(catalog_profile.check_valid_filter),排除 0% 或 100% 的不收,收下的 note 写上排除 -比例;没开剖析、连不上的照收,note 写没有核对的原因。 +校验见 catalog._from_model(有效行过滤按方言解析,不过就丢掉;存的是整理过的写法:列换成表结构里的真名、按方言加引号, +剖析核对的也是这一条)。开了数据剖析的源(catalog_profile.profile_settings):目录里记下的码值交给模型做背景(遮罩的列 +不给,「已确认的目录项」里遮罩列的码值同样不给),表级那段写完后经剖析的只读守卫数一数有效行过滤排除了多少行 +(catalog_profile.check_valid_filter),排除 0% 或 100% 的不收,收下的 note 写上排除比例;没开剖析、连不上的照收, +note 写没有核对的原因。 交给模型的是表结构(列、类型、主键、外键、唯一约束、注释)和使用者补充的背景;开了数据剖析的源另给目录里记下的码值 (剖析取过的取值和含义,遮罩的列不给),别的数据值一概不发。写给模型的文字都在 catalog.py 名字带 _SYSTEM / _PROMPT diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 1e9ec5bb..f905945c 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -1428,7 +1428,8 @@ async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | Non except ValueError as e: return _unchecked(str(e)) cap = max(int(settings.sample_size), int(settings.max_scan_rows)) - cond = sqlcheck.render_valid_filter(parsed, quoted=lambda name: dialect.quote(name) != name) + # 和存进目录的是同一条(catalog._model_facts 存的就是这个写法):列换成真名、按方言加引号,别的照原文 + cond = sqlcheck.canonical_valid_filter(parsed, quote=dialect.quote) cols = ", ".join(dialect.quote(c) for c in parsed.columns) inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap) sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s" diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index 8cd6e440..2a300acd 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -1142,7 +1142,9 @@ VALID_FILTER_FUNCTIONS = ("UPPER", "LOWER", "TRIM", "COALESCE") #: 模板占位:条件里不该有,有了也说不清最终拼出来的是什么(注释在解析之后按语法树认,字符串里的 -- 不算) _FILTER_TEMPLATE = re.compile(r"\{\{|\{%") -_FILTER_PREFIX = "有效行过滤" +#: 不合规的说法以这一项在界面上的叫法开头(catalog.TABLE_FIELD_LABEL:「有效记录条件」):没收下的原因会记进起草任务 +#: 逐表的日志,界面上看得到 +_FILTER_PREFIX = catalog.TABLE_FIELD_LABEL["valid_filter"] @dataclass(frozen=True) @@ -1150,7 +1152,9 @@ class ValidFilter: """一段有效行过滤解析的结果。 expression:sqlglot 的条件(不合规时为 None);columns:条件里用到的列(表结构里的真名,按出现的先后去重); - names:小写列名 → 表结构里的真名;problem:不合规时给人看的一句话,合规是 None;dialect:解析用的 sqlglot 方言。 + names:条件里写的列名 → 表结构里的真名;problem:不合规时给人看的一句话,合规是 None;dialect:解析用的 sqlglot + 方言。text:解析的原文(去掉首尾空白);spans:每处列引用(连同表名前缀)在原文里的位置 (起, 止(含), 真名),按 + 书写的先后,有一处拿不到位置时为 None(canonical_valid_filter 退回按语法树重写)。 """ expression: Any = None @@ -1158,6 +1162,8 @@ class ValidFilter: names: Mapping[str, str] = field(default_factory=dict) problem: str | None = None dialect: str | None = None + text: str = "" + spans: tuple[tuple[int, int, str], ...] | None = None def _bad_filter(problem: str, dialect: str | None = None) -> ValidFilter: @@ -1237,7 +1243,10 @@ def parse_valid_filter(condition: Any, meta: Mapping[str, Any] | None, *, kind: 规矩:一段文字、不超过 VALID_FILTER_MAX 字;不能有分号、模板占位、注释;解析出来恰好是一个条件,只由比较、IN(值的 列表)、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE 组成(不能有子查询、聚合、窗口、 - 别的函数、算术、参数占位);列不带表名或带这张表的表名(table),都在表结构里(不分大小写),至少用到一列。 + 别的函数、算术、参数占位);列不带表名或带这张表的表名(table),都在表结构里,至少用到一列。 + + 列名先照原样对表结构,对不上再不分大小写对;表结构里有只差大小写的几列(PostgreSQL 上 "Status" 和 status 可以 + 并存)、写法又和哪一列都不一样时说不清是哪一列,不收。 """ from app.data.names import name_key as _key @@ -1268,24 +1277,55 @@ def parse_valid_filter(condition: Any, meta: Mapping[str, Any] | None, *, kind: return _bad_filter(problem, dialect) from sqlglot import exp - lookup = {str(c.get("name")).lower(): str(c.get("name")) for c in (meta or {}).get("columns") or [] - if isinstance(c, Mapping) and c.get("name") is not None} + real_names = [str(c.get("name")) for c in (meta or {}).get("columns") or [] + if isinstance(c, Mapping) and c.get("name") is not None] + exact = set(real_names) + by_lower: dict[str, list[str]] = {} + for name in real_names: + by_lower.setdefault(name.lower(), []).append(name) own = {_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified")) if n} used: dict[str, str] = {} + spans: list[tuple[int, int, str]] | None = [] for column in expression.find_all(exp.Column, bfs=False): # 按书写的先后 if column.is_star: return _bad_filter("里不能写 *", dialect) qualifier = column.table if column.args.get("db") or column.args.get("catalog") or (qualifier and _key(qualifier) not in own): return _bad_filter(f"只能引用这张表的列,不能引用 {column.sql(dialect=dialect)}", dialect) - real = lookup.get(column.name.lower()) - if real is None: - return _bad_filter(f"里的列「{column.name}」不在表结构里", dialect) - used.setdefault(column.name.lower(), real) + written = column.name + if written in exact: + real = written + else: + found = by_lower.get(written.lower()) or [] + if not found: + return _bad_filter(f"里的列「{written}」不在表结构里", dialect) + if len(found) > 1: + return _bad_filter(f"里的列「{written}」在表结构里有只差大小写的几列({'、'.join(found)})," + "要照表结构的写法写清是哪一列", dialect) + real = found[0] + used.setdefault(written, real) + span = _column_span(column, text) + spans = None if spans is None or span is None else [*spans, (*span, real)] if not used: return _bad_filter("没有用到这张表的任何列", dialect) return ValidFilter(expression=expression, columns=tuple(dict.fromkeys(used.values())), names=used, - dialect=dialect) + dialect=dialect, text=text, spans=tuple(spans) if spans is not None else None) + + +def _column_span(column: Any, text: str) -> tuple[int, int] | None: + """一处列引用(连同表名前缀)在原文里的位置 (起, 止(含)):sqlglot 在标识符的 meta 里记下了它在原文里的起止。 + 拿不到、或者那一截原文里认不出这个列名时为 None。""" + ident = column.this + meta = getattr(ident, "meta", None) or {} + start, end = meta.get("start"), meta.get("end") + prefix = column.args.get("table") + if prefix is not None: + start = (getattr(prefix, "meta", None) or {}).get("start") + if not (isinstance(start, int) and isinstance(end, int) and 0 <= start <= end < len(text)): + return None + if not text[start:end + 1].rstrip("\"`]").lower().endswith(str(column.name).lower()): + return None + return start, end def valid_filter_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, @@ -1296,7 +1336,8 @@ def valid_filter_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind def render_valid_filter(parsed: ValidFilter, *, quoted: Any) -> str: """把解析好的条件按数据源的方言重新写出来,列换成表结构里的真名、不带表名;quoted(真名) 为真的加引号(和剖析查询 - 其余部分的写法一致:SqlDialect.quote 要加引号的才加)。拼进剖析的核对查询用,不改目录里存的原文。""" + 其余部分的写法一致:SqlDialect.quote 要加引号的才加)。按语法树重写,NOT IN、IS NOT NULL 会写成 NOT x IN、 + NOT x IS NULL:canonical_valid_filter 拿不到原文里的位置时才退回用它。""" from sqlglot import exp if parsed.expression is None: @@ -1304,14 +1345,60 @@ def render_valid_filter(parsed: ValidFilter, *, quoted: Any) -> str: def swap(node: Any) -> Any: if isinstance(node, exp.Column): - real = parsed.names.get(node.name.lower(), node.name) + real = parsed.names.get(node.name, node.name) return exp.Column(this=exp.Identifier(this=real, quoted=bool(quoted(real)))) return node return parsed.expression.copy().transform(swap).sql(dialect=parsed.dialect) +def canonical_valid_filter(parsed: ValidFilter, *, quote: Any) -> str: + """有效行过滤存进目录、拼进剖析核对查询的写法(两处用的是同一条):原文照旧,只把每处列引用换成 quote(真名)——表结构 + 里的真名、不带表名,按数据源的需要加引号(quote 是 catalog_profile.SqlDialect.quote,和剖析查询其余部分的写法一致)。 + + 以前目录里存的是模型的原文,核对时却按真名加了引号:PostgreSQL 上混合大小写的列("OrderStatus")不加引号会被折成 + 小写,存下的 orderstatus <> 'X' 一跑就报列不存在,核对过的却能跑通,SQL 检查还会让模型照抄存下的那一句。 + + 拿不到原文里的位置、换完以后认不出原来那几列时,退回按语法树重写(render_valid_filter)。""" + if parsed.expression is None: + raise ValueError(parsed.problem or "有效行过滤不合规") + if parsed.spans: + text = parsed.text + for start, end, real in sorted(parsed.spans, reverse=True): + text = text[:start] + quote(real) + text[end + 1:] + if _same_columns(text, parsed): + return text + return render_valid_filter(parsed, quoted=lambda name: quote(name) != name) + + +def _same_columns(text: str, parsed: ValidFilter) -> bool: + """换过列名的原文还是一个条件,列依次是表结构里的那几列、都不带表名。""" + import sqlglot + from sqlglot import exp + + try: + tree = sqlglot.parse_one(text, read=parsed.dialect) + except Exception: # noqa: BLE001 - 换坏了就退回按语法树重写 + return False + columns = list(tree.find_all(exp.Column, bfs=False)) + return ([c.name for c in columns] == [real for _, _, real in parsed.spans or ()] + and not any(c.table for c in columns)) + + +def stored_valid_filter(parsed: ValidFilter, kind: str | None) -> str: + """模型提议的有效行过滤存进目录的写法:canonical_valid_filter,按这个数据源的方言加引号(catalog_profile.SqlDialect: + SQLAlchemy 的写法,要加才加——PostgreSQL、SQLite 的 "OrderStatus",MySQL 的 `OrderStatus`,Oracle 探查出来的小写名 + 不加)。""" + from app.data.catalog_profile import SqlDialect + + try: + dialect = SqlDialect(kind) + except ValueError: + return parsed.text + return canonical_valid_filter(parsed, quote=dialect.quote) + + __all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "VALID_FILTER_FUNCTIONS", - "VALID_FILTER_MAX", "ValidFilter", "check_sql", "dialect_of", "graph_checks", "graph_sources", - "load_checkers", "load_checkers_by_name", "parse_valid_filter", "render_valid_filter", "tool_query", - "valid_filter_problem"] + "VALID_FILTER_MAX", "ValidFilter", "canonical_valid_filter", "check_sql", "dialect_of", "graph_checks", + "graph_sources", "load_checkers", "load_checkers_by_name", "parse_valid_filter", "render_valid_filter", + "stored_valid_filter", "tool_query", "valid_filter_problem"] diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py index de942c4e..139ba8ca 100644 --- a/backend/tests/test_catalog_draft_facts.py +++ b/backend/tests/test_catalog_draft_facts.py @@ -3,6 +3,7 @@ - 提示词写清两项只在表结构、注释、码值能看出来时才写;结构化输出、纯文本两条路都认这两项; - 有效行过滤按数据源的方言用 sqlglot 解析成一个条件:只能引用这张表的列,不能有子查询、聚合、窗口函数、分号,只允许 比较、IN、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE,不超过 500 字。不过就丢掉; + 存的是整理过的写法:原文照旧,列换成表结构里的真名、不带表名、按方言加引号(剖析核对的也是这一条); - 开了数据剖析、能连上的源:经剖析的只读守卫数出排除比例,排除 0% 或 100% 的不收,收下的在 note 里写剖析核对的结果; 没开剖析、连不上的照收,note 写没有核对的原因; - 开了剖析的源,目录里记下的码值交给模型做背景(遮罩的列不给;「已确认的目录项」里遮罩列的码值也不给); @@ -154,6 +155,54 @@ def test_the_condition_is_rendered_with_the_columns_quoted_like_the_source(): assert rendered == "`isTest` = 0 AND status <> 'C'" +#: 带一列混合大小写列名的订单表(ORM 建的 PostgreSQL 库里常见) +MIXED = {**META, "columns": [*META["columns"], _col("OrderStatus", "TEXT")]} + + +@pytest.mark.parametrize("kind, condition, stored", [ + # PostgreSQL 把没加引号的名字折成小写:混合大小写的列一律加引号,不论模型写的大小写对不对 + ("postgresql", "orderstatus <> 'X' AND STATUS NOT IN ('C', 'X')", + "\"OrderStatus\" <> 'X' AND status NOT IN ('C', 'X')"), + ("postgresql", "OrderStatus <> 'X'", "\"OrderStatus\" <> 'X'"), + ("postgresql", "orders.\"OrderStatus\" IS NOT NULL", "\"OrderStatus\" IS NOT NULL"), + ("mysql", "orderstatus <> 'X'", "`OrderStatus` <> 'X'"), + ("oracle", "ORDERS.STATUS <> 'C'", "status <> 'C'"), + # 不用改的照原文:NOT IN、IS NOT NULL 这些写法不被改写 + ("sqlite", "status NOT IN ('C', 'X') AND updated_at IS NOT NULL", + "status NOT IN ('C', 'X') AND updated_at IS NOT NULL"), +]) +def test_the_stored_filter_names_its_columns_the_way_the_source_needs(kind, condition, stored): + """存进目录的有效记录条件:原文照旧,只把列换成表结构里的真名、不带表名,按数据源的需要加引号(和剖析核对查询的写法 + 一致)。以前存的是模型的原文:PostgreSQL 上 orderstatus 会被当成另一列,核对时却按真名加了引号,存下的和核对过的 + 不是同一条,SQL 检查还会让模型照抄这一句。""" + entry = {"table": "orders", "valid_filter": {"condition": condition, "reason": "注释写明"}} + notes = catalog._from_model(entry, MIXED, system=False, source_kind=kind, table="orders") + assert notes["valid_filter"]["value"] == stored + again = sqlcheck.parse_valid_filter(stored, MIXED, kind=kind, table="orders") + assert again.problem is None and sqlcheck.stored_valid_filter(again, kind) == stored, "存下的再整理一遍不变" + + +def test_a_filter_that_grows_past_the_limit_once_quoted_is_dropped_not_written(): + """整理时加了引号、超过 500 字的不收:目录里这一项最多 500 字,写进去会让整张表写入失败。""" + condition = "orderstatus IN (" + ", ".join(f"'{i:03d}'" for i in range(66)) + ") OR updated_at IS NULL" + assert len(condition) == 499 + notes = catalog._from_model({"table": "orders", "label": "订单", "valid_filter": condition}, MIXED, system=False, + source_kind="postgresql", table="orders") + assert "valid_filter" not in notes and notes["label"]["value"] == "订单" + assert not catalog.validate_notes(notes) + + +def test_columns_that_differ_only_in_case_must_be_named_exactly(): + """表结构里有只差大小写的两列:写得和其中一列一模一样的认那一列,否则说不清是哪一列,不收。""" + meta = {**META, "columns": [*META["columns"], _col("Status", "TEXT")]} + problem = sqlcheck.valid_filter_problem("STATUS = 'C'", meta, kind="postgresql", table="orders") + assert problem == ("有效记录条件里的列「STATUS」在表结构里有只差大小写的几列(status、Status)," + "要照表结构的写法写清是哪一列") + for condition, stored in (("\"Status\" = 'C'", "\"Status\" = 'C'"), ("status = 'C'", "status = 'C'")): + parsed = sqlcheck.parse_valid_filter(condition, meta, kind="postgresql", table="orders") + assert parsed.problem is None and sqlcheck.stored_valid_filter(parsed, "postgresql") == stored + + # -------------------------------------------------------------------------- # 模型给的两项:校验、依据 # -------------------------------------------------------------------------- @@ -318,6 +367,25 @@ async def test_the_check_counts_excluded_rows_through_the_read_only_guard(scenic assert "COUNT(*)" in seen[0][0] and "status = 1" in seen[0][0] +async def test_the_check_runs_the_condition_as_it_is_stored(scenic, monkeypatch): + """剖析核对的就是存进目录的那一条:表名去掉、列换成表结构里的真名,别的照原文(NOT IN 不被改写成 NOT x IN)。""" + seen: list[str] = [] + real = catalog_profile.guard.check + + def spy(sql, **kw): + seen.append(sql) + return real(sql, **kw) + + monkeypatch.setattr(catalog_profile.guard, "check", spy) + source = await scenic() + reply = {"label": "入园记录", "valid_filter": {"condition": "VISITS.STATUS NOT IN (0)", "reason": "0 是作废的检票"}} + got = await D.run_model_draft(FakeModel({"visits": reply}), source, ["visits"]) + vf = got["visits"].notes["valid_filter"] + assert vf["value"] == "status NOT IN (0)" + assert len(seen) == 1 and f"CASE WHEN {vf['value']} THEN 0 ELSE 1 END" in seen[0] + assert "剖析核对:排除" in vf["note"] + + @pytest.mark.parametrize("condition", ["status IN (0, 1)", "status = 7"]) async def test_a_filter_that_excludes_nothing_or_everything_is_dropped(scenic, condition): source = await scenic()