From ee7b8b2f97ecab1aba081d4fd3a2d0cbdae94d62 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 10:25:28 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E5=89=96=E6=9E=90=E4=B8=8E=E6=8E=A2?= =?UTF-8?q?=E6=9F=A5=E7=9A=84=E6=96=B9=E8=A8=80=E4=BF=AE=E6=AD=A3=EF=BC=88?= =?UTF-8?q?C1=EF=BC=89=E5=92=8C=E6=B2=A1=E6=9C=89=E7=9C=9F=E5=BA=93?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E6=96=B9=E8=A8=80=E9=AA=8C=E8=AF=81=EF=BC=88?= =?UTF-8?q?C2=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么：剖析是全站唯一主动查业务库的地方，以前只在守卫放行和字符串片段两层上测过，从没在真库上跑过。 本机四个容器（PG 16、MySQL 8.4、MariaDB 11、Oracle Free 23）实测，旧的 IN 列表写法在 PostgreSQL 上整数子列 对文字主键直接报 operator does not exist: character varying = integer，关系一条都核对不了。 C1 方言修正： - IN 列表的字面量按被指向列的类型族写（key_literals）：父列文字写 '12'，父列数字只收整数写法的值，父列是时间、 对不上的不拼并在备注里写「两端类型不同」；子表是 CHAR(n) 时去掉补齐的尾随空格。 - 列类型改存按方言编译的完整写法（VARCHAR2(20 CHAR)、NUMBER(10, 0)、ENUM('a','b')、TIMESTAMP WITH TIME ZONE）， 编译不了退回 str()。SQLAlchemy 把 Oracle 的 VARCHAR2 反射成通用 VARCHAR（容器实测），照数据字典写成 VARCHAR2。 ENUM / SET 另记 enum_values；Oracle 小写带引号的表和列记 quote_name；各索引（含主键）的首列记进 index_first_columns（下一波取值观察判断能不能对大表发窗口查询）。新探查的快照哈希会变，旧运行封存的不受影响。 - catalog._type_family 认 ENUM / SET 是文字、INTERVAL 不是数字，去掉引号里的取值再认；新增 column_family （有 enum_values 的是文字，PostgreSQL 枚举编译出来只有类型名）；is_date_column 不再把 ENUM('date') 认成日期列。 - 码值候选：ENUM / SET 不看列名直接算候选，ENUM 列定义里这次没出现的取值也写进码值（含义待填）；VARCHAR2(20 CHAR) 的长度按 20 算；CHAR 码值去尾随空格；MySQL 上写明按排序规则分组、比较（可能不分大小写）。 - 统计信息估出的行数不超过整表统计上限时再跑一次有上限的计数确认，统计过期的大表不再被当成小表做整表统计。 - 日期列的最小值、最大值只看 0001-01-02 到 9999-12-31 之间的值，零日期、占位日期、PG 的 ±infinity 不计入， 照实写「另有 N 个无效日期未计入」，日期范围里带 invalid 个数（界面显示）。 - 新增取值观察用的方言方法 date_literal / day_trunc / sample_from，以及按日行数、时间窗口里的取值分布、块采样三类 查询的拼法（窗口上端截在 9999-12-31 之前，不让 infinity 多出一天）。 - 说明文字不再说「抽样」：大表看的是「按存储顺序取前 N 行」；设置项「抽样键值数」改叫「核对键值数」。 C2 验证： - tests/test_profile_dialect_golden.py：五种方言 × 全部剖析和观察查询的完整 SQL 钉在 tests/fixtures/dialect/， sqlglot 逐条解析、过只读守卫，再用 qualify + annotate_types 核对 IN 两边类型族一致（附旧写法的对照）。 - scripts/dialect-it.sh（不进 CI）：本机起四个容器，建虚构夹具（保留字列、Oracle 小写带引号的表、CHAR 码值、 MySQL ENUM、零日期 / infinity / 占位日期、做旧的统计信息、超上限的大表、带索引的日期列），用 AgentLab 自己的 探查和剖析代码跑全部查询，断言零报错、结论和手算一致。四种库都通过；MariaDB 插入时自己更新行数估算，统计信息 做不旧，照实打印。把 IN 列表改回旧写法回跑，PG 上照样报出上面那条错误。跑完删容器、留镜像。 --- backend/app/data/catalog.py | 40 +- backend/app/data/catalog_profile.py | 346 +++++++++++++--- backend/app/data/introspect.py | 86 +++- backend/tests/fixtures/dialect/mariadb.sql | 27 ++ backend/tests/fixtures/dialect/mysql.sql | 27 ++ backend/tests/fixtures/dialect/oracle.sql | 27 ++ backend/tests/fixtures/dialect/postgresql.sql | 27 ++ backend/tests/fixtures/dialect/sqlite.sql | 27 ++ backend/tests/test_catalog_draft_facts.py | 2 +- backend/tests/test_catalog_profile.py | 4 +- backend/tests/test_catalog_profile_units.py | 12 +- backend/tests/test_catalog_semantics.py | 2 +- backend/tests/test_profile_dialect_golden.py | 178 ++++++++ backend/tests/test_profile_dialect_units.py | 296 +++++++++++++ frontend/src/lib/terms.ts | 14 +- frontend/src/pages/catalog/ProfileDialog.tsx | 5 + .../src/pages/catalog/ProfileSettings.tsx | 2 +- frontend/src/types.ts | 3 +- scripts/check-catalog-profile.mjs | 6 +- scripts/dialect-it.sh | 62 +++ scripts/dialect_it.py | 388 ++++++++++++++++++ 21 files changed, 1499 insertions(+), 82 deletions(-) create mode 100644 backend/tests/fixtures/dialect/mariadb.sql create mode 100644 backend/tests/fixtures/dialect/mysql.sql create mode 100644 backend/tests/fixtures/dialect/oracle.sql create mode 100644 backend/tests/fixtures/dialect/postgresql.sql create mode 100644 backend/tests/fixtures/dialect/sqlite.sql create mode 100644 backend/tests/test_profile_dialect_golden.py create mode 100644 backend/tests/test_profile_dialect_units.py create mode 100755 scripts/dialect-it.sh create mode 100644 scripts/dialect_it.py diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 6ca00287..678f249b 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1365,7 +1365,9 @@ def is_date_column(name: str, type_name: str | None) -> bool: 数据剖析（表里只有一个日期类列时提议它作业务日期）和 SQL 检查（按业务日期以外的时间列统计）共用这一个判断。 以前两边各写一套：剖析认的「唯一日期列」，SQL 检查可能认为还有别的时间列，反过来也一样，两边对不上。 """ - t = (type_name or "").upper() + t = _type_words(type_name) + if t.startswith(("ENUM", "SET")) or "INTERVAL" in t: + return False if "DATE" in t or "TIMESTAMP" in t: return True if t and not any(k in t for k in _TEXT_TYPES): @@ -1400,11 +1402,29 @@ def _name_index(tables: Mapping[str, Any]) -> dict[tuple[str, str], set[str]]: return index +#: 类型写法里带引号的部分（ENUM('int','date') 的取值）：认类型时去掉，取值里的词不能让一列变成数字或时间 +_QUOTED_IN_TYPE = re.compile(r"'(?:[^']|'')*'") + + +def _type_words(type_name: str | None) -> str: + """认类型用的写法：大写、去掉引号里的取值。探查存的是按方言编译的完整写法（ENUM('paid','refund')、 + VARCHAR2(20 CHAR)、TIMESTAMP WITH TIME ZONE），不再是 SQLAlchemy 的通用写法。""" + return _QUOTED_IN_TYPE.sub("''", (type_name or "").upper()).strip() + + def _type_family(type_name: str | None) -> str | None: - """列类型的大类：数字、文字、时间；认不出来返回 None（不据此排除）。""" - t = (type_name or "").upper() + """列类型的大类：数字、文字、时间；认不出来返回 None（不据此排除）。 + + 认的是按方言编译的完整写法：MySQL 的 ENUM / SET 是文字（取值列表里的词不算），Oracle 的 INTERVAL DAY TO SECOND + 是时长、不算数字（以前按子串「INT」认成了数字）。PostgreSQL 的枚举编译出来是类型名本身，只看类型认不出来， + 要看列上的取值列表（column_family）。""" + t = _type_words(type_name) if not t: return None + if t.startswith(("ENUM", "SET")): + return "text" + if "INTERVAL" in t: + return None if any(k in t for k in ("CHAR", "TEXT", "CLOB", "STRING", "UUID")): return "text" if any(k in t for k in ("INT", "NUMBER", "NUMERIC", "DECIMAL", "SERIAL", "REAL", "FLOAT", "DOUBLE")): @@ -1414,6 +1434,16 @@ def _type_family(type_name: str | None) -> str | None: return None +def column_family(col: Mapping[str, Any] | None) -> str | None: + """表结构里一列的类型大类：有取值列表（探查记下的 enum_values：MySQL 的 ENUM / SET、PostgreSQL 的枚举）的是 + 文字，其余按类型写法认（_type_family）。""" + if not isinstance(col, Mapping): + return None + if col.get("enum_values"): + return "text" + return _type_family(col.get("type")) + + def _id_stem(column: str) -> str | None: m = _SNAKE_ID.match(column) or _CAMEL_ID.match(column) return m.group("stem") if m else None @@ -1469,8 +1499,8 @@ def infer_name_relations(schema_cache: Mapping[str, Any] | None, table: str, *, pk = list(target_meta.get("primary_key") or []) if len(pk) != 1: continue - pk_type = next((c.get("type") for c in target_meta.get("columns") or [] if c.get("name") == pk[0]), None) - mine, theirs = _type_family(col.get("type")), _type_family(pk_type) + pk_col = next((c for c in target_meta.get("columns") or [] if c.get("name") == pk[0]), None) + mine, theirs = column_family(col), column_family(pk_col) if mine and theirs and mine != theirs: continue out.append(_relation(table, [name], target, pk, "name", cardinality=_cardinality(meta, [name]))) diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 8eee4488..cc9b1062 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -41,7 +41,7 @@ import re import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import date as _date, datetime, timezone from decimal import Decimal from typing import Any @@ -99,7 +99,7 @@ _NUMBER_FIELDS: dict[str, tuple[str, float, float, bool]] = { "table_max_queries": ("每张表最多查询数", 1, 200, True), "table_max_s": ("每张表时长上限（秒）", 10, 600, False), "query_timeout_s": ("单条查询时限（秒）", 1, 60, False), - "sample_size": ("抽样键值数", 10, 10_000, True), + "sample_size": ("核对键值数", 10, 10_000, True), "max_scan_rows": ("整表统计行数上限", 0, 10_000_000, True), } _ENABLED_LABEL = "开启数据剖析" @@ -241,16 +241,82 @@ def sql_literal(value: Any, kind: str | None) -> str | None: return "'" + value.replace("'", "''") + "'" +#: 定长文字类型（CHAR(2)、NCHAR、bpchar、CHARACTER(2)）：PostgreSQL、Oracle 取回的值带尾随空格补齐 +_FIXED_CHAR = re.compile(r"^\s*(?:N?CHAR|BPCHAR|CHARACTER)\b(?!\s+VARYING)", re.IGNORECASE) + + +def is_fixed_char(type_name: str | None) -> bool: + """定长文字列（CHAR(n)）：值的尾随空格是补齐出来的，不是取值的一部分（SQL 比较时也不算）。""" + return bool(type_name) and bool(_FIXED_CHAR.match(str(type_name))) + + +def key_literals(values: list[Any], kind: str | None, *, parent: str | None, + child_type: str | None) -> tuple[list[str], int]: + """从子表取回的键值 → 去父表里对的 IN 列表字面量（去重、保持顺序）：(字面量, 因为两端类型不同没拼的个数)。 + + 字面量按**被指向列**的类型族写（catalog.column_family），不是按子表取回的值： + - 父列是文字、子列是数：写成 '123'。PostgreSQL 上 varchar 列 IN (123) 直接报「operator does not exist」； + - 父列是数、子列是文字：只收整数写法的值（'0012' 也照 12 写），别的（'A1'）计入不同类型、不拼。Oracle 上 + 数字列 IN ('A1') 会把字面量转成数报 ORA-01722，MySQL 会把 'A1' 转成 0 去比、对上不该对上的行； + - 父列是时间：键不核对（各家时间字面量写法不同、精度不同），全部计入不同类型； + - 父列类型认不出：照子表取回的值的类型写（sql_literal，以前的做法）。 + 子表是定长文字（CHAR(n)）时先去掉尾随空格：PG、Oracle 取回的 'A ' 拿去和 VARCHAR 父列比就对不上，而按 CHAR 的 + 语义尾随空格本来就不算。不安全的值（反斜杠、「:名字」、控制字符、太长）照 sql_literal 的规矩跳过，不算类型不同。 + """ + fixed = is_fixed_char(child_type) + out: list[str] = [] + mismatched = 0 + for value in values: + if fixed and isinstance(value, str): + value = value.rstrip(" ") + if parent == "time": + mismatched += 1 + continue + if parent == "text": + if isinstance(value, bool): + continue + if isinstance(value, (int, float, Decimal)): + lit = sql_literal(value, "number") + lit = None if lit is None else "'" + lit + "'" + else: + lit = sql_literal(value, "text") + elif parent == "number": + if isinstance(value, str): + text = value.strip() + if not _INT_TEXT.fullmatch(text): + mismatched += 1 + continue + lit = str(int(text)) + else: + lit = sql_literal(value, "number") + else: + lit = sql_literal(value, kind) + if lit is not None: + out.append(lit) + return list(dict.fromkeys(out)), mismatched + + _DIALECT_MODULES = {"sqlite": sqlite, "postgres": postgresql, "postgresql": postgresql, "mysql": mysql, "mariadb": mysql, "oracle": oracle} +#: 日期列的最小值、最大值只看这个区间里的值（含下限、不含上限）：0000-00-00（MySQL 关了严格模式写进去的零日期）、 +#: 0001-01-01、9999-12-31（「没有」「永久」的占位日期）、PostgreSQL 的 -infinity / infinity 都落在区间外。区间外的 +#: 值照实数出来（「有 N 个无效日期未计入」），不让一个占位日期把业务日期的范围撑成两千年。两端整天排除：带时刻的 +#: 9999-12-31 23:59:59、文字存的「0001-01-01 00:00:00」也算占位 +VALID_DATE_FROM = "0001-01-02" +VALID_DATE_BEFORE = "9999-12-31" +_ISO_DAY = re.compile(r"\d{4}-\d{2}-\d{2}") + + class SqlDialect: - """剖析 SQL 的方言差异：标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里。只拼字符串，不连库。 + """剖析 SQL 的方言差异：标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里、日期字面量和按天截断 + 怎么写、有没有块采样。只拼字符串，不连库。 标识符交给 SQLAlchemy 各方言的 identifier_preparer：需要时才加引号（memberTags 加、visits 不加）， MySQL 用反引号。Oracle 尤其不能一律加双引号：探查拿到的表名是 SQLAlchemy 规整过的小写，库里其实是 - 大写，"visits" 加了引号就成了另一张不存在的表；不加引号 Oracle 自己转大写才对得上。 + 大写，"visits" 加了引号就成了另一张不存在的表；不加引号 Oracle 自己转大写才对得上。反过来，库里真是小写的 + 对象（建表时写了 "visits"）探查时记了 quote_name，这种要强制加引号、不转大写（table、column、stats_sql）。 """ def __init__(self, kind: str | None) -> None: @@ -262,13 +328,64 @@ class SqlDialect: self._prep = self._dialect.identifier_preparer self.oracle = self.kind == "oracle" - def quote(self, name: str) -> str: - return self._prep.quote(name) + def quote(self, name: str, *, force: bool = False) -> str: + """标识符。force：探查记了 quote_name 的名字，原样加引号（Oracle 小写带引号的对象）。""" + return self._prep.quote_identifier(name) if force else self._prep.quote(name) def table(self, meta: dict[str, Any] | None, name: str) -> str: """写进 FROM 的表名：探查时指定了 schema 的带上它（Oracle 只读账号名下没有对象，全靠 schema 前缀）。""" schema = (meta or {}).get("schema") - return f"{self._prep.quote_schema(schema)}.{self.quote(name)}" if schema else self.quote(name) + quoted = self.quote(name, force=(meta or {}).get("quote_name") is True) + return f"{self._prep.quote_schema(schema)}.{quoted}" if schema else quoted + + def column(self, meta: dict[str, Any] | None, name: str) -> str: + """写进 SQL 的列名：表结构里这一列记了 quote_name 的强制加引号。""" + cols = (meta or {}).get("columns") or [] + entry = next((c for c in cols if isinstance(c, dict) and c.get("name") == name), None) + return self.quote(name, force=bool(entry and entry.get("quote_name") is True)) + + def date_literal(self, value: Any, *, text: bool = False) -> str: + """日期字面量（取值观察的时间窗口、日期列的有效区间用）。PostgreSQL、MySQL、Oracle 都认 DATE 'YYYY-MM-DD'； + SQLite 没有日期类型，日期存成文字，写字符串。text：列是文字存的日期（VARCHAR），按字符串比——PostgreSQL 上 + varchar 和 date 比较会直接报错。只收真实存在的日期（date 或 YYYY-MM-DD），别的抛 ValueError：拼进 SQL 的 + 只能是我们自己算出来的日子。""" + if isinstance(value, datetime): + value = value.date() + if isinstance(value, _date): + day = value.isoformat() + elif isinstance(value, str) and _ISO_DAY.fullmatch(value): + try: + day = _date.fromisoformat(value).isoformat() + except ValueError: + raise ValueError(f"不是有效的日期：{value}") from None + else: + raise ValueError(f"不是有效的日期：{value!r}") + if text or self.kind == "sqlite": + return f"'{day}'" + return f"DATE '{day}'" + + def day_trunc(self, column: str) -> str: + """时间列截到天（按日行数用）。PostgreSQL CAST(x AS DATE)；MySQL DATE(x)；Oracle 必须用 TRUNC(x)：Oracle 的 + DATE 带时分秒，CAST(ts AS DATE) 截不掉；SQLite 的日期是文字，取前 10 个字符。""" + if self.kind in ("postgres", "postgresql"): + return f"CAST({column} AS DATE)" + if self.kind in ("mysql", "mariadb"): + return f"DATE({column})" + if self.oracle: + return f"TRUNC({column})" + return f"substr({column}, 1, 10)" + + def sample_from(self, table: str, percent: float) -> str | None: + """块采样的 FROM：PostgreSQL TABLESAMPLE SYSTEM (p)、Oracle SAMPLE BLOCK (p)，按数据块随机取约 p% 的行； + MySQL、SQLite 没有块采样，返回 None（调用方退回「按存储顺序的前 N 行」，并照实写口径）。""" + p = float(percent) + if not (0 < p < 100): + raise ValueError(f"采样比例要在 0 到 100 之间：{percent}") + if self.kind in ("postgres", "postgresql"): + return f"{table} TABLESAMPLE SYSTEM ({p:g})" + if self.oracle: + return f"{table} SAMPLE BLOCK ({p:g})" + return None def limit(self, sql: str, n: int) -> str: """只取前 n 行。Oracle 11g 没有 LIMIT 也没有 FETCH FIRST，一律套一层 ROWNUM。""" @@ -304,7 +421,8 @@ class SqlDialect: if self.oracle: denorm = self._dialect.denormalize_name owner = sql_literal(denorm(schema), "text") if schema else "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')" - table = sql_literal(denorm(name), "text") + # 小写带引号的对象（quote_name）在数据字典里就是小写，不能转成大写去找 + table = sql_literal(name if (meta or {}).get("quote_name") is True else denorm(name), "text") if not (owner and table): return None return f"SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = {owner} AND TABLE_NAME = {table}" @@ -335,8 +453,49 @@ class SqlDialect: f"WHERE {column} IS NOT NULL GROUP BY {column} ORDER BY cnt DESC, {column}") return self.limit(sql, n) - def min_max_sql(self, table: str, columns: list[str]) -> str: - return f"SELECT {', '.join(f'MIN({c}), MAX({c})' for c in columns)} FROM {table}" + def min_max_sql(self, table: str, columns: list[tuple[str, bool]]) -> str: + """日期列的最小值、最大值和无效日期的个数：每列三个值（MIN、MAX、区间外的个数）。columns 是 (写进 SQL 的列, + 是不是文字存的日期)。只看有效区间（VALID_DATE_FROM ≤ x < VALID_DATE_BEFORE）里的值，见 VALID_DATE_FROM。 + 整表统计，只对小表用。""" + parts = [] + for c, text in columns: + lo = self.date_literal(VALID_DATE_FROM, text=text) + hi = self.date_literal(VALID_DATE_BEFORE, text=text) + valid = f"{c} >= {lo} AND {c} < {hi}" + parts.append(f"MIN(CASE WHEN {valid} THEN {c} END), MAX(CASE WHEN {valid} THEN {c} END), " + f"SUM(CASE WHEN {c} < {lo} OR {c} >= {hi} THEN 1 ELSE 0 END)") + return f"SELECT {', '.join(parts)} FROM {table}" + + # ---- 取值观察（下一轮第 3 波用）：窗口、按日行数、块采样。这里先把各方言的写法定下来，金样和容器集成测试都覆盖 + + def _window(self, column: str, since: Any, text: bool) -> str: + """时间窗口的条件：since 起（含），并且早于占位日期 9999-12-31——PostgreSQL 的 infinity、Oracle 的 9999-12-31 + 都大于任何一天，不截掉上端就会在「最近 35 天」里多出一天。""" + return (f"{column} >= {self.date_literal(since, text=text)} " + f"AND {column} < {self.date_literal(VALID_DATE_BEFORE, text=text)}") + + def day_volume_sql(self, table: str, column: str, since: Any, *, text: bool = False) -> str: + """最近若干天按天计数：since 起（含）每天几行。只在日期列打头有索引时对大表发（走索引的范围扫描）。""" + day = self.day_trunc(column) + return (f"SELECT {day} AS d, COUNT(*) AS n FROM {table} " + f"WHERE {self._window(column, since, text)} GROUP BY {day} ORDER BY d") + + def window_value_counts_sql(self, table: str, column: str, date_column: str, since: Any, n: int, *, + text: bool = False) -> str: + """时间窗口里的取值分布：业务日期 since 起（含）的行里，出现最多的 n 个取值和各自的行数。""" + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {table} " + f"WHERE {self._window(date_column, since, text)} AND {column} IS NOT NULL " + f"GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) + + def sample_value_counts_sql(self, table: str, column: str, percent: float, n: int) -> str | None: + """块采样里的取值分布；没有块采样的方言返回 None。""" + source = self.sample_from(table, percent) + if source is None: + return None + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {source} WHERE {column} IS NOT NULL " + f"GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) # ========================================================================== @@ -581,8 +740,10 @@ async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: Sql """先读统计信息；读不到（SQLite、没分析过的表）再数到 max_scan_rows + 1 行为止。 统计信息为 0 或负数（PostgreSQL 没分析过的表是 -1，MySQL 的估算值可能是 0）当作不知道：一张其实很大的表 - 要是被当成空表，后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完（或者任务止损） -时抛 _Stop。 + 要是被当成空表，后面就会对它做整表统计。统计信息说的行数不超过 max_scan_rows 时也再数一次（多一条有上限的 + 计数）：统计信息可能是很久以前的（表后来涨到了几千万行，ANALYZE 没再跑过），照它把大表当小表，就会对它做 + 整表的 COUNT(DISTINCT)、MIN/MAX。数过的结果是准数（count），数到上限还没数完就按大表算（at_least）。 + 返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完（或者任务止损）时抛 _Stop。 """ skips: list[_Skip] = [] sql = dialect.stats_sql(meta, name) @@ -592,7 +753,9 @@ async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: Sql except _Skip as e: skips.append(e) else: - if rows is not None and rows > 0: + if rows is not None and rows > max(max_scan_rows, 0): + return TableSize(rows, "stats"), skips + if rows is not None and rows > 0 and max_scan_rows <= 0: return TableSize(rows, "stats"), skips if max_scan_rows <= 0: return TableSize(None, "unknown"), skips @@ -752,6 +915,8 @@ class _CodesOutcome: rows: int note: str complete: bool = False + #: 列定义（ENUM）列出的取值：这次没出现的也写进码值（含义待填），码值是「可能的取值」 + declared: list[str] = field(default_factory=list) @dataclass @@ -901,6 +1066,11 @@ def _column(meta: dict[str, Any], name: str) -> str | None: if isinstance(c, dict) and str(c.get("name", "")).lower() == lowered), None) +def _column_meta(meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """表结构里这一列（类型、取值列表、quote_name）；name 是表结构里的写法（_column 的返回值）。""" + return next((c for c in meta.get("columns") or [] if isinstance(c, dict) and c.get("name") == name), None) + + def _constraint(meta: dict[str, Any], column: str) -> str | None: """这一列单独是主键或某个唯一约束时返回「主键」/「唯一约束」，否则 None。""" wanted = [column.lower()] @@ -950,7 +1120,7 @@ async def _unique(cx: _TableCx, table: str, meta: dict[str, Any], column: str, if not size.within(cx.settings.max_scan_rows): return None, "too_large" d = cx.dialect - res = await cx.run(d.unique_check_sql(d.table(meta, table), d.quote(column))) + res = await cx.run(d.unique_check_sql(d.table(meta, table), d.column(meta, column))) row = res.rows[0] if res.rows else [None, None] total, distinct = _as_int(row[0]), _as_int(row[1]) if total is None or distinct is None: @@ -969,20 +1139,38 @@ def _fk_note(child: tuple[bool | None, str]) -> str: return f"{catalog.PROFILE_NOTE_PREFIX}外键约束已保证引用完整，未抽样核对覆盖率；子表一侧{basis}：{shape}" +#: MySQL 上比较文字按列的排序规则：常见的排序规则不分大小写（_ci），PAD SPACE 的还忽略尾随空格。覆盖率、唯一性、 +#: 码值的分组都按它算，写进说明，人才知道「A01」和「a01」在这里算同一个值 +MYSQL_COLLATION_NOTE = "MySQL 按列的排序规则比较文字（常见的排序规则不分大小写），覆盖率和唯一性按它计算" +MYSQL_CODES_COLLATION_NOTE = "MySQL 按列的排序规则分组（常见的排序规则不分大小写），大小写不同的写法可能并成一个取值" + + +def _collation_note(dialect: SqlDialect, family: str | None, note: str) -> str | None: + """MySQL、MariaDB 上文字列（或类型认不出的列）的排序规则说明；别的方言、数字列没有。""" + return note if dialect.kind in ("mysql", "mariadb") and family in ("text", None) else None + + def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int | None, skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None, - holds: bool) -> str: - """关系的剖析备注：抽样规模、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 - 不写日期：什么时候剖析的看这一项的 updated_at（界面按本地时间显示）。""" + holds: bool, mismatched: int = 0, collation: str | None = None) -> str: + """关系的剖析备注：取了多少个键值、看了多少行、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 + 不写日期：什么时候剖析的看这一项的 updated_at（界面按本地时间显示）。 + + 不说「抽样」：DISTINCT … LIMIT 取的是数据库先给出的那几个值，大表上是按存储顺序的前若干行里的值，不是随机抽的 + （多半是最老的数据）。mismatched：和被指向列类型不同、没拼进核对的键值个数（key_literals）。""" parts: list[str] = [] low = coverage < catalog.PROFILE_VERIFY_COVERAGE if low: - parts.append(f"抽样覆盖率 {_percent(coverage)}，可能不是这条关系") - scope = f"（只看前 {scan_cap} 行）" if scan_cap else "" - sampled = f"子表抽样 {sample} 个不同键值{scope}，父表对上 {matched} 个" + parts.append(f"覆盖率 {_percent(coverage)}，可能不是这条关系") + scope = f"（按存储顺序只看前 {scan_cap} 行）" if scan_cap else "" + sampled = f"子表取了 {sample} 个不同键值{scope}，父表对上 {matched} 个" parts.append(sampled if low else f"{sampled}，覆盖率 {_percent(coverage)}") if skipped_values: parts.append(f"另有 {skipped_values} 个键值的类型不支持核对，未计入") + if mismatched: + parts.append(f"另有 {mismatched} 个键值和被指向列的类型不同（两端类型不同），未计入") + if collation: + parts.append(collation) unique, how = parent if unique is True: parts.append(f"被指向列是{how}" if how in ("主键", "唯一约束") else "被指向列经计数核对唯一") @@ -1029,18 +1217,25 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) child_table, parent_table = d.table(meta, table), d.table(pmeta, parent) - res = await cx.run(d.distinct_sample_sql(child_table, d.quote(col), s.sample_size, scan_cap=scan_cap), + res = await cx.run(d.distinct_sample_sql(child_table, d.column(meta, col), s.sample_size, scan_cap=scan_cap), max_rows=s.sample_size) kind = (res.column_types or {}).get(res.columns[0]) if res.columns else None values = [row[0] for row in res.rows if row] - literals = list(dict.fromkeys(lit for v in values if (lit := sql_literal(v, kind)) is not None)) + child_col, parent_col = _column_meta(meta, col), _column_meta(pmeta, pcol) + parent_family = catalog.column_family(parent_col) + literals, mismatched = key_literals(values, kind, parent=parent_family, + child_type=(child_col or {}).get("type")) if not values: raise _Skip("no_data", f"{table}.{col} 没有非空值，无法核对") + if not literals and mismatched: + their = {"number": "数字", "text": "文字", "time": "日期时间"}.get(parent_family or "", "其他类型") + raise _Skip("unsupported", f"两端类型不同：{parent}.{pcol} 是{their}，{table}.{col} 取到的值对不上这种类型，" + "无法核对") if not literals: raise _Skip("unsupported", f"{table}.{col} 的取值类型不支持核对（只核对整数和文字）") matched = 0 for start in range(0, len(literals), IN_CHUNK): - sql = d.match_count_sql(parent_table, d.quote(pcol), literals[start:start + IN_CHUNK]) + sql = d.match_count_sql(parent_table, d.column(pmeta, pcol), literals[start:start + IN_CHUNK]) matched += _as_int(_first(await cx.run(sql))) or 0 matched = min(matched, len(literals)) coverage = round(matched / len(literals), 4) @@ -1051,9 +1246,11 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T if child_unique is not None: cardinality = "one_to_one" if child_unique[0] is True else "many_to_one" holds = coverage >= catalog.PROFILE_VERIFY_COVERAGE and cardinality is not None + unsafe = len(values) - mismatched - len(literals) note = _relation_note(sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap, - skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique, - holds=holds) + skipped_values=max(unsafe, 0), parent=parent_unique, child=child_unique, holds=holds, + mismatched=mismatched, + collation=_collation_note(d, parent_family, MYSQL_COLLATION_NOTE)) checked = child_unique is not None and child_unique[0] is not None out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=coverage, cardinality=cardinality, holds=holds, sample=len(literals), @@ -1076,17 +1273,27 @@ _NOT_CODE_TAILS = frozenset({"id", "no", "num", "number", "name", "desc", "descr "at", "date", "time", "on", "amount", "price", "count", "qty", "rate", "url"}) #: 像日期的取值：2026-07-01、2026/07/01，后面可以跟时间 _DATE_TEXT = re.compile(r"^\d{4}[-/]\d{2}[-/]\d{2}") +#: 日期列最小值、最大值不计入的值（见 VALID_DATE_FROM）在说明里的说法 +INVALID_DATES = "零日期 0000-00-00、0001-01-01、9999-12-31 这类占位日期或无穷大" -def _code_type_ok(type_name: str | None) -> bool: - """整数，或者短文字（定长不超过 MAX_CODE_LEN；SQLite 的 TEXT 不带长度，也算）。""" - t = (type_name or "").upper() +def _is_enum(type_name: str | None, enum_values: Any = None) -> bool: + """取值列表有限的列：MySQL 的 ENUM / SET（类型写法以它们开头），或者探查记了取值列表（PostgreSQL 的枚举）。""" + return bool(enum_values) or catalog._type_words(type_name).startswith(("ENUM", "SET")) + + +def _code_type_ok(type_name: str | None, enum_values: Any = None) -> bool: + """ENUM / SET（取值列表有限，直接算码值候选）、整数，或者短文字（定长不超过 MAX_CODE_LEN；SQLite 的 TEXT 不带 + 长度，也算）。类型是按方言编译的完整写法：VARCHAR2(20 CHAR) 的长度按 20 算。""" + if _is_enum(type_name, enum_values): + return True + t = catalog._type_words(type_name) if not t or any(k in t for k in ("CLOB", "BLOB", "BINARY", "JSON", "DATE", "TIME", "INTERVAL", "REAL", "FLOAT", "DOUBLE", "BOOL")): return False if "INT" in t: return True - size = re.search(r"\(\s*(\d+)\s*(?:,\s*(\d+)\s*)?\)", t) + size = re.search(r"\(\s*(\d+)(?:\s+(?:CHAR|BYTE))?\s*(?:,\s*(\d+)\s*)?\)", t) if any(k in t for k in ("NUMBER", "NUMERIC", "DECIMAL")): return size is None or not size.group(2) or int(size.group(2)) == 0 if any(k in t for k in ("CHAR", "TEXT", "STRING")): @@ -1100,9 +1307,9 @@ def _looks_like_code(name: str) -> bool: def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: - """要取值分布的列：目录里度量类型记为状态的、剖析以前提过码值候选的（再核一次）、或者类型和命名像状态、 - 类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算；码值人工确认或驳回过的不再取。 - 被遮罩的列也列出来，由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" + """要取值分布的列：目录里度量类型记为状态的、剖析以前提过码值候选的（再核一次）、ENUM / SET 列（取值列表有限， + 不看名字）、或者类型和命名像状态、类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算；码值人工确认 + 或驳回过的不再取。被遮罩的列也列出来，由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" pk = [c.lower() for c in meta.get("primary_key") or []] unique = {str(g[0]).lower() for g in meta.get("unique") or [] if isinstance(g, list) and len(g) == 1} keys = {str(c).lower() for r in notes.get("relations") or [] if isinstance(r, dict) for c in r.get("columns") or []} @@ -1117,23 +1324,28 @@ def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: status_measure = (isinstance(measure, dict) and measure.get("value") == "status" and measure.get("status") != "rejected") profiled = isinstance(codes, dict) and codes.get("source") == "profile" - if not (status_measure or profiled or _looks_like_code(name)): + enum = _is_enum(col.get("type"), col.get("enum_values")) + if not (status_measure or profiled or enum or _looks_like_code(name)): continue lowered = name.lower() - if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type")): + if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type"), + col.get("enum_values")): continue out.append(name) return out -def _code_key(value: Any) -> str | None: - """取值分布里的一个取值 → 码值（文字）。只认整数和不太长的文字。""" +def _code_key(value: Any, *, fixed: bool = False) -> str | None: + """取值分布里的一个取值 → 码值（文字）。只认整数和不太长的文字。fixed：定长文字列（CHAR(n)），去掉补齐的尾随 + 空格（PG、Oracle 取回的是 'A '，WHERE status = 'A' 照样对得上，码值写 'A'）。""" if value is None or isinstance(value, bool): return None if isinstance(value, int): return str(value) if isinstance(value, float): return str(int(value)) if value.is_integer() else None + if isinstance(value, str) and fixed: + value = value.rstrip(" ") if isinstance(value, str) and value.strip() and len(value) <= MAX_CODE_LEN and not _CONTROL.search(value): return value return None @@ -1151,29 +1363,48 @@ async def _check_codes(cx: _TableCx, table: str, column: str, out: _TableOutcome if cx.is_masked(column): raise _Skip("masked", f"{table}.{column} 在数据源设置中被遮罩，不取值", carry=False) d, meta = cx.dialect, cx.tables[table] + col_meta = _column_meta(meta, column) or {} size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) - res = await cx.run(d.value_counts_sql(d.table(meta, table), d.quote(column), CODES_MAX + 1, scan_cap=scan_cap), + res = await cx.run(d.value_counts_sql(d.table(meta, table), d.column(meta, column), CODES_MAX + 1, + scan_cap=scan_cap), max_rows=CODES_MAX + 1) if not res.rows: raise _Skip("no_data", f"{table}.{column} 没有非空值，无法取值分布") if len(res.rows) > CODES_MAX: raise _Skip("high_cardinality", f"{table}.{column} 的取值超过 {CODES_MAX} 个，不作为码值候选", carry=False) - values: list[tuple[str, int]] = [] + fixed = is_fixed_char(col_meta.get("type")) + enum = _is_enum(col_meta.get("type"), col_meta.get("enum_values")) + counts: dict[str, int] = {} for row in res.rows: - key = _code_key(row[0]) + key = _code_key(row[0], fixed=fixed) if key is None: raise _Skip("unsupported", f"{table}.{column} 的取值不是整数或短文字，不作为码值候选", carry=False) - values.append((key, _as_int(row[1]) or 0)) - if len(values) >= 3 and all(n == 1 for _, n in values): - # 每个取值只出现一次（票种编码 T01…T09）：这是一列标识，不是状态码 + counts[key] = counts.get(key, 0) + (_as_int(row[1]) or 0) + values = list(counts.items()) + if len(values) >= 3 and all(n == 1 for _, n in values) and not enum: + # 每个取值只出现一次（票种编码 T01…T09）：这是一列标识，不是状态码。ENUM 的取值列表是列定义给的，不按这条判 raise _Skip("high_cardinality", f"{table}.{column} 的取值各不相同，像标识而不是码值，不作为码值候选", carry=False) rows = sum(n for _, n in values) - scope = f"按前 {scan_cap} 行统计" if scan_cap else "统计全表" + # 不说「抽样」：大表看的是按存储顺序的前若干行（多半是最老的数据），不是随机抽的 + scope = f"按存储顺序取前 {scan_cap} 行统计" if scan_cap else "统计全表" spread = "、".join(f"{k}（{_share(n, rows)}）" for k, n in values) - note = f"{NOTE_PREFIX}{scope}，{rows} 行非空值共 {len(values)} 个取值：{spread}。" - out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None)) + extra: list[str] = [] + # ENUM 的取值列表（SET 的是成员，取值是成员的组合，不算）：这次没出现的也是可能的取值 + is_set = catalog._type_words(col_meta.get("type")).startswith("SET") + declared = [] if is_set else [str(v) for v in col_meta.get("enum_values") or []] + unseen = [v for v in declared if v not in counts] + if unseen: + extra.append(f"列定义里另有 {len(unseen)} 个取值这次没有出现：{'、'.join(unseen[:CODES_MAX])}") + family = "text" if enum else catalog.column_family(col_meta) + collation = _collation_note(d, family, MYSQL_CODES_COLLATION_NOTE) + if collation: + extra.append(collation) + tail = "".join(f"；{x}" for x in extra) + note = f"{NOTE_PREFIX}{scope}，{rows} 行非空值共 {len(values)} 个取值：{spread}{tail}。" + out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None, + declared=unseen)) def _is_date_column(col: dict[str, Any]) -> bool: @@ -1190,20 +1421,31 @@ async def _check_dates(cx: _TableCx, table: str, columns: list[str], total: int, if not size.within(cx.settings.max_scan_rows): raise _Skip("too_large", f"{table} 的行数超过整表统计的行数上限（{cx.settings.max_scan_rows} 行），" "不取日期列的最小值和最大值") - res = await cx.run(d.min_max_sql(d.table(meta, table), [d.quote(c) for c in columns])) + # 文字存的日期（SQLite 的 TEXT、业务库里的 VARCHAR 日期）按字符串和有效区间比；日期类型的按日期字面量比 + texts = [catalog.column_family(_column_meta(meta, c)) != "time" for c in columns] + res = await cx.run(d.min_max_sql(d.table(meta, table), [(d.column(meta, c), t) for c, t in zip(columns, texts)])) row = res.rows[0] if res.rows else [] + invalid: dict[str, int] = {} for i, column in enumerate(columns): - low, high = (row[2 * i], row[2 * i + 1]) if len(row) >= 2 * i + 2 else (None, None) + low, high, bad = (row[3 * i], row[3 * i + 1], row[3 * i + 2]) if len(row) >= 3 * i + 3 else (None, None, 0) + invalid[column] = _as_int(bad) or 0 if low is None or high is None or not (_DATE_TEXT.match(str(low)) and _DATE_TEXT.match(str(high))): continue - result.date_ranges.append({"column": column, "min": str(low), "max": str(high)}) + found: dict[str, Any] = {"column": column, "min": str(low), "max": str(high)} + if invalid[column]: + found["invalid"] = invalid[column] + result.date_ranges.append(found) if total != 1: return if not result.date_ranges: + if invalid.get(columns[0]): + raise _Skip("no_data", f"{table}.{columns[0]} 只有 {invalid[columns[0]]} 个无效日期（{INVALID_DATES}），" + "没有可识别的日期值") raise _Skip("no_data", f"{table}.{columns[0]} 没有可识别的日期值") found = result.date_ranges[0] + dropped = (f"另有 {found['invalid']} 个无效日期（{INVALID_DATES}）未计入，" if found.get("invalid") else "") note = (f"{NOTE_PREFIX}表里只有这一个日期类列，取值从 {str(found['min'])[:19]} 到 " - f"{str(found['max'])[:19]}，可作为业务日期的参考。") + f"{str(found['max'])[:19]}，{dropped}可作为业务日期的参考。") out.business_date = _DateOutcome(column=found["column"], low=found["min"], high=found["max"], note=note) @@ -1241,6 +1483,8 @@ def _codes_value(c: _CodesOutcome, item: dict[str, Any] | None) -> dict[str, str 沿用它的含义；它写了、数据里没出现的取值也留着——码值是「可能的取值」，没出现不等于不存在。""" old = item.get("value") if isinstance(item, dict) and isinstance(item.get("value"), dict) else {} value = {key: str(old.get(key) or "") for key, _ in c.values} + for key in c.declared: + value.setdefault(key, str(old.get(key) or "")) for key, meaning in old.items(): if key not in value and isinstance(meaning, str) and meaning.strip(): value[key] = meaning @@ -1744,8 +1988,8 @@ def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> Profile #: 有效行过滤核对的说法（写进这一项的 note；前面是模型给的依据） FILTER_CHECKED = "剖析核对：排除 {share} 的行（共 {rows} 行）" -FILTER_SAMPLED = "剖析核对：按前 {rows} 行统计，排除 {share} 的行" -FILTER_SAMPLED_EDGE = "剖析核对：按前 {rows} 行统计，{what}，请人工核对" +FILTER_SAMPLED = "剖析核对：按存储顺序取前 {rows} 行统计，排除 {share} 的行" +FILTER_SAMPLED_EDGE = "剖析核对：按存储顺序取前 {rows} 行统计，{what}，请人工核对" FILTER_UNCHECKED = "没有核对排除比例（{why}）" FILTER_NOT_ENABLED = "数据源没有开数据剖析" diff --git a/backend/app/data/introspect.py b/backend/app/data/introspect.py index ea8875be..30fbfda2 100644 --- a/backend/app/data/introspect.py +++ b/backend/app/data/introspect.py @@ -165,6 +165,74 @@ def _unique_groups(constraints: list[dict[str, Any]], indexes: list[dict[str, An return groups +def type_text(col_type: Any, dialect: Any) -> str: + """反射到的列类型 → 存进表结构的写法：按这个数据源的方言编译的完整写法（VARCHAR2(20 CHAR)、NUMBER(10, 0)、 + ENUM('paid','refund')、TIMESTAMP WITH TIME ZONE），编译不了的退回 str()。 + + 以前存的是 str(类型)，即 SQLAlchemy 的通用写法：Oracle 的 VARCHAR2、NUMBER 丢了长度和精度，MySQL 的 ENUM 丢了 + 取值列表，PostgreSQL 的 TIMESTAMP 丢了时区。类型写不全，表结构前后比对就看不出类型变了，ENUM 状态列也进不了 + 码值候选。这一改会让新探查的表结构快照哈希变化（提示词里的类型写法变长），旧运行封存的快照不受影响。""" + if col_type is None: + return "" + try: + compiled = str(col_type.compile(dialect=dialect)) + except Exception: # noqa: BLE001 - 编译不了（NullType 等）就用通用写法，不让一列拖垮整次探查 + return str(col_type) + if getattr(dialect, "name", "") == "oracle" and compiled.startswith("VARCHAR("): + # SQLAlchemy 把 Oracle 的 VARCHAR2 反射成通用的 VARCHAR，按 Oracle 方言编译出来是 VARCHAR(10 CHAR)（容器集成 + # 测试实测）。Oracle 的数据字典里只有 VARCHAR2（建表写 VARCHAR 也存成 VARCHAR2），照数据字典写 + compiled = "VARCHAR2" + compiled[len("VARCHAR"):] + return compiled + + +def _enum_values(col_type: Any) -> list[str] | None: + """ENUM（MySQL、PostgreSQL 的枚举，都是 sqltypes.Enum 的子类）的取值列表，MySQL SET 的成员列表；别的类型 None。""" + from sqlalchemy.dialects.mysql import SET + from sqlalchemy.types import Enum + + if isinstance(col_type, Enum): + values = list(col_type.enums or []) + elif isinstance(col_type, SET): + values = list(col_type.values or []) + else: + return None + return [str(v) for v in values] + + +def _forced_quote(name: Any) -> bool: + """名字必须原样加引号：Oracle 反射回来的全小写对象（建表时写了 "visits"）是 quoted_name(quote=True)。 + 它存进 JSON 就成了普通字符串，剖析拼 SQL 时不加引号，Oracle 把它转成大写 VISITS，查的是另一个不存在的对象。""" + return getattr(name, "quote", None) is True + + +def column_entry(col: Mapping[str, Any], dialect: Any) -> dict[str, Any]: + """反射到的一列 → 表结构里的一列：名字、按方言编译的类型、可空、注释；ENUM / SET 另记取值列表（enum_values）， + 要强制加引号的名字另记 quote_name。只在有的时候写这两个键：其余列和以前一字不差。""" + entry: dict[str, Any] = { + "name": str(col["name"]), + "type": type_text(col.get("type"), dialect), + "nullable": bool(col.get("nullable", True)), + "comment": col.get("comment") or None, + } + values = _enum_values(col.get("type")) + if values: + entry["enum_values"] = values + if _forced_quote(col.get("name")): + entry["quote_name"] = True + return entry + + +def index_first_columns(indexes: list[dict[str, Any]], pk: list[str]) -> list[str]: + """哪些列打头有索引：各索引（含主键）的第一列，去重、按名字排序。取值观察看大表的最新业务日期、按日行数时， + 只在业务日期列打头有索引时才发窗口查询（按它走索引，不扫全表）。表达式索引（列名是 None）不算。""" + first = {str(pk[0])} if pk else set() + for ix in indexes: + cols = ix.get("column_names") or [] + if cols and cols[0] is not None: + first.add(str(cols[0])) + return sorted(first) + + async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any]: """探查整个库的结构，结果直接存进 DataSource.schema_cache。 @@ -266,24 +334,20 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any # 只读账号名下没有这些对象，全靠跨 schema 授权 "qualified": f"{target_schema}.{name}" if target_schema else name, "schema": target_schema, - "columns": [ - { - "name": c["name"], - "type": str(c.get("type") or ""), - "nullable": bool(c.get("nullable", True)), - "comment": c.get("comment") or None, - } - for c in columns - ], + "columns": [column_entry(c, sync_conn.dialect) for c in columns], "primary_key": pk, "comment": comment, "is_view": name in views, } + if _forced_quote(name): + tables[name]["quote_name"] = True if name in fk_ok: tables[name]["foreign_keys"] = _foreign_keys(fk_by.get(key) or [], target_schema, default_schema) if name in uq_ok: tables[name]["unique"] = _unique_groups(uq_by.get(key) or [], (ix_by.get(key) or []) if name in ix_ok else [], pk) + if name in ix_ok: + tables[name]["index_first_columns"] = index_first_columns(ix_by.get(key) or [], pk) payload: dict[str, Any] = {"tables": tables, "truncated": truncated, "total": len(names) + len(views)} if failures and not tables: @@ -471,6 +535,10 @@ def describe_table(source: Any, table: str, *, catalog: Mapping[str, Mapping[str marks.append("非空") if col.get("comment"): marks.append(col["comment"]) + values = col.get("enum_values") + if values and "(" not in str(col.get("type") or ""): + # PostgreSQL 的枚举类型写法只有类型名（order_status）：取值列表另外写出来，模型才知道能写哪些值 + marks.append("取值：" + "、".join(str(v) for v in values[:30]) + ("…" if len(values) > 30 else "")) suffix = f" {'、'.join(marks)}" if marks else "" lines.append(f" {col['name']} {col['type']}{suffix}") if catalog: diff --git a/backend/tests/fixtures/dialect/mariadb.sql b/backend/tests/fixtures/dialect/mariadb.sql new file mode 100644 index 00000000..0130c4d9 --- /dev/null +++ b/backend/tests/fixtures/dialect/mariadb.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：mariadb（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'sales' AND TABLE_NAME = 'orders'; +-- stats_quoted +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT(`level`) AS n, COUNT(DISTINCT `level`) AS d FROM `visits`; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY DATE(paid_at) ORDER BY d; +-- sample +-- （这种方言没有这类查询） diff --git a/backend/tests/fixtures/dialect/mysql.sql b/backend/tests/fixtures/dialect/mysql.sql new file mode 100644 index 00000000..1b046e36 --- /dev/null +++ b/backend/tests/fixtures/dialect/mysql.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：mysql（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'sales' AND TABLE_NAME = 'orders'; +-- stats_quoted +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT(`level`) AS n, COUNT(DISTINCT `level`) AS d FROM `visits`; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY DATE(paid_at) ORDER BY d; +-- sample +-- （这种方言没有这类查询） diff --git a/backend/tests/fixtures/dialect/oracle.sql b/backend/tests/fixtures/dialect/oracle.sql new file mode 100644 index 00000000..c338ead9 --- /dev/null +++ b/backend/tests/fixtures/dialect/oracle.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：oracle（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = 'SALES' AND TABLE_NAME = 'ORDERS'; +-- stats_quoted +SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT * FROM (SELECT 1 AS one FROM sales.orders) WHERE ROWNUM <= 100001) s; +-- distinct +SELECT * FROM (SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL) WHERE ROWNUM <= 2000; +-- distinct_capped +SELECT * FROM (SELECT DISTINCT store_code FROM (SELECT * FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL) WHERE ROWNUM <= 100000) s) WHERE ROWNUM <= 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- value_counts_capped +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM (SELECT * FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL) WHERE ROWNUM <= 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- day_volume +SELECT TRUNC(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY TRUNC(paid_at) ORDER BY d; +-- sample +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders SAMPLE BLOCK (1) WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; diff --git a/backend/tests/fixtures/dialect/postgresql.sql b/backend/tests/fixtures/dialect/postgresql.sql new file mode 100644 index 00000000..1416da24 --- /dev/null +++ b/backend/tests/fixtures/dialect/postgresql.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：postgresql（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +SELECT c.reltuples FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'sales' AND c.relname = 'orders'; +-- stats_quoted +SELECT c.reltuples FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = current_schema() AND c.relname = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT CAST(paid_at AS DATE) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY CAST(paid_at AS DATE) ORDER BY d; +-- sample +SELECT status, COUNT(*) AS cnt FROM sales.orders TABLESAMPLE SYSTEM (1) WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; diff --git a/backend/tests/fixtures/dialect/sqlite.sql b/backend/tests/fixtures/dialect/sqlite.sql new file mode 100644 index 00000000..8b10405f --- /dev/null +++ b/backend/tests/fixtures/dialect/sqlite.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：sqlite（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +-- （这种方言没有这类查询） +-- stats_quoted +-- （这种方言没有这类查询） +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= '0001-01-02' AND paid_at < '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= '0001-01-02' AND paid_at < '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < '0001-01-02' OR paid_at >= '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT substr(paid_at, 1, 10) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '9999-12-31' GROUP BY substr(paid_at, 1, 10) ORDER BY d; +-- sample +-- （这种方言没有这类查询） diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py index 37f3705b..c65eff02 100644 --- a/backend/tests/test_catalog_draft_facts.py +++ b/backend/tests/test_catalog_draft_facts.py @@ -402,7 +402,7 @@ async def test_a_big_table_is_checked_on_its_first_rows(scenic): found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], "status = 1") assert found.sampled and found.rows == 100 - assert found.note.startswith("剖析核对：按前 100 行统计，") + assert found.note.startswith("剖析核对：按存储顺序取前 100 行统计，") async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index fee64c5f..2a04fe76 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -240,7 +240,7 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert rel["to_table"] == parent assert (rel["source"], rel["status"]) == ("profile", "verified") assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" - assert "数据剖析" in rel["note"] and "抽样" in rel["note"] + assert "数据剖析" in rel["note"] and "子表取了" in rel["note"] and "抽样" not in rel["note"] # 子表一侧数过 COUNT 和 COUNT(DISTINCT)、确有重复：基数是用数据核实的（A6，一对多关联的检查据此给 error） assert rel["cardinality_checked"] is True # A7：说明里不写日期，时间看这一项的 updated_at（界面按本地时间显示） @@ -318,7 +318,7 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") assert rel["status"] == "proposed" and rel["source"] == "profile" assert rel["coverage"] == 0.15 - assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] + assert "覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] finding = _table(body, "parking_records")["findings"][0] assert finding["status"] == "proposed" and finding["coverage"] == 0.15 # 两个日期类列：不提议业务日期，取值范围只在剖析结果里 diff --git a/backend/tests/test_catalog_profile_units.py b/backend/tests/test_catalog_profile_units.py index 4fa1472f..22933754 100644 --- a/backend/tests/test_catalog_profile_units.py +++ b/backend/tests/test_catalog_profile_units.py @@ -40,7 +40,7 @@ def test_settings_default_off_and_parse_leniently(): ({"table_max_s": 9}, "每张表时长上限"), ({"table_max_s": 601}, "每张表时长上限"), ({"query_timeout_s": 61}, "单条查询时限"), - ({"sample_size": 5}, "抽样键值数"), + ({"sample_size": 5}, "核对键值数"), ({"max_scan_rows": -1}, "整表统计行数上限"), ({"enabled": "true"}, "开启数据剖析"), ({"table_max_queries": True}, "每张表最多查询数"), @@ -143,8 +143,16 @@ def test_profile_sql_passes_readonly_guard_on_every_dialect(kind): d.unique_check_sql(tbl, col), d.value_counts_sql(tbl, col, 21, scan_cap=None), d.value_counts_sql(tbl, col, 21, scan_cap=100_000), - d.min_max_sql(tbl, [col]), + d.min_max_sql(tbl, [(col, False)]), + d.min_max_sql(tbl, [(col, True)]), + d.day_volume_sql(tbl, col, "2026-08-01"), + d.window_value_counts_sql(tbl, col, col, "2026-08-01", 21), ] + sampled = d.sample_value_counts_sql(tbl, col, 1, 21) + if kind in ("postgresql", "oracle"): + sqls.append(sampled) + else: + assert sampled is None # MySQL、SQLite 没有块采样 for sql in sqls: assert guard.check(sql, readonly=True, dialect=kind) == sql, sql diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index 25eb9aad..5501a4d8 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -365,7 +365,7 @@ async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(c capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"]) await _profile(client, capped, ["visits"]) codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"] - assert "按前 10 行统计" in codes["note"] and "complete" not in codes + assert "按存储顺序取前 10 行统计" in codes["note"] and "complete" not in codes # ========================================================================== diff --git a/backend/tests/test_profile_dialect_golden.py b/backend/tests/test_profile_dialect_golden.py new file mode 100644 index 00000000..2eb0615c --- /dev/null +++ b/backend/tests/test_profile_dialect_golden.py @@ -0,0 +1,178 @@ +"""剖析 SQL 的方言金样（下一轮 03 细案 C2）：没有真库时挡住方言写法的回退。 + +三层，各挡各的： + +1. **金样**：每种方言（mysql、mariadb、postgresql、oracle、sqlite）× 每类查询（统计信息、有上限的计数、取不同键值、 + 对键值、唯一性、取值分布、日期最小最大、时间窗口里的取值分布、按日行数、块采样），完整 SQL 钉在 + tests/fixtures/dialect/<方言>.sql，人审过一次；以后拼法有任何改动都会出 diff。要更新金样： + ``UPDATE_DIALECT_GOLDEN=1 pytest tests/test_profile_dialect_golden.py``，再逐行看 git diff。 +2. **sqlglot 逐条解析**：按方言读一遍（mariadb 按 mysql 读），挡住语法写坏了（少括号、关键字拼错）。 +3. **IN 两边类型族一致**：用 sqlglot 的 qualify + annotate_types，配上表结构里的类型，核对 IN 列表的字面量和被指向 + 列是同一类型族（PostgreSQL 上 varchar 父列配整数子列时，字面量必须带引号）。 + +挡不住的照实写：sqlglot 不知道真库的统计信息是不是过期、零日期驱动怎么返回、Oracle 小写对象是否存在、排序规则 +怎么比——这些要靠 scripts/dialect-it.sh 在本机容器里跑真库（不进 CI）。 +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import sqlglot +from sqlglot import exp +from sqlglot.optimizer.annotate_types import annotate_types +from sqlglot.optimizer.qualify import qualify + +from app.data import catalog, guard +from app.data.catalog_profile import IN_CHUNK, SqlDialect, key_literals + +GOLDEN = Path(__file__).parent / "fixtures" / "dialect" +DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite") +#: sqlglot 的方言名 +READ = {"mysql": "mysql", "mariadb": "mysql", "postgresql": "postgres", "oracle": "oracle", "sqlite": "sqlite"} + +#: 夹具表结构：一张带 schema 的订单表、一张门店表（编码是文字），Oracle 的一张小写带引号的表和一个小写带引号的列 +META = { + "orders": {"schema": "sales", "columns": [ + {"name": "id", "type": "INTEGER"}, {"name": "store_code", "type": "INTEGER"}, + {"name": "status", "type": "CHAR(2)"}, {"name": "paid_at", "type": "TIMESTAMP"}, + {"name": "biz_day", "type": "VARCHAR(10)"}, {"name": "level", "type": "INTEGER"}], + "primary_key": ["id"]}, + "stores": {"schema": None, "columns": [{"name": "code", "type": "VARCHAR(10)"}], "primary_key": ["code"]}, + "visits": {"schema": None, "quote_name": True, "columns": [ + {"name": "id", "type": "INTEGER"}, {"name": "level", "type": "INTEGER", "quote_name": True}], + "primary_key": ["id"]}, +} + + +def _queries(kind: str) -> list[tuple[str, str | None]]: + """一种方言的全部剖析、观察查询：(名字, SQL)。没有这种查询（SQLite 没有统计信息、MySQL 没有块采样）的 SQL 为 None。""" + d = SqlDialect(kind) + orders, stores, visits = d.table(META["orders"], "orders"), d.table(META["stores"], "stores"), \ + d.table(META["visits"], "visits") + status, code = d.column(META["orders"], "status"), d.column(META["stores"], "code") + level = d.column(META["visits"], "level") + keys, _ = key_literals([12, 7], "number", parent="text", child_type="INTEGER") + out: list[tuple[str, str | None]] = [ + ("stats", d.stats_sql(META["orders"], "orders")), + ("stats_quoted", d.stats_sql(META["visits"], "visits")), + ("bounded", d.bounded_count_sql(orders, 100_001)), + ("distinct", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, scan_cap=None)), + ("distinct_capped", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, + scan_cap=100_000)), + ("match", d.match_count_sql(stores, code, keys)), + ("unique", d.unique_check_sql(visits, level)), + ("value_counts", d.value_counts_sql(orders, status, 21, scan_cap=None)), + ("value_counts_capped", d.value_counts_sql(orders, status, 21, scan_cap=100_000)), + ("min_max", d.min_max_sql(orders, [(d.column(META["orders"], "paid_at"), False), + (d.column(META["orders"], "biz_day"), True)])), + ("date_window", d.window_value_counts_sql(orders, status, d.column(META["orders"], "paid_at"), + "2026-08-27", 21)), + ("day_volume", d.day_volume_sql(orders, d.column(META["orders"], "paid_at"), "2026-08-27")), + ("sample", d.sample_value_counts_sql(orders, status, 1, 21)), + ] + return out + + +def _render(kind: str) -> str: + lines = [f"-- 剖析与取值观察的 SQL 金样：{kind}（tests/test_profile_dialect_golden.py 生成，改动要人审）"] + for name, sql in _queries(kind): + lines.append(f"-- {name}") + lines.append(f"{sql};" if sql is not None else "-- （这种方言没有这类查询）") + return "\n".join(lines) + "\n" + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_profile_sql_matches_the_golden_file(kind): + path = GOLDEN / f"{kind}.sql" + text = _render(kind) + if os.environ.get("UPDATE_DIALECT_GOLDEN") == "1": + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + assert path.read_text(encoding="utf-8") == text, f"{kind} 的剖析 SQL 和金样不一致：看 diff，确认无误后重新生成金样" + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_every_profile_query_parses_and_passes_the_guard(kind): + for name, sql in _queries(kind): + if sql is None: + continue + parsed = sqlglot.parse(sql, read=READ[kind], error_level=sqlglot.ErrorLevel.RAISE) + assert len(parsed) == 1 and parsed[0] is not None, (kind, name) + assert guard.check(sql, readonly=True, dialect=kind) == sql, (kind, name) + + +@pytest.mark.parametrize("kind", ["postgresql", "oracle", "mysql"]) +def test_observation_queries_use_the_right_day_and_sample_functions(kind): + """按天截断和块采样各家写法不同，解析成语法树再看用的是哪个函数，不只比字符串。""" + sql = dict(_queries(kind)) + day = sqlglot.parse_one(sql["day_volume"], read=READ[kind]) + if kind == "oracle": + assert any(isinstance(n, (exp.DateTrunc, exp.TimestampTrunc, exp.Anonymous)) or n.key == "trunc" + for n in day.walk()), sql["day_volume"] + assert "TRUNC(paid_at)" in sql["day_volume"] + elif kind == "postgresql": + assert any(isinstance(n, exp.Cast) for n in day.walk()) + else: + assert "DATE(paid_at)" in sql["day_volume"] + if kind == "mysql": + assert sql["sample"] is None + else: + sampled = sqlglot.parse_one(sql["sample"], read=READ[kind]) + assert any(isinstance(n, exp.TableSample) for n in sampled.walk()), sql["sample"] + + +# ---------------------------------------------------------------- IN 两边的类型族 + + +_FAMILY_OF = [(exp.DataType.TEXT_TYPES, "text"), (exp.DataType.NUMERIC_TYPES, "number"), + (exp.DataType.TEMPORAL_TYPES, "time")] + + +def _sg_family(t: exp.DataType | None) -> str | None: + if t is None: + return None + for group, name in _FAMILY_OF: + if t.this in group: + return name + return None + + +def _in_families(sql: str, kind: str, schema: dict[str, dict[str, str]]) -> list[tuple[str | None, set[str | None]]]: + """sql 里每个 IN：(被比较的列的类型族, 字面量的类型族集合)，按 annotate_types 配上表结构的类型推出来。""" + tree = qualify(sqlglot.parse_one(sql, read=READ[kind]), schema=schema, dialect=READ[kind], + validate_qualify_columns=False) + tree = annotate_types(tree, schema=schema, dialect=READ[kind]) + return [(_sg_family(node.this.type), {_sg_family(x.type) for x in node.expressions}) + for node in tree.find_all(exp.In)] + + +@pytest.mark.parametrize("kind", DIALECTS) +@pytest.mark.parametrize("parent_type, child_values, child_type", [ + ("VARCHAR(10)", [12, 7, 30], "INTEGER"), # 父列文字、子列数：字面量要带引号 + ("INTEGER", ["12", "0007", "A1"], "VARCHAR(10)"), # 父列数、子列文字：只收整数写法，写成数字 + ("VARCHAR(10)", ["A ", "BC"], "CHAR(2)"), # 定长子列：去掉补齐的空格 + ("INTEGER", [1, 2], "INTEGER"), +]) +def test_in_list_literals_have_the_parent_columns_family(kind, parent_type, child_values, child_type): + d = SqlDialect(kind) + parent = {"name": "code", "type": parent_type} + literals, _ = key_literals(child_values, None, parent=catalog.column_family(parent), child_type=child_type) + assert literals + sql = d.match_count_sql("stores", d.quote("code"), literals[:IN_CHUNK]) + found = _in_families(sql, kind, {"stores": {"code": parent_type}}) + assert found, sql + for column_family, literal_families in found: + assert literal_families == {column_family}, (kind, sql, column_family, literal_families) + + +def test_the_old_way_would_mismatch_on_postgres(): + """对照：以前按子表取回的值写字面量，varchar 父列配整数子列在 sqlglot 推出的类型上就两边不同——PostgreSQL 真库上 + 这是「operator does not exist: character varying = integer」。""" + from app.data.catalog_profile import sql_literal + + old = [lit for v in (12, 7) if (lit := sql_literal(v, "number"))] + sql = SqlDialect("postgresql").match_count_sql("stores", "code", old) + [(column_family, literal_families)] = _in_families(sql, "postgresql", {"stores": {"code": "VARCHAR(10)"}}) + assert column_family == "text" and literal_families == {"number"} diff --git a/backend/tests/test_profile_dialect_units.py b/backend/tests/test_profile_dialect_units.py new file mode 100644 index 00000000..db01b746 --- /dev/null +++ b/backend/tests/test_profile_dialect_units.py @@ -0,0 +1,296 @@ +"""剖析与探查的方言修正（下一轮 03 细案 C1）：不连库的部分。 + +- 列类型按方言编译的完整写法：类型族、日期列、码值候选都认得 ENUM / SET、VARCHAR2(20 CHAR)、INTERVAL 这些写法； +- IN 列表的字面量按被指向列的类型族生成（PG 上文字列配整数子列会直接报错，Oracle 会把父列转成数报 ORA-01722）； +- Oracle 小写带引号的对象（quote_name）强制加引号、不转大写； +- 统计信息估出的行数不超过整表统计上限时再数一次确认； +- 日期列的最小值、最大值排除零日期和哨兵日期，照实写排除了几个； +- 取值观察要用的三个方言方法（date_literal / day_trunc / sample_from）； +- 说明文字不再说「抽样」：大表看的是按存储顺序的前 N 行。 +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +from typing import Any + +import pytest +from sqlalchemy import VARCHAR +from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite +from sqlalchemy.sql.elements import quoted_name + +from app.data import catalog, catalog_profile, introspect +from app.data.catalog_profile import ProfileSettings, SqlDialect, TableSize +from app.data.engine import QueryResult + +DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite") + + +# ---------------------------------------------------------------- 类型的写法 + + +@pytest.mark.parametrize("type_name, family", [ + ("ENUM('paid','refund')", "text"), + ("ENUM('int','date')", "text"), # 取值里的词不能让它变成数字或时间 + ("SET('a','b')", "text"), + ("VARCHAR2(20 CHAR)", "text"), + ("NVARCHAR2(10)", "text"), + ("VARCHAR(20) COLLATE utf8mb4_bin", "text"), + ("NUMBER(10, 0)", "number"), + ("INTEGER(11)", "number"), + ("BIGINT UNSIGNED", "number"), + ("DOUBLE PRECISION", "number"), + ("TIMESTAMP WITH TIME ZONE", "time"), + ("TIMESTAMP(3)", "time"), + ("INTERVAL DAY(2) TO SECOND(6)", None), # 时长不是数也不是时刻；以前按「INT」认成了数字 + ("order_status", None), + ("", None), +]) +def test_type_family_reads_dialect_full_types(type_name, family): + assert catalog._type_family(type_name) == family + + +def test_type_family_of_a_column_uses_its_enum_values(): + """PostgreSQL 的枚举类型编译出来是类型名（order_status），只有取值列表能说明它是文字。""" + assert catalog.column_family({"name": "status", "type": "order_status", "enum_values": ["a", "b"]}) == "text" + assert catalog.column_family({"name": "n", "type": "INTEGER"}) == "number" + assert catalog.column_family({"name": "x"}) is None + + +def test_enum_values_never_make_a_date_column(): + assert not catalog.is_date_column("status", "ENUM('date','time')") + assert catalog.is_date_column("paid_at", "TIMESTAMP WITH TIME ZONE") + assert catalog.is_date_column("biz_dt", "VARCHAR2(10 CHAR)") # 文字存日期、名字像日期 + assert not catalog.is_date_column("gap", "INTERVAL DAY(2) TO SECOND(6)") + + +@pytest.mark.parametrize("col, ok", [ + ({"type": "VARCHAR2(20 CHAR)"}, True), + ({"type": "VARCHAR2(200 CHAR)"}, False), + ({"type": "CHAR(2)"}, True), + ({"type": "ENUM('paid','refund')"}, True), + ({"type": "SET('a','b')"}, True), + ({"type": "order_status", "enum_values": ["a", "b"]}, True), + ({"type": "NUMBER(10, 0)"}, True), + ({"type": "NUMBER(10, 2)"}, False), + ({"type": "TIMESTAMP(3)"}, False), + ({"type": "INTERVAL DAY(2) TO SECOND(6)"}, False), +]) +def test_code_type_accepts_enum_and_char_semantics(col, ok): + assert catalog_profile._code_type_ok(col.get("type"), col.get("enum_values")) is ok + + +def test_enum_columns_are_code_candidates_whatever_their_name(): + """ENUM 的取值列表有限，直接算码值候选：pay_way 这种名字不像码值的列以前被漏掉。""" + meta = {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "pay_way", "type": "ENUM('cash','card')", + "enum_values": ["cash", "card"]}, + {"name": "pay_note", "type": "VARCHAR(20)"}], + "primary_key": ["id"]} + assert catalog_profile._code_columns(meta, {}) == ["pay_way"] + + +# ---------------------------------------------------------------- 探查：类型写法、取值列表、引号、索引首列 + + +@pytest.mark.parametrize("dialect, col_type, text", [ + (mysql.dialect(), mysql.ENUM("paid", "refund"), "ENUM('paid','refund')"), + (mysql.dialect(), mysql.TINYINT(display_width=1), "TINYINT(1)"), + (oracle.dialect(), oracle.VARCHAR2(20), "VARCHAR2(20 CHAR)"), + # Oracle 的 VARCHAR2 反射回来是通用的 VARCHAR（容器集成测试实测）：照数据字典写成 VARCHAR2 + (oracle.dialect(), VARCHAR(10), "VARCHAR2(10 CHAR)"), + (oracle.dialect(), oracle.NUMBER(10, 0), "NUMBER(10, 0)"), + (postgresql.dialect(), postgresql.ENUM("a", "b", name="order_status"), "order_status"), + (sqlite.dialect(), sqlite.DATETIME(), "DATETIME"), +]) +def test_reflected_type_is_compiled_for_the_dialect(dialect, col_type, text): + assert introspect.type_text(col_type, dialect) == text + + +def test_type_that_cannot_compile_falls_back_to_str(): + from sqlalchemy.types import NullType + + assert introspect.type_text(NullType(), sqlite.dialect()) == "NULL" + assert introspect.type_text(None, sqlite.dialect()) == "" + + +def test_column_entry_records_enum_values_and_oracle_quoted_names(): + ora = oracle.dialect() + entry = introspect.column_entry({"name": quoted_name("level", quote=True), "type": oracle.VARCHAR2(10), + "nullable": True, "comment": None}, ora) + assert entry == {"name": "level", "type": "VARCHAR2(10 CHAR)", "nullable": True, "comment": None, + "quote_name": True} + plain = introspect.column_entry({"name": "status", "type": mysql.ENUM("a", "b"), "nullable": False}, mysql.dialect()) + assert plain["enum_values"] == ["a", "b"] and "quote_name" not in plain + members = introspect.column_entry({"name": "tags", "type": mysql.SET("x", "y")}, mysql.dialect()) + assert members["enum_values"] == ["x", "y"] and members["type"] == "SET('x','y')" + + +def test_index_first_columns_are_recorded(): + indexes = [{"column_names": ["paid_at", "id"]}, {"column_names": [None]}, {"column_names": ["store_id"]}, + {"column_names": ["paid_at"]}] + assert introspect.index_first_columns(indexes, ["id"]) == ["id", "paid_at", "store_id"] + + +# ---------------------------------------------------------------- IN 列表的字面量按被指向列的类型族 + + +def test_key_literals_follow_the_parent_family(): + lits, mismatched = catalog_profile.key_literals([12, 7], "number", parent="text", child_type="INTEGER") + assert lits == ["'12'", "'7'"] and mismatched == 0 + lits, mismatched = catalog_profile.key_literals(["12", "A1", "-3"], "text", parent="number", + child_type="VARCHAR(10)") + assert lits == ["12", "-3"] and mismatched == 1 + lits, mismatched = catalog_profile.key_literals([1, 2], "number", parent="time", child_type="INTEGER") + assert lits == [] and mismatched == 2 + # 父列类型不知道：照子表取回的值的类型写（以前的做法） + assert catalog_profile.key_literals([3, "x"], None, parent=None, child_type=None) == (["3", "'x'"], 0) + + +def test_char_keys_lose_their_padding(): + """PG、Oracle 的 CHAR(n) 取回来带尾随空格：拼成 'A ' 去和 VARCHAR 父列比就对不上。""" + lits, _ = catalog_profile.key_literals(["A ", "BC"], "text", parent="text", child_type="CHAR(2)") + assert lits == ["'A'", "'BC'"] + lits, _ = catalog_profile.key_literals(["A "], "text", parent="text", child_type="VARCHAR(2)") + assert lits == ["'A '"] + + +# ---------------------------------------------------------------- Oracle 小写带引号的对象 + + +def test_oracle_quote_name_forces_quotes_and_keeps_case(): + ora = SqlDialect("oracle") + meta = {"schema": "sales", "quote_name": True, + "columns": [{"name": "level", "type": "VARCHAR2(10 CHAR)", "quote_name": True}, + {"name": "amount", "type": "NUMBER(10, 2)"}]} + assert ora.table(meta, "visits") == 'sales."visits"' + assert ora.column(meta, "level") == '"level"' and ora.column(meta, "amount") == "amount" + stats = ora.stats_sql(meta, "visits") + assert "TABLE_NAME = 'visits'" in stats and "OWNER = 'SALES'" in stats + # 没有 quote_name 的照旧：不加引号，Oracle 自己转大写 + assert ora.table({"schema": None}, "visits") == "visits" + assert "TABLE_NAME = 'VISITS'" in ora.stats_sql({}, "visits") + + +# ---------------------------------------------------------------- 统计信息过期：再数一次确认 + + +def _result(value: Any) -> QueryResult: + return QueryResult(columns=["n"], rows=[[value]], row_count=1, truncated=False, elapsed_ms=1, sql="") + + +def _runner(answers: list[Any], seen: list[str]): + async def run(sql: str, **_kw: Any) -> QueryResult: + seen.append(sql) + return _result(answers.pop(0)) + return run + + +@pytest.mark.parametrize("stats, counted, expected", [ + (500, 500, TableSize(500, "count")), # 统计信息说小、数过也小：准数 + (500, 100_001, TableSize(None, "count", at_least=100_001)), # 统计信息过期：其实是大表 + (5_000_000, None, TableSize(5_000_000, "stats")), # 统计信息说大：不必再数 +]) +def test_small_stats_are_confirmed_with_a_bounded_count(stats, counted, expected): + seen: list[str] = [] + answers = [stats] + ([counted] if counted is not None else []) + size, skips = asyncio.run(catalog_profile.estimate_size(_runner(answers, seen), SqlDialect("postgresql"), + {"schema": None}, "visits", max_scan_rows=100_000)) + assert size == expected and skips == [] + assert len(seen) == (2 if counted is not None else 1) + if counted is not None: + assert "LIMIT 100001" in seen[1] + + +# ---------------------------------------------------------------- 日期列：排除零日期和哨兵日期 + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_min_max_excludes_invalid_dates_and_counts_them(kind): + d = SqlDialect(kind) + sql = d.min_max_sql("t", [("paid_at", False), ("biz_day", True)]) + # 日期类型的列按日期字面量比，文字存的日期按字符串比（PG 上 varchar 和 date 比较直接报错） + lo, hi = d.date_literal("0001-01-02"), d.date_literal("9999-12-31") + assert f"paid_at >= {lo}" in sql and f"paid_at < {hi}" in sql + assert "biz_day >= '0001-01-02'" in sql and "biz_day < '9999-12-31'" in sql + assert sql.count("SUM(CASE WHEN") == 2 + + +def _dates_cx(row: list[Any], kind: str = "mysql"): + settings = ProfileSettings(enabled=True) + source = type("S", (), {"id": "s1", "name": "shop", "kind": kind, "options": {}})() + job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect(kind), + runner=None, tables={"orders": {"columns": [ + {"name": "paid_at", "type": "DATETIME"}]}}, masked=set()) + job.sizes["orders"] = TableSize(10, "count") + result = catalog_profile.TableResult(table="orders") + cx = catalog_profile._TableCx(job, "orders", result, catalog_profile.TableHooks()) + + async def run(sql: str, **_kw: Any) -> QueryResult: + return QueryResult(columns=["a", "b", "c"], rows=[row], row_count=1, truncated=False, elapsed_ms=1, sql=sql) + cx.run = run # type: ignore[method-assign] + return cx, result + + +def test_dates_note_says_how_many_invalid_values_were_left_out(): + cx, result = _dates_cx(["2026-01-03 08:00:00", "2026-09-30 21:00:00", 4]) + out = catalog_profile._TableOutcome() + asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, out, result)) + assert result.date_ranges == [{"column": "paid_at", "min": "2026-01-03 08:00:00", "max": "2026-09-30 21:00:00", + "invalid": 4}] + assert "另有 4 个无效日期" in out.business_date.note and "0000-00-00" in out.business_date.note + + +def test_only_invalid_dates_is_no_data(): + cx, result = _dates_cx([None, None, 9]) + with pytest.raises(catalog_profile._Skip) as e: + asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, catalog_profile._TableOutcome(), + result)) + assert e.value.reason == "no_data" and "9 个无效日期" in e.value.detail + + +# ---------------------------------------------------------------- 取值观察要用的方言方法 + + +@pytest.mark.parametrize("kind, literal, day, sample", [ + ("mysql", "DATE '2026-09-01'", "DATE(paid_at)", None), + ("mariadb", "DATE '2026-09-01'", "DATE(paid_at)", None), + ("postgresql", "DATE '2026-09-01'", "CAST(paid_at AS DATE)", "orders TABLESAMPLE SYSTEM (1.5)"), + ("oracle", "DATE '2026-09-01'", "TRUNC(paid_at)", "orders SAMPLE BLOCK (1.5)"), + ("sqlite", "'2026-09-01'", "substr(paid_at, 1, 10)", None), +]) +def test_observation_dialect_methods(kind, literal, day, sample): + d = SqlDialect(kind) + assert d.date_literal(_dt.date(2026, 9, 1)) == literal == d.date_literal("2026-09-01") + assert d.date_literal("2026-09-01", text=True) == "'2026-09-01'" + assert d.day_trunc("paid_at") == day + assert d.sample_from("orders", 1.5) == sample + + +@pytest.mark.parametrize("bad", ["2026-9-1", "2026-09-01'; --", "0000-00-00", 20260901, None]) +def test_date_literal_refuses_anything_but_a_real_date(bad): + with pytest.raises(ValueError): + SqlDialect("postgresql").date_literal(bad) + + +@pytest.mark.parametrize("bad", [0, -1, 100, 101, float("nan")]) +def test_sample_percent_must_be_between_0_and_100(bad): + with pytest.raises(ValueError): + SqlDialect("postgresql").sample_from("orders", bad) + + +# ---------------------------------------------------------------- 说明文字 + + +def test_notes_say_storage_order_not_sampling(): + note = catalog_profile._relation_note(sample=2000, matched=2000, coverage=1.0, scan_cap=100_000, skipped_values=0, + mismatched=0, parent=(True, "主键"), child=(False, "count"), holds=True, + collation=None) + assert "按存储顺序只看前 100000 行" in note and "抽样" not in note + assert catalog_profile.FILTER_SAMPLED.format(rows="100", share="1%").startswith("剖析核对：按存储顺序取前 100 行统计") + + +def test_mysql_notes_mention_the_collation(): + note = catalog_profile._relation_note(sample=10, matched=10, coverage=1.0, scan_cap=None, skipped_values=0, + mismatched=0, parent=(True, "主键"), child=(False, "count"), holds=True, + collation=catalog_profile.MYSQL_COLLATION_NOTE) + assert "排序规则" in note and "不分大小写" in note diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 51fb6dba..ed0661cf 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2771,7 +2771,7 @@ export const SCHEMA_PARTIAL_TEXT = { // - 剖析是后台任务：逐张表进行，每张表剖析完就写进目录；查询一条一条地发，可以随时停止。关掉对话框照常进行。 // 目录只在整张表结束时写一次：停止、关停服务时正在做的那张来得及收尾，做完的项照常写进去；进程被杀时那张表没来得及收尾， // 这次做完的项一项都没写进去。说停下以后写进了什么，按逐表的状态照实说（cancelledNote / interruptedNote），不一律说「已写进」。 -// - 预算按表算：「每张表最多查询数」「每张表时长上限」用完只停这张表，下一张表重新计数；另有单条查询时限、抽样键值数、 +// - 预算按表算：「每张表最多查询数」「每张表时长上限」用完只停这张表，下一张表重新计数；另有单条查询时限、核对键值数、 // 整表统计行数上限。标签和服务端校验报错里的叫法一致（catalog_profile._NUMBER_FIELDS），保存被拒时按机读码认出是哪一项。 // - 没有整次总量上限：任务自己在连续失败、多数超时时停下（止损），写明原因。 // - 「没查成」≠ 结论错了：因为次数、时长、超时、出错没查成的项保留剖析以前的结论，可以「重试没做完的表」。 @@ -2783,7 +2783,7 @@ export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, st table_max_queries: '每张表最多查询数', table_max_s: '每张表时长上限', query_timeout_s: '单条查询时限', - sample_size: '抽样键值数', + sample_size: '核对键值数', max_scan_rows: '整表统计行数上限', } @@ -2799,7 +2799,7 @@ export const PROFILE_FIELD_HINT: Record = { table_max_queries: '剖析一张表最多发出的查询条数，用完只停这张表，下一张表重新计数', table_max_s: '剖析一张表最多用的时长，到时只停这张表，下一张表重新计时', query_timeout_s: '超过即中断这条查询；不超过数据源自身的查询时限，也不超过这张表剩下的时长', - sample_size: '核对一条关联关系时，从本表抽取的不同键值个数', + sample_size: '核对一条关联关系时，从本表取的不同键值个数：取的是数据库先给出的那些，不是随机抽样', max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计', } @@ -2863,7 +2863,7 @@ export const PROFILE_TEXT = { budgetPerTable: (queries: number, secs: number) => `每张表最多 ${formatNumber(queries)} 条查询、${formatNumber(secs)} 秒，用完只停这张表，下一张表重新计数`, budgetQuery: (secs: number) => `单条查询不超过 ${formatNumber(secs)} 秒`, - budgetSample: (n: number) => `每条关联关系抽样 ${formatNumber(n)} 个键值`, + budgetSample: (n: number) => `每条关联关系取 ${formatNumber(n)} 个不同键值核对`, budgetScan: (n: number) => (n > 0 ? `行数超过 ${formatNumber(n)} 的表不做整表统计` : '不做整表统计'), /** 上限的估算：表数 × 每张表的上限 */ budgetTotal: (tables: number, queries: number, span: string) => @@ -3002,7 +3002,7 @@ export const PROFILE_TEXT = { relationLowCoverage: '覆盖率不足，可能不是这条关系', relationNotUnique: '被指向列未核实唯一', coverage: (pct: string) => `覆盖率 ${pct}`, - sampled: (sample: number, matched: number) => `抽样 ${formatNumber(sample)} 个键值，对上 ${formatNumber(matched)} 个`, + sampled: (sample: number, matched: number) => `取了 ${formatNumber(sample)} 个不同键值，对上 ${formatNumber(matched)} 个`, cardinalityUnknown: '基数未核实', codeRows: (n: number) => `${formatNumber(n)} 行`, codesSummary: (values: number, rows: number) => `${formatNumber(values)} 个取值，共 ${formatNumber(rows)} 行`, @@ -3012,10 +3012,12 @@ export const PROFILE_TEXT = { dateProposal: (col: string) => `提议按 ${col} 作为业务日期`, dateSpan: (min: string, max: string) => `取值从 ${min} 到 ${max}`, dateRanges: '日期列的取值范围', + /** 日期列的最小值、最大值不计入的值（catalog_profile.VALID_DATE_FROM）：零日期、0001-01-01、9999-12-31、无穷大 */ + dateInvalid: (n: number) => `另有 ${formatNumber(n)} 个无效日期未计入（零日期或占位日期）`, skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`, moreSkipped: (n: number) => `…另有 ${formatNumber(n)} 项`, queriesTitle: (n: number) => `看发过的查询（${formatNumber(n)} 条）`, - queriesHint: 'SQL 里的取值都换成了「?」：抽样到的键值不保存在 AgentLab 中；遮罩的列不出现在任何查询里', + queriesHint: 'SQL 里的取值都换成了「?」：取到的键值不保存在 AgentLab 中；遮罩的列不出现在任何查询里', queryKind: { row_estimate: '行数估算', relation: '关系', codes: '码值', date: '日期' } as Record, queryOutcome: { ok: '成功', timeout: '超时', error: '失败', rejected: '未通过只读守卫' } as Record, queryRows: (n: number) => `${formatNumber(n)} 行`, diff --git a/frontend/src/pages/catalog/ProfileDialog.tsx b/frontend/src/pages/catalog/ProfileDialog.tsx index 321ad9c3..b14641fc 100644 --- a/frontend/src/pages/catalog/ProfileDialog.tsx +++ b/frontend/src/pages/catalog/ProfileDialog.tsx @@ -780,6 +780,10 @@ export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = nul {PT.dateProposal(date.column)} ，{PT.dateSpan(date.min.slice(0, 19), date.max.slice(0, 19))} + {(() => { + const invalid = d.date_ranges.find((r) => r.column === date.column)?.invalid + return invalid ? ，{PT.dateInvalid(invalid)} : null + })()} )} @@ -788,6 +792,7 @@ export function ProfileTaskBody({ detail: d, mode, filled = {}, focusAfter = nul {d.date_ranges.map((r) => (
  • {r.column}：{PT.dateSpan(r.min.slice(0, 19), r.max.slice(0, 19))} + {r.invalid ? `，${PT.dateInvalid(r.invalid)}` : null}
    • ))} diff --git a/frontend/src/pages/catalog/ProfileSettings.tsx b/frontend/src/pages/catalog/ProfileSettings.tsx index ddd9a3ec..91f13ee1 100644 --- a/frontend/src/pages/catalog/ProfileSettings.tsx +++ b/frontend/src/pages/catalog/ProfileSettings.tsx @@ -14,7 +14,7 @@ import type { ProfileForm } from './profile' // 数据剖析的设置：开关、五项预算、风险说明。数据源编辑框里是其中一段；导入表格的源没有编辑框，数据目录页在 // 剖析被拒（未开启）时也要就地打开它，所以另有一个只改这一项的弹窗（options 的其余键原样带上）。 // 剖析会对业务库发查询，开关默认关着；开启后五项预算才展开，风险说明写的是此刻填的数。 -// 预算按表算：每张表最多查询数、每张表时长上限（用完只停这张表），另有单条查询时限、抽样键值数、整表统计行数上限； +// 预算按表算：每张表最多查询数、每张表时长上限（用完只停这张表），另有单条查询时限、核对键值数、整表统计行数上限； // 没有整次剖析的总量上限（任务自己在连续失败、多数超时时停下）。 // =========================================================================== diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0bd204f6..f44637d7 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -3528,7 +3528,8 @@ export interface CatalogProfileTaskDetail extends CatalogProfileTaskRow { findings: CatalogProfileFinding[] skipped: CatalogProfileSkip[] /** 日期类列的取值范围 */ - date_ranges: { column: string; min: string; max: string }[] + /** invalid：零日期、占位日期这类不计入最小值和最大值的个数（没有时不给） */ + date_ranges: { column: string; min: string; max: string; invalid?: number }[] query_log: CatalogProfileQuery[] } diff --git a/scripts/check-catalog-profile.mjs b/scripts/check-catalog-profile.mjs index 66efe68e..5c7f6cde 100644 --- a/scripts/check-catalog-profile.mjs +++ b/scripts/check-catalog-profile.mjs @@ -695,7 +695,7 @@ await section('剖析 · 设置：每张表的两项预算、被拒时报错落 await sec.locator('[data-profile-enabled]').check() const labels = await sec.locator('label.label').allInnerTexts() check('开启后五项预算展开：每张表最多查询数、每张表时长上限在前，没有整次的查询次数上限、总时长上限', - labels.join('、') === '每张表最多查询数、每张表时长上限、单条查询时限、抽样键值数、整表统计行数上限', labels.join('、')) + labels.join('、') === '每张表最多查询数、每张表时长上限、单条查询时限、核对键值数、整表统计行数上限', labels.join('、')) const units = await sec.locator('[data-profile-field] span').allInnerTexts() check('……每项带单位', units.join(',') === '条,秒,秒,个,行', units.join(',')) const hints = await sec.locator('[id$="-hint"]').allInnerTexts() @@ -783,8 +783,8 @@ await section('剖析 · 开始前：范围、个数与预算', async () => { check('……写明需要剖析的口径（含上次是空表的不算）、全部不含视图', (await inBox(page, '[data-profile-scope="needed"]')).includes('从没剖析过、上次没做完，或剖析之后又出现了待核实关联关系、要取码值或日期范围的列的表。上次是空表的不算') && (await inBox(page, '[data-profile-scope="all"]')).includes('不含视图')) const budget = await inBox(page, '[data-profile-budget]') - check('预算按表写：每张表最多 40 条、60 秒，单条 10 秒，抽样、整表统计门槛，按哪个源的设置', - ['每张表最多 40 条查询、60 秒，用完只停这张表', '单条查询不超过 10 秒', '抽样 2,000 个键值', '行数超过 100,000 的表不做整表统计', `按「${NAME1}」的数据剖析设置`] + check('预算按表写：每张表最多 40 条、60 秒，单条 10 秒，核对键值数、整表统计门槛，按哪个源的设置', + ['每张表最多 40 条查询、60 秒，用完只停这张表', '单条查询不超过 10 秒', '取 2,000 个不同键值核对', '行数超过 100,000 的表不做整表统计', `按「${NAME1}」的数据剖析设置`] .every((x) => budget.includes(x)), budget) check('……按这次的表数估上限：「这次 5 张表，最多约 200 条查询、5 分钟」', (await inBox(page, '[data-profile-budget-total]')) === '这次 5 张表，最多约 200 条查询、5 分钟', await inBox(page, '[data-profile-budget-total]')) diff --git a/scripts/dialect-it.sh b/scripts/dialect-it.sh new file mode 100755 index 00000000..0b7319ec --- /dev/null +++ b/scripts/dialect-it.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# 方言的容器集成测试（下一轮 03 细案 C2）：本机 docker 起 PostgreSQL 16、MySQL 8.4、MariaDB 11、Oracle Free 23 四个容器， +# 建虚构的夹具（保留字列、Oracle 小写带引号的表、CHAR 码值、MySQL ENUM、零日期 / 无穷大 / 占位日期、做旧的统计信息、 +# 超过整表统计上限的大表、带索引的日期列），用 AgentLab 自己的探查和剖析代码跑全部剖析查询和取值观察的新方法， +# 断言零报错、结论和手算的一致。 +# +# 为什么不进 CI：要拉几 GB 的镜像、Oracle 起一次要一两分钟。要用户同意才在本机跑。跑完删容器、留镜像。 +# +# 用法：scripts/dialect-it.sh 四种库都跑 +# DIALECT_IT_ONLY=pg,mysql scripts/dialect-it.sh 只跑其中几种（pg、mysql、mariadb、oracle） +# PYTHON=/path/to/python scripts/dialect-it.sh 指定装着后端依赖的 Python（缺省 python3） +# +# 夹具全是虚构的通用名字（stores、orders、events、visits），不连任何业务库。 +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PY="${PYTHON:-python3}" +ONLY="${DIALECT_IT_ONLY:-pg,mysql,mariadb,oracle}" +PREFIX="agentlab-dialect-it" +PASS="itpass" + +started=() +cleanup() { + for name in "${started[@]}"; do + docker rm -f "$name" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +want() { [[ ",$ONLY," == *",$1,"* ]]; } + +run() { + local name="$PREFIX-$1"; shift + docker rm -f "$name" >/dev/null 2>&1 || true + docker run -d --name "$name" "$@" >/dev/null + started+=("$name") +} + +host_port() { docker port "$PREFIX-$1" "$2" | head -1 | sed 's/.*://'; } + +if want pg; then + run pg -e POSTGRES_PASSWORD="$PASS" -e POSTGRES_DB=it -p 127.0.0.1::5432 postgres:16 +fi +if want mysql; then + run mysql -e MYSQL_ROOT_PASSWORD="$PASS" -e MYSQL_DATABASE=it -p 127.0.0.1::3306 mysql:8.4 +fi +if want mariadb; then + run mariadb -e MARIADB_ROOT_PASSWORD="$PASS" -e MARIADB_DATABASE=it -p 127.0.0.1::3306 mariadb:11 +fi +if want oracle; then + run oracle -e ORACLE_PASSWORD="$PASS" -e APP_USER=it -e APP_USER_PASSWORD="$PASS" -p 127.0.0.1::1521 \ + gvenzl/oracle-free:23-slim +fi + +export DIALECT_IT_PASSWORD="$PASS" DIALECT_IT_ONLY="$ONLY" +want pg && export DIALECT_IT_PG_PORT="$(host_port pg 5432/tcp)" +want mysql && export DIALECT_IT_MYSQL_PORT="$(host_port mysql 3306/tcp)" +want mariadb && export DIALECT_IT_MARIADB_PORT="$(host_port mariadb 3306/tcp)" +want oracle && export DIALECT_IT_ORACLE_PORT="$(host_port oracle 1521/tcp)" + +# 等库起来由 Python 那边按连接重试（Oracle 最慢，最多等 6 分钟） +PYTHONPATH="$ROOT/backend" "$PY" "$ROOT/scripts/dialect_it.py" diff --git a/scripts/dialect_it.py b/scripts/dialect_it.py new file mode 100644 index 00000000..ff153dd1 --- /dev/null +++ b/scripts/dialect_it.py @@ -0,0 +1,388 @@ +"""方言容器集成测试的执行端（由 scripts/dialect-it.sh 起好容器后调用，不进 CI）。 + +对每种库： +1. 用管理账号建虚构夹具（见 _ddl / _rows）； +2. 用 AgentLab 自己的探查（app.data.introspect）读结构，核对类型写法、ENUM 取值、Oracle 小写带引号的名字、索引首列； +3. 用 AgentLab 自己的剖析（catalog_profile.profile_table，目录读写换成内存里的）逐表剖析，经只读连接、守卫、单条时限， + 和线上走的是同一条路； +4. 直接跑取值观察要用的新查询（按日行数、时间窗口里的取值分布、块采样）； +5. 断言：所有查询零报错（留痕里没有超时、失败、守卫拒），每个结论和下面按夹具定义手算的一致。 + +夹具的数（手算的依据）：orders 40 行，id 1–40。 +- store_code = id % 5 + 1（整数子列，指向文字主键 stores.code，'1'–'5' 都在）； +- status CHAR(2)：id 是 3 的倍数为 'A'，否则 'B'（13 / 27）；type：奇数 'web'、偶数 'shop'（20 / 20）； + level = id % 3 + 1（1: 13、2: 14、3: 13）；pay_way（只 MySQL / MariaDB，ENUM('cash','card','coupon')）：奇数 cash、偶数 card； +- paid_at = 2026-09-01 10:00 加 (id % 10) 天，id 39、40 是无效日期（MySQL 零日期、PG ±infinity、Oracle 0001-01-01 和 + 9999-12-31 23:59:59）；date（日期列）同 paid_at 的日子，id 39、40 为空； +- events 先插 10 行、收集统计信息，再插 1500 行（统计信息做旧：说 10 行，其实 1510 行，超过整表统计上限 1000）； +- Oracle 另有小写带引号的表 "visits"（"id"、"level"），3 行，level 1、1、2。 +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +import os +import sys +import tempfile +import time +from collections import Counter +from types import SimpleNamespace +from typing import Any + +# AgentLab 自己的库放到临时目录：不碰开发数据目录（必须在导入 app 之前设） +os.environ["AGENTLAB_DATA_DIR"] = tempfile.mkdtemp(prefix="agentlab-dialect-it-") + +from sqlalchemy import text # noqa: E402 +from sqlalchemy.ext.asyncio import create_async_engine # noqa: E402 + +from app.core.crypto import encrypt # noqa: E402 +from app.data import catalog, catalog_profile, introspect # noqa: E402 +from app.data import engine as data_engine # noqa: E402 +from app.data.catalog_profile import ProfileSettings, SqlDialect # noqa: E402 + +PASSWORD = os.environ.get("DIALECT_IT_PASSWORD", "itpass") +SETTINGS = ProfileSettings(enabled=True, table_max_queries=40, table_max_s=60, query_timeout_s=10, sample_size=100, + max_scan_rows=1000) +N_ORDERS, N_EVENTS_OLD, N_EVENTS_NEW = 40, 10, 1500 +BASE = _dt.datetime(2026, 9, 1, 10, 0, 0) +WINDOW_FROM = "2026-09-05" + +failures: list[str] = [] + + +def check(ok: bool, what: str) -> None: + print((" ✓ " if ok else " ✗ ") + what) + if not ok: + failures.append(what) + + +# ---------------------------------------------------------------- 手算 + + +def _order(i: int) -> dict[str, Any]: + return {"id": i, "store_code": i % 5 + 1, "status": "A" if i % 3 == 0 else "B", + "type": "web" if i % 2 else "shop", "level": i % 3 + 1, "pay_way": "cash" if i % 2 else "card", + "paid_at": None if i >= 39 else BASE + _dt.timedelta(days=i % 10), + "date": None if i >= 39 else (BASE + _dt.timedelta(days=i % 10)).date()} + + +ORDERS = [_order(i) for i in range(1, N_ORDERS + 1)] +VALID = [o for o in ORDERS if o["paid_at"] is not None] +IN_WINDOW = [o for o in VALID if o["paid_at"].date().isoformat() >= WINDOW_FROM] +EXPECT = { + "status": Counter(o["status"] for o in ORDERS), + "type": Counter(o["type"] for o in ORDERS), + "level": Counter(str(o["level"]) for o in ORDERS), + "pay_way": Counter(o["pay_way"] for o in ORDERS), + "paid_at": (min(o["paid_at"] for o in VALID), max(o["paid_at"] for o in VALID)), + "day_volume": Counter(o["paid_at"].date().isoformat() for o in IN_WINDOW), + "window_status": Counter(o["status"] for o in IN_WINDOW), +} + + +# ---------------------------------------------------------------- 各库的夹具 + + +class Target: + def __init__(self, key: str, kind: str, port: int) -> None: + self.key, self.kind, self.port = key, kind, port + self.oracle = kind == "oracle" + self.mysql = kind in ("mysql", "mariadb") + self.pg = kind == "postgresql" + + def admin_url(self) -> str: + if self.pg: + return f"postgresql+asyncpg://postgres:{PASSWORD}@127.0.0.1:{self.port}/it" + if self.mysql: + return f"mysql+aiomysql://root:{PASSWORD}@127.0.0.1:{self.port}/it" + return f"oracle+oracledb://it:{PASSWORD}@127.0.0.1:{self.port}/?service_name=FREEPDB1" + + def source(self) -> SimpleNamespace: + user = {"postgresql": "postgres", "mysql": "root", "mariadb": "root", "oracle": "it"}[self.kind] + return SimpleNamespace(id=f"it-{self.key}", name=f"it_{self.key}", kind=self.kind, host="127.0.0.1", + port=self.port, database="FREEPDB1" if self.oracle else "it", username=user, + password=encrypt(PASSWORD), options={}, readonly=True, description="", + schema_cache={}, origin="manual", enabled=True) + + +def _q(t: Target, name: str) -> str: + """建表时的列名：Oracle 上 TYPE、LEVEL、DATE 是保留字，按大写加引号建（库里就是大写的普通名字）。""" + if t.oracle and name in ("type", "level", "date"): + return f'"{name.upper()}"' + if t.mysql: + return f"`{name}`" + return f'"{name}"' if name in ("type", "level", "date") else name + + +def _ddl(t: Target) -> list[str]: + ts = "TIMESTAMP" if not t.mysql else "DATETIME" + intg = "NUMBER(10)" if t.oracle else "INTEGER" + vc = "VARCHAR2" if t.oracle else "VARCHAR" + cols = [f"id {intg} PRIMARY KEY", f"store_code {intg}", "status CHAR(2)", f"{_q(t, 'type')} {vc}(10)", + f"{_q(t, 'level')} {intg}", f"paid_at {ts}", f"{_q(t, 'date')} DATE"] + if t.mysql: + cols.append("pay_way ENUM('cash','card','coupon')") + out = [f"CREATE TABLE stores (code {vc}(10) PRIMARY KEY, name {vc}(20))", + f"CREATE TABLE orders ({', '.join(cols)})", + "CREATE INDEX ix_orders_paid_at ON orders (paid_at)"] + events = f"CREATE TABLE events (id {intg} PRIMARY KEY, kind {vc}(10))" + if t.pg: + events += " WITH (autovacuum_enabled = false)" + if t.mysql: + events += " STATS_PERSISTENT=1 STATS_AUTO_RECALC=0" + out.append(events) + if t.oracle: + out.append('CREATE TABLE "visits" ("id" NUMBER(10) PRIMARY KEY, "level" NUMBER(10))') + return out + + +def _paid(t: Target, o: dict[str, Any]) -> str: + if o["paid_at"] is not None: + stamp = o["paid_at"].strftime("%Y-%m-%d %H:%M:%S") + return f"TIMESTAMP '{stamp}'" if t.oracle else f"'{stamp}'" + # 无效日期：各家能存进去的那种 + if t.mysql: + return "'0000-00-00 00:00:00'" + if t.pg: + return "'infinity'" if o["id"] == 39 else "'-infinity'" + return "TIMESTAMP '9999-12-31 23:59:59'" if o["id"] == 39 else "TIMESTAMP '0001-01-01 00:00:00'" + + +def _rows(t: Target) -> list[str]: + out = [f"INSERT INTO stores (code, name) VALUES ('{c}', 'store {c}')" for c in ("1", "2", "3", "4", "5", "A1")] + for o in ORDERS: + day = f"DATE '{o['date'].isoformat()}'" if o["date"] is not None else "NULL" + cols = ["id", "store_code", "status", _q(t, "type"), _q(t, "level"), "paid_at", _q(t, "date")] + vals = [str(o["id"]), str(o["store_code"]), f"'{o['status']}'", f"'{o['type']}'", str(o["level"]), + _paid(t, o), day] + if t.mysql: + cols.append("pay_way") + vals.append(f"'{o['pay_way']}'") + out.append(f"INSERT INTO orders ({', '.join(cols)}) VALUES ({', '.join(vals)})") + if t.oracle: + out += ['INSERT INTO "visits" ("id", "level") VALUES (1, 1)', 'INSERT INTO "visits" ("id", "level") VALUES (2, 1)', + 'INSERT INTO "visits" ("id", "level") VALUES (3, 2)'] + return out + + +def _events(start: int, n: int) -> list[str]: + return [f"INSERT INTO events (id, kind) VALUES ({i}, '{'open' if i % 4 else 'close'}')" + for i in range(start, start + n)] + + +def _analyze(t: Target, table: str) -> list[str]: + if t.pg: + return [f"ANALYZE {table}"] + if t.mysql: + return [f"ANALYZE TABLE {table}"] + return [f"BEGIN DBMS_STATS.GATHER_TABLE_STATS(USER, '{table.upper()}'); END;"] + + +async def _wait(t: Target, deadline_s: float = 360) -> Any: + engine = create_async_engine(t.admin_url()) + started = time.monotonic() + while True: + try: + async with engine.connect() as conn: + await conn.execute(text("SELECT 1 FROM DUAL" if t.oracle else "SELECT 1")) + return engine + except Exception as e: # noqa: BLE001 - 容器还没起来：等 + if time.monotonic() - started > deadline_s: + raise RuntimeError(f"{t.key} 在 {deadline_s:.0f} 秒内没起来：{e}") from e + await asyncio.sleep(3) + + +async def _setup(t: Target) -> None: + engine = await _wait(t) + try: + async with engine.begin() as conn: + if t.mysql: + # 关掉严格模式才写得进零日期（老库里常见的情形） + await conn.execute(text("SET SESSION sql_mode = ''")) + for sql in _ddl(t) + _rows(t) + _events(1, N_EVENTS_OLD): + await conn.exec_driver_sql(sql) + async with engine.begin() as conn: + for sql in _analyze(t, "events") + _analyze(t, "orders") + _analyze(t, "stores"): + await conn.exec_driver_sql(sql) + if t.oracle: + await conn.exec_driver_sql("""BEGIN DBMS_STATS.GATHER_TABLE_STATS(USER, '"visits"'); END;""") + async with engine.begin() as conn: + if t.mysql: + await conn.execute(text("SET SESSION sql_mode = ''")) + for sql in _events(N_EVENTS_OLD + 1, N_EVENTS_NEW): + await conn.exec_driver_sql(sql) + finally: + await engine.dispose() + + +# ---------------------------------------------------------------- 剖析：目录换成内存里的 + + +async def _profile(source: Any, tables: list[str], notes: dict[str, dict[str, Any]]) -> dict[str, Any]: + cx = catalog_profile.make_context(source, SETTINGS, options={}) + captured: dict[str, Any] = {} + + async def db(fn: Any, *args: Any) -> Any: + if fn is catalog_profile._read_notes: + return notes.get(args[1], {}) + if fn is catalog_profile._record: + _sid, table, out, result, _actor, _rec = args + catalog_profile._settle_status(result) + captured[table] = (out, result) + return None + raise AssertionError(f"剖析要了没预料的库操作：{fn}") + + for table in tables: + await catalog_profile.profile_table(cx, table, db=db) + return captured + + +def _key(meta: dict[str, Any], name: str) -> str: + """表结构里这个名字的写法（Oracle 上普通名字是小写，TYPE 这类保留字是大写）。""" + return next(str(c["name"]) for c in meta["columns"] if str(c["name"]).lower() == name) + + +def _day(v: Any) -> str: + return str(v)[:10] + + +async def run_target(t: Target) -> None: + print(f"\n== {t.key}（{t.kind}，端口 {t.port}）") + await _setup(t) + source = t.source() + try: + cache = await introspect.introspect(source) + source.schema_cache = cache + tables = cache["tables"] + orders, events, stores = tables["orders"], tables["events"], tables["stores"] + types = {c["name"].lower(): c for c in orders["columns"]} + check(not cache.get("failed") and {"orders", "events", "stores"} <= set(tables), "探查读到三张夹具表") + check("paid_at" in [c.lower() for c in orders.get("index_first_columns") or []], + f"索引首列记进了表结构：{orders.get('index_first_columns')}") + if t.oracle: + check(stores["columns"][0]["type"] == "VARCHAR2(10 CHAR)", f"Oracle 文字类型写全了长度和单位：" + f"{stores['columns'][0]['type']}") + check(types["id"]["type"] == "NUMBER(10, 0)", f"Oracle 数字类型写全了精度：{types['id']['type']}") + check("visits" in tables and tables["visits"].get("quote_name") is True, + "Oracle 小写带引号的表记了 quote_name") + check(all(c.get("quote_name") is True for c in tables["visits"]["columns"]), + "Oracle 小写带引号的列记了 quote_name") + check(_key(orders, "level") == "LEVEL" and "quote_name" not in types["level"], + "Oracle 的保留字列 LEVEL 是大写的普通名字，不记 quote_name") + if t.mysql: + check(types["pay_way"]["type"].startswith("ENUM(") and + types["pay_way"].get("enum_values") == ["cash", "card", "coupon"], + f"MySQL ENUM 记下了取值列表：{types['pay_way']}") + if t.pg: + check(types["paid_at"]["type"] == "TIMESTAMP WITHOUT TIME ZONE", + f"PostgreSQL 时间类型写全了时区：{types['paid_at']['type']}") + + store_col, code_col = _key(orders, "store_code"), _key(stores, "code") + rel = catalog.relation_id("orders", [store_col], "stores", [code_col]) + notes = {"orders": {"relations": [{"id": rel, "columns": [store_col], "to_table": "stores", + "to_columns": [code_col], "cardinality": "many_to_one", "source": "name", + "status": "proposed"}]}} + names = ["orders", "events", "stores"] + (["visits"] if t.oracle else []) + got = await _profile(source, names, notes) + + logs = [q for _, r in got.values() for q in r.query_log] + bad = [f"{q['kind']} {q['target']}：{q['outcome']} {q['detail']}" for q in logs if q["outcome"] != "ok"] + check(not bad, f"剖析的 {len(logs)} 条查询零报错" + (f"：{bad[:3]}" if bad else "")) + + out, result = got["orders"] + check(result.status == "ok", f"orders 剖析完成：{result.status} {result.error or ''}") + [r] = out.relations or [None] + check(r is not None and r.coverage == 1.0 and r.sample == 5 and r.matched == 5 and r.holds, + f"整数子列对文字主键：取 5 个键值全对上、升为有确证（{r and r.note}）") + codes = {c.column.lower(): c for c in out.codes} + for col in ("status", "type", "level") + (("pay_way",) if t.mysql else ()): + c = codes.get(col) + check(c is not None and Counter(dict(c.values)) == EXPECT[col], + f"{col} 的取值分布和手算一致：{c and c.values}") + if t.oracle or t.pg: + check(codes["status"].values and all(not k.endswith(" ") for k, _ in codes["status"].values), + "CHAR(2) 码值去掉了补齐的空格") + if t.mysql: + check(codes["pay_way"].declared == ["coupon"] and "coupon" in codes["pay_way"].note, + "ENUM 列定义里没出现的取值也进了码值") + check(catalog_profile.MYSQL_CODES_COLLATION_NOTE in codes["status"].note, "MySQL 码值说明写了排序规则") + check(catalog_profile.MYSQL_COLLATION_NOTE in r.note, "MySQL 关系说明写了排序规则") + ranges = {d["column"].lower(): d for d in result.date_ranges} + paid = ranges.get("paid_at") + lo, hi = EXPECT["paid_at"] + check(paid is not None and paid["min"][:19].replace("T", " ") == str(lo) and + paid["max"][:19].replace("T", " ") == str(hi) and paid.get("invalid") == 2, + f"paid_at 的范围排除了 2 个无效日期：{paid}") + day = ranges.get("date") + check(day is not None and _day(day["min"]) == lo.date().isoformat() and _day(day["max"]) == + hi.date().isoformat() and not day.get("invalid"), f"date 列（保留字）的范围：{day}") + + out_e, result_e = got["events"] + size = result_e.row_estimate + d = SqlDialect(t.kind) + stats = await data_engine.run_query(source, d.stats_sql(events, "events")) + estimated = int(float(stats.rows[0][0] or 0)) if stats.rows else 0 + if estimated <= SETTINGS.max_scan_rows: + check(size is not None and size.rows is None and size.at_least == SETTINGS.max_scan_rows + 1, + f"统计信息做旧（说 {estimated} 行，其实 {N_EVENTS_OLD + N_EVENTS_NEW} 行）也认出了大表：{size}") + else: + # 有的库插入时自己就更新了行数估算，夹具做不旧：照实说，只核对按统计信息认出了大表 + print(f" · 统计信息没能做旧（库自己更新成了 {estimated} 行），只核对按统计信息认出大表") + check(size is not None and size.method == "stats" and (size.rows or 0) > SETTINGS.max_scan_rows, + f"按统计信息认出了大表：{size}") + kind_codes = next((c for c in out_e.codes if c.column.lower() == "kind"), None) + check(kind_codes is not None and not kind_codes.complete and "按存储顺序取前" in kind_codes.note, + f"大表的码值按存储顺序的前若干行，不标已列全：{kind_codes and kind_codes.note}") + + if t.oracle: + out_v, result_v = got["visits"] + lv = next((c for c in out_v.codes if c.column == "level"), None) + check(result_v.status == "ok" and lv is not None and Counter(dict(lv.values)) == Counter({"1": 2, "2": 1}), + f"小写带引号的表和列照样剖析：{result_v.status} {lv and lv.values}") + vstats = await data_engine.run_query(source, d.stats_sql(tables["visits"], "visits")) + check(bool(vstats.rows) and vstats.rows[0][0] is not None, f"小写表的统计信息按原名查得到：{vstats.rows}") + + # ---- 取值观察的新方法 + tbl = d.table(orders, "orders") + paid_col = d.column(orders, _key(orders, "paid_at")) + res = await data_engine.run_query(source, d.day_volume_sql(tbl, paid_col, WINDOW_FROM)) + got_days = Counter({_day(row[0]): int(row[1]) for row in res.rows}) + check(got_days == EXPECT["day_volume"], f"按日行数和手算一致（无效日期不算进窗口）：{dict(got_days)}") + status_col = d.column(orders, _key(orders, "status")) + res = await data_engine.run_query(source, d.window_value_counts_sql(tbl, status_col, paid_col, WINDOW_FROM, 21)) + got_window = Counter({str(row[0]).rstrip(" "): int(row[1]) for row in res.rows}) + check(got_window == EXPECT["window_status"], f"时间窗口里的取值分布和手算一致：{dict(got_window)}") + sampled = d.sample_value_counts_sql(d.table(events, "events"), d.column(events, _key(events, "kind")), 50, 21) + if t.pg or t.oracle: + res = await data_engine.run_query(source, sampled) + total = sum(int(row[1]) for row in res.rows) + check(total <= N_EVENTS_OLD + N_EVENTS_NEW, f"块采样跑得通（取到 {total} 行）") + else: + check(sampled is None, "MySQL / MariaDB 没有块采样，照实返回 None") + finally: + await data_engine.engines.invalidate(source.id) + + +async def main() -> int: + only = set((os.environ.get("DIALECT_IT_ONLY") or "pg,mysql,mariadb,oracle").split(",")) + plan = [("pg", "postgresql", "DIALECT_IT_PG_PORT"), ("mysql", "mysql", "DIALECT_IT_MYSQL_PORT"), + ("mariadb", "mariadb", "DIALECT_IT_MARIADB_PORT"), ("oracle", "oracle", "DIALECT_IT_ORACLE_PORT")] + for key, kind, env in plan: + if key not in only: + continue + port = os.environ.get(env) + if not port: + failures.append(f"{key}：没有拿到容器端口（{env}）") + continue + try: + await run_target(Target(key, kind, int(port))) + except Exception as e: # noqa: BLE001 - 一种库出错不挡别的库，照实记下 + print(f" ✗ {key} 中途出错：{type(e).__name__}: {e}") + failures.append(f"{key}：{type(e).__name__}: {e}") + await data_engine.engines.close() + print("\n" + ("全部通过" if not failures else f"{len(failures)} 项不通过：\n- " + "\n- ".join(failures))) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) From a1371821450ec00cf9a39bb69eea669375f18f4d Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 11:09:39 +0800 Subject: [PATCH 2/7] =?UTF-8?q?Excel=20=E6=9C=9F=205=EF=BC=9A=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E6=8C=89=E6=9C=9F=E7=B4=AF=E7=A7=AF=E3=80=81=E7=A9=BF?= =?UTF-8?q?=E6=8F=92=E7=9A=84=E5=B0=8F=E8=AE=A1=E3=80=81=E8=A1=A8=E6=A0=BC?= =?UTF-8?q?=E5=AF=B9=E8=B1=A1=E8=B5=B7=E8=8D=89=E3=80=81CSV=20=E8=B5=B0?= =?UTF-8?q?=E9=85=8D=E6=96=B9=EF=BC=88E=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么：每月一份的明细导出是最常见的场景，期 4 之前列表只能每期替换；夹在中间的小计会让规则起草失败； 表格对象这个现成的边界信号没用上；CSV 不能走配方。四项都复用现有的执行器、核对、累积、溯源，不另起一套。 - 列表按期累积：ListBlock.period 二选一——{column, check: within_context} 按日期列分期，试运行多一条结构类核对 D（每行的日期都在本期统计期内，空着也不行）；{synthesize: 统计期} 每行加「统计期起」「统计期止」两列，列表在 统计期定下来之前就抽完了，这两格先放占位、写库时换成真值。accumulate_blockers 放行绑定了统计期的列表；合计行、 小计行另存的表各期主键分不开，照旧不能累积；同一订单两期都出现由并集的主键核对拒收。U3 按列表的分期方式核对。 说明里「按哪一列筛选」用分期列，加的两列有列说明；空行之后文字行的说法补了并集版本（列表能累积以后要用到）。 - 穿插的小计：ListRows.subtotals {label_column, pick, scope: since_previous | group_column, group_column, keep_as}。 小计行不进数据，每个数字格一条合计格（分段键是块 id + ·小计），核对模块按它所管那组明细的 rowid 区间用 SQL 求和比对（新 T 编号，标题「小计行按各组明细求和」），整行记进排除的行（reason=subtotal），可以另存成 reported_total 表（小计项、分组取值、各数字列）；数字格全空报 subtotal_blank（结构类）；合计行的公式引用了小计格 时按它管的明细换算。静态校验：标签列、分组列要在列里，按组小计要给分组列，小计和合计的开头词不能互为前缀。 规则起草把第一列同一个合计字样、下面还有明细的行提议成小计（以前记 stray_totals 失败），框选同一条规则。 - 表格对象起草：工作表有表格对象时按它出列表块：表头行、列范围、数据行、汇总行（映射成 total_row，标签取汇总行 第一列的文字）都由表格对象给，锚点仍按表头文字、不记坐标；表名优先用表格对象的名字（含数字的不用）。不调用 AI。 - CSV 走配方：新增 csv_grid，按内容认出 CSV（不是 zip、不是 OLE、开头没有 NUL），编码和分隔符复用 tabular 读 CSV 的那一段，适配成一张名为「CSV」的工作表（SheetScan.path=csv:，WorkbookScan.format=csv）；只有写法毫无歧义的 整数、小数转成数，千分位、前导零、日期写法一律留作文字交给配方的规则。坐标沿用「工作表!A1」（偏离细案的 R{行}C{列}，理由见 csv_grid 的模块说明）。CSV 不做稀疏判定。导入向导和简单上传的结构提示都放行 CSV。 - 每个列表一道题「每期怎么更新」（q_list_period）：每期替换 / 按期累积并按某个日期列核对 / 按期累积并加统计期 起止两列，后两种同时把导入模式改成按期累积、缺统计期的工作表一并配上（问题集合按草稿生成、回答时不变，草稿里 的列表还不符合累积资格，q_mode 不会出现，所以并成一道题）。确认项新增 subtotals:<块>、list_period:<块>。 - 前端配方面板：列表块多「小计行」（标签列、开头词、管哪些明细、分组列、另存）和「按期累积时每一行属于哪一期」； 回执「排除的行」认 subtotal。 验证：新增 tests/test_csv_grid.py、test_recipe_p5.py（执行器、核对、校验、资格、并集物化、起草）、 test_recipe_p5_e2e.py（经接口：三份月度 CSV 累积、部分重叠拒收、同一期替换要确认；带穿插小计的 Excel 起草、 核对、提交）。去掉小计识别、统计期回填、D 核对分别回跑，新测试 10 条失败。旧测试里「夹在中间的小计一律失败」 「列表只能每期替换」的断言按新行为改写。 --- backend/app/data/csv_grid.py | 154 +++++++ backend/app/data/recipe.py | 80 +++- backend/app/data/recipe_accumulate.py | 39 +- backend/app/data/recipe_checks.py | 46 +- backend/app/data/recipe_confirm.py | 27 +- backend/app/data/recipe_engine.py | 166 ++++++- backend/app/data/recipe_imports.py | 5 +- backend/app/data/recipe_notes.py | 75 +++- backend/app/data/recipe_select.py | 15 + backend/app/data/recipe_suggest.py | 203 ++++++++- backend/app/data/recipe_types.py | 125 +++++- backend/app/data/xlsx_cells.py | 13 +- backend/app/data/xlsx_scan.py | 6 +- backend/tests/test_csv_grid.py | 111 +++++ backend/tests/test_recipe_contract.py | 9 +- backend/tests/test_recipe_engine_messages.py | 13 + backend/tests/test_recipe_notes_p3.py | 5 +- backend/tests/test_recipe_p5.py | 435 +++++++++++++++++++ backend/tests/test_recipe_p5_e2e.py | 191 ++++++++ backend/tests/test_recipe_select.py | 18 +- backend/tests/test_recipe_suggest.py | 17 +- backend/tests/test_recipe_validate.py | 2 +- backend/tests/test_recipe_validate_p3.py | 2 +- frontend/src/lib/terms.ts | 21 +- frontend/src/pages/DataSourcesTab.tsx | 4 +- frontend/src/pages/import/ImportWizard.tsx | 3 +- frontend/src/pages/import/RecipePanel.tsx | 101 ++++- frontend/src/types.ts | 1 + scripts/check-manage.mjs | 11 +- 29 files changed, 1796 insertions(+), 102 deletions(-) create mode 100644 backend/app/data/csv_grid.py create mode 100644 backend/tests/test_csv_grid.py create mode 100644 backend/tests/test_recipe_p5.py create mode 100644 backend/tests/test_recipe_p5_e2e.py diff --git a/backend/app/data/csv_grid.py b/backend/app/data/csv_grid.py new file mode 100644 index 00000000..83e6177c --- /dev/null +++ b/backend/app/data/csv_grid.py @@ -0,0 +1,154 @@ +"""CSV 走配方（Excel 期 5 第 4 项）：把 CSV 适配成配方读取层认的样子——一张可见工作表的扫描结果（WorkbookScan） +和网格（Grid / 逐行流），之后的执行器、核对、说明、溯源、按期累积全部复用，不另写一套 CSV 的解析（否则会和配方 +语言分叉，违背「一种说法」）。 + +**怎么认出是 CSV。** 看内容不看扩展名（配方的几个入口只拿得到原件字节）：不是 zip（xlsx）、不是旧版 .xls 的 OLE +文件头、前 64 KB 里没有 NUL 字节（图片、PDF 这类二进制文件都有），就按 CSV 读。编码识别、分隔符嗅探复用 tabular +读 CSV 的那一段（UTF-8 / GB18030 / 带 BOM 的 UTF-16，逗号、分号、制表符、竖线）。 + +**坐标沿用「工作表!A1」。** 细案原写「格子坐标记成 R{行}C{列}」，这里没这么做（见交付说明的偏离项）：坐标在执行器、 +核对、修复按钮、框选、溯源、裁判摘录、前端网格里一共有十来处解析和拼写，引入第二种写法就要处处认两种，漏一处就是 +着色错位、下钻找不到格子；期 1 读 CSV 时（tabular._shape_reasons）用的也是 A1。CSV 没有工作表，名字固定为 +CSV_SHEET：每月文件名不同（orders_2026-08.csv、orders_2026-09.csv），配方按工作表名认，名字必须稳定。行号是第几条 +记录（引号里的换行不另起一行），列号是第几个字段，和用 Excel 打开这个 CSV 看到的 A1 坐标一致。 + +**格子的值。** CSV 里只有文字。只有写法毫无歧义的才转成数（-12、80.5，不超过 15 位有效数字），其余一律原样留作文字： +「1,234」（千分位，交给配方的 text_number 决定）、「007」（前导零是编码的一部分）、「+5」「1e5」「12%」、日期写法 +（执行器按 DATE 列解析文字日期，起草按文字认日期）、「=SUM(…)」（CSV 里不是公式，Excel 打开时才会当公式算）。 +空字段、只有空白的字段是空格（和 xlsx_scan 的口径一致）。没有公式、合并单元格、隐藏行列。 +""" +from __future__ import annotations + +import csv +import io +import re +from dataclasses import dataclass +from typing import Any, Iterator + +from app.data.recipe_types import GridCell +from app.data.xlsx_scan import Bounds, SheetScan, UnsupportedTable, WorkbookScan + +__all__ = ["CSV_PATH", "CSV_SHEET", "MAX_COLS", "MAX_ROWS", "CsvTable", "cell_value", "looks_like_csv", "parse", + "rows", "scan", "scan_any"] + +#: CSV 的「工作表名」：配方按它认（SheetMatch.name），每月的文件名不同也认得出 +CSV_SHEET = "CSV" +#: SheetScan.path 的取值：读取层据此知道这张「工作表」要从 CSV 里读，不去压缩包里找 XML +CSV_PATH = "csv:" +#: 和 Excel 的上限一致：超过的 CSV 用 Excel 打开也装不下，交给模型的格子坐标也写不出来 +MAX_ROWS = 1_048_576 +MAX_COLS = 16_384 +#: 嗅探看多少字节 +_SNIFF = 64 * 1024 +#: 写法毫无歧义的整数、小数（不带正号、不带前导零、不用科学计数法） +_INT = re.compile(r"-?(?:0|[1-9][0-9]*)") +_DEC = re.compile(r"-?(?:0|[1-9][0-9]*)\.[0-9]+") +#: 超过这么多位有效数字的不转成数：转成浮点会丢位（19 位卡号撞值），留作文字 +_DIGITS_MAX = 15 + + +def looks_like_csv(raw: bytes) -> bool: + """这份原件按 CSV 读：不是 zip、不是 OLE（旧版 .xls）、开头没有 NUL 字节、不是空文件。带 BOM 的 UTF-16 例外 + （每个字符都带 NUL，Excel「Unicode 文本」另存出来的就是它）。""" + if not raw or raw[:4] == b"PK\x03\x04" or raw[:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1": + return False + if raw[:2] in (b"\xff\xfe", b"\xfe\xff"): + return True + return b"\x00" not in raw[:_SNIFF] + + +def cell_value(text: str) -> Any: + """CSV 的一个字段 → 格子的值：空 / 只有空白是 None（空格），写法毫无歧义的整数、小数转成数，其余原样（文字）。""" + t = text.strip() + if not t: + return None + digits = len(t.lstrip("-").replace(".", "").lstrip("0")) or 1 + if digits <= _DIGITS_MAX: + if _INT.fullmatch(t): + return int(t) + if _DEC.fullmatch(t): + return float(t) + return text + + +@dataclass +class CsvTable: + """解析好的 CSV：records[i] 是第 i + 1 条记录的字段（原文）。""" + + records: list[list[str]] + delimiter: str + + +def parse(raw: bytes) -> CsvTable: + """原件 → 记录。编码、分隔符照 tabular 读 CSV 的做法；字段太长、引号不配对读不了时抛 UnsupportedTable。""" + from app.data import tabular + + text = tabular._decode_csv(raw) + try: + delimiter = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|").delimiter + except csv.Error: + delimiter = "," # 嗅不出来就按最常见的来 + records: list[list[str]] = [] + try: + for record in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter): + if len(records) >= MAX_ROWS: + raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行，超出可以导入的上限") + if len(record) > MAX_COLS: + raise UnsupportedTable(f"CSV 文件第 {len(records) + 1} 行有 {len(record):,} 列，超过 {MAX_COLS:,} 列的上限") + records.append(record) + except csv.Error as e: + raise UnsupportedTable(f"无法按 CSV 读取这个文件：第 {len(records) + 1} 行的写法不正确（{e}）。" + "请在 Excel 中重新另存为 CSV 后上传") from e + return CsvTable(records=records, delimiter=delimiter) + + +def _cells(table: CsvTable) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + for r, record in enumerate(table.records, 1): + out = [(c, GridCell(v)) for c, field in enumerate(record, 1) if (v := cell_value(field)) is not None] + if out: + yield r, out + + +def scan(raw: bytes) -> WorkbookScan: + """CSV 原件 → 扫描结果：一张可见工作表（名字 CSV_SHEET、路径 CSV_PATH），边界、非空格数、逐行非空格数的游程。 + 没有合并区、隐藏行列、公式、错误值、离群格（CSV 里每一格都是写出来的，没有残留格式）。""" + table = parse(raw) + sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH) + lo_r = lo_c = hi_r = hi_c = 0 + runs: list[tuple[int, int, int]] = [] + for r, cells in _cells(table): + n = len(cells) + sheet.nonempty += n + first, last = cells[0][0], cells[-1][0] + if not lo_r: + lo_r, lo_c, hi_c = r, first, last + lo_c, hi_c, hi_r = min(lo_c, first), max(hi_c, last), r + if runs and runs[-1][1] == r - 1 and runs[-1][2] == n: + runs[-1] = (runs[-1][0], r, n) + else: + runs.append((r, r, n)) + if lo_r: + sheet.bounds = Bounds(lo_r, lo_c, hi_r, hi_c) + sheet.dimension = sheet.bounds.a1() + sheet.row_runs = runs + return WorkbookScan(sheets=[sheet], format="csv") + + +def scan_any(raw: bytes) -> WorkbookScan: + """配方导入的扫描入口：CSV 走这里的适配，其余照旧交给 xlsx_scan（读不了照旧拒收，文案不变）。""" + from app.data import xlsx_scan + + if looks_like_csv(raw): + return scan(raw) + return xlsx_scan.scan(raw) + + +def rows(table: CsvTable, *, min_row: int, max_row: int, min_col: int, + max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + """按行流式给一个矩形里的非空格：(行号, [(列号, 格)…])，口径同 xlsx_cells.sheet_rows（整行为空的不给）。""" + for r in range(max(min_row, 1), min(max_row, len(table.records)) + 1): + record = table.records[r - 1] + out = [(c, GridCell(v)) for c in range(max(min_col, 1), min(max_col, len(record)) + 1) + if (v := cell_value(record[c - 1])) is not None] + if out: + yield r, out diff --git a/backend/app/data/recipe.py b/backend/app/data/recipe.py index 6e4d4a6d..a4835456 100644 --- a/backend/app/data/recipe.py +++ b/backend/app/data/recipe.py @@ -43,6 +43,9 @@ from app.data.recipe_parsers import ( text_label, ) from app.data.recipe_types import ( + LIST_PERIOD_END, + LIST_PERIOD_START, + LIST_SUBTOTAL_DIM, LIST_TOTAL_DIM, ColumnOut, CrosstabBlock, @@ -271,6 +274,8 @@ _FIELD_LABEL = { "parts": "组成列", "claims": "认领", "a": "一方", "b": "另一方", "reason": "理由", "ignore_rows": "按行标签忽略的行", "ignore_columns": "按表头忽略的列", "ignore_outside": "按同一行文字忽略的数字", "label": "标签", "anchor": "同一行的文字", + "subtotals": "小计行", "scope": "小计管哪些明细", "group_column": "分组的列", "period": "每行属于哪一期", + "column": "日期列", "check": "核对方式", "synthesize": "加统计期列", } @@ -608,15 +613,25 @@ class _Checker: for si, bi, block in self.lists: base = ("sheets", si, "blocks", bi) cols = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns)] + if block.period is not None and block.period.synthesize: + # 期 5：加上的统计期起止两列也是这张表的列，和表头推出的列名不能撞 + pp = base + ("period", "synthesize") + cols += [(LIST_PERIOD_START, pp), (LIST_PERIOD_END, pp)] out.append(_Writer(block.table, base + ("table",), "data", cols, origin=_pointer(base))) + numeric = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns) if c.type in _NUMERIC] total = block.rows.total_row if total is not None and total.keep_as: tp = base + ("rows", "total_row", "keep_as") - tcols = [(LIST_TOTAL_DIM, tp)] - tcols += [(c.name, base + ("columns", n, "name")) - for n, c in enumerate(block.columns) if c.type in _NUMERIC] - out.append(_Writer(total.keep_as, tp, "reported_total", tcols, + out.append(_Writer(total.keep_as, tp, "reported_total", [(LIST_TOTAL_DIM, tp)] + numeric, origin=_pointer(base + ("rows", "total_row")))) + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + sp = base + ("rows", "subtotals", "keep_as") + scols = [(LIST_SUBTOTAL_DIM, sp)] + if sub.scope == "group_column" and sub.group_column: + scols.append((sub.group_column, base + ("rows", "subtotals", "group_column"))) + out.append(_Writer(sub.keep_as, sp, "reported_total", scols + numeric, + origin=_pointer(base + ("rows", "subtotals")))) return out # ---- 8. 名字 ---- @@ -680,6 +695,10 @@ class _Checker: if sheet.context: continue for bi, block in enumerate(sheet.blocks): + if isinstance(block, ListBlock) and block.period is not None: + # 期 5：每行属于哪一期要拿统计期来核对（按日期列）或填进去（加起止两列） + self.add(("sheets", si, "blocks", bi, "period"), "covers_without_context", + "要按统计期核对或填写每一行的统计期，但这个工作表没有配置统计期") if not isinstance(block, CrosstabBlock): continue ap = ("sheets", si, "blocks", bi, "axis") @@ -873,6 +892,10 @@ class _Checker: if total is not None and any(ch.isdigit() for ch in total.pick): self.add(("sheets", si, "blocks", bi, "rows", "total_row", "pick"), "period_literal", f"合计行的开头词「{total.pick}」不能含数字（配方里不能写死年份或期次）") + sub = block.rows.subtotals + if sub is not None and any(ch.isdigit() for ch in sub.pick): + self.add(("sheets", si, "blocks", bi, "rows", "subtotals", "pick"), "period_literal", + f"小计行的开头词「{sub.pick}」不能含数字（配方里不能写死年份或期次）") # ---- 4、9、13. 单位：键是本表列名，值在词表里，与原表标签、表头里的单位一致 ---- @@ -902,10 +925,13 @@ class _Checker: self.unit_against(seg.keep_as.table, seg.keep_as.value, target.locate.title, "分段标题") for _si, _bi, block in self.lists: total = block.rows.total_row + sub = block.rows.subtotals for c in block.columns: self.unit_against(block.table, c.name, c.header, "表头") if total is not None and total.keep_as and c.type in _NUMERIC: self.unit_against(total.keep_as, c.name, c.header, "表头") + if sub is not None and sub.keep_as and c.type in _NUMERIC: + self.unit_against(sub.keep_as, c.name, c.header, "表头") def after_target(self, si: int, bi: int, seg: DerivedSegment) -> DimensionSegment | None: """合计分段紧跟的那个 dimension 分段（同一块里按 id 找）；找不到或不是 dimension 段时已报 derived_after_invalid。""" @@ -998,6 +1024,8 @@ class _Checker: if total is not None and total.label_column not in names: self.add(base + ("rows", "total_row", "label_column"), "column_unknown", f"合计标签所在的列「{total.label_column}」不在这个列表的列里{self._near(total.label_column, names)}") + self.check_subtotals(base, block, names) + self.check_list_period(base, block, names) seen: dict[str, str] = {} for n, c in enumerate(block.columns): key = match_key(c.header) @@ -1006,6 +1034,50 @@ class _Checker: f"表头「{c.header}」与「{seen[key]}」是同一个表头") seen.setdefault(key, c.header) + def check_subtotals(self, base: tuple[Any, ...], block: ListBlock, names: list[str]) -> None: + """期 5：小计行的标签列、分组列要是这个列表的列；按组小计要给分组列；小计和合计的开头词不能互为前缀 + （「合计」「合计小计」：一行就分不清是小计还是合计）。""" + sub = block.rows.subtotals + if sub is None: + return + sp = base + ("rows", "subtotals") + if sub.label_column not in names: + self.add(sp + ("label_column",), "column_unknown", + f"小计标签所在的列「{sub.label_column}」不在这个列表的列里{self._near(sub.label_column, names)}") + if sub.scope == "group_column": + if not sub.group_column: + self.add(sp + ("group_column",), "subtotal_group_missing", "按组小计时要指明按哪一列分组") + elif sub.group_column not in names: + self.add(sp + ("group_column",), "column_unknown", + f"小计分组的列「{sub.group_column}」不在这个列表的列里{self._near(sub.group_column, names)}") + elif sub.group_column: + self.add(sp + ("group_column",), "subtotal_group_missing", + "小计管「上一行小计之后的全部明细」时不按列分组，请去掉分组的列，或改为按组小计") + total = block.rows.total_row + if total is not None: + a, b = match_key(sub.pick), match_key(total.pick) + if a.startswith(b) or b.startswith(a): + self.add(sp + ("pick",), "subtotal_pick_conflict", + f"小计行的开头词「{sub.pick}」和合计行的「{total.pick}」一个是另一个的开头，分不清是小计还是合计") + + def check_list_period(self, base: tuple[Any, ...], block: ListBlock, names: list[str]) -> None: + """期 5：每行属于哪一期，按日期列核对和加起止两列只能二选一；按日期列时那一列要是这个列表的日期列。""" + period = block.period + if period is None: + return + pp = base + ("period",) + if (period.column is None) == (period.synthesize is None): + self.add(pp, "list_period_invalid", "每一行属于哪一期只能二选一：按一个日期列核对，或加统计期起止两列") + return + if period.column is not None: + col = next((c for c in block.columns if c.name == period.column), None) + if col is None: + self.add(pp + ("column",), "column_unknown", + f"核对统计期的列「{period.column}」不在这个列表的列里{self._near(period.column, names)}") + elif col.type != "DATE": + self.add(pp + ("column",), "list_period_invalid", + f"核对统计期的列「{period.column}」的类型要是日期") + # ---- 期 3：忽略规则、表头上方的标题（P3-SPEC 9.2） ---- def check_ignores(self) -> None: diff --git a/backend/app/data/recipe_accumulate.py b/backend/app/data/recipe_accumulate.py index 5f6a8b54..ac689387 100644 --- a/backend/app/data/recipe_accumulate.py +++ b/backend/app/data/recipe_accumulate.py @@ -30,6 +30,7 @@ from app.data.recipe_parsers import hour_range, hour_range_total, match_key, tex from app.data.recipe_types import ( UNION_VER, AccumulatePlan, ChangeClass, CheckResult, ColumnOut, CrosstabBlock, DerivedSegment, DimensionSegment, PartInfo, Recipe, UnionPart, UnionReport, accumulate_blockers, derive_tables, + list_period_columns, ) from app.data.table_versions import settle_journal, sha_json @@ -508,14 +509,19 @@ def union_options(parts: list[tuple[str, str, str]], target_recipe_sha: str, *, class _Table: - """并集里的一张表：列（目标配方的在前、退役的按给定顺序追加在后）、主键、日期轴列。""" + """并集里的一张表：列（目标配方的在前、退役的按给定顺序追加在后）、主键、日期轴列。期 5：列表写入的表按 period + 分期（list_period_columns）：按日期列的，U3 核对那一列落在该期统计期内；加统计期起止两列的，核对两列就是该期。""" - def __init__(self, name: str, cols: list[ColumnOut], grain: list[str]) -> None: + def __init__(self, name: str, cols: list[ColumnOut], grain: list[str], + period: tuple[str, ...] | None = None) -> None: self.name = name self.cols = cols self.grain = grain self.axis = next((c.name for c in cols if c.role == "axis"), None) self.names = [c.name for c in cols] + self.period = period if period is None or all(c in self.names for c in period[1:]) else None + if self.axis is None and self.period is not None and self.period[0] == "column": + self.axis = self.period[1] def _copy_col(col: ColumnOut) -> ColumnOut: @@ -551,6 +557,7 @@ def _layout(target: Recipe, parts: list[UnionPart], out: dict[str, _Table] = {} retired = retired or {} + periods = list_period_columns(target) for name, cols in tables.items(): mine = [_copy_col(c) for c in cols] known = {c.name for c in mine} @@ -561,7 +568,7 @@ def _layout(target: Recipe, parts: list[UnionPart], mine.append(_copy_col(found) if found is not None else _copy_col(col)) known.add(col.name) spec = specs.get(name) - out[name] = _Table(name, mine, list(spec.grain) if spec is not None else []) + out[name] = _Table(name, mine, list(spec.grain) if spec is not None else [], periods.get(name)) for name, cols in retired.items(): if name in out: continue @@ -831,20 +838,36 @@ def _finish(conn: sqlite3.Connection, layout: dict[str, _Table], parts: list[Uni u2.append(f"表「{table.name}」有 {dup} 组主键重复") checks.append(_check("U2", "union_pk", "累积后主键不重复", checked=len(keyed), failed=u2, sql=sql2)) - # U3 统计期：第 k 期的 rowid 区间里，日期都在 [start_k, end_k] 之内 + # U3 统计期：第 k 期的 rowid 区间里，日期都在 [start_k, end_k] 之内。期 5：按日期列分期的列表，日期为空也算不在； + # 加统计期起止两列的列表，两列必须正好是这一期 u3: list[str] = [] sql3, params3, checked3 = None, [], 0 for k, (part, ranges) in enumerate(zip(parts, part_rows), 1): for name, span in ranges.items(): - axis = layout[name].axis - if axis is None or axis not in part_cols[k - 1].get(name, []): - continue + table = layout[name] + have = part_cols[k - 1].get(name, []) a, b = span["union"] + if table.period is not None and table.period[0] == "synthesize": + start_col, end_col = table.period[1], table.period[2] + if start_col not in have or end_col not in have or b < a: + continue + checked3 += 1 + sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " + f"AND ({_q(start_col)} IS NOT ? OR {_q(end_col)} IS NOT ?)") + params3 = [a, b, part.start, part.end] + bad = conn.execute(sql3, params3).fetchone()[0] + if bad: + u3.append(f"第 {k} 期（{_period_text(part.start, part.end)}）的「{name}」有 {bad} 行的统计期不是这一期") + continue + axis = table.axis + if axis is None or axis not in have: + continue if b < a: continue checked3 += 1 + nulls = f"{_q(axis)} IS NULL OR " if table.period is not None else "" sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " - f"AND ({_q(axis)} < ? OR {_q(axis)} > ?)") + f"AND ({nulls}{_q(axis)} < ? OR {_q(axis)} > ?)") params3 = [a, b, part.start, part.end] bad = conn.execute(sql3, params3).fetchone()[0] if bad: diff --git a/backend/app/data/recipe_checks.py b/backend/app/data/recipe_checks.py index e095ada7..79a21b96 100644 --- a/backend/app/data/recipe_checks.py +++ b/backend/app/data/recipe_checks.py @@ -23,6 +23,7 @@ from typing import Any, Callable, Iterable from app.data.engine import open_checked_sqlite from app.data.recipe_types import ( + SUBTOTAL_KEY, CheckResult, ColumnOut, CrosstabBlock, @@ -34,6 +35,7 @@ from app.data.recipe_types import ( Recipe, SumEq, derive_tables, + list_period_columns, ) #: 无法核对的原因（契约 CheckResult.reasons 的键）的固定顺序：细节、说明都按这个顺序列 @@ -71,24 +73,33 @@ class CheckPlan: k: dict[str, str] = field(default_factory=dict) #: derived 分段键 → G 编号（只给有公式格的 derived 段） g: dict[str, str] = field(default_factory=dict) - #: 列表块键 → T 编号（每个有合计行的列表块一条） + #: 列表块键 → T 编号（每个有合计行的列表块一条）；期 5：块键 + SUBTOTAL_KEY → 这个块的小计核对（一条） t: dict[str, str] = field(default_factory=dict) items: dict[str, list[DerivedItem]] = field(default_factory=dict) def _recipe_keys(recipe: Recipe) -> tuple[list[str], list[str]]: - """配方里 derived 分段键、带合计行的列表块键，按配方里的先后。""" + """配方里 derived 分段键、带合计行的列表块键，按配方里的先后。期 5：带小计的列表块另有一个键（块键 + + SUBTOTAL_KEY），排在这个块的合计行后面：同一块的小计和合计各编一个 T，结论分开写。""" derived: list[str] = [] totals: list[str] = [] for sheet in recipe.sheets: for block in sheet.blocks: if isinstance(block, CrosstabBlock): derived += [s.id for s in block.segments if isinstance(s, DerivedSegment)] - elif block.rows.total_row is not None: + continue + if block.rows.total_row is not None: totals.append(block.id) + if block.rows.subtotals is not None: + totals.append(f"{block.id}{SUBTOTAL_KEY}") return derived, totals +def is_subtotal_key(key: str) -> bool: + """T 编号的键是不是一个块的小计核对（期 5）。""" + return key.endswith(SUBTOTAL_KEY) + + def _ordered(recipe_keys: list[str], items: list[DerivedItem]) -> list[str]: """先按配方里的先后；配方里找不到的分段（不该出现）按执行结果里第一次出现的先后排在后面，不丢。""" present: list[str] = [] @@ -570,6 +581,25 @@ def _check_pk(sql: _Sql, cid: str, title: str, table: str, grain: list[str]) -> # ---------------------------------------------------------------- 入口 +def _check_period_rows(sql: _Sql, cid: str, title: str, table: str, column: str, period: Any, + lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult: + """期 5：列表按日期列分期（ListPeriod.column、check=within_context）时，每行的这一列都落在本期统计期内。 + 日期为空的行也算不在：看不出它属于哪一期。不在的行按期累积会混进别的期（统计期互不重叠的前提就破了）， + 所以是结构类、不能写理由接受。统计期没定下来（要人录入）时不会走到这里：那时不建库。""" + if period is None or not period.start or not period.end: + raise _Broken("本期没有统计期，无法核对每行的日期") + q = (f"SELECT rowid FROM {_q(table)} WHERE {_q(column)} IS NULL OR {_q(column)} < ? OR {_q(column)} > ? " + "ORDER BY rowid") + params = [period.start, period.end] + bad = [r[0] for r in sql.all(q, params)] + total = sql.one(f"SELECT COUNT(*) FROM {_q(table)}")[0] + cells = [c for rowid in bad[:_DETAIL_MAX] if (c := lineage_cell(lineage, table, column, rowid))] + details = [f"有 {len(bad)} 行的「{column}」为空或不在 {period.start} 至 {period.end} 之内"] if bad else [] + return CheckResult(id=cid, kind="period_rows", title=title, status="mismatch" if bad else "passed", + category="structure", checked=int(total), failed=len(bad), sql=q, params=params, + details=details, cells=cells) + + def _axis_names(recipe: Recipe) -> dict[str, str]: """derived 分段键 → 所在交叉表块的轴列名（K 的铺满判定不按日期分）。""" out: dict[str, str] = {} @@ -629,7 +659,9 @@ def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[Che jobs.append((cid, "formula_refs", f"合计公式引用（分段「{seg}」）", lambda t, seg=seg, cid=cid: _check_g(sql, cid, t, plan.items[seg]))) for block, cid in plan.t.items(): - jobs.append((cid, "column_sum", f"合计行按列求和（块「{block}」）", + title = (f"小计行按各组明细求和（块「{block.removesuffix(SUBTOTAL_KEY)}」）" if is_subtotal_key(block) + else f"合计行按列求和（块「{block}」）") + jobs.append((cid, "column_sum", title, lambda t, block=block, cid=cid: _check_t(sql, cid, t, plan.items[block], full_calc))) for rel in recipe.relations: if isinstance(rel, SumEq): @@ -646,6 +678,12 @@ def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[Che for i, table in enumerate([t for t in tables if grains.get(t)], 1): jobs.append((f"P{i}", "pk_unique", f"主键唯一：「{table}」", lambda t, table=table, i=i: _check_pk(sql, f"P{i}", t, table, grains[table]))) + period = extraction.period + dated = [(t, spec[1]) for t, spec in list_period_columns(recipe).items() if spec[0] == "column" and t in tables] + for i, (table, column) in enumerate(dated, 1): + jobs.append((f"D{i}", "period_rows", f"「{table}」每行的「{column}」都在本期统计期内", + lambda t, table=table, column=column, i=i: _check_period_rows( + sql, f"D{i}", t, table, column, period, extraction.lineage))) try: return list(extraction.context_checks) + [_guarded(sql, *job) for job in jobs] finally: diff --git a/backend/app/data/recipe_confirm.py b/backend/app/data/recipe_confirm.py index f2a9419c..3271eed9 100644 --- a/backend/app/data/recipe_confirm.py +++ b/backend/app/data/recipe_confirm.py @@ -43,7 +43,7 @@ from app.data.recipe_diff import ( ) from app.data.recipe_parsers import UNITS, match_key, period_residue, split_unit_suffix from app.data.recipe_types import ( - ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, + LIST_PERIOD_END, LIST_PERIOD_START, SUBTOTAL_KEY, ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, SheetRecipe, SumEq, derive_tables, ) @@ -523,6 +523,12 @@ def _signatures(recipe: Recipe, sheet_name: Callable[[SheetRecipe], str]) -> dic total = block.rows.total_row out[f"total_row:{block.id}"] = _sig( None if total is None else (total.label_column, total.pick, total.keep_as)) + # 期 5：穿插的小计、每行属于哪一期 + subt = block.rows.subtotals + out[f"subtotals:{block.id}"] = _sig( + None if subt is None else (subt.label_column, subt.pick, subt.scope, subt.group_column, subt.keep_as)) + period = block.period + out[f"list_period:{block.id}"] = _sig(None if period is None else (period.column, period.synthesize)) for r in recipe.relations: if isinstance(r, SumEq): out[f"relation:{r.id}"] = _sig("sum_eq", r.table, r.total, sorted(r.parts)) @@ -1058,6 +1064,25 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *, detail = (f"按明细核对的列：{'、'.join(nums)}；其余列合计行里的内容不核对" if nums else "这个块没有数字列，合计行里的内容不核对") bag.add(f"total_row:{b.id}", f"{prefix}改作核对{keep}", detail) + for b in lists: + # 期 5：穿插在明细中间的小计行：不进数据，按各组明细核对。要人确认：这几行从此不在本表里 + subt = b.rows.subtotals + if subt is not None: + rows = rows_text(derived_rows.get(f"{b.id}{SUBTOTAL_KEY}")) + prefix = f"「{b.id}」中间的小计行" + (f"（{rows}）" if rows else "") + keep = f"，原值另存表「{subt.keep_as}」" if subt.keep_as else "，原值不另存" + scope = ("每行小计核对上一行小计之后的全部明细" if subt.scope == "since_previous" + else f"每行小计核对紧挨在它上面、「{subt.group_column}」相同的那几行明细") + nums = [c.name for c in b.columns if c.type in ("INTEGER", "REAL")] + detail = f"{scope}；按明细核对的列：{'、'.join(nums)}" if nums else f"{scope}；这个块没有数字列，不核对" + bag.add(f"subtotals:{b.id}", f"{prefix}改作核对、不进数据{keep}", detail) + period = b.period + if period is not None and period.column: + bag.add(f"list_period:{b.id}", f"「{b.id}」每一行的「{period.column}」都要在本期统计期内", + "按期累积时各期的行按这一列区分；有一行不在本期统计期内（或为空），这一期整份拒收") + elif period is not None and period.synthesize: + bag.add(f"list_period:{b.id}", f"「{b.id}」每一行加上本期统计期的起、止两列", + f"列名「{LIST_PERIOD_START}」「{LIST_PERIOD_END}」，取自表内的统计期或人工录入的统计期") # 年份取自统计期：表头是日期格时年份来自格子本身，不算 forms: dict[str, set[str]] = {} diff --git a/backend/app/data/recipe_engine.py b/backend/app/data/recipe_engine.py index 7c33e010..f44803da 100644 --- a/backend/app/data/recipe_engine.py +++ b/backend/app/data/recipe_engine.py @@ -39,6 +39,14 @@ pk_duplicate 同时出现）。块级的定位失败（找不到日期表头、 个矩形。 - 回执里的「排除的行」rows_excluded（H2：排除的行要写进回执）：隐藏行、列表跳过的空行、按规则忽略的行、 停止之后没导入的文字行、只核对不另存的合计行。 + +**期 5（下一轮 03 细案 E）又加了两件事，同样没用新字段的配方写出的库逐字节不变：** +- 列表中间穿插的小计（ListRows.subtotals）：小计行不进数据，每个数字格一条合计格，按它所管的那组明细（上一行小计 + 之后的全部明细，或紧挨在上面、分组列相同的那几行）交给核对模块用 SQL 求和比对；整行记进排除的行（subtotal）。 +- 列表按期累积时由系统加的统计期起止两列（ListPeriod.synthesize）：列表在统计期定下来之前就抽完了，这两格先放 + 占位，统计期定下来以后写库时换成真值（_Sink.fill_period）；按日期列分期的（ListPeriod.column）由核对模块的 D + 核对查每行都在本期统计期内。 +CSV 原件经 csv_grid 适配成一张「工作表」，这里照常按网格或逐行流读（xlsx_cells 按 SheetScan.path 分派）。 """ from __future__ import annotations @@ -64,10 +72,10 @@ from app.data.recipe_parsers import ( period_residue, text_label, thousands_number, ) from app.data.recipe_types import ( - AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, DimensionSegment, - ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, + SUBTOTAL_KEY, AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, + DimensionSegment, ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, PeriodInput, PeriodOut, Problem, Recipe, RegionMark, SegmentLabels, SheetRecipe, TableOut, TableSpec, - derive_tables, + derive_tables, list_period_columns, ) from app.data.xlsx_scan import MAX_MERGED, SheetScan, WorkbookScan, cell_ref, col_letter @@ -101,7 +109,7 @@ PROBLEM_CODES: dict[str, str] = { # 列表定位 "header_not_found": "structure", "header_ambiguous": "structure", "column_missing": "structure", "column_extra": "structure", "rows_after_stop": "confirm", "total_row_missing": "structure", - "total_row_blank": "structure", "list_empty": "confirm", + "total_row_blank": "structure", "list_empty": "confirm", "subtotal_blank": "structure", # 值 "value_not_integer": "structure", "value_text_number": "structure", "value_blank": "structure", "value_error": "structure", "value_not_number": "structure", "value_not_date": "structure", @@ -143,7 +151,7 @@ PLACEHOLDER_TEXTS_MAX = 8 PLACEHOLDER_TEXT_LEN = 8 #: 排除的行在回执里的顺序（同一工作表内按原因、块、锚点、首行排） _EXCLUDED_ORDER = ("hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", - "total_not_kept") + "total_not_kept", "subtotal") #: 公式区域展开的上限：更大的区域不可能全落在基表里，直接判「引用了基表之外的格」 REF_CELLS_MAX = 200_000 @@ -613,11 +621,18 @@ class _Problems: class _Sink: """一张表的行：主键去重、计数、溯源、批量写库（有库时）。rowid 就是插入序号（新库、1 起）。""" - def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None) -> None: + def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None, + synth: tuple[str, str] | None = None) -> None: self.eng = eng self.name = name self.cols = cols names = [c.name for c in cols] + #: 期 5：由系统加的统计期起止两列（ListPeriod.synthesize）在一行里的位置。列表是在统计期定下来之前抽完的， + #: 这两格先放占位（_PERIOD_START / _PERIOD_END），统计期定下来以后（fill_period）写库时换成真值；在那之前 + #: 不分批写库（flush 等着），整张表留在内存里 + self.synth_idx = (names.index(synth[0]), names.index(synth[1])) \ + if synth and synth[0] in names and synth[1] in names else None + self.period: tuple[str | None, str | None] | None = None self.grain = [g for g in (spec.grain if spec else []) if g in names] self.grain_idx = [names.index(g) for g in self.grain] self.check_pk = bool(self.grain_idx) and not eng.partial @@ -682,9 +697,22 @@ class _Sink: self.eng.problems.issue("value_blank", sheet, f"工作表「{sheet}」的表「{self.name}」的主键列{_quoted(names)}", coords, text=_PK_NULL, group="pk_null") - def flush(self) -> None: + def fill_period(self, start: str | None, end: str | None) -> None: + """统计期定下来了：加的统计期起止两列从此写真值（start、end 是 YYYY-MM-DD）。""" + self.period = (start, end) + + def flush(self, *, final: bool = False) -> None: + if self.synth_idx is not None and self.period is None and not final: + return # 统计期还没定：加的两列还是占位，先不写 if self.buffer and self.eng.conn is not None: - self.eng.conn.executemany(self.insert_sql, self.buffer) + rows = self.buffer + if self.synth_idx is not None: + a, b = self.synth_idx + start, end = self.period or (None, None) + fill = {a: start, b: end} + rows = [tuple(fill.get(i, v) if v is _PERIOD_START or v is _PERIOD_END else v + for i, v in enumerate(row)) for row in rows] + self.eng.conn.executemany(self.insert_sql, rows) self.buffer.clear() @@ -695,6 +723,12 @@ class _Sink: _FAIL = object() #: 列表数据区里不能填充的合并单元格覆盖的空格（已经报了 merged_in_values） _MERGED = object() +#: 期 5：加的统计期起止两列在统计期定下来之前的占位（_Sink.fill_period）。主键判重时它们和自己相等： +#: 同一期的行统计期都一样，判重只看其余的主键列，结论不变 +_PERIOD_START = object() +_PERIOD_END = object() +#: 小计按组时「还没有组」 +_NO_GROUP = object() # -------------------------------------------------------------------------- @@ -1625,6 +1659,22 @@ class _ListRun: self.ignore_keys = {k for rule in block.ignore_columns if (k := match_key(rule.header))} total = block.rows.total_row self.total_idx = next((i for i, c in enumerate(self.cols) if total and c.name == total.label_column), None) + # 期 5：穿插的小计行（ListRows.subtotals）。sub_first：上一行小计（或表头）之后第一行明细的 rowid；按组小计 + # 时 run_first / run_value 是紧挨在上面、分组列取值相同的那一段明细的首个 rowid 和取值。sub_rows：小计所在的 + # 行 → 它管的明细的 rowid（合计行的公式引用了小计格时，按它换算回明细） + self.sub = block.rows.subtotals + sub = self.sub + self.sub_idx = next((i for i, c in enumerate(self.cols) if sub and c.name == sub.label_column), None) + self.sub_key = match_key(sub.pick) if sub else "" + self.group_idx = next((i for i, c in enumerate(self.cols) + if sub and sub.scope == "group_column" and c.name == sub.group_column), None) + self.sub_first: int | None = None + self.run_first: int | None = None + self.run_value: Any = _NO_GROUP + self.sub_hidden = False + self.sub_rows: dict[int, list[int]] = {} + #: 期 5：加统计期起止两列（ListPeriod.synthesize）：每行末尾两格占位，统计期定下来以后由 _Sink 换成真值 + self.synth = block.period is not None and block.period.synthesize is not None # ---------------- 喂入 ---------------- @@ -2001,6 +2051,7 @@ class _ListRun: fills = self._merges_at(r, inb) if self.sh.hidden_row(r): self.hidden_seen = True + self.sub_hidden = True if self.sh.sr.hidden.rows == "exclude": self.excluded_rows.add(r) self.sh.layout(r, self.layout_hidden) @@ -2012,6 +2063,11 @@ class _ListRun: if label is not None and match_key(_text(label.value)).startswith(pick): self._total_row(r, inb, cells, label) return + if self.sub_idx is not None: + label = inb.get(self.colmap.get(self.sub_idx, -1)) + if label is not None and match_key(_text(label.value)).startswith(self.sub_key): + self._subtotal_row(r, inb, cells, label) + return self._record(r, inb, cells, fills) def _plan(self) -> list[tuple[Any, int, bool, str]]: @@ -2073,6 +2129,8 @@ class _ListRun: v = None values.append(v) lineage.append((col.name, r, c)) + if self.synth: + values += [_PERIOD_START, _PERIOD_END] ignored = self.ignored claims = {c: ("ignored_column" if c in ignored else "value") for c in inb} self.sh.layout(r, self.layout_data) @@ -2084,6 +2142,12 @@ class _ListRun: if self.rowid_first is None: self.rowid_first = rowid self.rowid_last = rowid + if self.sub_first is None: + self.sub_first = rowid + if self.group_idx is not None: + group = values[self.group_idx] + if self.run_value is _NO_GROUP or group != self.run_value: + self.run_first, self.run_value = rowid, group self.emit(r, cells, claims, True) def _total_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], @@ -2142,7 +2206,67 @@ class _ListRun: self.stop = ("total", r) self.emit(r, cells, claims, True) - def _ref_lookup(self, rc: tuple[int, int]) -> str | int: + def _subtotal_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], + label: GridCell) -> None: + """期 5：穿插在明细中间的一行小计。不进数据：标签格和数字格的去向是 total_label / total_value（同合计行）， + 每个数字格一条合计格（DerivedItem，kind=column_sum，分段键是块 id + SUBTOTAL_KEY），核对模块按它所管的那组 + 明细（rowid 闭区间）用 SQL 逐列求和比对，执行器不做加法（H1）；整行记进排除的行（reason=subtotal）。配了 + keep_as 的另存一行（小计项、按组时的分组取值、各数字列）。 + + 它管哪几行：since_previous 是上一行小计（或表头）之后的全部明细；group_column 是其中紧挨在它上面、分组列 + 取值相同的那一段。一行明细都没有时区间为空，核对判「无法核对」（合计行之上没有明细行）。数字格全空的小计 + 没有一列能核对，报 subtotal_blank（结构类）：不报的话这一行就悄悄没了核对。""" + eng, sh, block, sub = self.eng, self.sh, self.block, self.sub + assert sub is not None + claims = {c: ("ignored_column" if c in self.ignored else "total_label") for c in inb} + label_raw = _text(label.value) + first = self.sub_first + if sub.scope == "group_column" and first is not None and self.run_first is not None: + first = max(first, self.run_first) + last = self.rowid_last if first is not None else None + numeric = [(i, col) for i, col in enumerate(self.cols) if col.type in ("INTEGER", "REAL")] + if numeric and not any(self.colmap[i] in inb for i, _ in numeric): + coord = sh.coord(r, self.colmap[self.sub_idx]) # type: ignore[index] + where = f"{self.where}：小计行（{coord.split('!')[-1]}）的数字格全部为空" + eng.problems.add( + "subtotal_blank", + f"{where}，无法按这一组明细核对。请在文件中补上小计后重新上传；原表不再有小计时，在配方面板的" + "「小计行」中取消「有小计行」", + f"{where}，无法核对", [coord], sheet=sh.name) + key = f"{block.id}{SUBTOTAL_KEY}" + keep_values: list[Any] = [] + for i, col in numeric: + c = self.colmap[i] + cell = inb.get(c) + if cell is None: + keep_values.append(None) + continue + claims[c] = "total_value" + got = eng.convert_number(cell, col.type, block.values, sh.name, f"{self.where}的小计行", r, c, + data_area=False) + keep_values.append(got) + item = DerivedItem( + kind="column_sum", segment=key, sheet=sh.name, cell=sh.coord(r, c), label_raw=label_raw, + label=canon(label_raw), value=None if got is _FAIL else got, is_formula=cell.formula is not None, + formula=cell.formula, base_table=block.table, base_value=col.name, rowid_first=first, + rowid_last=last, hidden_in_range=self.sub_hidden, keep_table=sub.keep_as) + if item.is_formula: + eng.resolve_refs(item, cell.formula or "", self._ref_lookup) + eng.ex.derived.append(item) + if sub.keep_as and sub.keep_as in eng.sinks: + ks = eng.sinks[sub.keep_as] + ks.sources.append(f"{sh.sr.id}/{block.id}") + group = [None if self.run_value is _NO_GROUP else self.run_value] if self.group_idx is not None else [] + lin = [(col.name, r, self.colmap[i]) for i, col in numeric] + ks.add([canon(label_raw), *group, *keep_values], sheet=sh.name, + at=(r, self.colmap[self.sub_idx]), cells=lin, fails=True) # type: ignore[index] + n = sum(1 for role in claims.values() if role in ("total_label", "total_value")) + sh.exclude("subtotal", r, r, n, block=block.id, anchor=sub.pick) + self.sub_rows[r] = list(range(first, last + 1)) if first is not None and last is not None else [] + self.sub_first, self.run_first, self.run_value, self.sub_hidden = None, None, _NO_GROUP, False + self.emit(r, cells, claims, True) + + def _ref_lookup(self, rc: tuple[int, int]) -> str | int | list[int]: r, c = rc if self.c1 <= c <= self.c2: rowid = self.row_rowid.get(r) @@ -2150,6 +2274,9 @@ class _ListRun: return rowid if r in self.excluded_rows: return "hidden" + if r in self.sub_rows: + # 合计行的公式引用了小计格（=C5+C9）：小计格代表它管的那组明细 + return self.sub_rows[r] return "outside" @@ -2343,7 +2470,8 @@ class _Engine: self.canon_log[key] = CanonEntry(table, col.name, text, stored, 1, _coord(sheet, r, c)) return stored - def resolve_refs(self, item: DerivedItem, formula: str, lookup: Callable[[tuple[int, int]], str | int]) -> None: + def resolve_refs(self, item: DerivedItem, formula: str, + lookup: Callable[[tuple[int, int]], str | int | list[int]]) -> None: """合计公式的引用经溯源：全在基表 → ref_rowids；有隐藏行或分类汇总函数 → hidden； 有不在基表的 → outside；形状认不出 → unrecognized。执行器不做加法（H1）。""" shape = formula_refs(formula) @@ -2357,7 +2485,9 @@ class _Engine: outside = not cells for rc in cells: hit = lookup(rc) - if isinstance(hit, int): + if isinstance(hit, list): + rowids.extend(hit) # 期 5：引用了小计格，换算成它管的明细 + elif isinstance(hit, int): rowids.append(hit) elif hit == "hidden": hidden = True @@ -2607,8 +2737,9 @@ class _Engine: self.ex.problems = self.problems.finish(set()) return self.ex self.cols = cols + synth = {t: (spec[1], spec[2]) for t, spec in list_period_columns(self.recipe).items() if spec[0] == "synthesize"} for name, cs in cols.items(): - self.sinks[name] = _Sink(self, name, cs, self.specs.get(name)) + self.sinks[name] = _Sink(self, name, cs, self.specs.get(name), synth.get(name)) if self.db_path is not None: _remove_db(self.db_path) self.conn = _open_sqlite(self.db_path, readonly=False) @@ -2622,6 +2753,11 @@ class _Engine: crosstabs.extend(self._phase1(sh)) t2 = time.perf_counter() period, period_reported, _ = self._period() + if self.ex.period is not None: + # 期 5：列表加的统计期起止两列现在才有值（列表在统计期定下来之前就抽完了） + for sink in self.sinks.values(): + if sink.synth_idx is not None: + sink.fill_period(self.ex.period.start, self.ex.period.end) for ct in crosstabs: ct.complete_axis(period, period_reported) ct.extract() @@ -2650,7 +2786,7 @@ class _Engine: _remove_db(self.db_path) # type: ignore[arg-type] else: for sink in self.sinks.values(): - sink.flush() + sink.flush(final=True) self.conn.execute("COMMIT") self.conn.close() self.conn = None @@ -2748,7 +2884,9 @@ class _Engine: sh.failed = True return [] b = s.bounds - if b is not None and not self.partial and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): + # 稀疏判定挡的是 Excel 里误输入、残留格式撑大的已用区域；CSV 的每一格都是写出来的，空字段多只是数据稀疏 + if b is not None and not self.partial and self.scan.format != "csv" \ + and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): far = "、".join(list(dict.fromkeys(s.far_cells))[:3]) msg = (f"工作表「{sh.name}」的已用区域 {b.a1()} 共 {b.area:,} 个单元格，但只有 {s.nonempty:,} 个单元格有内容；" f"远处的单元格（如 {far}）可能是误输入或残留格式。请在 Excel 中删除这些单元格后重新上传") diff --git a/backend/app/data/recipe_imports.py b/backend/app/data/recipe_imports.py index 40dba2b1..e715a08c 100644 --- a/backend/app/data/recipe_imports.py +++ b/backend/app/data/recipe_imports.py @@ -225,7 +225,7 @@ class Pipeline: def default_pipeline() -> Pipeline: """接上真模块。函数体里惰性导入：这些模块（openpyxl、执行器、起草器）导入不便宜，接口模块加载时不必付。""" from app.data import ( - recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, + csv_grid, recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, recipe_engine, recipe_fixes, recipe_notes, recipe_select, recipe_suggest, xlsx_cells, xlsx_scan, ) @@ -234,7 +234,8 @@ def default_pipeline() -> Pipeline: recipe_sha256=recipe.recipe_sha256, canonical=recipe.canonical_recipe, apply_patch=recipe.apply_patch, - scan=xlsx_scan.scan, + # CSV / TSV 也走配方：csv_grid 适配成一张「工作表」，其余照旧交给 xlsx_scan（Excel 期 5） + scan=csv_grid.scan_any, scan_to_json=xlsx_scan.scan_to_json, scan_from_json=xlsx_scan.scan_from_json, read_grid=xlsx_cells.read_grid, diff --git a/backend/app/data/recipe_notes.py b/backend/app/data/recipe_notes.py index b4464cf7..48c8da20 100644 --- a/backend/app/data/recipe_notes.py +++ b/backend/app/data/recipe_notes.py @@ -27,6 +27,9 @@ from app.data.names import canon from app.data.recipe_checks import date_columns, plan_checks, split_coord from app.data.recipe_parsers import UNITS from app.data.recipe_types import ( + LIST_PERIOD_END, + LIST_PERIOD_START, + SUBTOTAL_KEY, Acceptance, CheckResult, ColumnOut, @@ -46,6 +49,7 @@ from app.data.recipe_types import ( TableNote, UnionNotePart, derive_tables, + list_period_columns, prose_problems, render_prose, ) @@ -134,6 +138,13 @@ NOTE_TEXT = { "ignored_rows_union": "部分期的原表另有按配方忽略的行，内容见证据面板", "ignored_cells_union": "部分期的原表另有按配方忽略的单元格，内容见证据面板", "dim_values_vary": "各期的 {dim} 取值不完全相同：跨期比较前请先确认两期都有的取值", + # ---- 期 5：列表按期累积、穿插的小计。列表可以累积以后，空行之后的文字行也要有并集说法 + "rows_after_stop_union": "部分期的原表在空行之后另有未导入的文字行，内容见证据面板", + "subtotal_kept": "原表穿插的小计在 {table}，不要与本表相加", + "subtotal_not_kept": "原表穿插的小计行未导入本表", + "list_subtotal_passed": "原表穿插的小计行的值，已按各组明细求和核对一致：不要与 {base} 相加，也不要彼此相加", + "list_subtotal_unverified": "原表穿插的小计行的值，按原表保存的值存储。本期部分小计未能核对：不要与 {base} 相加，" + "也不要彼此相加", } #: 列说明的片段模板 @@ -148,6 +159,9 @@ COLUMN_TEXT = { "col_partial_periods": "部分期没有这一列的数据，为空值", "col_absent": "当前版本的数据中没有这一列，为空值", "col_retired": "这一列自某一期起不再导入，此后各期为空值", + # ---- 期 5：列表按期累积时由系统加的统计期起止两列（ListPeriod.synthesize）。不含数字（说明不许报数） + "period_start": "本期统计期的第一天，取自表内的统计期或人工录入的统计期，格式 YYYY-MM-DD", + "period_end": "本期统计期的最后一天，取自表内的统计期或人工录入的统计期，格式 YYYY-MM-DD", } #: 「逐日 / 逐行」「个别日期 / 个别行」：粒度只有一列日期时按日说，否则按行说 @@ -161,7 +175,7 @@ _K_REASON = {r: f"total_k_{r}" for r in ("uncached", "blank", "null_detail", "ti _T_REASON = {r: f"list_total_{r}" for r in ("uncached", "blank", "null_detail", "tiling", "hidden")} #: 允许出现「一致」的模板：只在对应的 K / T 通过时才会选用 CONSISTENT_KEYS = frozenset({"total_k_passed", "total_k_passed_g_unverifiable", "list_total_passed", - "list_total_passed_partial"}) + "list_total_passed_partial", "list_subtotal_passed"}) def _tok(kind: str, name: str) -> str: @@ -202,6 +216,8 @@ class _Writers: lists: list[ListBlock] = field(default_factory=list) #: 把合计行另存进本表的列表块 list_totals: list[ListBlock] = field(default_factory=list) + #: 期 5：把穿插的小计另存进本表的列表块 + list_subtotals: list[ListBlock] = field(default_factory=list) def add_sheet(self, sheet: SheetRecipe) -> None: if all(s is not sheet for s in self.sheets): @@ -242,6 +258,9 @@ def _writers(recipe: Recipe) -> dict[str, _Writers]: total = block.rows.total_row if total is not None and total.keep_as: w(total.keep_as, sheet).list_totals.append(block) + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + w(sub.keep_as, sheet).list_subtotals.append(block) return out @@ -395,6 +414,15 @@ def _base_total_fragments(w: _Writers) -> list[NoteFragment]: b.rows.total_row is not None and not b.rows.total_row.keep_as for b in w.lists) if dropped: out.append(NoteFragment("total_not_kept", None, NOTE_TEXT["total_not_kept"])) + # 期 5：穿插的小计行不在本表里：另存在哪，或者没导入 + for b in w.lists: + sub = b.rows.subtotals + if sub is None: + continue + if sub.keep_as: + _add_unique(out, _frag("subtotal_kept", sub.keep_as, NOTE_TEXT["subtotal_kept"].format(table=_tab(sub.keep_as)))) + elif all(f.key != "subtotal_not_kept" for f in out): + out.append(NoteFragment("subtotal_not_kept", None, NOTE_TEXT["subtotal_not_kept"])) return out @@ -450,6 +478,13 @@ def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[NoteFragme else: frag = _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table))) _add_unique(out, frag) + for block in w.list_subtotals: + # 期 5：另存的小计：T 通过才说「核对一致」，没核对上的（无法核对）照实说部分未能核对；不一致的整份拒收 + t = ctx.check(table, ctx.plan.t.get(f"{block.id}{SUBTOTAL_KEY}"), f"块「{block.id}」的小计行") + if t is None or _blocking(t): + continue + key = "list_subtotal_passed" if t.status == "passed" else "list_subtotal_unverified" + _add_unique(out, _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table)))) return out @@ -653,10 +688,17 @@ def _date_key(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> str | None: return None +def _period_column_key(col: ColumnOut) -> str | None: + """期 5：系统加的统计期起止两列（role=period）用哪条列说明。""" + if col.role != "period": + return None + return "period_start" if col.name == LIST_PERIOD_START else "period_end" if col.name == LIST_PERIOD_END else None + + def _null_eligible(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> bool: """这一列要不要考虑空值说明：值列；主键不会是空值；表内合计表的空值已由表说明（未能核对）交代。""" value_col = col.role in ("measure", "value") and col.type in ("INTEGER", "REAL") - is_total = bool(w.kept_from or w.list_totals) + is_total = bool(w.kept_from or w.list_totals or w.list_subtotals) return value_col and not is_total and col.name not in set(ctx.grains.get(table, [])) @@ -668,6 +710,8 @@ def _column_templates(ctx: _Ctx, table: str, null_counts: Mapping[str, Mapping[s frags: list[str] = [] if (date := _date_key(ctx, w, table, col)) is not None: frags.append(COLUMN_TEXT[date]) + if (period := _period_column_key(col)) is not None: + frags.append(COLUMN_TEXT[period]) if col.unit: frags.append(COLUMN_TEXT["unit"].format(unit=_tok("单位", col.unit))) has_nulls = counts is None or counts.get(col.name, 0) > 0 @@ -793,22 +837,27 @@ _FAMILY: dict[str, str] = { "hidden_excluded": "hidden_excluded", "hidden_included": "hidden_included", "ignored_columns": "ignored_columns", "ignored_rows": "ignored_rows", "ignored_cells": "ignored_cells", "rows_after_stop": "rows_after_stop", + "subtotal_kept": "subtotal_kept", "subtotal_not_kept": "subtotal_not_kept", + "list_subtotal_passed": "list_total", "list_subtotal_unverified": "list_total", "period_conflict_accepted": "period_conflict", "period_conflict_pending": "period_conflict", "period_human": "period_human", "outside_digits": "outside_digits", "note": "note", } #: 合并后的拼接顺序（与 _table_fragments 一致；累积特有的几句排在用户写的表说明之前） -_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", "total_k", - "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", +_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", + "subtotal_kept", "subtotal_not_kept", "total_k", "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", "ignored_rows", "ignored_cells", "rows_after_stop", "period_conflict", "period_human", "outside_digits", "union", "note") #: 与核对结论无关、只描述表结构的片段族：用目标配方那一期的；只在早期出现的 subject（退役的表、另存表）用 #: 最后一个含它的那一期的 -_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note"}) +_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note", + "subtotal_kept", "subtotal_not_kept"}) #: 「任一期出现就写」的片段族 → 并集里的说法 _ANY_UNION = {"hidden_excluded": "hidden_excluded_union", "hidden_included": "hidden_included_union", "ignored_columns": "ignored_columns_union", "ignored_rows": "ignored_rows_union", - "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union"} + "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union", + # 期 5：列表可以按期累积了，空行之后的文字行任一期有就写 + "rows_after_stop": "rows_after_stop_union"} _K_PASSED = frozenset({"total_k_passed", "total_k_passed_g_unverifiable"}) @@ -943,20 +992,26 @@ def _merge_group(table: str, family: str, subject: str | None, group: dict[int, if family == "period_human": key = "period_human_all" if len(present) == len(having) else "period_human_union" return NoteFragment(key, subject, NOTE_TEXT[key]) - # list_total、rows_after_stop 只有列表才有，而列表不能按期累积（P3-SPEC 2.2）；不认识的片段族同样没有并集说法。 + # list_total 只在列表的合计行、小计行另存的表上，这种表不能按期累积（accumulate_blockers）；不认识的片段族同样 + # 没有并集说法。 # 记 problems（不能保存），绝不沿用某一期的单期说法。消息给人看，不露片段族的键名 problems.append(f"表「{table}」：{_UNMERGEABLE.get(family, '有一段说明')}不能按期合并，无法生成说明") return None #: 没有并集说法的片段族 → 给人看的叫法（problems 里不露键名） -_UNMERGEABLE = {"list_total": "列表合计行的核对结论", "rows_after_stop": "列表空行之后的文字行"} +_UNMERGEABLE = {"list_total": "列表合计行、小计行的核对结论"} def _axis_column(recipe: Recipe, table: str) -> str | None: + """各期按哪一列区分（说明里「请按 {col} 筛选」）：交叉表的日期轴；期 5 起列表按 period 绑定的日期列，或者 + 系统加的统计期起那一列。""" tables, _ = derive_tables(recipe) axes = [c.name for c in tables.get(table, []) if c.role == "axis"] - return axes[0] if len(axes) == 1 else None + if len(axes) == 1: + return axes[0] + spec = list_period_columns(recipe).get(table) + return spec[1] if spec else None def _union_table_fragments(table: str, having: list[int], parts: list[_UPart], ref: int, target: Recipe, @@ -1037,6 +1092,8 @@ def _union_column_templates(table: str, columns: list[ColumnOut], having: list[i frags.append(COLUMN_TEXT["date_year_from_period"]) elif "date" in dates or (not with_col and col.role == "axis"): frags.append(COLUMN_TEXT["date"]) + if (period := _period_column_key(col)) is not None: + frags.append(COLUMN_TEXT[period]) if target_cols is not None and col.name in target_cols: unit = target_cols[col.name].unit elif with_col: diff --git a/backend/app/data/recipe_select.py b/backend/app/data/recipe_select.py index daa7332c..34697950 100644 --- a/backend/app/data/recipe_select.py +++ b/backend/app/data/recipe_select.py @@ -427,6 +427,21 @@ def _bottom_box(ctx: _Ctx, plan: Any, first: int) -> bool: ctx.fail("selection_bottom_not_anchorable", f"框里只有表头和合计行（第 {last} 行），没有数据行：请把框扩大到数据所在的行") return False + # 期 5：框里夹在中间、第一列以合计字样开头、下面紧接着还有同一张表的明细的行（起草器 _scan_block 的同一条 + # 规则），照起草器提议成小计（_settle_subtotals）；提议不成的照旧报错 + mid = [r for r in data[1:] if S._total_word(S._label_of(grid.get(r, c1))) is not None] + rest = [r for r in data if r not in mid] + if mid and rest and not any(grid.get(r, c) is not None and S._total_word(grid.get(r, c).value) is not None + for r in data for c in range(c1 + 1, c2 + 1)): + trial = S._ListPlan(header_rows=plan.header_rows, cols=plan.cols, headers=plan.headers, rows=rest, + subtotal_rows=list(mid), + subtotal_words=[S._total_word(S._label_of(grid.get(r, c1))) or "" for r in mid]) + types = S._types_of(grid, rest, plan.cols) + if all(S._more_rows(grid, r, trial, S._bottom(grid), types) for r in mid): + S._settle_subtotals(grid, trial) + if trial.subtotal_word is not None: + plan.subtotal_rows, plan.subtotal_word = trial.subtotal_rows, trial.subtotal_word + data = [r for r in data if r not in mid] if not _no_stray_totals(ctx, data, c1, c2): return False bottom = S._bottom(grid) diff --git a/backend/app/data/recipe_suggest.py b/backend/app/data/recipe_suggest.py index 53314072..9a176511 100644 --- a/backend/app/data/recipe_suggest.py +++ b/backend/app/data/recipe_suggest.py @@ -12,6 +12,11 @@ validate）和执行器干跑（注入的 dry_run），再由人逐条确认之 哪些是区域外文字、哪些格没有去处）直接用执行器的结论：同一件事只有执行器一套判法，这里自己定位只是没有 干跑时的近似。 +**期 5（下一轮 03 细案 E）。** 有 Excel 表格对象（插入 → 表格）的工作表优先按表格对象出列表块（_lists_from_tables： +表头、列范围、汇总行都是表格对象给的，锚点仍按表头文字）；夹在明细中间、第一列以同一个合计字样开头的行提议成 +穿插的小计（_settle_subtotals），不再一律记起草失败；每个列表都问一道「每期怎么更新」（q_list_period），选按期 +累积时同时绑定每一行属于哪一期（按日期列核对，或加统计期起止两列）。 + **问题的 effects 怎么写。** 每个选项是一组 JSON Patch，路径按传进来的配方写；接口层从 answers_base 起、按 问题顺序重放全部回答（P2-SPEC 7.2）。因此： - 关系的两个选项都用整条 replace（「登记」写回完整的 sum_eq / not_comparable，「不登记」写 dismissed）， @@ -63,7 +68,7 @@ from app.data.recipe_types import ( derive_tables, ) from app.data.tabular import DATE_HEADER_MIN, EXCEL_ERRORS -from app.data.xlsx_scan import WorkbookScan, cell_ref, col_letter, parse_ref +from app.data.xlsx_scan import TableScan, WorkbookScan, cell_ref, col_letter, parse_ref logger = logging.getLogger(__name__) @@ -612,8 +617,15 @@ class _ListPlan: #: 按数据行收进来并记起草失败 adjacent: list[tuple[int, list[int]]] = field(default_factory=list) #: 不在末尾第一列的合计字样：(行, 列)。第一列在中间或最上面、或者落在别的列上，都无法按合计行核对， - #: 记起草失败；不记就会把合计当明细导入，求和翻倍（H2） + #: 记起草失败；不记就会把合计当明细导入，求和翻倍（H2）。期 5：第一列里同一个字样的穿插行先记进 subtotal_rows， + #: 收完整块再判能不能提议成小计（_settle_subtotals）；判不了的才回到这里 stray_totals: list[tuple[int, int]] = field(default_factory=list) + #: 期 5：穿插在明细中间、第一列以合计字样开头的行（不进 rows）和它们的字样；能提议成小计时 subtotal_word 非空 + subtotal_rows: list[int] = field(default_factory=list) + subtotal_words: list[str] = field(default_factory=list) + subtotal_word: str | None = None + #: 期 5：按 Excel 的表格对象起草的（xlsx_scan.TableScan 的名字）：表名优先用它，卡片里写明 + table_object: str | None = None # 命名阶段填 id: str = "" table: str = "" @@ -808,13 +820,44 @@ def _scan_block(grid: Grid, plan: _ListPlan, bottom: int) -> None: if word is not None and plan.rows and not _more_rows(grid, r, plan, bottom, types): plan.total_row, plan.total_word = r, word break + others = [(r, c) for c, cell in zip(cols[1:], cells[1:]) + if cell is not None and _total_word(cell.value) is not None] + plan.stray_totals.extend(others) + if word is not None and plan.rows and not others: + # 期 5：夹在明细中间的合计字样，先当小计候选：不进数据行，收完整块再判（_settle_subtotals） + plan.subtotal_rows.append(r) + plan.subtotal_words.append(word) + r += 1 + continue if word is not None: plan.stray_totals.append((r, cols[0])) - plan.stray_totals.extend((r, c) for c, cell in zip(cols[1:], cells[1:]) - if cell is not None and _total_word(cell.value) is not None) plan.rows.append(r) acc.add(r) r += 1 + _settle_subtotals(grid, plan) + + +def _settle_subtotals(grid: Grid, plan: _ListPlan) -> None: + """期 5：夹在明细中间的合计字样能不能提议成小计（ListRows.subtotals，规则起草不再把它们记成失败）。 + + 能的条件：都在第一列、是同一个字样，每一行至少有一格数（没有数的小计核对不了）。末尾那一行和它们字样相同时 + （表以最后一组的小计结束，没有总计），它也是小计，不是合计行。不能的（字样不同、有一行没有数）照旧记进 + stray_totals（起草失败，交给人或 AI），这些行回到数据行里——和期 4 的结果一样。""" + if not plan.subtotal_rows: + return + words = set(plan.subtotal_words) + numeric = [c for c in plan.cols[1:] if any(_value_num(grid.get(r, c)) is not None for r in plan.rows)] + ok = len(words) == 1 and all(any(_value_num(grid.get(r, c)) is not None for c in numeric) + for r in plan.subtotal_rows) + if ok: + plan.subtotal_word = next(iter(words)) + if plan.total_row is not None and plan.total_word == plan.subtotal_word: + plan.subtotal_rows.append(plan.total_row) + plan.total_row = plan.total_word = None + return + plan.stray_totals.extend((r, plan.cols[0]) for r in plan.subtotal_rows) + plan.rows = sorted(set(plan.rows) | set(plan.subtotal_rows)) + plan.subtotal_rows, plan.subtotal_words = [], [] def _list_failures(sheet: str, grid: Grid, plan: _ListPlan) -> list[_Failure]: @@ -898,7 +941,8 @@ def _analyze_lists(grid: Grid) -> _Lists: _finish_block(grid, plan) out.failures.extend(_list_failures(grid.sheet, grid, plan)) out.blocks.append(plan) - end = max((p.total_row or (p.rows[-1] if p.rows else p.header_rows[-1])) for p in made) + end = max(max([p.total_row or 0, *p.subtotal_rows, p.rows[-1] if p.rows else p.header_rows[-1]]) + for p in made) while r_idx < len(rows) and rows[r_idx] <= end: r_idx += 1 return out @@ -945,7 +989,7 @@ def _finish_block(grid: Grid, plan: _ListPlan) -> None: break if cells: break - last = plan.total_row or rows[-1] + last = max([plan.total_row or 0, rows[-1], *plan.subtotal_rows]) for r in range(plan.header_rows[0], last + 1): for c in cols: if grid.get(r, c) is not None or r in plan.header_rows: @@ -975,15 +1019,77 @@ class _Sheet: hidden_cols: list[int] = field(default_factory=list) -def _analyze_sheet(grid: Grid, sid: str) -> _Sheet: +def _lists_from_tables(grid: Grid, objects: list[TableScan]) -> _Lists | None: + """期 5：按 Excel 的表格对象（插入 → 表格）起草列表块。表格对象明确给出了表头行、列范围、数据行和汇总行，比按 + 「像不像表头」猜可靠：数字格空着的行不会被当成表头，紧挨着的两张表也分得开。 + + 出来的块和按规则猜的一样只记文字：锚点仍按表头文字（不记坐标，下一期表格挪了位置照样认得出），汇总行按第一列 + 的文字映射成合计行（total_row），夹在中间的合计字样按小计提议（_settle_subtotals）。表格对象的 ref 写坏了、 + 没有表头行时返回 None，退回按规则猜。""" + out = _Lists(sheet=grid.sheet) + spans = sorted(((t.span(), t) for t in objects), key=lambda x: (x[0] or (0, 0, 0, 0))[:2]) + for span, t in spans: + if span is None or t.header_row_count < 1: + return None + r1, c1, r2, c2 = span + hrows = list(range(r1, r1 + t.header_row_count)) + cols = list(range(c1, c2 + 1)) + texts = _header_texts(grid, hrows, cols) + if set(texts) != set(cols): + blank = [col_letter(c) for c in cols if c not in texts] + msg = (f"工作表「{grid.sheet}」：表格「{t.name}」的表头有空着的列（{'、'.join(blank)} 列），" + "按列表导入要每一列都有表头文字") + out.failures.append(_Failure(msg, msg)) + continue + plan = _ListPlan(header_rows=hrows, cols=cols, headers={c: texts[c] for c in cols}, table_object=t.name) + last_data = r2 - max(t.totals_row_count, 0) + rows = [r for r in grid.row_numbers() if hrows[-1] < r <= last_data and any(grid.get(r, c) is not None + for c in cols)] + prev = hrows[-1] + for r in rows: + if r > prev + 1: + plan.skip_blank = True + plan.skipped.extend(range(prev + 1, r)) + prev = r + word = _total_word(_label_of(grid.get(r, c1))) + if word is not None and plan.rows: + plan.subtotal_rows.append(r) + plan.subtotal_words.append(word) + continue + plan.rows.append(r) + if not plan.rows: + msg = f"工作表「{grid.sheet}」：表格「{t.name}」的表头下面没有数据行" + out.failures.append(_Failure(msg, msg)) + continue + if t.totals_row_count > 0: + tr = last_data + 1 + label = _label_of(grid.get(tr, c1)) + word = _total_word(label) or (label if label and len(label) <= 12 and not any( + ch.isdigit() for ch in label) else None) + if word is not None: + plan.total_row, plan.total_word = tr, word + else: + why = ("起草只读了表格前面的部分，没读到汇总行" if grid.truncated and tr > max(grid.row_numbers()) + else "汇总行的第一列没有可用作合计标签的文字（不能含数字、不超过 12 个字）") + msg = f"工作表「{grid.sheet}」：表格「{t.name}」的汇总行（第 {tr} 行）无法按合计行核对：{why}" + out.failures.append(_Failure(msg, msg)) + _settle_subtotals(grid, plan) + _finish_block(grid, plan) + out.failures.extend(_list_failures(grid.sheet, grid, plan)) + out.blocks.append(plan) + return out + + +def _analyze_sheet(grid: Grid, sid: str, objects: list[TableScan] | None = None) -> _Sheet: sh = _Sheet(sheet=grid.sheet, sid=sid) axis = _axis_row(grid) + from_objects = _lists_from_tables(grid, objects) if objects and axis is None else None if axis is not None: sh.cross = _analyze_crosstab(grid, *axis) sh.region = set(sh.cross.region) sh.failures.extend(sh.cross.failures) else: - sh.lists = _analyze_lists(grid) + sh.lists = from_objects if from_objects is not None else _analyze_lists(grid) sh.failures.extend(sh.lists.failures) for b in sh.lists.blocks: sh.region |= b.region @@ -1111,7 +1217,8 @@ def _measure_column(label: str, pos: int) -> tuple[str, str | None, str | None]: def _analyze(scan: WorkbookScan, grids: dict[str, Grid]) -> _Analysis: visible = _visible_grids(scan, grids) hidden = [s.name for s in scan.sheets if s.state != "visible"] - sheets = [_analyze_sheet(g, f"s{i + 1}") for i, g in enumerate(visible)] + sheets = [_analyze_sheet(g, f"s{i + 1}", [t for t in scan.tables if t.sheet == g.sheet]) + for i, g in enumerate(visible)] failures = [f for sh in sheets for f in sh.failures] tables_used: set[str] = set() @@ -1266,7 +1373,9 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] 合计表名撞了已用的名字时照起草器的老行为：不另建表定义，keep_as 仍写这个名字（交给静态校验报）。""" b.id = block_id if table is None: - title = b.title if b.title and not any(ch.isdigit() for ch in b.title) else None + # 期 5：按表格对象起草的，表名优先用表格对象的名字（不含数字时；Excel 默认的「表1」不用） + named = b.table_object if b.table_object and not any(ch.isdigit() for ch in b.table_object) else None + title = named or (b.title if b.title and not any(ch.isdigit() for ch in b.title) else None) want = _ENUM_PREFIX.sub("", P.split_unit_suffix(title)[0]) if title else sheet n_fallback = len(tables_used) + 1 b.table = _unique(to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}", tables_used) @@ -1293,6 +1402,11 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] total_spec = {"name": keep, "grain": ["合计项"], "kind": "reported_total", "units": num_units, "note": ""} b.keep_table = keep total = {"label_column": b.names[b.cols[0]], "pick": b.total_word, "keep_as": keep} + subtotals = None + if b.subtotal_word is not None: + # 期 5：穿插的小计只核对、不另存（另存的小计表不能按期累积，要另存由人在配方面板里打开） + subtotals = {"label_column": b.names[b.cols[0]], "pick": b.subtotal_word, "scope": "since_previous", + "group_column": None, "keep_as": None} grain = [b.names[b.grain_col]] if b.grain_col is not None else [] main = {"name": b.table, "grain": grain, "kind": "data", "units": dict(b.units), "note": ""} fill = bool(b.merged_fill) and all(r2 > r1 and c1 == c2 and b.types.get(c1) == "TEXT" @@ -1300,7 +1414,8 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] block = { "id": b.id, "layout": "list", "table": b.table, "header_rows": len(b.header_rows), "after_title": None, "columns": cols_out, "extra_columns": "reject", - "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total}, + "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total, + **({"subtotals": subtotals} if subtotals is not None else {})}, "values": {"placeholders": [{"text": e[0], "meaning": "无数据"} for e in b.placeholders.values()], "blank": "null", "text_number": "reject", "formula": "accept_cached"}, "merged_data": "fill" if fill else "reject", @@ -1797,6 +1912,8 @@ class _ListLoc: rows: list[int] total_row: int | None region: set[tuple[int, int]] = field(default_factory=set) + #: 期 5：穿插的小计行 + subtotal_rows: list[int] = field(default_factory=list) def _find_grid(name: str, grids: dict[str, Grid]) -> Grid | None: @@ -1943,19 +2060,26 @@ def _locate_list(block: ListBlock, grid: Grid, taken: set[int]) -> _ListLoc | No bottom = _bottom(grid) rr = r + 1 pick = P.match_key(block.rows.total_row.pick) if block.rows.total_row else None + sub_pick = P.match_key(block.rows.subtotals.pick) if block.rows.subtotals else None + subs: list[int] = [] while rr <= bottom: if all(grid.get(rr, c) is None for c in cols): if block.rows.blank_rows == "skip": rr += 1 continue break - if pick and P.match_key(_label_of(grid.get(rr, cols[0])) or "").startswith(pick): + label = P.match_key(_label_of(grid.get(rr, cols[0])) or "") + if pick and label.startswith(pick): total = rr break + if sub_pick and label.startswith(sub_pick): + subs.append(rr) + rr += 1 + continue rows.append(rr) rr += 1 - loc = _ListLoc(hrows, cols, rows, total) - for row in [*hrows, *rows, *([total] if total else [])]: + loc = _ListLoc(hrows, cols, rows, total, subtotal_rows=subs) + for row in [*hrows, *rows, *subs, *([total] if total else [])]: loc.region.update((row, c) for c in cols) return loc @@ -2130,6 +2254,9 @@ def questions_for(recipe: dict[str, Any], facts: DraftFacts, grids: dict[str, Gr if ext is not None and ext.sheets.skipped_hidden: for nm in _dedupe([item.get("sheet", "") for item in ext.sheets.skipped_hidden]): cards.append(_hidden_sheet_card(nm)) + # 期 5：列表每期怎么更新的问题排在其余问题之后、q_mode 之前：它只和按期累积有关，不该顶掉期 2 起排在前面的问题 + questions = ([q for q in questions if not q.id.startswith("q_list_period:")] + + [q for q in questions if q.id.startswith("q_list_period:")]) blockers = accumulate_blockers(parsed) cards.append(_mode_card(blockers)) if not blockers: @@ -2543,9 +2670,57 @@ def _list_cards(si: int, bi: int, sheet: Any, block: ListBlock, loc: _ListLoc | reason += f"，原值另存表「{total.keep_as}」" if total.keep_as else "，只核对不另存" cells = _coords(name, [(loc.total_row, loc.cols[0])]) if loc is not None and loc.total_row else [] cards.append(Card(f"total_row:{block.id}", f"{where}是合计行，改作核对", reason, cells)) + subt = block.rows.subtotals + if subt is not None: + rows = loc.subtotal_rows if loc is not None else [] + where = _rows_text(rows) if rows else f"「{subt.pick}」开头的行" + scope = ("上一行小计之后的全部明细" if subt.scope == "since_previous" + else f"紧挨在它上面、「{subt.group_column}」相同的那几行明细") + reason = f"每行小计按列求和与{scope}核对，小计行不进数据" + reason += f"，原值另存表「{subt.keep_as}」" if subt.keep_as else "，只核对不另存" + cells = _coords(name, [(r, loc.cols[0]) for r in rows]) if loc is not None else [] + cards.append(Card(f"subtotals:{block.id}", f"{where}是穿插的小计行，改作核对", reason, cells)) + q = _list_period_question(si, bi, sheet, block, recipe) + questions.append(q) + cards.append(Card(f"list_period:{block.id}", "这份明细每期怎么更新", + "每期替换：当前版本只含最新一期。按期累积：每期一份明细加进当前版本、可以跨期查询，要说明每一行" + "属于哪一期——选一个日期列，核对每行都在本期统计期内；明细里没有能代表统计期的日期列时，" + "选「每行加上本期统计期的起止两列」", [], q.id)) return cards, questions +def _list_period_question(si: int, bi: int, sheet: Any, block: ListBlock, recipe: dict[str, Any]) -> Question: + """期 5：列表每期怎么更新、按期累积时每一行属于哪一期（ListBlock.period），一道题定：每期替换（不绑定）；按期 + 累积并按某个日期列核对（每个日期列一个选项）；按期累积并给每行加统计期起止两列。后两种同时把导入模式改成按期 + 累积，所在工作表没配统计期的一并加上统计期（回执里统计期取自表内，没有就请人录入）。 + + 为什么不分成「先绑定、再问 q_mode」两道题：问题集合按起草的草稿生成、回答时不变（recipe_imports.answer）， + 列表的草稿还没绑定统计期，不符合按期累积的资格，q_mode 根本不会出现。几个列表各答各的：有的选累积、有的选替换 + 时，静态校验照实报出没绑定的那个列表不能累积。""" + base = ["sheets", si, "blocks", bi, "period"] + context = [] if sheet.context else _add_ops(recipe, ["sheets", si, "context"], [ + {"id": "统计期", "kind": "period", "parser": "cn_date_range", "prefer_prefix": None, "cross_check": "filename"}]) + mode = _add_ops(recipe, ["mode"], "accumulate") + # 「每期替换」排第一个：它就是草稿本来的样子（期 4 起各处取第一个选项的地方都不会因此改了配方） + options: list[dict[str, Any]] = [{"value": "none", "label": "每期替换", "needs_reason": False}] + effects: dict[str, list[dict[str, Any]]] = {"none": _add_ops(recipe, base, None)} + for c in block.columns: + if c.type != "DATE": + continue + value = f"column:{c.name}" + options.append({"value": value, "label": f"按期累积，按「{c.name}」核对每行都在本期统计期内", + "needs_reason": False}) + effects[value] = context + mode + _add_ops(recipe, base, {"column": c.name, "check": "within_context", + "synthesize": None}) + options.append({"value": "synthesize", "label": "按期累积，每行加上本期统计期的起止两列", "needs_reason": False}) + effects["synthesize"] = context + mode + _add_ops(recipe, base, {"column": None, "check": "within_context", + "synthesize": "统计期"}) + current = block.period + default = ("synthesize" if current is not None and current.synthesize else + f"column:{current.column}" if current is not None and current.column else "none") + return Question(f"q_list_period:{block.id}", f"列表「{block.id}」每期怎么更新", options, default, effects) + + # ========================================================================== # 期 3：按规则重新起草，并把名字对齐到现行配方（P3-SPEC 6.3） # ========================================================================== diff --git a/backend/app/data/recipe_types.py b/backend/app/data/recipe_types.py index 7643144f..c4b8d62c 100644 --- a/backend/app/data/recipe_types.py +++ b/backend/app/data/recipe_types.py @@ -300,11 +300,29 @@ class TotalRow(_M): keep_as: str | None = None +class Subtotals(_M): + """穿插在明细中间的小计行：不作数据，按它所管的那一组明细逐列求和核对；keep_as 给了就另存一张表。""" + + #: 小计标签所在的列（列名） + label_column: str + #: 小计行的标签以这个词开头（match_key 比较）。取自实际格子文字的候选词，不能含数字 + pick: str = Field(min_length=1, max_length=12) + #: 一行小计管哪几行明细：since_previous 上一行小计（或表头）之后的全部明细；group_column 紧挨在它上面、 + #: group_column 这一列取值相同的那几行明细 + scope: Literal["since_previous", "group_column"] = "since_previous" + #: scope=group_column 时按哪一列分组（列名）；since_previous 时为 None + group_column: str | None = None + #: 原表的小计另存成哪张表（kind=reported_total）；None 只核对不另存 + keep_as: str | None = None + + class ListRows(_M): #: 数据中间的空行：stop 到此为止（默认）；skip 跳过继续（回执记个数） blank_rows: Literal["stop", "skip"] = "stop" #: 合计行：到合计行为止，合计行按列求和核对 total_row: TotalRow | None = None + #: 期 5：穿插在明细中间的小计行（Subtotals）。默认 None = 期 4 的行为（像小计的行照常是数据或起草失败） + subtotals: Subtotals | None = None class ListValues(_M): @@ -317,6 +335,18 @@ class ListValues(_M): formula: Literal["accept_cached", "reject"] = "accept_cached" +class ListPeriod(_M): + """列表按期累积时，每一行属于哪一期，二选一：column 按表里的一个日期列，每行的日期都要落在本期的统计期内； + synthesize 每行加上本期统计期的起、止两列（明细里没有日期列，或者日期不代表它属于哪一期时）。""" + + #: 按这个日期列（ListColumn.name，类型 DATE）核对；和 synthesize 二选一 + column: str | None = None + #: 怎么核对：within_context 每行的日期都要落在本期统计期内（空着的也不行） + check: Literal["within_context"] = "within_context" + #: 加统计期起、止两列（LIST_PERIOD_START、LIST_PERIOD_END）；和 column 二选一 + synthesize: Literal["统计期"] | None = None + + class ListBlock(_M): id: str = Field(min_length=1, max_length=32) layout: Literal["list"] @@ -333,6 +363,8 @@ class ListBlock(_M): merged_data: Literal["reject", "fill"] = "reject" #: 期 3：按表头忽略的列（IgnoreColumn），只针对这些列；其余多出的列照 extra_columns。默认空 = 期 2 的行为 ignore_columns: list[IgnoreColumn] = Field(default_factory=list, max_length=50) + #: 期 5：按期累积时每一行属于哪一期（ListPeriod）。默认 None = 期 4 的行为（列表只能每期替换） + period: ListPeriod | None = None Block = Annotated[Union[CrosstabBlock, ListBlock], Field(discriminator="layout")] @@ -462,13 +494,14 @@ CheckStatus = Literal["passed", "mismatch", "unverifiable", "info"] @dataclass class CheckResult: - #: 稳定 id：K1… 合计核对、G1… 公式引用、T1… 列表合计、R1… 关系（沿用配方里的 id）、C1 统计期、C2 文件名、 - #: N1… 行数、P1… 主键；期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期（物化时跑，都是结构类，P3-SPEC 2.7）。 + #: 稳定 id：K1… 合计核对、G1… 公式引用、T1… 列表合计（期 5 起也含列表中间的小计）、R1… 关系（沿用配方里的 + #: id）、C1 统计期、C2 文件名、N1… 行数、P1… 主键、D1… 列表的日期落在统计期内（期 5，ListPeriod.column）； + #: 期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期（物化时跑，都是结构类，P3-SPEC 2.7）。 #: F 编号留给系统发现的事实（Fact.id），两者不混用 id: str kind: Literal["derived_sum", "formula_refs", "column_sum", "relation_sum_eq", "relation_not_comparable", "context_agree", "filename_period", "row_count", "pk_unique", - "union_rows", "union_pk", "union_period"] + "union_rows", "union_pk", "union_period", "period_rows"] title: str status: CheckStatus #: 不通过时的类别：structure / data_quality / info @@ -759,9 +792,10 @@ class SheetsOut: #: hidden_excluded = hidden.rows=exclude 排除的隐藏行；blank_skipped = 列表 blank_rows=skip 跳过的空行； #: ignored_rows = 交叉表 ignore_rows 忽略的行；ignored_outside = ignore_outside 忽略了区域外数字格的行； #: after_stop = 列表停止之后没导入的文字行；total_not_kept = 参与核对、但不另存的合计行（交叉表 keep_as 为空的 -#: derived 段、列表 total_row.keep_as 为空的合计行） +#: derived 段、列表 total_row.keep_as 为空的合计行）；期 5：subtotal = 列表中间的小计行（参与核对、不进数据， +#: 另存与否都记在这里：它们都不在本表里） ExcludedReason = Literal["hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", - "total_not_kept"] + "total_not_kept", "subtotal"] @dataclass @@ -1031,6 +1065,15 @@ def prose_problems(template: str, *, known: dict[str, set[str]] | None = None) - #: 列表合计行另存成表时，合计项列的固定列名 LIST_TOTAL_DIM = "合计项" +#: 期 5：列表的小计行另存成表时，小计项列的固定列名 +LIST_SUBTOTAL_DIM = "小计项" +#: 期 5：列表按期累积、由系统给每行加统计期时（ListPeriod.synthesize）的两列：本期统计期的第一天、最后一天， +#: 写法 YYYY-MM-DD（同 DATE 列，存 TEXT） +LIST_PERIOD_START = "统计期起" +LIST_PERIOD_END = "统计期止" +#: 期 5：小计核对的键（核对编号 T、合计格 DerivedItem.segment）= 块 id 加这个后缀。块 id 在整份配方内唯一， +#: 同一块的合计行用块 id 本身，两者不相混 +SUBTOTAL_KEY = "·小计" _SQL_TYPE = {"INTEGER": "INTEGER", "REAL": "REAL", "TEXT": "TEXT", "DATE": "TEXT"} @@ -1107,16 +1150,42 @@ def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[Reci cols = [ColumnOut(c.name, _SQL_TYPE[c.type], header=c.header, role="measure" if c.type in ("INTEGER", "REAL") else "text") for c in block.columns] + if block.period is not None and block.period.synthesize: + # 期 5：每行加上本期统计期的起止（role=period：没有格子来历，取值来自统计期） + cols += [ColumnOut(LIST_PERIOD_START, "TEXT", role="period"), + ColumnOut(LIST_PERIOD_END, "TEXT", role="period")] put(block.table, cols, base) total = block.rows.total_row + numeric = [c for c in cols if c.type in ("INTEGER", "REAL")] if total is not None and total.keep_as: tcols = [ColumnOut(LIST_TOTAL_DIM, "TEXT", role="dim")] - tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") - for c in cols if c.type in ("INTEGER", "REAL")] + tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") for c in numeric] put(total.keep_as, tcols, f"{base}/rows/total_row") + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + # 期 5：小计另存：小计项（标签的规范写法）、按组小计时的分组列、各数字列 + scols = [ColumnOut(LIST_SUBTOTAL_DIM, "TEXT", role="dim")] + if sub.scope == "group_column" and sub.group_column: + scols.append(ColumnOut(sub.group_column, "TEXT", role="dim")) + scols += [ColumnOut(c.name, c.type, header=c.header, role="value") for c in numeric] + put(sub.keep_as, scols, f"{base}/rows/subtotals") return out, problems +def list_period_columns(recipe: Recipe) -> dict[str, tuple[str, ...]]: + """期 5：列表写入的表 → 它按什么分期：("column", 日期列) 或 ("synthesize", 统计期起列, 统计期止列)。只列配了 + ListBlock.period 的列表；累积计划的 U3、单期的 D 核对、说明里「按哪一列筛选」共用这一份。""" + out: dict[str, tuple[str, ...]] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock) and block.period is not None: + if block.period.synthesize: + out.setdefault(block.table, ("synthesize", LIST_PERIOD_START, LIST_PERIOD_END)) + elif block.period.column: + out.setdefault(block.table, ("column", block.period.column)) + return out + + # ========================================================================== # 期 3：按期累积的资格（静态校验、起草器、累积计划共用同一份，P3-SPEC 2.2） # ========================================================================== @@ -1130,10 +1199,13 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: 1. 至少一个工作表配置了统计期上下文（SheetRecipe.context 非空），否则报 accumulate_needs_period（path=/mode）； 2. derive_tables 推出的每张表都「按统计期分得开」，否则逐表报 accumulate_unkeyed（path=/tables/{ti}，表没在 tables 里声明时指向第一个写入它的块），消息写明哪一条不满足： - - 只由交叉表块写入（measures、dimension 段，或者 derived 段的 keep_as）——列表写入的表一律不满足； - - 写入它的每个交叉表块，axis.checks 都含 covers_context（日期恰好落在统计期内）； - - 这张表的 grain 含那个块的 axis.name。 - 两条都成立时，各期的日期集合互不相交（统计期不重叠由累积计划保证），主键里不需要另加统计期列。 + - 交叉表写入的（measures、dimension 段，或者 derived 段的 keep_as）：写入它的每个交叉表块，axis.checks 都含 + covers_context（日期恰好落在统计期内），并且这张表的 grain 含那个块的 axis.name； + - 期 5：列表写入的明细表：写入它的每个列表块都配了 period（按日期列核对，或加统计期起止两列），并且所在的 + 工作表配了统计期。列表的合计行、小计行另存的表不能累积：各期的合计项、小计项相同，主键分不开各期； + - 同一张表不能既由交叉表又由列表写入。 + 都成立时，各期的行按统计期分得开（统计期不重叠由累积计划保证；列表明细跨期重复的同一条记录由并集的主键核对 + U2 拒收）。 """ problems: list[RecipeProblem] = [] if not any(sheet.context for sheet in recipe.sheets): @@ -1144,6 +1216,11 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: writers: dict[str, list[CrosstabBlock | ListBlock]] = {} first_path: dict[str, str] = {} + #: 列表的合计行、小计行另存的表 → 写入它的块 + totals: dict[str, list[ListBlock]] = {} + #: 列表块 → 所在工作表有没有配统计期 + ctx_of: dict[int, bool] = {} + def _add(table: str, block: CrosstabBlock | ListBlock, path: str) -> None: got = writers.setdefault(table, []) if all(b is not block for b in got): @@ -1161,16 +1238,32 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: _add(seg.keep_as.table, block, path) else: _add(block.table, block, path) - total = block.rows.total_row - if total is not None and total.keep_as: - _add(total.keep_as, block, path) + ctx_of[id(block)] = bool(sheet.context) + for kept in (block.rows.total_row.keep_as if block.rows.total_row else None, + block.rows.subtotals.keep_as if block.rows.subtotals else None): + if kept: + _add(kept, block, path) + totals.setdefault(kept, []).append(block) index = {t.name: ti for ti, t in enumerate(recipe.tables)} grains = {t.name: list(t.grain) for t in recipe.tables} for name in tables: blocks = writers.get(name, []) - if any(isinstance(b, ListBlock) for b in blocks): - reasons = ["列表形态的表目前只支持每期替换"] + lists = [b for b in blocks if isinstance(b, ListBlock)] + if lists and len(lists) != len(blocks): + reasons = ["同一张表既由交叉表又由列表写入，各期的行分不开"] + elif name in totals: + reasons = ["列表的合计行、小计行另存的表不能按期累积：各期的合计项、小计项相同，主键分不开各期" + "（可以改为只核对、不另存）"] + elif lists: + reasons = [] + unbound = [b.id for b in lists if b.period is None] + if unbound: + reasons.append("、".join(f"列表「{x}」" for x in unbound) + + "没有指定每一行属于哪一期（按日期列核对，或加统计期起止两列）") + no_ctx = [b.id for b in lists if b.period is not None and not ctx_of.get(id(b))] + if no_ctx: + reasons.append("、".join(f"列表「{x}」" for x in no_ctx) + "所在的工作表没有配置统计期") else: reasons = [] uncovered = [b.id for b in blocks if isinstance(b, CrosstabBlock) and "covers_context" not in b.axis.checks] diff --git a/backend/app/data/xlsx_cells.py b/backend/app/data/xlsx_cells.py index 28f137e0..c270f8bb 100644 --- a/backend/app/data/xlsx_cells.py +++ b/backend/app/data/xlsx_cells.py @@ -24,7 +24,7 @@ from collections.abc import Iterator from typing import Any from xml.etree import ElementTree as ET -from app.data import tabular +from app.data import csv_grid, tabular from app.data.recipe_types import Grid, GridCell from app.data.xlsx_scan import SheetScan, UnsupportedTable, WorkbookScan, _broken, cell_ref, parse_ref @@ -70,6 +70,13 @@ class Workbooks: self.raw = raw self._values: Any = None self._formulas: Any = None + self._csv: Any = None + + def csv(self) -> Any: + """CSV 原件解析好的记录（csv_grid.CsvTable），第一次要读时才解析，同一次执行里复用。""" + if self._csv is None: + self._csv = csv_grid.parse(self.raw) + return self._csv def values(self) -> Any: if self._values is None: @@ -126,6 +133,10 @@ def sheet_rows(books: Workbooks, sheet: SheetScan, *, min_row: int, max_row: int 行数和每行的宽度都一样。文件内容写坏了（见 _READ_ERRORS）抛 UnsupportedTable，日志只记异常类型。 """ try: + if sheet.path == csv_grid.CSV_PATH: + # CSV 适配出来的「工作表」：从解析好的记录里取，口径同下（只给非空格、整行为空的不给） + yield from csv_grid.rows(books.csv(), min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col) + return yield from _sheet_rows(books, sheet, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col) except UnsupportedTable: raise diff --git a/backend/app/data/xlsx_scan.py b/backend/app/data/xlsx_scan.py index 4910c6ef..5acd20fd 100644 --- a/backend/app/data/xlsx_scan.py +++ b/backend/app/data/xlsx_scan.py @@ -216,6 +216,9 @@ class WorkbookScan: tables: list[TableScan] = field(default_factory=list) #: 定义名称 → 原文。工作表级的名称记作「工作表名!名称」，免得同名互相覆盖 defined_names: dict[str, str] = field(default_factory=dict) + #: 原件的格式：xlsx，或者 csv（csv_grid.scan 适配出来的一张「工作表」，读取层据 SheetScan.path 从 CSV 里读）。 + #: 老暂存区的扫描结果里没有这个键，按 xlsx 读 + format: str = "xlsx" def sheet(self, name: str) -> SheetScan | None: return next((s for s in self.sheets if s.name == name), None) @@ -1090,4 +1093,5 @@ def scan_from_json(data: dict[str, Any]) -> WorkbookScan: for t in tables: t.columns = list(t.columns) return WorkbookScan(sheets=sheets, full_calc_on_load=bool(data.get("full_calc_on_load")), tables=tables, - defined_names=dict(data.get("defined_names") or {})) + defined_names=dict(data.get("defined_names") or {}), + format=str(data.get("format") or "xlsx")) diff --git a/backend/tests/test_csv_grid.py b/backend/tests/test_csv_grid.py new file mode 100644 index 00000000..30a5f78a --- /dev/null +++ b/backend/tests/test_csv_grid.py @@ -0,0 +1,111 @@ +"""CSV 走配方（Excel 期 5 第 4 项）：CSV 适配成网格，后面的执行器、核对、溯源全部复用。 + +格子坐标沿用「工作表!A1」（见 csv_grid 的模块说明）：CSV 没有工作表，名字固定是「CSV」，每月的文件名变了也认得出。 +""" +from __future__ import annotations + +import pytest + +from app.data import csv_grid, recipe_engine, xlsx_cells +from app.data.recipe_types import Recipe +from app.data.xlsx_scan import UnsupportedTable, scan_from_json, scan_to_json + +CSV = ("订单号,下单日期,门店,金额,备注\n" + "A001,2026-08-01,一号店,120,\n" + "A002,2026-08-02,二号店,80.5,\"带,逗号\"\n" + "\n" + "A003,2026-08-03,一号店,1,234,多出的一格\n").encode("utf-8") + + +def test_text_is_sniffed_as_csv_and_binary_is_not(): + assert csv_grid.looks_like_csv(CSV) + assert csv_grid.looks_like_csv("a;b\n1;2\n".encode("gb18030")) + assert csv_grid.looks_like_csv(b"\xff\xfe" + "a\tb\n".encode("utf-16-le")) + assert not csv_grid.looks_like_csv(b"PK\x03\x04rest-of-zip") + assert not csv_grid.looks_like_csv(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100) # 旧版 .xls + assert not csv_grid.looks_like_csv(b"\x89PNG\r\n\x1a\n\x00\x00") + assert not csv_grid.looks_like_csv(b"") + + +def test_scan_gives_one_visible_sheet_with_bounds_and_row_counts(): + scan = csv_grid.scan(CSV) + assert scan.format == "csv" and len(scan.sheets) == 1 + sh = scan.sheets[0] + assert sh.name == csv_grid.CSV_SHEET and sh.state == "visible" and sh.path == csv_grid.CSV_PATH + assert (sh.bounds.min_row, sh.bounds.min_col, sh.bounds.max_row, sh.bounds.max_col) == (1, 1, 5, 6) + # 空字段不算非空格；第 4 行是空行 + assert sh.row_runs == [(1, 1, 5), (2, 2, 4), (3, 3, 5), (5, 5, 6)] + assert sh.nonempty == 20 + again = scan_from_json(scan_to_json(scan)) + assert again.format == "csv" and again.sheets[0].path == csv_grid.CSV_PATH + + +def test_cell_values_are_typed_only_when_unambiguous(): + """整数、小数写成数；千分位、前导零、正号、科学计数法、日期都留作文字，交给配方的规则（text_number、类型）判。""" + v = csv_grid.cell_value + assert v("120") == 120 and v(" 80.5 ") == 80.5 and v("-3") == -3 and v("0") == 0 + for text in ("1,234", "007", "+5", "1e5", "2026-08-01", "12%", "TRUE", "=SUM(A1:A2)", "1234567890123456"): + assert v(text) == text, text + assert v("") is None and v(" ") is None + + +def test_grid_and_rows_read_the_same_cells(): + scan = csv_grid.scan(CSV) + grid = xlsx_cells.read_grid(CSV, scan, csv_grid.CSV_SHEET) + assert grid.get(2, 4).value == 120 and grid.get(3, 4).value == 80.5 and grid.get(3, 5).value == "带,逗号" + assert grid.get(2, 5) is None and grid.get(1, 1).value == "订单号" + assert grid.get(5, 4).value == 1 and grid.get(5, 5).value == 234 # 没加引号的千分位被逗号拆开了：照实读 + rows = list(xlsx_cells.iter_rows(CSV, scan, csv_grid.CSV_SHEET, min_row=1, max_row=5, min_col=1, max_col=6)) + assert [r for r, _ in rows] == [1, 2, 3, 5] + assert sum(len(cells) for _, cells in rows) == len(grid.cells) == scan.sheets[0].nonempty + small = xlsx_cells.read_grid(CSV, scan, csv_grid.CSV_SHEET, max_rows=2) + assert small.truncated and set(r for r, _ in small.cells) == {1, 2} + + +def test_semicolon_and_tab_delimiters_and_gbk(): + scan = csv_grid.scan("名称;数量\n甲;3\n".encode("gb18030")) + grid = xlsx_cells.read_grid("名称;数量\n甲;3\n".encode("gb18030"), scan, csv_grid.CSV_SHEET) + assert grid.get(2, 1).value == "甲" and grid.get(2, 2).value == 3 + raw = "a\tb\n1\t2\n".encode("utf-8") + assert xlsx_cells.read_grid(raw, csv_grid.scan(raw), csv_grid.CSV_SHEET).get(2, 2).value == 2 + + +def test_scan_any_dispatches_and_keeps_the_xlsx_refusal(): + assert csv_grid.scan_any(CSV).format == "csv" + with pytest.raises(UnsupportedTable): + csv_grid.scan_any(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100) + + +def test_too_many_columns_is_refused(): + raw = (",".join(["x"] * (csv_grid.MAX_COLS + 1)) + "\n").encode() + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan(raw) + assert "列" in str(e.value) + + +LIST_RECIPE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "CSV"}, + "blocks": [{"id": "列表1", "layout": "list", "table": "订单", + "columns": [{"header": "订单号", "name": "订单号", "type": "TEXT"}, + {"header": "下单日期", "name": "下单日期", "type": "DATE"}, + {"header": "门店", "name": "门店", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "REAL"}], + "rows": {"blank_rows": "stop"}}]}], + "tables": [{"name": "订单", "grain": ["订单号"]}], +} + + +def test_engine_runs_a_list_recipe_on_csv(tmp_path): + raw = "订单号,下单日期,门店,金额\nA001,2026-08-01,一号店,120\nA002,2026-08-02,二号店,80.5\n".encode() + scan = csv_grid.scan(raw) + db = tmp_path / "t.sqlite" + ex = recipe_engine.execute(Recipe.model_validate(LIST_RECIPE), raw, "orders_2026-08.csv", scan, str(db)) + assert ex.ok, [p.message for p in ex.problems] + assert ex.tables[0].rows == 2 and ex.tables[0].sheet == "CSV" + # 溯源照常：订单号这一列从 CSV!A2 起往下 2 格 + run = ex.lineage["订单"]["订单号"][0] + assert run[1:] == ["CSV", "A2", 2, "down"] + import sqlite3 + rows = sqlite3.connect(db).execute('SELECT "订单号", "下单日期", "金额" FROM "订单" ORDER BY rowid').fetchall() + assert rows == [("A001", "2026-08-01", 120.0), ("A002", "2026-08-02", 80.5)] diff --git a/backend/tests/test_recipe_contract.py b/backend/tests/test_recipe_contract.py index 34800e71..50b1f0b9 100644 --- a/backend/tests/test_recipe_contract.py +++ b/backend/tests/test_recipe_contract.py @@ -370,10 +370,12 @@ def test_accumulate_blockers_reference_recipe_is_eligible(): def test_accumulate_blockers_list_shape_is_unkeyed(): data = json.loads(json.dumps(C01_LIKE)) - data["sheets"][0]["context"] = [{}] # 即使配了统计期，列表写入的表也不能累积 + data["sheets"][0]["context"] = [{}] # 配了统计期，列表却没说每一行属于哪一期：不能累积 got = T.accumulate_blockers(T.Recipe.model_validate(data)) assert [(p.code, p.path) for p in got] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] - assert all("列表形态的表目前只支持每期替换" in p.message for p in got) + assert "没有指定每一行属于哪一期" in got[0].message + # 合计行另存的表：各期的合计项相同，主键分不开各期（期 5 列表可以累积以后仍然不行） + assert "合计行、小计行另存的表不能按期累积" in got[1].message assert "「月报」" in got[0].message and "「月报_表内合计」" in got[1].message # c01 的草稿本来就没有统计期：两条都报，统计期那条在前（起草卡片取第一条的人话） got = T.accumulate_blockers(T.Recipe.model_validate(C01_LIKE)) @@ -479,7 +481,8 @@ def test_closed_value_sets(): assert {"union_rows", "union_pk", "union_period"} <= set(_literal(T.CheckResult, "kind")) assert "ignored" in typing.get_args(T.Role) and "ignored_column" in typing.get_args(T.Role) assert typing.get_args(T.ExcludedReason) == ( - "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept") + "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept", + "subtotal") assert _literal(T.FixAnchor, "kind") == ("problem", "recipe_problem", "sheet_renamed") assert _literal(T.Anchor, "kind") == ( "header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text") diff --git a/backend/tests/test_recipe_engine_messages.py b/backend/tests/test_recipe_engine_messages.py index 26dd3d84..fef58344 100644 --- a/backend/tests/test_recipe_engine_messages.py +++ b/backend/tests/test_recipe_engine_messages.py @@ -318,6 +318,18 @@ def c_total_row_blank(mp): return _run(sales_book(edit=edit), LIST), [N] +def c_subtotal_blank(mp): + """期 5：穿插的小计行数字格全空：没有一列能核对，结构类。""" + data = list_recipe() + data["sheets"][0]["blocks"][0]["rows"]["subtotals"] = {"label_column": "地区", "pick": "小计"} + + def edit(ws): + ws["D4"] = N + ws.insert_rows(6) + ws["A6"] = "小计" # 第 6 行是小计：标签在，数字格全空 + return _run(sales_book(edit=edit), data), [N] + + def c_value_not_integer(mp): return _run(_flow(_set(C11=9876.54)), FLOW), [9876.54] @@ -423,6 +435,7 @@ NEXT_STEPS = { "hidden_cols": "在配方面板的「隐藏列」中选择「照常导入」", "total_row_missing": "在配方面板的「合计行」中取消「有合计行」", "total_row_blank": "在配方面板的「合计行」中取消「有合计行」", + "subtotal_blank": "在配方面板的「小计行」中取消「有小计行」", } diff --git a/backend/tests/test_recipe_notes_p3.py b/backend/tests/test_recipe_notes_p3.py index ba454c55..cfc9a6e8 100644 --- a/backend/tests/test_recipe_notes_p3.py +++ b/backend/tests/test_recipe_notes_p3.py @@ -532,7 +532,8 @@ def test_problems_for_inconsistent_inputs(): def test_list_tables_cannot_be_merged_and_are_problems(): - """列表不能按期累积（P3-SPEC 2.2）：真把两期列表并起来，列表合计没有并集说法，记 problems，绝不沿用单期说法。""" + """列表合计行另存的表不能按期累积（accumulate_blockers）：真把两期这样的列表并起来，列表合计没有并集说法，记 + problems，绝不沿用单期说法。""" data = deepcopy(LIST_RECIPE) def lpart(span): @@ -544,7 +545,7 @@ def test_list_tables_cannot_be_merged_and_are_problems(): tgt = Recipe.model_validate(data) notes = build_union_notes(tgt, [lpart(AUG), lpart(SEP)], union_tables=derive_tables(tgt)[0], null_counts=None, part_null_counts=None, added=[], retired=[], label_sets=[], gaps=False) - assert any("列表合计行的核对结论不能按期合并" in p for p in notes.problems) + assert any("列表合计行、小计行的核对结论不能按期合并" in p for p in notes.problems) assert not any("list_total" in p for p in notes.problems) # 不露键名 assert "一致" not in notes.tables["销售_表内合计"].comment diff --git a/backend/tests/test_recipe_p5.py b/backend/tests/test_recipe_p5.py new file mode 100644 index 00000000..b635a014 --- /dev/null +++ b/backend/tests/test_recipe_p5.py @@ -0,0 +1,435 @@ +"""Excel 期 5（下一轮 03 细案 E）：列表按期累积、穿插的小计、表格对象起草、CSV 走配方。 + +不连接口的部分：执行器、核对、静态校验、累积资格、并集物化、起草。夹具全是合成的通用名字（门店一、分区甲、 +订单 A001……），数字是手写的小数，手算得出。接口的端到端在 test_recipe_p5_e2e.py。 +""" +from __future__ import annotations + +import copy +import io +import os +import sqlite3 +import tempfile +from pathlib import Path +from typing import Any + +import pytest +from openpyxl import Workbook +from openpyxl.worksheet.table import Table + +from app.data import csv_grid, raw_store, recipe_checks, recipe_engine, recipe_suggest, table_versions, xlsx_cells +from app.data import xlsx_scan +from app.data.recipe import validate_recipe +from app.data.recipe_accumulate import materialize_union +from app.data.recipe_types import ( + LIST_PERIOD_END, LIST_PERIOD_START, SUBTOTAL_KEY, Recipe, UnionPart, accumulate_blockers, derive_tables, +) + + +def _book(rows: list[tuple[Any, ...]], *, sheet: str = "明细", top: list[tuple[str, Any]] | None = None, + start: int = 1) -> bytes: + wb = Workbook() + ws = wb.active + ws.title = sheet + for ref, value in top or []: + ws[ref] = value + for r, row in enumerate(rows, start=start): + for c, v in enumerate(row, start=1): + if v is not None: + ws.cell(r, c, v) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def _run(raw: bytes, recipe: dict[str, Any], db: str | None = None, filename: str = "明细.xlsx", **kw: Any): + scan = csv_grid.scan_any(raw) + return recipe_engine.execute(Recipe.model_validate(recipe), raw, filename, scan, db, **kw) + + +def _rows(db: str, sql: str) -> list[tuple[Any, ...]]: + conn = sqlite3.connect(db) + try: + return conn.execute(sql).fetchall() + finally: + conn.close() + + +def _checks(db: str, ex: Any, recipe: dict[str, Any]) -> dict[str, Any]: + return {c.id: c for c in recipe_checks.run_checks(db, ex, Recipe.model_validate(recipe))} + + +def _list(columns: list[tuple[str, str]], *, table: str = "明细", sheet: str = "明细", context: bool = False, + **block: Any) -> dict[str, Any]: + out = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": sheet}, + "context": [{"id": "统计期"}] if context else [], + "blocks": [{"id": "列表1", "layout": "list", "table": table, + "columns": [{"header": h, "name": h, "type": t} for h, t in columns], **block}]}], + "tables": [{"name": table, "grain": []}], + } + return out + + +# ========================================================================== +# 穿插的小计 +# ========================================================================== + +STORES = [("地区", "门店", "金额", "数量"), + ("分区甲", "门店一", 100, 1), ("分区甲", "门店二", 200, 2), ("小计", None, 300, 3), + ("分区乙", "门店三", 150, 4), ("分区乙", "门店四", 150, 5), ("小计", None, 300, 9), + ("合计", None, 600, 12)] +STORE_COLS = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER"), ("数量", "INTEGER")] + + +def _stores_recipe(**sub: Any) -> dict[str, Any]: + return _list(STORE_COLS, rows={"total_row": {"label_column": "地区", "pick": "合计"}, + "subtotals": {"label_column": "地区", "pick": "小计", **sub}}) + + +def test_subtotal_rows_are_checked_against_their_group_and_kept_out_of_the_data(tmp_path): + db = str(tmp_path / "s.db") + recipe = _stores_recipe() + ex = _run(_book(STORES), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + # 只有 4 行明细进了数据；两行小计、一行合计都不在 + assert _rows(db, 'SELECT "门店", "金额" FROM "明细" ORDER BY rowid') == [ + ("门店一", 100), ("门店二", 200), ("门店三", 150), ("门店四", 150)] + subs = [d for d in ex.derived if d.segment == f"列表1{SUBTOTAL_KEY}"] + assert [(d.cell, d.rowid_first, d.rowid_last) for d in subs] == [ + ("明细!C4", 1, 2), ("明细!D4", 1, 2), ("明细!C7", 3, 4), ("明细!D7", 3, 4)] + [excluded] = [x for x in ex.rows_excluded if x.reason == "subtotal"] + assert excluded.rows == [[4, 4], [7, 7]] and excluded.anchor == "小计" and excluded.block == "列表1" + checks = _checks(db, ex, recipe) + assert checks["T1"].title == "合计行按列求和（块「列表1」）" and checks["T1"].status == "passed" + assert checks["T2"].title == "小计行按各组明细求和（块「列表1」）" and checks["T2"].status == "passed" + assert checks["T2"].checked == 4 + + +def test_a_wrong_subtotal_is_a_mismatch(tmp_path): + rows = list(STORES) + rows[6] = ("小计", None, 300, 10) # 4 + 5 ≠ 10 + db = str(tmp_path / "s.db") + recipe = _stores_recipe() + ex = _run(_book(rows), recipe, db) + checks = _checks(db, ex, recipe) + t2 = checks["T2"] + assert t2.status == "mismatch" and t2.failed == 1 and t2.cells == ["明细!D7"] + assert "按明细求和为 9" in t2.details[0] + + +def test_group_column_scope_only_covers_the_rows_of_the_same_group(tmp_path): + """按组小计：紧挨在小计上面、分组列取值相同的那几行。小计行的标签写在「门店」列，分组按「地区」。 + 同一份数据按「上一行小计之后的全部明细」核对就对不上（350 ≠ 50）。""" + rows = [("地区", "门店", "金额"), ("分区甲", "门店一", 100), ("分区甲", "门店二", 200), + ("分区乙", "门店三", 50), ("分区乙", "小计", 50)] + cols = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER")] + recipe = _list(cols, rows={"subtotals": {"label_column": "门店", "pick": "小计", "scope": "group_column", + "group_column": "地区", "keep_as": "明细_小计"}}) + recipe["tables"].append({"name": "明细_小计", "kind": "reported_total"}) + db = str(tmp_path / "g.db") + ex = _run(_book(rows), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + assert _checks(db, ex, recipe)["T1"].status == "passed" + assert _rows(db, 'SELECT * FROM "明细_小计"') == [("小计", "分区乙", 50)] + assert [c.name for c in derive_tables(Recipe.model_validate(recipe))[0]["明细_小计"]] == ["小计项", "地区", "金额"] + since = copy.deepcopy(recipe) + since["sheets"][0]["blocks"][0]["rows"]["subtotals"].update(scope="since_previous", group_column=None) + since["tables"][1]["name"] = "明细_小计" + db2 = str(tmp_path / "g2.db") + ex2 = _run(_book(rows), since, db2) + assert _checks(db2, ex2, since)["T1"].status == "mismatch" + + +def test_subtotal_validation(): + def problems(**sub: Any) -> list[str]: + data = _stores_recipe(**sub) + _, probs = validate_recipe(data, facts=None, origin="rules") + return [p.code for p in probs] + + assert problems() == [] + assert "subtotal_group_missing" in problems(scope="group_column") + assert "column_unknown" in problems(scope="group_column", group_column="城市") + assert "subtotal_group_missing" in problems(group_column="地区") + assert "period_literal" in problems(pick="2026小计") + bad = _stores_recipe() + bad["sheets"][0]["blocks"][0]["rows"]["subtotals"]["pick"] = "合计" + assert "subtotal_pick_conflict" in [p.code for p in validate_recipe(bad, facts=None, origin="rules")[1]] + + +# ========================================================================== +# 列表按期累积：每一行属于哪一期 +# ========================================================================== + +PERIOD_CELL = ("A1", "统计时间：2026年8月1日至2026年8月31日") +ORDERS = [("订单号", "下单日期", "金额"), ("A001", "2026-08-01", 120), ("A002", "2026-08-15", 80), + ("A003", "2026-08-31", 40)] +ORDER_COLS = [("订单号", "TEXT"), ("下单日期", "DATE"), ("金额", "INTEGER")] + + +def _orders(period: dict[str, Any], *, context: bool = True, grain: list[str] | None = None) -> dict[str, Any]: + data = _list(ORDER_COLS, table="订单", context=context, period=period) + data["tables"][0]["grain"] = grain if grain is not None else ["订单号"] + return data + + +def test_synthesized_period_columns_carry_this_periods_bounds(tmp_path): + recipe = _orders({"synthesize": "统计期"}, grain=["订单号", LIST_PERIOD_START]) + cols = derive_tables(Recipe.model_validate(recipe))[0]["订单"] + assert [(c.name, c.role) for c in cols][-2:] == [(LIST_PERIOD_START, "period"), (LIST_PERIOD_END, "period")] + db = str(tmp_path / "p.db") + ex = _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + assert _rows(db, f'SELECT "订单号", "{LIST_PERIOD_START}", "{LIST_PERIOD_END}" FROM "订单" ORDER BY rowid') == [ + ("A001", "2026-08-01", "2026-08-31"), ("A002", "2026-08-01", "2026-08-31"), + ("A003", "2026-08-01", "2026-08-31")] + assert ex.table_hashes["订单"] + # 统计期起止列没有格子来历：溯源只有表头推出来的三列 + assert set(ex.lineage["订单"]) == {"订单号", "下单日期", "金额"} + # 干跑（不建库）同样走得通 + assert _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe).ok + + +def test_synthesized_period_with_a_human_period(tmp_path): + import datetime as dt + + from app.data.recipe_types import PeriodInput + + recipe = _orders({"synthesize": "统计期"}) + raw = _book(ORDERS) + assert {p.code for p in _run(raw, recipe).problems} == {"period_missing"} + db = str(tmp_path / "h.db") + ex = _run(raw, recipe, db, context_inputs={"统计期": PeriodInput(dt.date(2026, 8, 1), dt.date(2026, 8, 31))}) + assert ex.ok and _rows(db, f'SELECT DISTINCT "{LIST_PERIOD_START}" FROM "订单"') == [("2026-08-01",)] + + +def test_rows_must_fall_inside_the_period_when_bound_to_a_date_column(tmp_path): + recipe = _orders({"column": "下单日期"}) + db = str(tmp_path / "w.db") + ex = _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, db) + assert ex.ok + d1 = _checks(db, ex, recipe)["D1"] + assert d1.kind == "period_rows" and d1.status == "passed" and d1.checked == 3 + late = list(ORDERS) + [("A004", "2026-09-02", 10)] + db2 = str(tmp_path / "w2.db") + ex2 = _run(_book(late, top=[PERIOD_CELL], start=3), recipe, db2) + d1 = _checks(db2, ex2, recipe)["D1"] + assert d1.status == "mismatch" and d1.category == "structure" and d1.failed == 1 + assert d1.cells == ["明细!B7"] and "2026-08-01 至 2026-08-31" in d1.details[0] + + +def test_list_period_validation(): + def codes(data: dict[str, Any]) -> list[str]: + return [p.code for p in validate_recipe(data, facts=None, origin="rules")[1]] + + assert codes(_orders({"column": "下单日期"})) == [] + assert "list_period_invalid" in codes(_orders({"column": "金额"})) + assert "column_unknown" in codes(_orders({"column": "付款日期"})) + assert "list_period_invalid" in codes(_orders({"column": "下单日期", "synthesize": "统计期"})) + assert "list_period_invalid" in codes(_orders({})) + assert "covers_without_context" in codes(_orders({"synthesize": "统计期"}, context=False)) + clash = _list(ORDER_COLS + [(LIST_PERIOD_START, "TEXT")], table="订单", context=True, + period={"synthesize": "统计期"}) + assert "name_collision" in codes(clash) + + +def test_lists_become_eligible_for_accumulation_once_bound(): + unbound = Recipe.model_validate({**_list(ORDER_COLS, table="订单", context=True), "mode": "accumulate"}) + [p] = accumulate_blockers(unbound) + assert "没有指定每一行属于哪一期" in p.message + for period in ({"column": "下单日期"}, {"synthesize": "统计期"}): + data = {**_orders(period), "mode": "accumulate"} + assert accumulate_blockers(Recipe.model_validate(data)) == [] + assert validate_recipe(data, facts=None, origin="rules")[1] == [] + kept = {**_orders({"column": "下单日期"}), "mode": "accumulate"} + kept["sheets"][0]["blocks"][0]["rows"] = {"total_row": {"label_column": "订单号", "pick": "合计", + "keep_as": "订单_表内合计"}} + kept["tables"].append({"name": "订单_表内合计", "grain": ["合计项"], "kind": "reported_total"}) + blockers = accumulate_blockers(Recipe.model_validate(kept)) + assert [p.path for p in blockers] == ["/tables/1"] and "合计行、小计行另存的表" in blockers[0].message + + +def _built(raw: bytes, recipe: dict[str, Any], tag: str) -> tuple[str, Any]: + folder = Path(tempfile.mkdtemp(prefix="p5-built-")) + db = folder / f"{tag}.db" + ex = _run(raw, recipe, str(db)) + assert ex.ok, [p.message for p in ex.problems] + table_versions.settle_journal(db) + return str(db), ex + + +def _part(db: str, ex: Any, recipe: dict[str, Any], **kw: Any) -> UnionPart: + return UnionPart(import_id=None, start=kw.get("start", ex.period.start), end=kw.get("end", ex.period.end), + db_path=db, recipe=Recipe.model_validate(recipe), db_sha256=raw_store.sha256_file(db), + table_hashes=dict(ex.table_hashes)) + + +@pytest.mark.parametrize("period", [{"column": "下单日期"}, {"synthesize": "统计期"}]) +def test_two_months_of_a_list_materialize_into_one_union(tmp_path, period): + recipe = {**_orders(period), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, "aug") + sep_rows = [ORDERS[0], ("B001", "2026-09-03", 70), ("B002", "2026-09-30", 30)] + sep = _built(_book(sep_rows, top=[("A1", "统计时间：2026年9月1日至2026年9月30日")], start=3), recipe, "sep") + out = tmp_path / "u.db" + report = materialize_union(out, target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe), _part(*sep, recipe)]) + assert report.ok, [c.details for c in report.checks] + assert report.rows == {"订单": 5} + assert {c.id: c.status for c in report.checks} == {"U1": "passed", "U2": "passed", "U3": "passed"} + assert report.checks[2].checked == 2 + # 登记的统计期和库里的对不上（把 8 月的库说成 9 月）：U3 拒收 + bad = materialize_union(tmp_path / "bad.db", target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe, start="2026-09-01", end="2026-09-30")]) + u3 = next(c for c in bad.checks if c.id == "U3") + assert not bad.ok and u3.status == "mismatch" and "第 1 期" in u3.details[0] + + +def test_the_same_order_in_two_periods_is_a_union_key_conflict(tmp_path): + """明细跨期重复（同一订单两期都出现）按日期列分期时主键冲突，U2 拒收（细案风险条：语义要在配方里显式声明）。""" + recipe = {**_orders({"synthesize": "统计期"}), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, "aug2") + sep = _built(_book(ORDERS, top=[("A1", "统计时间：2026年9月1日至2026年9月30日")], start=3), recipe, "sep2") + report = materialize_union(tmp_path / "u.db", target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe), _part(*sep, recipe)]) + u2 = next(c for c in report.checks if c.id == "U2") + assert not report.ok and u2.status == "mismatch" + # 主键带上统计期起就分得开：每月一份的快照式明细 + keyed = {**_orders({"synthesize": "统计期"}, grain=["订单号", LIST_PERIOD_START]), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), keyed, "aug3") + sep = _built(_book(ORDERS, top=[("A1", "统计时间：2026年9月1日至2026年9月30日")], start=3), keyed, "sep3") + assert materialize_union(tmp_path / "k.db", target=Recipe.model_validate(keyed), + parts=[_part(*aug, keyed), _part(*sep, keyed)]).rows == {"订单": 6} + + +# ========================================================================== +# 起草：表格对象、穿插的小计、每一行属于哪一期的问题 +# ========================================================================== + + +def _ok_validate(data, facts, origin): + from app.data.recipe import validate_recipe as v + + return v(data, facts=facts, origin=origin) + + +def _draft(raw: bytes, filename: str = "明细.xlsx"): + scan = csv_grid.scan_any(raw) + grids = {s.name: xlsx_cells.read_grid(raw, scan, s.name, max_rows=recipe_suggest.DRAFT_MAX_ROWS) + for s in scan.sheets if s.state == "visible" and s.nonempty} + + def dry(recipe: Recipe): + return recipe_engine.execute(recipe, raw, filename, scan, None) + return recipe_suggest.draft(scan, grids, filename, validate=_ok_validate, dry_run=dry), scan, grids + + +def _table_book(*, totals: bool = True, name: str = "门店销售") -> bytes: + """一个表格对象：B3:D7（表头、3 行明细、汇总行），表格上面有一行说明、右边有一个空格隔开的备注列。""" + wb = Workbook() + ws = wb.active + ws.title = "销售" + ws["B1"] = "门店销售明细" + rows = [("门店", "日期", "金额"), ("门店一", "2026-08-01", 10), ("门店二", "2026-08-02", 20), + (None, None, None), ("门店三", "2026-08-03", 30)] + if totals: + rows.append(("汇总", None, 60)) + for r, row in enumerate(rows, start=3): + for c, v in enumerate(row, start=2): + if v is not None: + ws.cell(r, c, v) + t = Table(displayName=name, ref=f"B3:D{2 + len(rows)}") + if totals: + t.totalsRowCount = 1 + ws.add_table(t) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def test_a_table_object_is_drafted_as_a_list_without_ai(): + raw = _table_book() + scan = xlsx_scan.scan(raw) + assert [(t.name, t.ref, t.totals_row_count) for t in scan.tables] == [("门店销售", "B3:D8", 1)] + draft, _, _ = _draft(raw) + assert draft.complete and draft.origin == "rules", draft.failures + [block] = draft.recipe["sheets"][0]["blocks"] + assert block["layout"] == "list" and block["table"] == "门店销售" + assert [c["header"] for c in block["columns"]] == ["门店", "日期", "金额"] + assert block["rows"]["total_row"] == {"label_column": "门店", "pick": "汇总", "keep_as": "门店销售_表内合计"} + assert block["rows"]["blank_rows"] == "skip" + # 锚点按表头文字，不记坐标 + assert "B3" not in str(block) + + +def test_a_table_object_with_digits_in_its_name_falls_back_to_the_title(): + draft, _, _ = _draft(_table_book(name="表1")) + assert draft.complete and draft.recipe["sheets"][0]["blocks"][0]["table"] == "门店销售明细" + + +def test_drafted_subtotals_run_and_verify(tmp_path): + raw = _book(STORES) + draft, scan, _ = _draft(raw) + assert draft.complete, draft.failures + block = draft.recipe["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"]["pick"] == "小计" and block["rows"]["total_row"]["pick"] == "合计" + db = str(tmp_path / "d.db") + recipe = Recipe.model_validate(draft.recipe) + ex = recipe_engine.execute(recipe, raw, "明细.xlsx", scan, db) + assert ex.ok and ex.tables[0].rows == 4 + assert all(c.status == "passed" for c in recipe_checks.run_checks(db, ex, recipe) if c.kind == "column_sum") + + +def test_period_question_binds_the_list_and_switches_to_accumulation(): + raw = _book(ORDERS, top=[PERIOD_CELL], start=3) + draft, _, grids = _draft(raw) + assert draft.complete, draft.failures + from app.data import recipe as R + + recipe = draft.recipe + cards, questions = recipe_suggest.questions_for(recipe, draft.facts, grids) + q = next(q for q in questions if q.id == "q_list_period:列表1") + assert [o["value"] for o in q.options] == ["none", "column:下单日期", "synthesize"] and q.default == "none" + assert not any(q.id == "q_mode" for q in questions) + bound = R.apply_patch(recipe, q.effects["column:下单日期"]) + assert bound["sheets"][0]["blocks"][0]["period"]["column"] == "下单日期" and bound["mode"] == "accumulate" + assert validate_recipe(bound, facts=None, origin="rules")[1] == [] + assert R.apply_patch(recipe, q.effects["none"])["mode"] == "replace" + # 工作表没配统计期的，选了以后一并加上 + plain = _book(ORDERS) + draft2, _, grids2 = _draft(plain) + assert draft2.recipe["sheets"][0]["context"] == [] + _, qs = recipe_suggest.questions_for(draft2.recipe, draft2.facts, grids2) + q2 = next(q for q in qs if q.id == "q_list_period:列表1") + bound2 = R.apply_patch(draft2.recipe, q2.effects["synthesize"]) + assert bound2["sheets"][0]["context"][0]["id"] == "统计期" + assert bound2["sheets"][0]["blocks"][0]["period"]["synthesize"] == "统计期" + assert accumulate_blockers(Recipe.model_validate(bound2)) == [] + + +# ========================================================================== +# CSV +# ========================================================================== + + +def test_csv_drafts_a_list_and_the_engine_runs_it(tmp_path): + raw = ("订单号,下单日期,门店,金额\n" + "A001,2026-08-01,一号店,120\n" + "A002,2026-08-02,二号店,80.5\n" + "小计,,,200.5\n" + "A003,2026-08-03,一号店,1\n" + "合计,,,201.5\n").encode("gb18030") + draft, scan, _ = _draft(raw, "orders_2026-08.csv") + assert draft.complete and draft.origin == "rules", draft.failures + sheet = draft.recipe["sheets"][0] + assert sheet["match"]["name"] == csv_grid.CSV_SHEET + block = sheet["blocks"][0] + assert [c["type"] for c in block["columns"]] == ["TEXT", "DATE", "TEXT", "REAL"] + assert block["rows"]["subtotals"]["pick"] == "小计" + db = str(tmp_path / "c.db") + recipe = Recipe.model_validate(draft.recipe) + ex = recipe_engine.execute(recipe, raw, "orders_2026-08.csv", scan, db) + assert ex.ok and ex.tables[0].rows == 3 + assert all(c.status == "passed" for c in recipe_checks.run_checks(db, ex, recipe) if c.kind == "column_sum") + # 回执的坐标照 Excel 打开这个 CSV 时的 A1 写 + [excluded] = [x for x in ex.rows_excluded if x.reason == "subtotal"] + assert excluded.sheet == "CSV" and excluded.rows == [[4, 4]] diff --git a/backend/tests/test_recipe_p5_e2e.py b/backend/tests/test_recipe_p5_e2e.py new file mode 100644 index 00000000..d5538bee --- /dev/null +++ b/backend/tests/test_recipe_p5_e2e.py @@ -0,0 +1,191 @@ +"""Excel 期 5 的端到端（真管线，经接口）：每月一份的 CSV 明细按期累积成一张表，重叠的那期被拒，同一期再传要确认 +替换；带穿插小计的 Excel 规则起草成功、小计核对通过、小计行不进数据。 + +夹具全部合成（订单号 A001…、门店一…），不调用任何模型（规则起草不完整时的 AI 可用性查询换成「不可用」的桩）。 +""" +from __future__ import annotations + +import io +import json +import uuid +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from openpyxl import Workbook + +from app.core.config import settings +from app.main import app + +API = "/api/datasources" +CSV_MIME = "text/csv" + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture(autouse=True) +def no_model(monkeypatch): + from app.data import recipe_ai + from app.data.recipe_types import AiAvailability + + async def unavailable(session): + return None, "", AiAvailability(False, reason="验收测试不接模型") + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", unavailable) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def _name() -> str: + return f"p5_{uuid.uuid4().hex[:10]}" + + +def month_csv(month: int, orders: list[tuple[str, int, int]]) -> tuple[bytes, str]: + """一个月的订单导出：订单号、下单日期、门店、金额。文件名带统计期（只作旁证，CSV 里没有统计期格）。""" + lines = ["订单号,下单日期,门店,金额"] + for oid, day, amount in orders: + lines.append(f"{oid},2026-{month:02d}-{day:02d},门店{'一二'[day % 2]},{amount}") + last = 31 if month in (8, 10) else 30 + return ("\n".join(lines) + "\n").encode("utf-8"), f"orders_2026-{month:02d}-01_2026-{month:02d}-{last}.csv" + + +AUG = [("A001", 1, 120), ("A002", 15, 80), ("A003", 31, 40)] +SEP = [("B001", 3, 70), ("B002", 30, 30)] + + +async def stage(client, raw: bytes, filename: str, name: str) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/stage", files={"file": (filename, raw, CSV_MIME)}, data={"name": name}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def answer(client, sid: str, answers: dict[str, Any]) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/answers", json={"answers": answers}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def trial(client, sid: str, **body: Any) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/trial", json=body) + assert resp.status_code == 200, resp.text + return resp.json() + + +def _period(month: int) -> dict[str, Any]: + last = 31 if month in (8, 10) else 30 + return {"统计期": {"start": f"2026-{month:02d}-01", "end": f"2026-{month:02d}-{last}"}} + + +async def commit(client, st: dict[str, Any], confirmations: list[str] | None = None): + ids = [i["id"] for i in st["trial"]["confirm_items"]] if confirmations is None else confirmations + return await client.post(f"{API}/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ids}) + + +async def rows(client, name: str, sql: str) -> list[list[Any]]: + resp = await client.post(f"/api/tools/db_query__{name}/run", json={"args": {"sql": sql}}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["ok"], out + return json.loads(out["result"])["rows"] + + +async def first_csv(client, name: str) -> dict[str, Any]: + raw, fn = month_csv(8, AUG) + st = await stage(client, raw, fn, name) + assert st["recipe"]["sheets"][0]["match"]["name"] == "CSV" + assert not st["recipe_problems"], st["recipe_problems"] + q = next(q for q in st["questions"] if q["id"].startswith("q_list_period:")) + # 列表的草稿还不能按期累积，没有 q_mode；「每期怎么更新」这一道题同时定下导入模式和每一行属于哪一期 + assert not any(x["id"] == "q_mode" for x in st["questions"]) + st = await answer(client, st["id"], {q["id"]: {"value": "synthesize"}}) + assert st["recipe"]["mode"] == "accumulate" and not st["recipe_problems"], st["recipe_problems"] + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + st = await trial(client, st["id"], context_inputs=_period(8)) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = [i["id"] for i in st["trial"]["confirm_items"]] + assert "list_period:列表1" in ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + return {**resp.json(), "table": st["recipe"]["sheets"][0]["blocks"][0]["table"]} + + +async def reupload(client, source_id: str, month: int, orders: list[tuple[str, int, int]], + period: dict[str, Any] | None = None) -> dict[str, Any]: + raw, fn = month_csv(month, orders) + resp = await client.post(f"{API}/{source_id}/reupload", files={"file": (fn, raw, CSV_MIME)}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + return await trial(client, st["id"], context_inputs=period or _period(month)) + + +async def test_monthly_csv_exports_accumulate_into_one_table(client): + name = _name() + first = await first_csv(client, name) + sid, table = first["source"]["id"], first["table"] + assert await rows(client, name, f'SELECT "订单号", "统计期起", "统计期止" FROM "{table}" ORDER BY rowid') == [ + ["A001", "2026-08-01", "2026-08-31"], ["A002", "2026-08-01", "2026-08-31"], ["A003", "2026-08-01", "2026-08-31"]] + st = await reupload(client, sid, 9, SEP) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert st["trial"]["accumulate"]["action"] == "append" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + got = await rows(client, name, f'SELECT "统计期起", COUNT(*) FROM "{table}" GROUP BY 1 ORDER BY 1') + assert got == [["2026-08-01", 3], ["2026-09-01", 2]] + # 部分重叠的一期被拒收 + st = await reupload(client, sid, 9, SEP, period={"统计期": {"start": "2026-08-15", "end": "2026-09-14"}}) + assert st["trial"]["status"] == "rejected" and any(p["code"] == "period_overlap" for p in st["trial"]["problems"]) + # 同一期再传（数不同）：要确认替换 + st = await reupload(client, sid, 9, [("B001", 3, 75)]) + assert st["trial"]["status"] == "passed" and st["trial"]["accumulate"]["action"] == "replace_period" + ids = [i["id"] for i in st["trial"]["confirm_items"]] + replace = "period_replace:2026-09-01~2026-09-30" + assert replace in ids + resp = await commit(client, st, [i for i in ids if i != replace]) + assert resp.status_code == 422 + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await rows(client, name, f'SELECT COUNT(*), SUM("金额") FROM "{table}"') == [[4, 315]] + + +async def test_excel_with_subtotals_drafts_checks_and_keeps_them_out(client): + wb = Workbook() + ws = wb.active + ws.title = "门店" + for row in [("门店", "金额"), ("门店一", 100), ("门店二", 200), ("小计", 300), ("门店三", 150), ("门店四", 150), + ("小计", 300), ("合计", 600)]: + ws.append(row) + buf = io.BytesIO() + wb.save(buf) + name = _name() + resp = await client.post(f"{API}/imports/stage", files={"file": ("门店.xlsx", buf.getvalue(), "application/octet-stream")}, + data={"name": name}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert not st["draft_problems"] and not st["recipe_problems"], (st["draft_problems"], st["recipe_problems"]) + block = st["recipe"]["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"]["pick"] == "小计" + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + checks = {c["id"]: c for c in st["trial"]["checks"]} + assert checks["T2"]["status"] == "passed" and "小计" in checks["T2"]["title"] + excluded = st["trial"]["receipt"]["rows_excluded"] + assert any(x["reason"] == "subtotal" and x["rows"] == [[4, 4], [7, 7]] for x in excluded), excluded + ids = [i["id"] for i in st["trial"]["confirm_items"]] + assert "subtotals:列表1" in ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + table = st["recipe"]["sheets"][0]["blocks"][0]["table"] + assert await rows(client, name, f'SELECT COUNT(*), SUM("金额") FROM "{table}"') == [[4, 600]] diff --git a/backend/tests/test_recipe_select.py b/backend/tests/test_recipe_select.py index 6824450b..9e408f16 100644 --- a/backend/tests/test_recipe_select.py +++ b/backend/tests/test_recipe_select.py @@ -210,10 +210,8 @@ def _subtotals() -> Book: @pytest.mark.parametrize("ref,bottom,where", [ - ("A1:C7", "box", "第 4 行像合计行"), # 小计夹在框中间 ("A1:C4", "box", "第 4 行像合计行"), # 框到小计为止：它下面紧接着还有明细，是夹在中间的小计 ("A1:C3", "box", "第 4 行像合计行"), # 框到小计上一行：下一行是小计，同上 - ("A1:C3", "auto", "第 4 行像合计行"), # 按规则推断：起草器在这里记失败，框选同样不收 ]) def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): b = _subtotals() @@ -222,6 +220,22 @@ def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): assert where in res.problems[0].message and "明细!A4" in res.problems[0].cells +@pytest.mark.parametrize("ref,bottom", [ + ("A1:C7", "box"), # 小计夹在框中间、合计在框底 + ("A1:C3", "auto"), # 按规则推断：起草器把夹在中间的小计提议成小计（期 5），框选同样 +]) +def test_subtotal_in_the_middle_becomes_subtotals_not_data(ref, bottom): + """期 5：夹在中间、第一列以合计字样开头的行提议成小计（ListRows.subtotals）：不进数据，按各组明细核对。""" + b = _subtotals() + res = selection_edit(None, b.grid, sel(ref, "list", "明细", bottom=bottom), extraction=None) + assert res.ok, res.problems + [op] = res.ops # 配方为空：一条根替换，新建只有这一个块的配方 + block = op["value"]["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"] == {"label_column": "地区", "pick": "小计", "scope": "since_previous", + "group_column": None, "keep_as": None} + assert block["rows"]["total_row"]["pick"] == "合计" + + def test_total_word_in_another_column_is_not_imported_as_data(): """合计字样写在第二列（第一列空着）：起草器记失败，框选同样报错，不当明细导入。""" wb = Workbook() diff --git a/backend/tests/test_recipe_suggest.py b/backend/tests/test_recipe_suggest.py index 40c62af1..44bd2164 100644 --- a/backend/tests/test_recipe_suggest.py +++ b/backend/tests/test_recipe_suggest.py @@ -951,7 +951,7 @@ def test_list_subtotal_rows_in_middle_fail(): assert [f for f in d.failures if "像合计行" in f] == [ "工作表「门店」：第 4 行像合计行（B 列「小计」），但合计字样不在第一列，无法按合计行核对", "工作表「门店」：第 7 行像合计行（B 列「小计」），但合计字样不在第一列，无法按合计行核对"] - # 小计写在第一列、但下面还有明细：不是末尾的合计行，整块照样收进来，不另报「数据区外的数字」 + # 期 5：小计写在第一列、下面还有明细：提议成穿插的小计（不进数据，按上一行小计之后的明细核对），不再记失败 wb2 = Workbook() ws2 = wb2.active ws2.title = "门店" @@ -960,10 +960,21 @@ def test_list_subtotal_rows_in_middle_fail(): ws2.append(row) sc2, grids2 = build(wb2) d2 = S.draft(sc2, grids2, "x.xlsx", validate=ok_validate()) - assert not d2.complete - assert d2.failures == ["工作表「门店」：第 4 行像合计行（A 列「小计」），但不在列表末尾，无法确定它汇总的是哪几行"] + assert d2.complete and d2.failures == [] block = d2.recipe["sheets"][0]["blocks"][0] assert block["rows"]["total_row"]["pick"] == "合计" + assert block["rows"]["subtotals"] == {"label_column": "门店", "pick": "小计", "scope": "since_previous", + "group_column": None, "keep_as": None} + # 字样不一样（「小计」「合计」都夹在中间）：分不清，照旧记失败 + wb3 = Workbook() + ws3 = wb3.active + ws3.title = "门店" + for row in [["门店", "金额"], ["门店一", 100], ["小计", 100], ["门店二", 200], ["合计", 200], ["门店三", 150], + ["总计", 450]]: + ws3.append(row) + sc3, grids3 = build(wb3) + d3 = S.draft(sc3, grids3, "x.xlsx", validate=ok_validate()) + assert not d3.complete and any("不在列表末尾" in f for f in d3.failures) # 末尾第一列的合计行照旧改作核对，不算失败（test_list_total_row_is_verified_and_kept） diff --git a/backend/tests/test_recipe_validate.py b/backend/tests/test_recipe_validate.py index 0d2f945e..3e9fe110 100644 --- a/backend/tests/test_recipe_validate.py +++ b/backend/tests/test_recipe_validate.py @@ -315,7 +315,7 @@ def test_accumulate_mode_follows_eligibility(): probs = run(lr, facts=None) assert [(p.code, p.path) for p in probs] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] - assert "列表形态的表目前只支持每期替换" in probs[0].message + assert "没有指定每一行属于哪一期" in probs[0].message # 同一份配方每期替换时不报 lr["mode"] = "replace" assert run(lr, facts=None) == [] diff --git a/backend/tests/test_recipe_validate_p3.py b/backend/tests/test_recipe_validate_p3.py index a2e2f775..b4f0acca 100644 --- a/backend/tests/test_recipe_validate_p3.py +++ b/backend/tests/test_recipe_validate_p3.py @@ -79,7 +79,7 @@ def test_accumulate_list_table_message_does_not_promise_future_versions(): r["sheets"][0]["context"] = [{}] _rec, problems = validate_recipe(r) assert [p.code for p in problems] == ["accumulate_unkeyed"] - assert "列表形态的表目前只支持每期替换" in problems[0].message and "后续版本" not in problems[0].message + assert "没有指定每一行属于哪一期" in problems[0].message and "后续版本" not in problems[0].message # ========================================================================== diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index ed0661cf..8d80043d 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1231,8 +1231,6 @@ export const UPLOAD_TEXT = { shapeTitle: '该表格不是一行一条记录的明细表', shapeLead: '检测到以下结构，默认不导入：', shapeRecipe: '这类表格可以按配方导入：指定表头、分段和合计行的处理方式，试运行核对、逐条确认后再启用。', - /** CSV 没有配方这条路（配方只认 Excel 的工作表结构） */ - shapeRecipeCsv: '按配方导入目前只支持 Excel（.xlsx）文件：可另存为 .xlsx 后再按配方导入。', shapeHeader: (row: number) => `如果表头不在第 ${row} 行，请取消后修改表头行号，再重新上传。`, /** 结构问题之外，预告的混合列：选按原样导入以后还要再选一次 */ shapeMixedLead: '另外，以下各列以数字为主，但混有非数字的值。选择按原样导入后，还需要选择这些值的处理方式：', @@ -1500,9 +1498,9 @@ export const RECIPE_TEXT = { descriptionLabel: '说明（供助手参考）', pickDrop: '拖入文件，或点击选择', /** 与选择框的 accept 一致 */ - pickFormats: '支持 Excel（.xlsx、.xlsm）', + pickFormats: '支持 Excel（.xlsx、.xlsm）和 CSV / TSV', pickAria: '选择表格文件', - pickWrongType: (name: string) => `「${name}」不是 Excel 文件：按配方导入只支持 .xlsx、.xlsm`, + pickWrongType: (name: string) => `「${name}」不是 Excel 或 CSV 文件：按配方导入支持 .xlsx、.xlsm、.csv、.tsv`, cancelUpload: '取消上传', cancel: '取消', close: '关闭', @@ -1637,6 +1635,18 @@ export const RECIPE_TEXT = { totalLabelColumn: '合计标签所在的列', totalPick: '合计行以此开头', totalKeep: '原表合计另存为表（留空则只核对）', + // 期 5：穿插的小计行（ListRows.subtotals）、列表每期怎么更新（ListBlock.period） + subtotals: '小计行', + subtotalsOn: '有小计行', + subtotalsHint: '夹在明细中间的小计不进数据，按它所管的那组明细逐列求和核对', + subtotalLabelColumn: '小计标签所在的列', + subtotalPick: '小计行以此开头', + subtotalScope: '一行小计管哪些明细', + subtotalGroupColumn: '按哪一列分组', + subtotalKeep: '原表小计另存为表（留空则只核对）', + listPeriod: '按期累积时每一行属于哪一期', + listPeriodColumn: '按哪个日期列核对', + listPeriodHint: '列表要按期累积，每一行都要说得出属于哪一期；所在工作表要配置统计期（表内没有时，每期试运行时录入）。导入模式另在「导入模式」中选', hiddenRows: '隐藏行', hiddenCols: '隐藏列', fallback: '工作表改名时', @@ -1994,6 +2004,7 @@ export const RECEIPT_BLOCK_TEXT = { ignored_outside: '按配方忽略了区域外数字的行', after_stop: '列表结束之后的文字行', total_not_kept: '只核对、不另存的合计行', + subtotal: '穿插的小计行（核对后不进数据）', } as Record, excludedOther: '其他原因', excludedTitle: '排除的行', @@ -2107,6 +2118,8 @@ export const RECIPE_CHOICE_LABEL = { colType: { TEXT: '文字', INTEGER: '整数', REAL: '小数', DATE: '日期' } as Record, colStore: { '': '默认', canonical: '规范写法', raw: '原文' } as Record, axisChecks: { contiguous: '逐日连续', covers_context: '恰好覆盖统计期' } as Record, + subtotalScope: { since_previous: '上一行小计之后的全部明细', group_column: '紧挨在上面、分组列取值相同的明细' } as Record, + listPeriod: { none: '不区分（只能每期替换）', column: '按日期列核对每行都在本期统计期内', synthesize: '每行加上本期统计期的起止两列' } as Record, } // =========================================================================== diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index a7585768..cfd0c97e 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -1773,7 +1773,7 @@ function UploadDecisionBody({ decision, choices, headerRow, sheetNoun, onRecipe )}
    -

    {onRecipe ? UPLOAD_TEXT.shapeRecipe : UPLOAD_TEXT.shapeRecipeCsv}

    +

    {UPLOAD_TEXT.shapeRecipe}

    {onRecipe && (