diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 60617db2..25ac4c8c 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -579,18 +579,48 @@ def _undated(note: str) -> str: return _PROFILE_NOTE_DAY.sub("数据剖析", note, count=1) +#: 剖析备注改过的说法：(升级前的写法, 现在的写法)。大表看的是按存储顺序的前 N 行（多半是最老的数据）、关系取的是 +#: 数据库先给出的那几个键值，都不是随机抽样，说法改成照实的（data/catalog_profile.py 的 _relation_note、码值的 +#: 「统计范围」、FILTER_SAMPLED）。只换这几处固定的句式，换完和现在的写法一字不差才算同一句 +_LEGACY_PROFILE_WORDING = ( + (re.compile(r"抽样覆盖率 (?=\S+?，可能不是这条关系)"), "覆盖率 "), + (re.compile(r"子表抽样 (?=\d+ 个不同键值)"), "子表取了 "), + (re.compile(r"（只看前 (?=\d+ 行）)"), "（按存储顺序只看前 "), + (re.compile(r"按前 (?=[\d,]+ 行统计)"), "按存储顺序取前 "), +) +#: 起草时核对有效行过滤写进备注的那句（catalog_profile.FILTER_SAMPLED / FILTER_SAMPLED_EDGE）：接在模型给的依据后面， +#: 整段不以「数据剖析：」开头，只换这一句 +_LEGACY_FILTER_WORDING = re.compile(r"剖析核对：按前 (?=[\d,]+ 行统计)") + + +def _current_wording(note: str) -> str: + """剖析备注里升级前的说法换成现在的写法（_LEGACY_PROFILE_WORDING）。只用来比较，不改写目录里存着的备注。""" + for pattern, now in _LEGACY_PROFILE_WORDING: + note = pattern.sub(now, note) + return note + + def same_profile_note(a: Any, b: Any) -> bool: """两段备注是否相同；数据剖析写的备注只差开头的日期（升级前每次都写当天的日期），或者一段带日期、一段是不带 - 日期的新写法，也算相同。 + 日期的新写法，或者只差改过的几处说法（_LEGACY_PROFILE_WORDING：「抽样覆盖率」→「覆盖率」之类），也算相同。 隔天重跑、数据没变，结论一字不差、只有日期变了：要是这也算「变了」，目录就升一个版本，发布时记下的目录版本 - 对不上，之后每次正式运行都提醒「目录有变化」。升级后第一次重新剖析也是一样：结论没变，只是新写法不带日期。 + 对不上，之后每次正式运行都提醒「目录有变化」。升级后第一次重新剖析也是一样：结论没变，只是新写法不带日期、 + 措辞改了。算相同时保留旧的那段（目录不动），旧说法要等结论真的变了、重写备注时才换掉。 + + 起草写的备注（有效行过滤）里接着的剖析核对那句同理，只认那一句的旧说法（_LEGACY_FILTER_WORDING）。 """ if a == b: return True - if not (is_profile_note(a) and is_profile_note(b)): + if not (isinstance(a, str) and isinstance(b, str)): return False - return _undated(a) == _undated(b) + profile_a, profile_b = is_profile_note(a), is_profile_note(b) + if profile_a and profile_b: + return _current_wording(_undated(a)) == _current_wording(_undated(b)) + if profile_a or profile_b: + return False + return (_LEGACY_FILTER_WORDING.sub("剖析核对：按存储顺序取前 ", a) + == _LEGACY_FILTER_WORDING.sub("剖析核对：按存储顺序取前 ", b)) def _same_core(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool: @@ -1378,14 +1408,17 @@ _DATE_WORDS = frozenset({"date", "time", "day", "dt", "ts", "timestamp", "dateti _TEXT_TYPES = ("CHAR", "TEXT", "STRING", "CLOB") -def is_date_column(name: str, type_name: str | None) -> bool: +def is_date_column(name: str, type_name: str | None, *, enum_values: Any = None) -> bool: """日期类列：类型是日期、时间戳（DATE、DATETIME、TIMESTAMP……）；或者类型是文字（或者不知道类型）、名字像日期 - （SQLite 的日期常存成文字）。只有时刻的 TIME、整数存的时间戳不算：取值不是日期。 + （SQLite 的日期常存成文字）。只有时刻的 TIME、整数存的时间戳不算：取值不是日期。有取值列表的（MySQL 的 ENUM / + SET、探查记下 enum_values 的 PostgreSQL 枚举）不算：PG 的枚举编译出来是类型名本身，update_reason 里也有「DATE」。 数据剖析（表里只有一个日期类列时提议它作业务日期）和 SQL 检查（按业务日期以外的时间列统计）共用这一个判断。 以前两边各写一套：剖析认的「唯一日期列」，SQL 检查可能认为还有别的时间列，反过来也一样，两边对不上。 """ - t = (type_name or "").upper() + t = _type_words(type_name) + if enum_values or _MYSQL_ENUM.match(t) or "INTERVAL" in t: + return False if "DATE" in t or "TIMESTAMP" in t: return True if t and not any(k in t for k in _TEXT_TYPES): @@ -1420,11 +1453,41 @@ def _name_index(tables: Mapping[str, Any]) -> dict[tuple[str, str], set[str]]: return index +#: 类型写法里带引号的部分（ENUM('int','date') 的取值）：认类型时去掉，取值里的词不能让一列变成数字或时间 +_QUOTED_IN_TYPE = re.compile(r"'(?:[^']|'')*'") + + +def _type_words(type_name: str | None) -> str: + """认类型用的写法：大写、去掉引号里的取值。探查存的是按方言编译的完整写法（ENUM('paid','refund')、 + VARCHAR2(20 CHAR)、TIMESTAMP WITH TIME ZONE），不再是 SQLAlchemy 的通用写法。""" + return _QUOTED_IN_TYPE.sub("''", (type_name or "").upper()).strip() + + +#: MySQL 的 ENUM / SET 编译出来的写法：ENUM('a','b')、SET('x','y')，只认带括号的。PostgreSQL 的枚举编译出来是类型名本身， +#: setup_status、enum_kind 这种以 set、enum 开头的名字不能按前缀认成 MySQL 的 SET / ENUM——PG 的枚举一律靠探查记下的 +#: enum_values 认（column_family） +_MYSQL_ENUM = re.compile(r"^(ENUM|SET)\s*\(") + + +def _mysql_enum_kind(type_name: str | None) -> str | None: + """类型写法是 MySQL 的 ENUM(…) / SET(…) 时返回 "ENUM" / "SET"，否则 None。""" + m = _MYSQL_ENUM.match(_type_words(type_name)) + return m.group(1) if m else None + + def _type_family(type_name: str | None) -> str | None: - """列类型的大类：数字、文字、时间；认不出来返回 None（不据此排除）。""" - t = (type_name or "").upper() + """列类型的大类：数字、文字、时间；认不出来返回 None（不据此排除）。 + + 认的是按方言编译的完整写法：MySQL 的 ENUM / SET 是文字（取值列表里的词不算），Oracle 的 INTERVAL DAY TO SECOND + 是时长、不算数字（以前按子串「INT」认成了数字）。PostgreSQL 的枚举编译出来是类型名本身，只看类型认不出来， + 要看列上的取值列表（column_family）。""" + t = _type_words(type_name) if not t: return None + if _MYSQL_ENUM.match(t): + return "text" + if "INTERVAL" in t: + return None if any(k in t for k in ("CHAR", "TEXT", "CLOB", "STRING", "UUID")): return "text" if any(k in t for k in ("INT", "NUMBER", "NUMERIC", "DECIMAL", "SERIAL", "REAL", "FLOAT", "DOUBLE")): @@ -1434,6 +1497,16 @@ def _type_family(type_name: str | None) -> str | None: return None +def column_family(col: Mapping[str, Any] | None) -> str | None: + """表结构里一列的类型大类：有取值列表（探查记下的 enum_values：MySQL 的 ENUM / SET、PostgreSQL 的枚举）的是 + 文字，其余按类型写法认（_type_family）。""" + if not isinstance(col, Mapping): + return None + if col.get("enum_values"): + return "text" + return _type_family(col.get("type")) + + def _id_stem(column: str) -> str | None: m = _SNAKE_ID.match(column) or _CAMEL_ID.match(column) return m.group("stem") if m else None @@ -1489,8 +1562,8 @@ def infer_name_relations(schema_cache: Mapping[str, Any] | None, table: str, *, pk = list(target_meta.get("primary_key") or []) if len(pk) != 1: continue - pk_type = next((c.get("type") for c in target_meta.get("columns") or [] if c.get("name") == pk[0]), None) - mine, theirs = _type_family(col.get("type")), _type_family(pk_type) + pk_col = next((c for c in target_meta.get("columns") or [] if c.get("name") == pk[0]), None) + mine, theirs = column_family(col), column_family(pk_col) if mine and theirs and mine != theirs: continue out.append(_relation(table, [name], target, pk, "name", cardinality=_cardinality(meta, [name]))) diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 8eee4488..17b59a39 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -41,9 +41,9 @@ import re import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone +from datetime import date as _date, datetime, timezone from decimal import Decimal -from typing import Any +from typing import Any, NamedTuple from sqlalchemy import select, update from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite @@ -99,7 +99,7 @@ _NUMBER_FIELDS: dict[str, tuple[str, float, float, bool]] = { "table_max_queries": ("每张表最多查询数", 1, 200, True), "table_max_s": ("每张表时长上限（秒）", 10, 600, False), "query_timeout_s": ("单条查询时限（秒）", 1, 60, False), - "sample_size": ("抽样键值数", 10, 10_000, True), + "sample_size": ("核对键值数", 10, 10_000, True), "max_scan_rows": ("整表统计行数上限", 0, 10_000_000, True), } _ENABLED_LABEL = "开启数据剖析" @@ -241,16 +241,96 @@ def sql_literal(value: Any, kind: str | None) -> str | None: return "'" + value.replace("'", "''") + "'" +#: 定长文字类型（CHAR(2)、NCHAR、bpchar、CHARACTER(2)）：PostgreSQL、Oracle 取回的值带尾随空格补齐 +_FIXED_CHAR = re.compile(r"^\s*(?:N?CHAR|BPCHAR|CHARACTER)\b(?!\s+VARYING)", re.IGNORECASE) + + +def is_fixed_char(type_name: str | None) -> bool: + """定长文字列（CHAR(n)）：值的尾随空格是补齐出来的，不是取值的一部分（SQL 比较时也不算）。""" + return bool(type_name) and bool(_FIXED_CHAR.match(str(type_name))) + + +class KeyLiterals(NamedTuple): + """key_literals 的结果。literals：去父表里对的 IN 列表字面量（去重、保持顺序）；unmatchable：按类型就对不上父键、 + 算作没对上的键值个数（去重后）；unsafe：值本身不能安全拼进 SQL、不计入核对的个数。""" + literals: list[str] + unmatchable: int + unsafe: int + + +def key_literals(values: list[Any], kind: str | None, *, parent: str | None, + child_type: str | None) -> KeyLiterals: + """从子表取回的键值 → 去父表里对的 IN 列表字面量（KeyLiterals）。 + + 字面量按**被指向列**的类型族写（catalog.column_family），不是按子表取回的值： + - 父列是文字、子列是数：写成 '123'。PostgreSQL 上 varchar 列 IN (123) 直接报「operator does not exist」； + - 父列是数、子列是文字：只拼整数写法的值（'0012' 也照 12 写）。别的（'A1'、'WALKIN'）不拼——Oracle 上数字列 + IN ('A1') 会把字面量转成数报 ORA-01722，MySQL 会把 'A1' 转成 0 去比、对上不该对上的行——但它们**照样是子表 + 的键值**：整数主键里永远没有 'WALKIN'，所以算作没对上，计入 unmatchable，由调用方留在覆盖率的分母里。要是 + 从分母里拿掉，散客代号占九成的一列只剩 1 个整数写法的键对上，就成了「覆盖率 100%」； + - 父列是时间：不在这里处理，_check_relation 先整条跳过（各家时间字面量写法不同、精度不同，键不核对）； + - 父列类型认不出：照子表取回的值的类型写（sql_literal，以前的做法）。 + 子表是定长文字（CHAR(n)）时先去掉尾随空格：PG、Oracle 取回的 'A ' 拿去和 VARCHAR 父列比就对不上，而按 CHAR 的 + 语义尾随空格本来就不算。不安全的值（反斜杠、「:名字」、控制字符、太长）照 sql_literal 的规矩跳过，记进 unsafe + （调用方另记「类型不支持核对，未计入」）。两个不同的取值写成同一个字面量（'0012' 和 '12'）只算一个，不算 unsafe。 + """ + if parent == "time": + raise ValueError("被指向列是日期时间的关系不核对键值（见 _check_relation）") + fixed = is_fixed_char(child_type) + out: list[str] = [] + unmatchable: set[str] = set() + unsafe = 0 + for value in values: + if fixed and isinstance(value, str): + value = value.rstrip(" ") + if parent == "text": + if isinstance(value, bool): + unsafe += 1 + continue + if isinstance(value, (int, float, Decimal)): + lit = sql_literal(value, "number") + lit = None if lit is None else "'" + lit + "'" + else: + lit = sql_literal(value, "text") + elif parent == "number": + if isinstance(value, str): + text = value.strip() + if not _INT_TEXT.fullmatch(text): + unmatchable.add(text) + continue + lit = str(int(text)) + else: + lit = sql_literal(value, "number") + else: + lit = sql_literal(value, kind) + if lit is None: + unsafe += 1 + else: + out.append(lit) + return KeyLiterals(list(dict.fromkeys(out)), len(unmatchable), unsafe) + + _DIALECT_MODULES = {"sqlite": sqlite, "postgres": postgresql, "postgresql": postgresql, "mysql": mysql, "mariadb": mysql, "oracle": oracle} +#: 日期列的最小值、最大值只看这个区间里的值（含下限、不含上限）：0000-00-00（MySQL 关了严格模式写进去的零日期）、 +#: 0001-01-01、9999-12-31（「没有」「永久」的占位日期）、PostgreSQL 的 -infinity / infinity 都落在区间外。区间外的 +#: 值照实数出来（「有 N 个无效日期未计入」），不让一个占位日期把业务日期的范围撑成两千年。两端整天排除：带时刻的 +#: 9999-12-31 23:59:59、文字存的「0001-01-01 00:00:00」也算占位 +VALID_DATE_FROM = "0001-01-02" +VALID_DATE_BEFORE = "9999-12-31" +_ISO_DAY = re.compile(r"\d{4}-\d{2}-\d{2}") + + class SqlDialect: - """剖析 SQL 的方言差异：标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里。只拼字符串，不连库。 + """剖析 SQL 的方言差异：标识符怎么加引号、怎么只取前 n 行、统计信息在哪个系统视图里、日期字面量和按天截断 + 怎么写、有没有块采样。只拼字符串，不连库。 标识符交给 SQLAlchemy 各方言的 identifier_preparer：需要时才加引号（memberTags 加、visits 不加）， MySQL 用反引号。Oracle 尤其不能一律加双引号：探查拿到的表名是 SQLAlchemy 规整过的小写，库里其实是 - 大写，"visits" 加了引号就成了另一张不存在的表；不加引号 Oracle 自己转大写才对得上。 + 大写，"visits" 加了引号就成了另一张不存在的表；不加引号 Oracle 自己转大写才对得上。反过来，库里真是小写的 + 对象（建表时写了 "visits"）探查时记了 quote_name，这种要强制加引号、不转大写（table、column、stats_sql）。 """ def __init__(self, kind: str | None) -> None: @@ -262,13 +342,64 @@ class SqlDialect: self._prep = self._dialect.identifier_preparer self.oracle = self.kind == "oracle" - def quote(self, name: str) -> str: - return self._prep.quote(name) + def quote(self, name: str, *, force: bool = False) -> str: + """标识符。force：探查记了 quote_name 的名字，原样加引号（Oracle 小写带引号的对象）。""" + return self._prep.quote_identifier(name) if force else self._prep.quote(name) def table(self, meta: dict[str, Any] | None, name: str) -> str: """写进 FROM 的表名：探查时指定了 schema 的带上它（Oracle 只读账号名下没有对象，全靠 schema 前缀）。""" schema = (meta or {}).get("schema") - return f"{self._prep.quote_schema(schema)}.{self.quote(name)}" if schema else self.quote(name) + quoted = self.quote(name, force=(meta or {}).get("quote_name") is True) + return f"{self._prep.quote_schema(schema)}.{quoted}" if schema else quoted + + def column(self, meta: dict[str, Any] | None, name: str) -> str: + """写进 SQL 的列名：表结构里这一列记了 quote_name 的强制加引号。""" + cols = (meta or {}).get("columns") or [] + entry = next((c for c in cols if isinstance(c, dict) and c.get("name") == name), None) + return self.quote(name, force=bool(entry and entry.get("quote_name") is True)) + + def date_literal(self, value: Any, *, text: bool = False) -> str: + """日期字面量（取值观察的时间窗口、日期列的有效区间用）。PostgreSQL、MySQL、Oracle 都认 DATE 'YYYY-MM-DD'； + SQLite 没有日期类型，日期存成文字，写字符串。text：列是文字存的日期（VARCHAR），按字符串比——PostgreSQL 上 + varchar 和 date 比较会直接报错。只收真实存在的日期（date 或 YYYY-MM-DD），别的抛 ValueError：拼进 SQL 的 + 只能是我们自己算出来的日子。""" + if isinstance(value, datetime): + value = value.date() + if isinstance(value, _date): + day = value.isoformat() + elif isinstance(value, str) and _ISO_DAY.fullmatch(value): + try: + day = _date.fromisoformat(value).isoformat() + except ValueError: + raise ValueError(f"不是有效的日期：{value}") from None + else: + raise ValueError(f"不是有效的日期：{value!r}") + if text or self.kind == "sqlite": + return f"'{day}'" + return f"DATE '{day}'" + + def day_trunc(self, column: str) -> str: + """时间列截到天（按日行数用）。PostgreSQL CAST(x AS DATE)；MySQL DATE(x)；Oracle 必须用 TRUNC(x)：Oracle 的 + DATE 带时分秒，CAST(ts AS DATE) 截不掉；SQLite 的日期是文字，取前 10 个字符。""" + if self.kind in ("postgres", "postgresql"): + return f"CAST({column} AS DATE)" + if self.kind in ("mysql", "mariadb"): + return f"DATE({column})" + if self.oracle: + return f"TRUNC({column})" + return f"substr({column}, 1, 10)" + + def sample_from(self, table: str, percent: float) -> str | None: + """块采样的 FROM：PostgreSQL TABLESAMPLE SYSTEM (p)、Oracle SAMPLE BLOCK (p)，按数据块随机取约 p% 的行； + MySQL、SQLite 没有块采样，返回 None（调用方退回「按存储顺序的前 N 行」，并照实写口径）。""" + p = float(percent) + if not (0 < p < 100): + raise ValueError(f"采样比例要在 0 到 100 之间：{percent}") + if self.kind in ("postgres", "postgresql"): + return f"{table} TABLESAMPLE SYSTEM ({p:g})" + if self.oracle: + return f"{table} SAMPLE BLOCK ({p:g})" + return None def limit(self, sql: str, n: int) -> str: """只取前 n 行。Oracle 11g 没有 LIMIT 也没有 FETCH FIRST，一律套一层 ROWNUM。""" @@ -304,7 +435,8 @@ class SqlDialect: if self.oracle: denorm = self._dialect.denormalize_name owner = sql_literal(denorm(schema), "text") if schema else "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')" - table = sql_literal(denorm(name), "text") + # 小写带引号的对象（quote_name）在数据字典里就是小写，不能转成大写去找 + table = sql_literal(name if (meta or {}).get("quote_name") is True else denorm(name), "text") if not (owner and table): return None return f"SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = {owner} AND TABLE_NAME = {table}" @@ -335,8 +467,49 @@ class SqlDialect: f"WHERE {column} IS NOT NULL GROUP BY {column} ORDER BY cnt DESC, {column}") return self.limit(sql, n) - def min_max_sql(self, table: str, columns: list[str]) -> str: - return f"SELECT {', '.join(f'MIN({c}), MAX({c})' for c in columns)} FROM {table}" + def min_max_sql(self, table: str, columns: list[tuple[str, bool]]) -> str: + """日期列的最小值、最大值和无效日期的个数：每列三个值（MIN、MAX、区间外的个数）。columns 是 (写进 SQL 的列, + 是不是文字存的日期)。只看有效区间（VALID_DATE_FROM ≤ x < VALID_DATE_BEFORE）里的值，见 VALID_DATE_FROM。 + 整表统计，只对小表用。""" + parts = [] + for c, text in columns: + lo = self.date_literal(VALID_DATE_FROM, text=text) + hi = self.date_literal(VALID_DATE_BEFORE, text=text) + valid = f"{c} >= {lo} AND {c} < {hi}" + parts.append(f"MIN(CASE WHEN {valid} THEN {c} END), MAX(CASE WHEN {valid} THEN {c} END), " + f"SUM(CASE WHEN {c} < {lo} OR {c} >= {hi} THEN 1 ELSE 0 END)") + return f"SELECT {', '.join(parts)} FROM {table}" + + # ---- 取值观察（下一轮第 3 波用）：窗口、按日行数、块采样。这里先把各方言的写法定下来，金样和容器集成测试都覆盖 + + def _window(self, column: str, since: Any, text: bool) -> str: + """时间窗口的条件：since 起（含），并且早于占位日期 9999-12-31——PostgreSQL 的 infinity、Oracle 的 9999-12-31 + 都大于任何一天，不截掉上端就会在「最近 35 天」里多出一天。""" + return (f"{column} >= {self.date_literal(since, text=text)} " + f"AND {column} < {self.date_literal(VALID_DATE_BEFORE, text=text)}") + + def day_volume_sql(self, table: str, column: str, since: Any, *, text: bool = False) -> str: + """最近若干天按天计数：since 起（含）每天几行。只在日期列打头有索引时对大表发（走索引的范围扫描）。""" + day = self.day_trunc(column) + return (f"SELECT {day} AS d, COUNT(*) AS n FROM {table} " + f"WHERE {self._window(column, since, text)} GROUP BY {day} ORDER BY d") + + def window_value_counts_sql(self, table: str, column: str, date_column: str, since: Any, n: int, *, + text: bool = False) -> str: + """时间窗口里的取值分布：业务日期 since 起（含）的行里，出现最多的 n 个取值和各自的行数。""" + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {table} " + f"WHERE {self._window(date_column, since, text)} AND {column} IS NOT NULL " + f"GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) + + def sample_value_counts_sql(self, table: str, column: str, percent: float, n: int) -> str | None: + """块采样里的取值分布；没有块采样的方言返回 None。""" + source = self.sample_from(table, percent) + if source is None: + return None + sql = (f"SELECT {column}, COUNT(*) AS cnt FROM {source} WHERE {column} IS NOT NULL " + f"GROUP BY {column} ORDER BY cnt DESC, {column}") + return self.limit(sql, n) # ========================================================================== @@ -581,8 +754,10 @@ async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: Sql """先读统计信息；读不到（SQLite、没分析过的表）再数到 max_scan_rows + 1 行为止。 统计信息为 0 或负数（PostgreSQL 没分析过的表是 -1，MySQL 的估算值可能是 0）当作不知道：一张其实很大的表 - 要是被当成空表，后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完（或者任务止损） -时抛 _Stop。 + 要是被当成空表，后面就会对它做整表统计。统计信息说的行数不超过 max_scan_rows 时也再数一次（多一条有上限的 + 计数）：统计信息可能是很久以前的（表后来涨到了几千万行，ANALYZE 没再跑过），照它把大表当小表，就会对它做 + 整表的 COUNT(DISTINCT)、MIN/MAX。数过的结果是准数（count），数到上限还没数完就按大表算（at_least）。 + 返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完（或者任务止损）时抛 _Stop。 """ skips: list[_Skip] = [] sql = dialect.stats_sql(meta, name) @@ -592,7 +767,9 @@ async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: Sql except _Skip as e: skips.append(e) else: - if rows is not None and rows > 0: + if rows is not None and rows > max(max_scan_rows, 0): + return TableSize(rows, "stats"), skips + if rows is not None and rows > 0 and max_scan_rows <= 0: return TableSize(rows, "stats"), skips if max_scan_rows <= 0: return TableSize(None, "unknown"), skips @@ -752,6 +929,8 @@ class _CodesOutcome: rows: int note: str complete: bool = False + #: 列定义（ENUM）列出的取值：这次没出现的也写进码值（含义待填），码值是「可能的取值」 + declared: list[str] = field(default_factory=list) @dataclass @@ -901,6 +1080,11 @@ def _column(meta: dict[str, Any], name: str) -> str | None: if isinstance(c, dict) and str(c.get("name", "")).lower() == lowered), None) +def _column_meta(meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """表结构里这一列（类型、取值列表、quote_name）；name 是表结构里的写法（_column 的返回值）。""" + return next((c for c in meta.get("columns") or [] if isinstance(c, dict) and c.get("name") == name), None) + + def _constraint(meta: dict[str, Any], column: str) -> str | None: """这一列单独是主键或某个唯一约束时返回「主键」/「唯一约束」，否则 None。""" wanted = [column.lower()] @@ -950,7 +1134,7 @@ async def _unique(cx: _TableCx, table: str, meta: dict[str, Any], column: str, if not size.within(cx.settings.max_scan_rows): return None, "too_large" d = cx.dialect - res = await cx.run(d.unique_check_sql(d.table(meta, table), d.quote(column))) + res = await cx.run(d.unique_check_sql(d.table(meta, table), d.column(meta, column))) row = res.rows[0] if res.rows else [None, None] total, distinct = _as_int(row[0]), _as_int(row[1]) if total is None or distinct is None: @@ -969,20 +1153,39 @@ def _fk_note(child: tuple[bool | None, str]) -> str: return f"{catalog.PROFILE_NOTE_PREFIX}外键约束已保证引用完整，未抽样核对覆盖率；子表一侧{basis}：{shape}" +#: MySQL 上比较文字按列的排序规则：常见的排序规则不分大小写（_ci），PAD SPACE 的还忽略尾随空格。覆盖率、唯一性、 +#: 码值的分组都按它算，写进说明，人才知道「A01」和「a01」在这里算同一个值 +MYSQL_COLLATION_NOTE = "MySQL 按列的排序规则比较文字（常见的排序规则不分大小写），覆盖率和唯一性按它计算" +MYSQL_CODES_COLLATION_NOTE = "MySQL 按列的排序规则分组（常见的排序规则不分大小写），大小写不同的写法可能并成一个取值" + + +def _collation_note(dialect: SqlDialect, family: str | None, note: str) -> str | None: + """MySQL、MariaDB 上文字列（或类型认不出的列）的排序规则说明；别的方言、数字列没有。""" + return note if dialect.kind in ("mysql", "mariadb") and family in ("text", None) else None + + def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int | None, skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None, - holds: bool) -> str: - """关系的剖析备注：抽样规模、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 - 不写日期：什么时候剖析的看这一项的 updated_at（界面按本地时间显示）。""" + holds: bool, unmatchable: int = 0, collation: str | None = None) -> str: + """关系的剖析备注：取了多少个键值、看了多少行、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 + 不写日期：什么时候剖析的看这一项的 updated_at（界面按本地时间显示）。 + + 不说「抽样」：DISTINCT … LIMIT 取的是数据库先给出的那几个值，大表上是按存储顺序的前若干行里的值，不是随机抽的 + （多半是最老的数据）。unmatchable：被指向列是数字、子表取到的却不是整数写法的键值个数（key_literals）——已经算在 + sample 里、算作没对上，这里说明为什么没去父表里对。""" parts: list[str] = [] low = coverage < catalog.PROFILE_VERIFY_COVERAGE if low: - parts.append(f"抽样覆盖率 {_percent(coverage)}，可能不是这条关系") - scope = f"（只看前 {scan_cap} 行）" if scan_cap else "" - sampled = f"子表抽样 {sample} 个不同键值{scope}，父表对上 {matched} 个" + parts.append(f"覆盖率 {_percent(coverage)}，可能不是这条关系") + scope = f"（按存储顺序只看前 {scan_cap} 行）" if scan_cap else "" + sampled = f"子表取了 {sample} 个不同键值{scope}，父表对上 {matched} 个" parts.append(sampled if low else f"{sampled}，覆盖率 {_percent(coverage)}") if skipped_values: parts.append(f"另有 {skipped_values} 个键值的类型不支持核对，未计入") + if unmatchable: + parts.append(f"其中 {unmatchable} 个不是整数写法，被指向列是数字，算作没对上") + if collation: + parts.append(collation) unique, how = parent if unique is True: parts.append(f"被指向列是{how}" if how in ("主键", "唯一约束") else "被指向列经计数核对唯一") @@ -1026,24 +1229,32 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T return d, s = cx.dialect, cx.settings + child_col, parent_col = _column_meta(meta, col), _column_meta(pmeta, pcol) + parent_family = catalog.column_family(parent_col) + if parent_family == "time": + # 时间键不核对：各家时间字面量写法、精度不同，拼出来的 IN 列表对不对得上说不准。不发子表的取值查询 + raise _Skip("unsupported", f"{parent}.{pcol} 是日期时间，各库的时间写法和精度不同，不核对这种关系的覆盖率") size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) child_table, parent_table = d.table(meta, table), d.table(pmeta, parent) - res = await cx.run(d.distinct_sample_sql(child_table, d.quote(col), s.sample_size, scan_cap=scan_cap), + res = await cx.run(d.distinct_sample_sql(child_table, d.column(meta, col), s.sample_size, scan_cap=scan_cap), max_rows=s.sample_size) kind = (res.column_types or {}).get(res.columns[0]) if res.columns else None values = [row[0] for row in res.rows if row] - literals = list(dict.fromkeys(lit for v in values if (lit := sql_literal(v, kind)) is not None)) + literals, unmatchable, unsafe = key_literals(values, kind, parent=parent_family, + child_type=(child_col or {}).get("type")) if not values: raise _Skip("no_data", f"{table}.{col} 没有非空值，无法核对") - if not literals: + if not literals and not unmatchable: raise _Skip("unsupported", f"{table}.{col} 的取值类型不支持核对（只核对整数和文字）") + # 分母是子表取到的全部键值：不是整数写法、按类型就对不上整数父键的（unmatchable）也在里面，算作没对上 + sample = len(literals) + unmatchable matched = 0 for start in range(0, len(literals), IN_CHUNK): - sql = d.match_count_sql(parent_table, d.quote(pcol), literals[start:start + IN_CHUNK]) + sql = d.match_count_sql(parent_table, d.column(pmeta, pcol), literals[start:start + IN_CHUNK]) matched += _as_int(_first(await cx.run(sql))) or 0 matched = min(matched, len(literals)) - coverage = round(matched / len(literals), 4) + coverage = round(matched / sample, 4) parent_unique = await _unique(cx, parent, pmeta, pcol, result) child_unique = await _unique(cx, table, meta, col, result) if parent_unique[0] else None @@ -1051,12 +1262,13 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T if child_unique is not None: cardinality = "one_to_one" if child_unique[0] is True else "many_to_one" holds = coverage >= catalog.PROFILE_VERIFY_COVERAGE and cardinality is not None - note = _relation_note(sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap, - skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique, - holds=holds) + note = _relation_note(sample=sample, matched=matched, coverage=coverage, scan_cap=scan_cap, + skipped_values=unsafe, parent=parent_unique, child=child_unique, holds=holds, + unmatchable=unmatchable, + collation=_collation_note(d, parent_family, MYSQL_COLLATION_NOTE)) checked = child_unique is not None and child_unique[0] is not None out.relations.append(_RelationOutcome(rid=str(rel["id"]), target=_relation_target(rel), coverage=coverage, - cardinality=cardinality, holds=holds, sample=len(literals), + cardinality=cardinality, holds=holds, sample=sample, matched=matched, note=note, cardinality_checked=checked)) @@ -1076,17 +1288,28 @@ _NOT_CODE_TAILS = frozenset({"id", "no", "num", "number", "name", "desc", "descr "at", "date", "time", "on", "amount", "price", "count", "qty", "rate", "url"}) #: 像日期的取值：2026-07-01、2026/07/01，后面可以跟时间 _DATE_TEXT = re.compile(r"^\d{4}[-/]\d{2}[-/]\d{2}") +#: 日期列最小值、最大值不计入的值（见 VALID_DATE_FROM）在说明里的说法 +INVALID_DATES = "零日期 0000-00-00、0001-01-01、9999-12-31 这类占位日期或无穷大" -def _code_type_ok(type_name: str | None) -> bool: - """整数，或者短文字（定长不超过 MAX_CODE_LEN；SQLite 的 TEXT 不带长度，也算）。""" - t = (type_name or "").upper() +def _is_enum(type_name: str | None, enum_values: Any = None) -> bool: + """取值列表有限的列：MySQL 的 ENUM(…) / SET(…)，或者探查记了取值列表（PostgreSQL 的枚举：类型写法只是类型名， + setup_status 这种以 set 开头的名字不算 SET，见 catalog._MYSQL_ENUM）。""" + return bool(enum_values) or catalog._mysql_enum_kind(type_name) is not None + + +def _code_type_ok(type_name: str | None, enum_values: Any = None) -> bool: + """ENUM / SET（取值列表有限，直接算码值候选）、整数，或者短文字（定长不超过 MAX_CODE_LEN；SQLite 的 TEXT 不带 + 长度，也算）。类型是按方言编译的完整写法：VARCHAR2(20 CHAR) 的长度按 20 算。""" + if _is_enum(type_name, enum_values): + return True + t = catalog._type_words(type_name) if not t or any(k in t for k in ("CLOB", "BLOB", "BINARY", "JSON", "DATE", "TIME", "INTERVAL", "REAL", "FLOAT", "DOUBLE", "BOOL")): return False if "INT" in t: return True - size = re.search(r"\(\s*(\d+)\s*(?:,\s*(\d+)\s*)?\)", t) + size = re.search(r"\(\s*(\d+)(?:\s+(?:CHAR|BYTE))?\s*(?:,\s*(\d+)\s*)?\)", t) if any(k in t for k in ("NUMBER", "NUMERIC", "DECIMAL")): return size is None or not size.group(2) or int(size.group(2)) == 0 if any(k in t for k in ("CHAR", "TEXT", "STRING")): @@ -1100,9 +1323,9 @@ def _looks_like_code(name: str) -> bool: def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: - """要取值分布的列：目录里度量类型记为状态的、剖析以前提过码值候选的（再核一次）、或者类型和命名像状态、 - 类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算；码值人工确认或驳回过的不再取。 - 被遮罩的列也列出来，由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" + """要取值分布的列：目录里度量类型记为状态的、剖析以前提过码值候选的（再核一次）、ENUM / SET 列（取值列表有限， + 不看名字）、或者类型和命名像状态、类型、渠道的整数或短文字列。主键、唯一约束列、关系里的键列不算；码值人工确认 + 或驳回过的不再取。被遮罩的列也列出来，由检查记为「被遮罩」跳过——要让人看得见为什么它没有码值。""" pk = [c.lower() for c in meta.get("primary_key") or []] unique = {str(g[0]).lower() for g in meta.get("unique") or [] if isinstance(g, list) and len(g) == 1} keys = {str(c).lower() for r in notes.get("relations") or [] if isinstance(r, dict) for c in r.get("columns") or []} @@ -1117,23 +1340,28 @@ def _code_columns(meta: dict[str, Any], notes: dict[str, Any]) -> list[str]: status_measure = (isinstance(measure, dict) and measure.get("value") == "status" and measure.get("status") != "rejected") profiled = isinstance(codes, dict) and codes.get("source") == "profile" - if not (status_measure or profiled or _looks_like_code(name)): + enum = _is_enum(col.get("type"), col.get("enum_values")) + if not (status_measure or profiled or enum or _looks_like_code(name)): continue lowered = name.lower() - if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type")): + if (pk == [lowered]) or lowered in unique or lowered in keys or not _code_type_ok(col.get("type"), + col.get("enum_values")): continue out.append(name) return out -def _code_key(value: Any) -> str | None: - """取值分布里的一个取值 → 码值（文字）。只认整数和不太长的文字。""" +def _code_key(value: Any, *, fixed: bool = False) -> str | None: + """取值分布里的一个取值 → 码值（文字）。只认整数和不太长的文字。fixed：定长文字列（CHAR(n)），去掉补齐的尾随 + 空格（PG、Oracle 取回的是 'A '，WHERE status = 'A' 照样对得上，码值写 'A'）。""" if value is None or isinstance(value, bool): return None if isinstance(value, int): return str(value) if isinstance(value, float): return str(int(value)) if value.is_integer() else None + if isinstance(value, str) and fixed: + value = value.rstrip(" ") if isinstance(value, str) and value.strip() and len(value) <= MAX_CODE_LEN and not _CONTROL.search(value): return value return None @@ -1151,34 +1379,53 @@ async def _check_codes(cx: _TableCx, table: str, column: str, out: _TableOutcome if cx.is_masked(column): raise _Skip("masked", f"{table}.{column} 在数据源设置中被遮罩，不取值", carry=False) d, meta = cx.dialect, cx.tables[table] + col_meta = _column_meta(meta, column) or {} size = await cx.size_with_data(table, result) scan_cap = cx.scan_cap(size) - res = await cx.run(d.value_counts_sql(d.table(meta, table), d.quote(column), CODES_MAX + 1, scan_cap=scan_cap), + res = await cx.run(d.value_counts_sql(d.table(meta, table), d.column(meta, column), CODES_MAX + 1, + scan_cap=scan_cap), max_rows=CODES_MAX + 1) if not res.rows: raise _Skip("no_data", f"{table}.{column} 没有非空值，无法取值分布") if len(res.rows) > CODES_MAX: raise _Skip("high_cardinality", f"{table}.{column} 的取值超过 {CODES_MAX} 个，不作为码值候选", carry=False) - values: list[tuple[str, int]] = [] + fixed = is_fixed_char(col_meta.get("type")) + enum = _is_enum(col_meta.get("type"), col_meta.get("enum_values")) + counts: dict[str, int] = {} for row in res.rows: - key = _code_key(row[0]) + key = _code_key(row[0], fixed=fixed) if key is None: raise _Skip("unsupported", f"{table}.{column} 的取值不是整数或短文字，不作为码值候选", carry=False) - values.append((key, _as_int(row[1]) or 0)) - if len(values) >= 3 and all(n == 1 for _, n in values): - # 每个取值只出现一次（票种编码 T01…T09）：这是一列标识，不是状态码 + counts[key] = counts.get(key, 0) + (_as_int(row[1]) or 0) + values = list(counts.items()) + if len(values) >= 3 and all(n == 1 for _, n in values) and not enum: + # 每个取值只出现一次（票种编码 T01…T09）：这是一列标识，不是状态码。ENUM 的取值列表是列定义给的，不按这条判 raise _Skip("high_cardinality", f"{table}.{column} 的取值各不相同，像标识而不是码值，不作为码值候选", carry=False) rows = sum(n for _, n in values) - scope = f"按前 {scan_cap} 行统计" if scan_cap else "统计全表" + # 不说「抽样」：大表看的是按存储顺序的前若干行（多半是最老的数据），不是随机抽的 + scope = f"按存储顺序取前 {scan_cap} 行统计" if scan_cap else "统计全表" spread = "、".join(f"{k}（{_share(n, rows)}）" for k, n in values) - note = f"{NOTE_PREFIX}{scope}，{rows} 行非空值共 {len(values)} 个取值：{spread}。" - out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None)) + extra: list[str] = [] + # ENUM 的取值列表（SET 的是成员，取值是成员的组合，不算）：这次没出现的也是可能的取值 + is_set = catalog._mysql_enum_kind(col_meta.get("type")) == "SET" + declared = [] if is_set else [str(v) for v in col_meta.get("enum_values") or []] + unseen = [v for v in declared if v not in counts] + if unseen: + extra.append(f"列定义里另有 {len(unseen)} 个取值这次没有出现：{'、'.join(unseen[:CODES_MAX])}") + family = "text" if enum else catalog.column_family(col_meta) + collation = _collation_note(d, family, MYSQL_CODES_COLLATION_NOTE) + if collation: + extra.append(collation) + tail = "".join(f"；{x}" for x in extra) + note = f"{NOTE_PREFIX}{scope}，{rows} 行非空值共 {len(values)} 个取值：{spread}{tail}。" + out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None, + declared=unseen)) def _is_date_column(col: dict[str, Any]) -> bool: """日期类列（catalog.is_date_column：和 SQL 检查认时间列是同一个判断）。""" - return catalog.is_date_column(str(col.get("name") or ""), col.get("type")) + return catalog.is_date_column(str(col.get("name") or ""), col.get("type"), enum_values=col.get("enum_values")) async def _check_dates(cx: _TableCx, table: str, columns: list[str], total: int, out: _TableOutcome, @@ -1190,20 +1437,31 @@ async def _check_dates(cx: _TableCx, table: str, columns: list[str], total: int, if not size.within(cx.settings.max_scan_rows): raise _Skip("too_large", f"{table} 的行数超过整表统计的行数上限（{cx.settings.max_scan_rows} 行），" "不取日期列的最小值和最大值") - res = await cx.run(d.min_max_sql(d.table(meta, table), [d.quote(c) for c in columns])) + # 文字存的日期（SQLite 的 TEXT、业务库里的 VARCHAR 日期）按字符串和有效区间比；日期类型的按日期字面量比 + texts = [catalog.column_family(_column_meta(meta, c)) != "time" for c in columns] + res = await cx.run(d.min_max_sql(d.table(meta, table), [(d.column(meta, c), t) for c, t in zip(columns, texts)])) row = res.rows[0] if res.rows else [] + invalid: dict[str, int] = {} for i, column in enumerate(columns): - low, high = (row[2 * i], row[2 * i + 1]) if len(row) >= 2 * i + 2 else (None, None) + low, high, bad = (row[3 * i], row[3 * i + 1], row[3 * i + 2]) if len(row) >= 3 * i + 3 else (None, None, 0) + invalid[column] = _as_int(bad) or 0 if low is None or high is None or not (_DATE_TEXT.match(str(low)) and _DATE_TEXT.match(str(high))): continue - result.date_ranges.append({"column": column, "min": str(low), "max": str(high)}) + found: dict[str, Any] = {"column": column, "min": str(low), "max": str(high)} + if invalid[column]: + found["invalid"] = invalid[column] + result.date_ranges.append(found) if total != 1: return if not result.date_ranges: + if invalid.get(columns[0]): + raise _Skip("no_data", f"{table}.{columns[0]} 只有 {invalid[columns[0]]} 个无效日期（{INVALID_DATES}），" + "没有可识别的日期值") raise _Skip("no_data", f"{table}.{columns[0]} 没有可识别的日期值") found = result.date_ranges[0] + dropped = (f"另有 {found['invalid']} 个无效日期（{INVALID_DATES}）未计入，" if found.get("invalid") else "") note = (f"{NOTE_PREFIX}表里只有这一个日期类列，取值从 {str(found['min'])[:19]} 到 " - f"{str(found['max'])[:19]}，可作为业务日期的参考。") + f"{str(found['max'])[:19]}，{dropped}可作为业务日期的参考。") out.business_date = _DateOutcome(column=found["column"], low=found["min"], high=found["max"], note=note) @@ -1226,7 +1484,7 @@ def _relation_finding(o: _RelationOutcome, rel: dict[str, Any], *, status: str, elif status == "verified": summary = f"覆盖率 {_percent(o.coverage)}，" + ("仍为有确证" if previous == "verified" else "升为有确证") elif o.coverage < catalog.PROFILE_VERIFY_COVERAGE: - summary = f"抽样覆盖率 {_percent(o.coverage)}，保持推断" + summary = f"覆盖率 {_percent(o.coverage)}，保持推断" else: summary = "被指向列未核实唯一，保持推断" return {"kind": "relation", "path": f"relations.{o.rid}", "target": o.target, @@ -1241,6 +1499,8 @@ def _codes_value(c: _CodesOutcome, item: dict[str, Any] | None) -> dict[str, str 沿用它的含义；它写了、数据里没出现的取值也留着——码值是「可能的取值」，没出现不等于不存在。""" old = item.get("value") if isinstance(item, dict) and isinstance(item.get("value"), dict) else {} value = {key: str(old.get(key) or "") for key, _ in c.values} + for key in c.declared: + value.setdefault(key, str(old.get(key) or "")) for key, meaning in old.items(): if key not in value and isinstance(meaning, str) and meaning.strip(): value[key] = meaning @@ -1744,8 +2004,8 @@ def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> Profile #: 有效行过滤核对的说法（写进这一项的 note；前面是模型给的依据） FILTER_CHECKED = "剖析核对：排除 {share} 的行（共 {rows} 行）" -FILTER_SAMPLED = "剖析核对：按前 {rows} 行统计，排除 {share} 的行" -FILTER_SAMPLED_EDGE = "剖析核对：按前 {rows} 行统计，{what}，请人工核对" +FILTER_SAMPLED = "剖析核对：按存储顺序取前 {rows} 行统计，排除 {share} 的行" +FILTER_SAMPLED_EDGE = "剖析核对：按存储顺序取前 {rows} 行统计，{what}，请人工核对" FILTER_UNCHECKED = "没有核对排除比例（{why}）" FILTER_NOT_ENABLED = "数据源没有开数据剖析" diff --git a/backend/app/data/csv_grid.py b/backend/app/data/csv_grid.py new file mode 100644 index 00000000..7fb8ece7 --- /dev/null +++ b/backend/app/data/csv_grid.py @@ -0,0 +1,206 @@ +"""CSV 走配方（Excel 期 5 第 4 项）：把 CSV 适配成配方读取层认的样子——一张可见工作表的扫描结果（WorkbookScan） +和网格（Grid / 逐行流），之后的执行器、核对、说明、溯源、按期累积全部复用，不另写一套 CSV 的解析（否则会和配方 +语言分叉，违背「一种说法」）。 + +**怎么认出是 CSV。** 和简单导入（tabular._CSV）一样按扩展名定：只有 .csv / .tsv 按 CSV 读，并且内容得是文字 +（不是 zip、不是旧版 .xls 的 OLE 文件头、前 64 KB 里没有 NUL 字节）。以前只看内容，企业系统常见的「HTML 表格冒充 +.xls」、误传的文本文件都会被拆成一张叫「CSV」的工作表去起草、核对，不再像以前那样明确拒收；现在别的扩展名照旧交给 +xlsx_scan 拒收，内容是网页或 XML 的另给一句能照着做的话。编码识别复用 tabular 读 CSV 的那一段（UTF-8 / GB18030 / +带 BOM 的 UTF-16）；.tsv 一律按制表符分隔（同 tabular），.csv 嗅探逗号、分号、制表符、竖线。扫描时定下的分隔符 +记进工作表的 path（CSV_PATH 后面跟分隔符），之后读格子用同一个，不再各嗅各的。 + +**坐标沿用「工作表!A1」。** 细案原写「格子坐标记成 R{行}C{列}」，这里没这么做（见交付说明的偏离项）：坐标在执行器、 +核对、修复按钮、框选、溯源、裁判摘录、前端网格里一共有十来处解析和拼写，引入第二种写法就要处处认两种，漏一处就是 +着色错位、下钻找不到格子；期 1 读 CSV 时（tabular._shape_reasons）用的也是 A1。CSV 没有工作表，名字固定为 +CSV_SHEET：每月文件名不同（orders_2026-08.csv、orders_2026-09.csv），配方按工作表名认，名字必须稳定。行号是第几条 +记录（引号里的换行不另起一行），列号是第几个字段，和用 Excel 打开这个 CSV 看到的 A1 坐标一致。 + +**格子的值。** CSV 里只有文字。只有写法毫无歧义的才转成数（-12、80.5，不超过 15 位有效数字），其余一律原样留作文字： +「1,234」（千分位，交给配方的 text_number 决定）、「007」（前导零是编码的一部分）、「+5」「1e5」「12%」、日期写法 +（执行器按 DATE 列解析文字日期，起草按文字认日期）、「=SUM(…)」（CSV 里不是公式，Excel 打开时才会当公式算）。 +空字段、只有空白的字段是空格（和 xlsx_scan 的口径一致）。没有公式、合并单元格、隐藏行列。 +""" +from __future__ import annotations + +import csv +import io +import re +from dataclasses import dataclass +from typing import Any, Iterator + +from app.data.recipe_types import GridCell +from app.data.xlsx_scan import Bounds, SheetScan, UnsupportedTable, WorkbookScan + +__all__ = ["CSV_PATH", "CSV_SHEET", "CSV_SUFFIXES", "MAX_COLS", "MAX_ROWS", "CsvTable", "cell_value", "is_csv_sheet", + "looks_like_csv", "looks_like_markup", "parse", "rows", "scan", "scan_any", "sheet_delimiter"] + +#: CSV 的「工作表名」：配方按它认（SheetMatch.name），每月的文件名不同也认得出 +CSV_SHEET = "CSV" +#: SheetScan.path 的开头：读取层据此知道这张「工作表」要从 CSV 里读，不去压缩包里找 XML。后面跟着扫描时定下的分隔符 +#: （"csv:,"、"csv:\t"）；只有「csv:」的（这一版之前的暂存区）读的时候再嗅一次 +CSV_PATH = "csv:" +#: 按 CSV 读的扩展名，和简单导入（tabular._CSV）一致 +CSV_SUFFIXES = (".csv", ".tsv") +#: 嗅探认的分隔符 +_DELIMITERS = ",;\t|" +#: 内容是网页或 XML（「HTML 表格冒充 .xls」、Excel 2003 的 XML 表格）时的说法 +MARKUP_MESSAGE = ("无法打开该文件：它其实是网页或 XML 格式（有的系统导出的「Excel」文件是这样），不是 Excel 工作簿。" + "请用 Excel 打开后另存为 .xlsx 或 .csv 再上传") +NOT_TEXT_MESSAGE = ("无法按 CSV 读取这个文件：扩展名是 CSV，内容却不是文字（可能是图片、PDF 之类的文件改了扩展名）。" + "请确认上传的文件，或在 Excel 中另存为 .csv 后重新上传") +#: 和 Excel 的上限一致：超过的 CSV 用 Excel 打开也装不下，交给模型的格子坐标也写不出来 +MAX_ROWS = 1_048_576 +MAX_COLS = 16_384 +#: 嗅探看多少字节 +_SNIFF = 64 * 1024 +#: 写法毫无歧义的整数、小数（不带正号、不带前导零、不用科学计数法） +_INT = re.compile(r"-?(?:0|[1-9][0-9]*)") +_DEC = re.compile(r"-?(?:0|[1-9][0-9]*)\.[0-9]+") +#: 超过这么多位有效数字的不转成数：转成浮点会丢位（19 位卡号撞值），留作文字 +_DIGITS_MAX = 15 + + +def looks_like_csv(raw: bytes) -> bool: + """这份原件按 CSV 读：不是 zip、不是 OLE（旧版 .xls）、开头没有 NUL 字节、不是空文件。带 BOM 的 UTF-16 例外 + （每个字符都带 NUL，Excel「Unicode 文本」另存出来的就是它）。""" + if not raw or raw[:4] == b"PK\x03\x04" or raw[:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1": + return False + if raw[:2] in (b"\xff\xfe", b"\xfe\xff"): + return True + return b"\x00" not in raw[:_SNIFF] + + +def looks_like_markup(raw: bytes) -> bool: + """内容是网页或 XML：去掉 BOM 和开头的空白后以「<」开头。真的工作簿以 zip（PK）或 OLE 文件头开头，不会是这样。""" + head = raw[:4096] + if head[:2] in (b"\xff\xfe", b"\xfe\xff"): + try: + text = head.decode("utf-16", errors="ignore") + except UnicodeDecodeError: + return False + else: + text = head.decode("latin-1") + return text.lstrip("\ufeff\xef\xbb\xbf \t\r\n").startswith("<") + + +def _suffix(filename: str | None) -> str: + name = str(filename or "").rsplit("/", 1)[-1] + return "." + name.rsplit(".", 1)[-1].lower() if "." in name else "" + + +def cell_value(text: str) -> Any: + """CSV 的一个字段 → 格子的值：空 / 只有空白是 None（空格），写法毫无歧义的整数、小数转成数，其余原样（文字）。""" + t = text.strip() + if not t: + return None + digits = len(t.lstrip("-").replace(".", "").lstrip("0")) or 1 + if digits <= _DIGITS_MAX: + if _INT.fullmatch(t): + return int(t) + if _DEC.fullmatch(t): + return float(t) + return text + + +@dataclass +class CsvTable: + """解析好的 CSV：records[i] 是第 i + 1 条记录的字段（原文）。""" + + records: list[list[str]] + delimiter: str + + +def parse(raw: bytes, delimiter: str | None = None) -> CsvTable: + """原件 → 记录。编码照 tabular 读 CSV 的做法；delimiter 不给时嗅探（同 tabular）。字段太长、引号不配对读不了时抛 + UnsupportedTable。""" + from app.data import tabular + + text = tabular._decode_csv(raw) + if delimiter is None: + try: + delimiter = csv.Sniffer().sniff(text[:8192], delimiters=_DELIMITERS).delimiter + except csv.Error: + delimiter = "," # 嗅不出来就按最常见的来 + records: list[list[str]] = [] + try: + for record in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter): + if len(records) >= MAX_ROWS: + raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行，超出可以导入的上限") + if len(record) > MAX_COLS: + raise UnsupportedTable(f"CSV 文件第 {len(records) + 1} 行有 {len(record):,} 列，超过 {MAX_COLS:,} 列的上限") + records.append(record) + except csv.Error as e: + raise UnsupportedTable(f"无法按 CSV 读取这个文件：第 {len(records) + 1} 行的写法不正确（{e}）。" + "请在 Excel 中重新另存为 CSV 后上传") from e + return CsvTable(records=records, delimiter=delimiter) + + +def _cells(table: CsvTable) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + for r, record in enumerate(table.records, 1): + out = [(c, GridCell(v)) for c, field in enumerate(record, 1) if (v := cell_value(field)) is not None] + if out: + yield r, out + + +def is_csv_sheet(sheet: Any) -> bool: + """这张「工作表」是 CSV 适配出来的（path 以 CSV_PATH 开头）。""" + return str(getattr(sheet, "path", "") or "").startswith(CSV_PATH) + + +def sheet_delimiter(sheet: Any) -> str | None: + """扫描时定下的分隔符（记在 path 里）；这一版之前的暂存区没记，返回 None（读的时候再嗅）。""" + tail = str(getattr(sheet, "path", "") or "")[len(CSV_PATH):] + return tail if len(tail) == 1 and tail in _DELIMITERS else None + + +def scan(raw: bytes, *, delimiter: str | None = None) -> WorkbookScan: + """CSV 原件 → 扫描结果：一张可见工作表（名字 CSV_SHEET、路径 CSV_PATH 加分隔符），边界、非空格数、逐行非空格数的 + 游程。没有合并区、隐藏行列、公式、错误值、离群格（CSV 里每一格都是写出来的，没有残留格式）。""" + table = parse(raw, delimiter) + sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + table.delimiter) + lo_r = lo_c = hi_r = hi_c = 0 + runs: list[tuple[int, int, int]] = [] + for r, cells in _cells(table): + n = len(cells) + sheet.nonempty += n + first, last = cells[0][0], cells[-1][0] + if not lo_r: + lo_r, lo_c, hi_c = r, first, last + lo_c, hi_c, hi_r = min(lo_c, first), max(hi_c, last), r + if runs and runs[-1][1] == r - 1 and runs[-1][2] == n: + runs[-1] = (runs[-1][0], r, n) + else: + runs.append((r, r, n)) + if lo_r: + sheet.bounds = Bounds(lo_r, lo_c, hi_r, hi_c) + sheet.dimension = sheet.bounds.a1() + sheet.row_runs = runs + return WorkbookScan(sheets=[sheet], format="csv") + + +def scan_any(raw: bytes, filename: str | None) -> WorkbookScan: + """配方导入的扫描入口：.csv / .tsv 走这里的适配（.tsv 按制表符分隔），其余照旧交给 xlsx_scan（读不了照旧拒收， + 文案不变）；内容是网页或 XML 的另给一句能照着做的话（MARKUP_MESSAGE）。扩展名是 CSV、内容却是工作簿（xlsx 改了 + 扩展名）的照工作簿读。""" + from app.data import xlsx_scan + + suffix = _suffix(filename) + if suffix in CSV_SUFFIXES: + if looks_like_csv(raw): + return scan(raw, delimiter="\t" if suffix == ".tsv" else None) + if raw[:4] != b"PK\x03\x04": + raise UnsupportedTable(NOT_TEXT_MESSAGE) + elif looks_like_markup(raw): + raise UnsupportedTable(MARKUP_MESSAGE) + return xlsx_scan.scan(raw) + + +def rows(table: CsvTable, *, min_row: int, max_row: int, min_col: int, + max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + """按行流式给一个矩形里的非空格：(行号, [(列号, 格)…])，口径同 xlsx_cells.sheet_rows（整行为空的不给）。""" + for r in range(max(min_row, 1), min(max_row, len(table.records)) + 1): + record = table.records[r - 1] + out = [(c, GridCell(v)) for c in range(max(min_col, 1), min(max_col, len(record)) + 1) + if (v := cell_value(record[c - 1])) is not None] + if out: + yield r, out diff --git a/backend/app/data/introspect.py b/backend/app/data/introspect.py index ea8875be..7d7f478b 100644 --- a/backend/app/data/introspect.py +++ b/backend/app/data/introspect.py @@ -165,6 +165,89 @@ def _unique_groups(constraints: list[dict[str, Any]], indexes: list[dict[str, An return groups +def _storage_type(col_type: Any) -> Any: + """PostgreSQL 的域（DOMAIN）换成它底下的类型（域套域的一路往下找）；别的类型原样返回。""" + from sqlalchemy.dialects.postgresql import DOMAIN + + seen = 0 + while isinstance(col_type, DOMAIN) and getattr(col_type, "data_type", None) is not None and seen < 8: + col_type, seen = col_type.data_type, seen + 1 + return col_type + + +def type_text(col_type: Any, dialect: Any) -> str: + """反射到的列类型 → 存进表结构的写法：按这个数据源的方言编译的完整写法（VARCHAR2(20 CHAR)、NUMBER(10, 0)、 + ENUM('paid','refund')、TIMESTAMP WITH TIME ZONE），编译不了的退回 str()。PostgreSQL 的域（DOMAIN）记它底下的类型： + 域编译出来只是域名（settlement_amount），看不出是数字还是文字，名字里的词（set、UPDATE 里的 DATE）还会让类型大类、 + 日期列、码值候选认错；模型照着写 SQL 也要知道它存的是什么。 + + 以前存的是 str(类型)，即 SQLAlchemy 的通用写法：Oracle 的 VARCHAR2、NUMBER 丢了长度和精度，MySQL 的 ENUM 丢了 + 取值列表，PostgreSQL 的 TIMESTAMP 丢了时区。类型写不全，表结构前后比对就看不出类型变了，ENUM 状态列也进不了 + 码值候选。这一改会让新探查的表结构快照哈希变化（提示词里的类型写法变长），旧运行封存的快照不受影响。""" + if col_type is None: + return "" + col_type = _storage_type(col_type) + try: + compiled = str(col_type.compile(dialect=dialect)) + except Exception: # noqa: BLE001 - 编译不了（NullType 等）就用通用写法，不让一列拖垮整次探查 + return str(col_type) + if getattr(dialect, "name", "") == "oracle" and compiled.startswith("VARCHAR("): + # SQLAlchemy 把 Oracle 的 VARCHAR2 反射成通用的 VARCHAR，按 Oracle 方言编译出来是 VARCHAR(10 CHAR)（容器集成 + # 测试实测）。Oracle 的数据字典里只有 VARCHAR2（建表写 VARCHAR 也存成 VARCHAR2），照数据字典写 + compiled = "VARCHAR2" + compiled[len("VARCHAR"):] + return compiled + + +def _enum_values(col_type: Any) -> list[str] | None: + """ENUM（MySQL、PostgreSQL 的枚举，都是 sqltypes.Enum 的子类）的取值列表，MySQL SET 的成员列表；别的类型 None。 + 套在域（DOMAIN）里的枚举也算（同 type_text）。""" + from sqlalchemy.dialects.mysql import SET + from sqlalchemy.types import Enum + + col_type = _storage_type(col_type) + if isinstance(col_type, Enum): + values = list(col_type.enums or []) + elif isinstance(col_type, SET): + values = list(col_type.values or []) + else: + return None + return [str(v) for v in values] + + +def _forced_quote(name: Any) -> bool: + """名字必须原样加引号：Oracle 反射回来的全小写对象（建表时写了 "visits"）是 quoted_name(quote=True)。 + 它存进 JSON 就成了普通字符串，剖析拼 SQL 时不加引号，Oracle 把它转成大写 VISITS，查的是另一个不存在的对象。""" + return getattr(name, "quote", None) is True + + +def column_entry(col: Mapping[str, Any], dialect: Any) -> dict[str, Any]: + """反射到的一列 → 表结构里的一列：名字、按方言编译的类型、可空、注释；ENUM / SET 另记取值列表（enum_values）， + 要强制加引号的名字另记 quote_name。只在有的时候写这两个键：其余列和以前一字不差。""" + entry: dict[str, Any] = { + "name": str(col["name"]), + "type": type_text(col.get("type"), dialect), + "nullable": bool(col.get("nullable", True)), + "comment": col.get("comment") or None, + } + values = _enum_values(col.get("type")) + if values: + entry["enum_values"] = values + if _forced_quote(col.get("name")): + entry["quote_name"] = True + return entry + + +def index_first_columns(indexes: list[dict[str, Any]], pk: list[str]) -> list[str]: + """哪些列打头有索引：各索引（含主键）的第一列，去重、按名字排序。取值观察看大表的最新业务日期、按日行数时， + 只在业务日期列打头有索引时才发窗口查询（按它走索引，不扫全表）。表达式索引（列名是 None）不算。""" + first = {str(pk[0])} if pk else set() + for ix in indexes: + cols = ix.get("column_names") or [] + if cols and cols[0] is not None: + first.add(str(cols[0])) + return sorted(first) + + async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any]: """探查整个库的结构，结果直接存进 DataSource.schema_cache。 @@ -266,24 +349,20 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any # 只读账号名下没有这些对象，全靠跨 schema 授权 "qualified": f"{target_schema}.{name}" if target_schema else name, "schema": target_schema, - "columns": [ - { - "name": c["name"], - "type": str(c.get("type") or ""), - "nullable": bool(c.get("nullable", True)), - "comment": c.get("comment") or None, - } - for c in columns - ], + "columns": [column_entry(c, sync_conn.dialect) for c in columns], "primary_key": pk, "comment": comment, "is_view": name in views, } + if _forced_quote(name): + tables[name]["quote_name"] = True if name in fk_ok: tables[name]["foreign_keys"] = _foreign_keys(fk_by.get(key) or [], target_schema, default_schema) if name in uq_ok: tables[name]["unique"] = _unique_groups(uq_by.get(key) or [], (ix_by.get(key) or []) if name in ix_ok else [], pk) + if name in ix_ok: + tables[name]["index_first_columns"] = index_first_columns(ix_by.get(key) or [], pk) payload: dict[str, Any] = {"tables": tables, "truncated": truncated, "total": len(names) + len(views)} if failures and not tables: @@ -471,6 +550,10 @@ def describe_table(source: Any, table: str, *, catalog: Mapping[str, Mapping[str marks.append("非空") if col.get("comment"): marks.append(col["comment"]) + values = col.get("enum_values") + if values and "(" not in str(col.get("type") or ""): + # PostgreSQL 的枚举类型写法只有类型名（order_status）：取值列表另外写出来，模型才知道能写哪些值 + marks.append("取值：" + "、".join(str(v) for v in values[:30]) + ("…" if len(values) > 30 else "")) suffix = f" {'、'.join(marks)}" if marks else "" lines.append(f" {col['name']} {col['type']}{suffix}") if catalog: diff --git a/backend/app/data/recipe.py b/backend/app/data/recipe.py index 6e4d4a6d..5bc4cad5 100644 --- a/backend/app/data/recipe.py +++ b/backend/app/data/recipe.py @@ -43,6 +43,9 @@ from app.data.recipe_parsers import ( text_label, ) from app.data.recipe_types import ( + LIST_PERIOD_END, + LIST_PERIOD_START, + LIST_SUBTOTAL_DIM, LIST_TOTAL_DIM, ColumnOut, CrosstabBlock, @@ -271,6 +274,8 @@ _FIELD_LABEL = { "parts": "组成列", "claims": "认领", "a": "一方", "b": "另一方", "reason": "理由", "ignore_rows": "按行标签忽略的行", "ignore_columns": "按表头忽略的列", "ignore_outside": "按同一行文字忽略的数字", "label": "标签", "anchor": "同一行的文字", + "subtotals": "小计行", "no_subtotals": "没有小计行", "scope": "小计管哪些明细", "group_column": "分组的列", "period": "每行属于哪一期", + "column": "日期列", "check": "核对方式", "synthesize": "加统计期列", } @@ -608,15 +613,25 @@ class _Checker: for si, bi, block in self.lists: base = ("sheets", si, "blocks", bi) cols = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns)] + if block.period is not None and block.period.synthesize: + # 期 5：加上的统计期起止两列也是这张表的列，和表头推出的列名不能撞 + pp = base + ("period", "synthesize") + cols += [(LIST_PERIOD_START, pp), (LIST_PERIOD_END, pp)] out.append(_Writer(block.table, base + ("table",), "data", cols, origin=_pointer(base))) + numeric = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns) if c.type in _NUMERIC] total = block.rows.total_row if total is not None and total.keep_as: tp = base + ("rows", "total_row", "keep_as") - tcols = [(LIST_TOTAL_DIM, tp)] - tcols += [(c.name, base + ("columns", n, "name")) - for n, c in enumerate(block.columns) if c.type in _NUMERIC] - out.append(_Writer(total.keep_as, tp, "reported_total", tcols, + out.append(_Writer(total.keep_as, tp, "reported_total", [(LIST_TOTAL_DIM, tp)] + numeric, origin=_pointer(base + ("rows", "total_row")))) + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + sp = base + ("rows", "subtotals", "keep_as") + scols = [(LIST_SUBTOTAL_DIM, sp)] + if sub.scope == "group_column" and sub.group_column: + scols.append((sub.group_column, base + ("rows", "subtotals", "group_column"))) + out.append(_Writer(sub.keep_as, sp, "reported_total", scols + numeric, + origin=_pointer(base + ("rows", "subtotals")))) return out # ---- 8. 名字 ---- @@ -680,6 +695,10 @@ class _Checker: if sheet.context: continue for bi, block in enumerate(sheet.blocks): + if isinstance(block, ListBlock) and block.period is not None: + # 期 5：每行属于哪一期要拿统计期来核对（按日期列）或填进去（加起止两列） + self.add(("sheets", si, "blocks", bi, "period"), "covers_without_context", + "要按统计期核对或填写每一行的统计期，但这个工作表没有配置统计期") if not isinstance(block, CrosstabBlock): continue ap = ("sheets", si, "blocks", bi, "axis") @@ -873,6 +892,10 @@ class _Checker: if total is not None and any(ch.isdigit() for ch in total.pick): self.add(("sheets", si, "blocks", bi, "rows", "total_row", "pick"), "period_literal", f"合计行的开头词「{total.pick}」不能含数字（配方里不能写死年份或期次）") + sub = block.rows.subtotals + if sub is not None and any(ch.isdigit() for ch in sub.pick): + self.add(("sheets", si, "blocks", bi, "rows", "subtotals", "pick"), "period_literal", + f"小计行的开头词「{sub.pick}」不能含数字（配方里不能写死年份或期次）") # ---- 4、9、13. 单位：键是本表列名，值在词表里，与原表标签、表头里的单位一致 ---- @@ -902,10 +925,13 @@ class _Checker: self.unit_against(seg.keep_as.table, seg.keep_as.value, target.locate.title, "分段标题") for _si, _bi, block in self.lists: total = block.rows.total_row + sub = block.rows.subtotals for c in block.columns: self.unit_against(block.table, c.name, c.header, "表头") if total is not None and total.keep_as and c.type in _NUMERIC: self.unit_against(total.keep_as, c.name, c.header, "表头") + if sub is not None and sub.keep_as and c.type in _NUMERIC: + self.unit_against(sub.keep_as, c.name, c.header, "表头") def after_target(self, si: int, bi: int, seg: DerivedSegment) -> DimensionSegment | None: """合计分段紧跟的那个 dimension 分段（同一块里按 id 找）；找不到或不是 dimension 段时已报 derived_after_invalid。""" @@ -998,6 +1024,8 @@ class _Checker: if total is not None and total.label_column not in names: self.add(base + ("rows", "total_row", "label_column"), "column_unknown", f"合计标签所在的列「{total.label_column}」不在这个列表的列里{self._near(total.label_column, names)}") + self.check_subtotals(base, block, names) + self.check_list_period(base, block, names) seen: dict[str, str] = {} for n, c in enumerate(block.columns): key = match_key(c.header) @@ -1006,6 +1034,53 @@ class _Checker: f"表头「{c.header}」与「{seen[key]}」是同一个表头") seen.setdefault(key, c.header) + def check_subtotals(self, base: tuple[Any, ...], block: ListBlock, names: list[str]) -> None: + """期 5：小计行的标签列、分组列要是这个列表的列；按组小计要给分组列；小计和合计的开头词不能互为前缀 + （「合计」「合计小计」：一行就分不清是小计还是合计）；配了小计又声明没有小计（no_subtotals）是矛盾的。""" + sub = block.rows.subtotals + if sub is None: + return + sp = base + ("rows", "subtotals") + if block.rows.no_subtotals: + self.add(base + ("rows", "no_subtotals"), "subtotals_conflict", + "配了小计行，又声明了这张列表没有小计行：二选一") + if sub.label_column not in names: + self.add(sp + ("label_column",), "column_unknown", + f"小计标签所在的列「{sub.label_column}」不在这个列表的列里{self._near(sub.label_column, names)}") + if sub.scope == "group_column": + if not sub.group_column: + self.add(sp + ("group_column",), "subtotal_group_missing", "按组小计时要指明按哪一列分组") + elif sub.group_column not in names: + self.add(sp + ("group_column",), "column_unknown", + f"小计分组的列「{sub.group_column}」不在这个列表的列里{self._near(sub.group_column, names)}") + elif sub.group_column: + self.add(sp + ("group_column",), "subtotal_group_missing", + "小计管「上一行小计之后的全部明细」时不按列分组，请去掉分组的列，或改为按组小计") + total = block.rows.total_row + if total is not None: + a, b = match_key(sub.pick), match_key(total.pick) + if a.startswith(b) or b.startswith(a): + self.add(sp + ("pick",), "subtotal_pick_conflict", + f"小计行的开头词「{sub.pick}」和合计行的「{total.pick}」一个是另一个的开头，分不清是小计还是合计") + + def check_list_period(self, base: tuple[Any, ...], block: ListBlock, names: list[str]) -> None: + """期 5：每行属于哪一期，按日期列核对和加起止两列只能二选一；按日期列时那一列要是这个列表的日期列。""" + period = block.period + if period is None: + return + pp = base + ("period",) + if (period.column is None) == (period.synthesize is None): + self.add(pp, "list_period_invalid", "每一行属于哪一期只能二选一：按一个日期列核对，或加统计期起止两列") + return + if period.column is not None: + col = next((c for c in block.columns if c.name == period.column), None) + if col is None: + self.add(pp + ("column",), "column_unknown", + f"核对统计期的列「{period.column}」不在这个列表的列里{self._near(period.column, names)}") + elif col.type != "DATE": + self.add(pp + ("column",), "list_period_invalid", + f"核对统计期的列「{period.column}」的类型要是日期") + # ---- 期 3：忽略规则、表头上方的标题（P3-SPEC 9.2） ---- def check_ignores(self) -> None: diff --git a/backend/app/data/recipe_accumulate.py b/backend/app/data/recipe_accumulate.py index 5f6a8b54..ac689387 100644 --- a/backend/app/data/recipe_accumulate.py +++ b/backend/app/data/recipe_accumulate.py @@ -30,6 +30,7 @@ from app.data.recipe_parsers import hour_range, hour_range_total, match_key, tex from app.data.recipe_types import ( UNION_VER, AccumulatePlan, ChangeClass, CheckResult, ColumnOut, CrosstabBlock, DerivedSegment, DimensionSegment, PartInfo, Recipe, UnionPart, UnionReport, accumulate_blockers, derive_tables, + list_period_columns, ) from app.data.table_versions import settle_journal, sha_json @@ -508,14 +509,19 @@ def union_options(parts: list[tuple[str, str, str]], target_recipe_sha: str, *, class _Table: - """并集里的一张表：列（目标配方的在前、退役的按给定顺序追加在后）、主键、日期轴列。""" + """并集里的一张表：列（目标配方的在前、退役的按给定顺序追加在后）、主键、日期轴列。期 5：列表写入的表按 period + 分期（list_period_columns）：按日期列的，U3 核对那一列落在该期统计期内；加统计期起止两列的，核对两列就是该期。""" - def __init__(self, name: str, cols: list[ColumnOut], grain: list[str]) -> None: + def __init__(self, name: str, cols: list[ColumnOut], grain: list[str], + period: tuple[str, ...] | None = None) -> None: self.name = name self.cols = cols self.grain = grain self.axis = next((c.name for c in cols if c.role == "axis"), None) self.names = [c.name for c in cols] + self.period = period if period is None or all(c in self.names for c in period[1:]) else None + if self.axis is None and self.period is not None and self.period[0] == "column": + self.axis = self.period[1] def _copy_col(col: ColumnOut) -> ColumnOut: @@ -551,6 +557,7 @@ def _layout(target: Recipe, parts: list[UnionPart], out: dict[str, _Table] = {} retired = retired or {} + periods = list_period_columns(target) for name, cols in tables.items(): mine = [_copy_col(c) for c in cols] known = {c.name for c in mine} @@ -561,7 +568,7 @@ def _layout(target: Recipe, parts: list[UnionPart], mine.append(_copy_col(found) if found is not None else _copy_col(col)) known.add(col.name) spec = specs.get(name) - out[name] = _Table(name, mine, list(spec.grain) if spec is not None else []) + out[name] = _Table(name, mine, list(spec.grain) if spec is not None else [], periods.get(name)) for name, cols in retired.items(): if name in out: continue @@ -831,20 +838,36 @@ def _finish(conn: sqlite3.Connection, layout: dict[str, _Table], parts: list[Uni u2.append(f"表「{table.name}」有 {dup} 组主键重复") checks.append(_check("U2", "union_pk", "累积后主键不重复", checked=len(keyed), failed=u2, sql=sql2)) - # U3 统计期：第 k 期的 rowid 区间里，日期都在 [start_k, end_k] 之内 + # U3 统计期：第 k 期的 rowid 区间里，日期都在 [start_k, end_k] 之内。期 5：按日期列分期的列表，日期为空也算不在； + # 加统计期起止两列的列表，两列必须正好是这一期 u3: list[str] = [] sql3, params3, checked3 = None, [], 0 for k, (part, ranges) in enumerate(zip(parts, part_rows), 1): for name, span in ranges.items(): - axis = layout[name].axis - if axis is None or axis not in part_cols[k - 1].get(name, []): - continue + table = layout[name] + have = part_cols[k - 1].get(name, []) a, b = span["union"] + if table.period is not None and table.period[0] == "synthesize": + start_col, end_col = table.period[1], table.period[2] + if start_col not in have or end_col not in have or b < a: + continue + checked3 += 1 + sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " + f"AND ({_q(start_col)} IS NOT ? OR {_q(end_col)} IS NOT ?)") + params3 = [a, b, part.start, part.end] + bad = conn.execute(sql3, params3).fetchone()[0] + if bad: + u3.append(f"第 {k} 期（{_period_text(part.start, part.end)}）的「{name}」有 {bad} 行的统计期不是这一期") + continue + axis = table.axis + if axis is None or axis not in have: + continue if b < a: continue checked3 += 1 + nulls = f"{_q(axis)} IS NULL OR " if table.period is not None else "" sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " - f"AND ({_q(axis)} < ? OR {_q(axis)} > ?)") + f"AND ({nulls}{_q(axis)} < ? OR {_q(axis)} > ?)") params3 = [a, b, part.start, part.end] bad = conn.execute(sql3, params3).fetchone()[0] if bad: diff --git a/backend/app/data/recipe_checks.py b/backend/app/data/recipe_checks.py index e095ada7..79a21b96 100644 --- a/backend/app/data/recipe_checks.py +++ b/backend/app/data/recipe_checks.py @@ -23,6 +23,7 @@ from typing import Any, Callable, Iterable from app.data.engine import open_checked_sqlite from app.data.recipe_types import ( + SUBTOTAL_KEY, CheckResult, ColumnOut, CrosstabBlock, @@ -34,6 +35,7 @@ from app.data.recipe_types import ( Recipe, SumEq, derive_tables, + list_period_columns, ) #: 无法核对的原因（契约 CheckResult.reasons 的键）的固定顺序：细节、说明都按这个顺序列 @@ -71,24 +73,33 @@ class CheckPlan: k: dict[str, str] = field(default_factory=dict) #: derived 分段键 → G 编号（只给有公式格的 derived 段） g: dict[str, str] = field(default_factory=dict) - #: 列表块键 → T 编号（每个有合计行的列表块一条） + #: 列表块键 → T 编号（每个有合计行的列表块一条）；期 5：块键 + SUBTOTAL_KEY → 这个块的小计核对（一条） t: dict[str, str] = field(default_factory=dict) items: dict[str, list[DerivedItem]] = field(default_factory=dict) def _recipe_keys(recipe: Recipe) -> tuple[list[str], list[str]]: - """配方里 derived 分段键、带合计行的列表块键，按配方里的先后。""" + """配方里 derived 分段键、带合计行的列表块键，按配方里的先后。期 5：带小计的列表块另有一个键（块键 + + SUBTOTAL_KEY），排在这个块的合计行后面：同一块的小计和合计各编一个 T，结论分开写。""" derived: list[str] = [] totals: list[str] = [] for sheet in recipe.sheets: for block in sheet.blocks: if isinstance(block, CrosstabBlock): derived += [s.id for s in block.segments if isinstance(s, DerivedSegment)] - elif block.rows.total_row is not None: + continue + if block.rows.total_row is not None: totals.append(block.id) + if block.rows.subtotals is not None: + totals.append(f"{block.id}{SUBTOTAL_KEY}") return derived, totals +def is_subtotal_key(key: str) -> bool: + """T 编号的键是不是一个块的小计核对（期 5）。""" + return key.endswith(SUBTOTAL_KEY) + + def _ordered(recipe_keys: list[str], items: list[DerivedItem]) -> list[str]: """先按配方里的先后；配方里找不到的分段（不该出现）按执行结果里第一次出现的先后排在后面，不丢。""" present: list[str] = [] @@ -570,6 +581,25 @@ def _check_pk(sql: _Sql, cid: str, title: str, table: str, grain: list[str]) -> # ---------------------------------------------------------------- 入口 +def _check_period_rows(sql: _Sql, cid: str, title: str, table: str, column: str, period: Any, + lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult: + """期 5：列表按日期列分期（ListPeriod.column、check=within_context）时，每行的这一列都落在本期统计期内。 + 日期为空的行也算不在：看不出它属于哪一期。不在的行按期累积会混进别的期（统计期互不重叠的前提就破了）， + 所以是结构类、不能写理由接受。统计期没定下来（要人录入）时不会走到这里：那时不建库。""" + if period is None or not period.start or not period.end: + raise _Broken("本期没有统计期，无法核对每行的日期") + q = (f"SELECT rowid FROM {_q(table)} WHERE {_q(column)} IS NULL OR {_q(column)} < ? OR {_q(column)} > ? " + "ORDER BY rowid") + params = [period.start, period.end] + bad = [r[0] for r in sql.all(q, params)] + total = sql.one(f"SELECT COUNT(*) FROM {_q(table)}")[0] + cells = [c for rowid in bad[:_DETAIL_MAX] if (c := lineage_cell(lineage, table, column, rowid))] + details = [f"有 {len(bad)} 行的「{column}」为空或不在 {period.start} 至 {period.end} 之内"] if bad else [] + return CheckResult(id=cid, kind="period_rows", title=title, status="mismatch" if bad else "passed", + category="structure", checked=int(total), failed=len(bad), sql=q, params=params, + details=details, cells=cells) + + def _axis_names(recipe: Recipe) -> dict[str, str]: """derived 分段键 → 所在交叉表块的轴列名（K 的铺满判定不按日期分）。""" out: dict[str, str] = {} @@ -629,7 +659,9 @@ def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[Che jobs.append((cid, "formula_refs", f"合计公式引用（分段「{seg}」）", lambda t, seg=seg, cid=cid: _check_g(sql, cid, t, plan.items[seg]))) for block, cid in plan.t.items(): - jobs.append((cid, "column_sum", f"合计行按列求和（块「{block}」）", + title = (f"小计行按各组明细求和（块「{block.removesuffix(SUBTOTAL_KEY)}」）" if is_subtotal_key(block) + else f"合计行按列求和（块「{block}」）") + jobs.append((cid, "column_sum", title, lambda t, block=block, cid=cid: _check_t(sql, cid, t, plan.items[block], full_calc))) for rel in recipe.relations: if isinstance(rel, SumEq): @@ -646,6 +678,12 @@ def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[Che for i, table in enumerate([t for t in tables if grains.get(t)], 1): jobs.append((f"P{i}", "pk_unique", f"主键唯一：「{table}」", lambda t, table=table, i=i: _check_pk(sql, f"P{i}", t, table, grains[table]))) + period = extraction.period + dated = [(t, spec[1]) for t, spec in list_period_columns(recipe).items() if spec[0] == "column" and t in tables] + for i, (table, column) in enumerate(dated, 1): + jobs.append((f"D{i}", "period_rows", f"「{table}」每行的「{column}」都在本期统计期内", + lambda t, table=table, column=column, i=i: _check_period_rows( + sql, f"D{i}", t, table, column, period, extraction.lineage))) try: return list(extraction.context_checks) + [_guarded(sql, *job) for job in jobs] finally: diff --git a/backend/app/data/recipe_confirm.py b/backend/app/data/recipe_confirm.py index f2a9419c..015abf54 100644 --- a/backend/app/data/recipe_confirm.py +++ b/backend/app/data/recipe_confirm.py @@ -43,7 +43,7 @@ from app.data.recipe_diff import ( ) from app.data.recipe_parsers import UNITS, match_key, period_residue, split_unit_suffix from app.data.recipe_types import ( - ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, + LIST_PERIOD_END, LIST_PERIOD_START, SUBTOTAL_KEY, ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, SheetRecipe, SumEq, derive_tables, ) @@ -523,6 +523,13 @@ def _signatures(recipe: Recipe, sheet_name: Callable[[SheetRecipe], str]) -> dic total = block.rows.total_row out[f"total_row:{block.id}"] = _sig( None if total is None else (total.label_column, total.pick, total.keep_as)) + # 期 5：穿插的小计、每行属于哪一期 + subt = block.rows.subtotals + out[f"subtotals:{block.id}"] = _sig( + None if subt is None else (subt.label_column, subt.pick, subt.scope, subt.group_column, subt.keep_as)) + out[f"no_subtotals:{block.id}"] = _sig(block.rows.no_subtotals) + period = block.period + out[f"list_period:{block.id}"] = _sig(None if period is None else (period.column, period.synthesize)) for r in recipe.relations: if isinstance(r, SumEq): out[f"relation:{r.id}"] = _sig("sum_eq", r.table, r.total, sorted(r.parts)) @@ -1058,6 +1065,29 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *, detail = (f"按明细核对的列：{'、'.join(nums)}；其余列合计行里的内容不核对" if nums else "这个块没有数字列，合计行里的内容不核对") bag.add(f"total_row:{b.id}", f"{prefix}改作核对{keep}", detail) + for b in lists: + # 期 5：穿插在明细中间的小计行：不进数据，按各组明细核对。要人确认：这几行从此不在本表里 + subt = b.rows.subtotals + if subt is not None: + rows = rows_text(derived_rows.get(f"{b.id}{SUBTOTAL_KEY}")) + prefix = f"「{b.id}」中间的小计行" + (f"（{rows}）" if rows else "") + keep = f"，原值另存表「{subt.keep_as}」" if subt.keep_as else "，原值不另存" + scope = ("每行小计核对上一行小计之后的全部明细" if subt.scope == "since_previous" + else f"每行小计核对紧挨在它上面、「{subt.group_column}」相同的那几行明细") + nums = [c.name for c in b.columns if c.type in ("INTEGER", "REAL")] + detail = f"{scope}；按明细核对的列：{'、'.join(nums)}" if nums else f"{scope}；这个块没有数字列，不核对" + bag.add(f"subtotals:{b.id}", f"{prefix}改作核对、不进数据{keep}", detail) + elif b.rows.no_subtotals: + # 声明没有小计行是放宽：以合计字样开头的行从此照常进数据，不再报「像小计」。配上（或改过）的那一次确认 + bag.add(f"no_subtotals:{b.id}", f"「{b.id}」没有小计行：以合计字样开头的行都按数据导入，不再提醒像小计的行", + "如果其实有穿插的小计，对这些列求和会把小计重复算进去") + period = b.period + if period is not None and period.column: + bag.add(f"list_period:{b.id}", f"「{b.id}」每一行的「{period.column}」都要在本期统计期内", + "按期累积时各期的行按这一列区分；有一行不在本期统计期内（或为空），这一期整份拒收") + elif period is not None and period.synthesize: + bag.add(f"list_period:{b.id}", f"「{b.id}」每一行加上本期统计期的起、止两列", + f"列名「{LIST_PERIOD_START}」「{LIST_PERIOD_END}」，取自表内的统计期或人工录入的统计期") # 年份取自统计期：表头是日期格时年份来自格子本身，不算 forms: dict[str, set[str]] = {} @@ -1235,6 +1265,14 @@ def _period_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], ctx: ConfirmCon bag.add(f"rows_after_stop:{blk}", "；".join(str(p.get("message") or "") for p in probs), "、".join(cells[:20]), source="outside") + # 期 5：没配小计的列表里像小计的数据行：照常导入了，要人确认它们确实是数据（不是的话用修复按钮改作小计核对） + for p in ex.get("problems") or []: + p = plain(p) + if isinstance(p, dict) and p.get("code") == "subtotal_suspect": + blk = str((p.get("fix_args") or {}).get("block") or "、".join(str(c) for c in p.get("cells") or [])) + bag.add(f"subtotal_suspect:{blk}", str(p.get("message") or ""), + "、".join(str(c) for c in (p.get("cells") or [])[:20]), source="outside") + # 列表块只有表头、没有数据行（AU-1）：问题对象里没有块键，按消息里的「列表「块」」认 for p in ex.get("problems") or []: p = plain(p) diff --git a/backend/app/data/recipe_engine.py b/backend/app/data/recipe_engine.py index 7c33e010..7f0ac181 100644 --- a/backend/app/data/recipe_engine.py +++ b/backend/app/data/recipe_engine.py @@ -39,6 +39,14 @@ pk_duplicate 同时出现）。块级的定位失败（找不到日期表头、 个矩形。 - 回执里的「排除的行」rows_excluded（H2：排除的行要写进回执）：隐藏行、列表跳过的空行、按规则忽略的行、 停止之后没导入的文字行、只核对不另存的合计行。 + +**期 5（下一轮 03 细案 E）又加了两件事，同样没用新字段的配方写出的库逐字节不变：** +- 列表中间穿插的小计（ListRows.subtotals）：小计行不进数据，每个数字格一条合计格，按它所管的那组明细（上一行小计 + 之后的全部明细，或紧挨在上面、分组列相同的那几行）交给核对模块用 SQL 求和比对；整行记进排除的行（subtotal）。 +- 列表按期累积时由系统加的统计期起止两列（ListPeriod.synthesize）：列表在统计期定下来之前就抽完了，这两格先放 + 占位，统计期定下来以后写库时换成真值（_Sink.fill_period）；按日期列分期的（ListPeriod.column）由核对模块的 D + 核对查每行都在本期统计期内。 +CSV 原件经 csv_grid 适配成一张「工作表」，这里照常按网格或逐行流读（xlsx_cells 按 SheetScan.path 分派）。 """ from __future__ import annotations @@ -59,15 +67,16 @@ from app.data import engine as _data_engine from app.data import tabular, xlsx_cells from app.data.names import canon from app.data.recipe_parsers import ( - AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month, filename_period, + TOTAL_WORDS, AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month, + filename_period, hour_range, hour_range_total, is_pure_period_cell, looks_numeric_text, match_key, month_day_or_date, period_residue, text_label, thousands_number, ) from app.data.recipe_types import ( - AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, DimensionSegment, - ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, + SUBTOTAL_KEY, AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, + DimensionSegment, ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, PeriodInput, PeriodOut, Problem, Recipe, RegionMark, SegmentLabels, SheetRecipe, TableOut, TableSpec, - derive_tables, + derive_tables, list_period_columns, ) from app.data.xlsx_scan import MAX_MERGED, SheetScan, WorkbookScan, cell_ref, col_letter @@ -101,7 +110,8 @@ PROBLEM_CODES: dict[str, str] = { # 列表定位 "header_not_found": "structure", "header_ambiguous": "structure", "column_missing": "structure", "column_extra": "structure", "rows_after_stop": "confirm", "total_row_missing": "structure", - "total_row_blank": "structure", "list_empty": "confirm", + "total_row_blank": "structure", "list_empty": "confirm", "subtotal_blank": "structure", + "subtotal_suspect": "confirm", # 值 "value_not_integer": "structure", "value_text_number": "structure", "value_blank": "structure", "value_error": "structure", "value_not_number": "structure", "value_not_date": "structure", @@ -125,6 +135,7 @@ PROBLEM_FIX: dict[str, str] = { "axis_extra_cells": "ignore_cells", "column_extra": "ignore_cells", "row_unclaimed": "ignore_cells", "outside_number": "ignore_cells", "hidden_rows": "declare_hidden", "hidden_cols": "declare_hidden", "sheet_missing": "rename_sheet", + "subtotal_suspect": "declare_subtotals", } #: 列表找表头的窗口：从 after_title 的下一行（或已用区域首行）起往下这么多行 @@ -143,7 +154,7 @@ PLACEHOLDER_TEXTS_MAX = 8 PLACEHOLDER_TEXT_LEN = 8 #: 排除的行在回执里的顺序（同一工作表内按原因、块、锚点、首行排） _EXCLUDED_ORDER = ("hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", - "total_not_kept") + "total_not_kept", "subtotal") #: 公式区域展开的上限：更大的区域不可能全落在基表里，直接判「引用了基表之外的格」 REF_CELLS_MAX = 200_000 @@ -613,11 +624,18 @@ class _Problems: class _Sink: """一张表的行：主键去重、计数、溯源、批量写库（有库时）。rowid 就是插入序号（新库、1 起）。""" - def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None) -> None: + def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None, + synth: tuple[str, str] | None = None) -> None: self.eng = eng self.name = name self.cols = cols names = [c.name for c in cols] + #: 期 5：由系统加的统计期起止两列（ListPeriod.synthesize）在一行里的位置。列表是在统计期定下来之前抽完的， + #: 这两格先放占位（_PERIOD_START / _PERIOD_END），统计期定下来以后（fill_period）写库时换成真值；在那之前 + #: 不分批写库（flush 等着），整张表留在内存里 + self.synth_idx = (names.index(synth[0]), names.index(synth[1])) \ + if synth and synth[0] in names and synth[1] in names else None + self.period: tuple[str | None, str | None] | None = None self.grain = [g for g in (spec.grain if spec else []) if g in names] self.grain_idx = [names.index(g) for g in self.grain] self.check_pk = bool(self.grain_idx) and not eng.partial @@ -682,9 +700,22 @@ class _Sink: self.eng.problems.issue("value_blank", sheet, f"工作表「{sheet}」的表「{self.name}」的主键列{_quoted(names)}", coords, text=_PK_NULL, group="pk_null") - def flush(self) -> None: + def fill_period(self, start: str | None, end: str | None) -> None: + """统计期定下来了：加的统计期起止两列从此写真值（start、end 是 YYYY-MM-DD）。""" + self.period = (start, end) + + def flush(self, *, final: bool = False) -> None: + if self.synth_idx is not None and self.period is None and not final: + return # 统计期还没定：加的两列还是占位，先不写 if self.buffer and self.eng.conn is not None: - self.eng.conn.executemany(self.insert_sql, self.buffer) + rows = self.buffer + if self.synth_idx is not None: + a, b = self.synth_idx + start, end = self.period or (None, None) + fill = {a: start, b: end} + rows = [tuple(fill.get(i, v) if v is _PERIOD_START or v is _PERIOD_END else v + for i, v in enumerate(row)) for row in rows] + self.eng.conn.executemany(self.insert_sql, rows) self.buffer.clear() @@ -695,6 +726,12 @@ class _Sink: _FAIL = object() #: 列表数据区里不能填充的合并单元格覆盖的空格（已经报了 merged_in_values） _MERGED = object() +#: 期 5：加的统计期起止两列在统计期定下来之前的占位（_Sink.fill_period）。主键判重时它们和自己相等： +#: 同一期的行统计期都一样，判重只看其余的主键列，结论不变 +_PERIOD_START = object() +_PERIOD_END = object() +#: 小计按组时「还没有组」 +_NO_GROUP = object() # -------------------------------------------------------------------------- @@ -1625,6 +1662,31 @@ class _ListRun: self.ignore_keys = {k for rule in block.ignore_columns if (k := match_key(rule.header))} total = block.rows.total_row self.total_idx = next((i for i, c in enumerate(self.cols) if total and c.name == total.label_column), None) + # 期 5：穿插的小计行（ListRows.subtotals）。sub_first：上一行小计（或表头）之后第一行明细的 rowid；按组小计 + # 时 run_first / run_value 是紧挨在上面、分组列取值相同的那一段明细的首个 rowid 和取值。sub_rows：小计所在的 + # 行 → 它管的明细的 rowid（合计行的公式引用了小计格时，按它换算回明细） + self.sub = block.rows.subtotals + sub = self.sub + self.sub_idx = next((i for i, c in enumerate(self.cols) if sub and c.name == sub.label_column), None) + self.sub_key = match_key(sub.pick) if sub else "" + self.group_idx = next((i for i, c in enumerate(self.cols) + if sub and sub.scope == "group_column" and c.name == sub.group_column), None) + self.sub_first: int | None = None + self.run_first: int | None = None + self.run_value: Any = _NO_GROUP + self.sub_hidden = False + self.sub_rows: dict[int, list[int]] = {} + #: 期 5：加统计期起止两列（ListPeriod.synthesize）：每行末尾两格占位，统计期定下来以后由 _Sink 换成真值 + self.synth = block.period is not None and block.period.synthesize is not None + #: 期 5：没配小计时，像夹在明细中间的小计的数据行（行号）和命中的合计字样：照常按数据导入，收尾时报 + #: subtotal_suspect 要人确认（带修复按钮：改作小计核对）。人声明过没有小计（no_subtotals）的不看 + self.suspects: list[int] = [] + self.suspect_word: str | None = None + self.watch_suspects = block.rows.subtotals is None and not block.rows.no_subtotals + #: 上一行普通明细在哪些文字、日期列（列序号，不含标签所在的首列）上有值；还没有明细时是空集 + self.prev_filled: frozenset[int] = frozenset() + #: 等着下一行明细确认的候选：(行号, 合计字样, 上一行明细有值、这一行空着的列) + self.pending: list[tuple[int, str, frozenset[int]]] = [] # ---------------- 喂入 ---------------- @@ -1692,6 +1754,7 @@ class _ListRun: if self.located: for i, text in self.headers_read.items(): self.eng.headers[(self.block.table, self.cols[i].name)] = text + self._report_suspects() # ---------------- 表头 ---------------- @@ -2001,6 +2064,7 @@ class _ListRun: fills = self._merges_at(r, inb) if self.sh.hidden_row(r): self.hidden_seen = True + self.sub_hidden = True if self.sh.sr.hidden.rows == "exclude": self.excluded_rows.add(r) self.sh.layout(r, self.layout_hidden) @@ -2010,10 +2074,85 @@ class _ListRun: label = inb.get(self.colmap.get(self.total_idx, -1)) pick = match_key(block.rows.total_row.pick) # type: ignore[union-attr] if label is not None and match_key(_text(label.value)).startswith(pick): + if self.watch_suspects: + self._settle_suspects(None) self._total_row(r, inb, cells, label) return + if self.sub_idx is not None: + label = inb.get(self.colmap.get(self.sub_idx, -1)) + if label is not None and match_key(_text(label.value)).startswith(self.sub_key): + self._subtotal_row(r, inb, cells, label) + return + elif self.watch_suspects: + self._watch_suspect(r, inb) self._record(r, inb, cells, fills) + def _watch_suspect(self, r: int, inb: dict[int, GridCell]) -> None: + """期 5：没配小计、也没声明没有小计时，认夹在明细中间的小计（照常按数据导入，收尾时报 subtotal_suspect）。 + + 要同时满足才算：首列以合计字样开头、其余文字和日期列都空着、至少一格有数（_suspect_word）；上一行明细在某个 + 文字或日期列上有值、这一行那一列空着；下一行明细在同一列上也有值，或者下面紧跟着合计行。所以候选先挂着， + 等下一行明细来了再定（_settle_suspects）。 + + 为什么这么严（复核反例）：只有「标签列 + 数字列」的键值清单（「累计收入 300」「汇总调整 5」），「其余文字列 + 为空」自然成立，什么也说明不了；放在最后、后面没有明细的行也不是「夹在中间」。报错了就是每期都要勾的 + 确认项，按配方导入一期的无人值守运行会被它挡住。""" + filled = frozenset(i for i, col in enumerate(self.cols) + if col.type in ("TEXT", "DATE") and (c := self.colmap.get(i)) is not None + and c != self.c1 and c in inb) + word = self._suspect_word(inb) if self.prev_filled else None + if word is not None: + self.pending.append((r, word, self.prev_filled)) + return + self._settle_suspects(filled) + self.prev_filled = filled + + def _settle_suspects(self, filled: frozenset[int] | None) -> None: + """挂着的候选有了下文：filled 是紧跟着的那行明细有值的文字、日期列（None 是紧跟着合计行）。上一行明细有值、 + 候选空着的列，下一行明细也有值，才算夹在明细中间。""" + for row, word, gap in self.pending: + if filled is None or gap & filled: + self.suspects.append(row) + self.suspect_word = self.suspect_word or word + self.pending = [] + + def _suspect_word(self, inb: dict[int, GridCell]) -> str | None: + """期 5：这一行本身像不像小计：首列是以合计字样（TOTAL_WORDS）开头的文字，其余文字、日期列都空着，至少一格 + 有数。像的话返回那个字样。以前这种行静默当数据导入，求和就把小计重复算了进去（H2）；现在照常导入，但收尾时 + 报出来要人确认，并给「改作小计核对」的修复按钮。合计行已经在前面认走了。上下文的条件见 _watch_suspect。""" + cell = inb.get(self.c1) + if cell is None or cell.formula is not None or not isinstance(cell.value, str): + return None + key = match_key(cell.value) + word = next((w for w in TOTAL_WORDS if key.startswith(w)), None) + if word is None or not any(self.colmap.get(i) == self.c1 for i in range(len(self.cols))): + return None + numeric = False + for i, col in enumerate(self.cols): + c = self.colmap.get(i) + if c is None or c == self.c1 or c not in inb: + continue + if col.type in ("TEXT", "DATE"): + return None + numeric = True + return word if numeric else None + + def _report_suspects(self) -> None: + """收尾：像小计的数据行合成一条 subtotal_suspect（需确认），修复参数带块 id、标签列、字样、行号和标签格。""" + if not self.suspects or not self.located: + return + label = next(col.name for i, col in enumerate(self.cols) if self.colmap.get(i) == self.c1) + cells = [self.sh.coord(r, self.c1) for r in self.suspects] + rows = _rows_text(self.suspects) + word = self.suspect_word or "" + where = (f"{self.where}：第 {rows} 行像夹在明细中间的小计（「{label}」列以「{word}」开头，上下明细都有值的" + "文字列在这一行空着），按数据导入了") + msg = (f"{where}。如果它们是小计，对这一列求和会把小计重复算进去：请用修复按钮改作小计核对；" + "确实是数据时，确认后照常导入；以后各期不想再确认，在配方的「小计行」里勾选「没有小计行」") + self.eng.problems.add("subtotal_suspect", msg, where, cells[:CELLS_MAX], sheet=self.sh.name, + fix_args={"block": self.block.id, "label_column": label, "pick": word, + "rows": list(self.suspects), "cells": cells[:CELLS_MAX]}) + def _plan(self) -> list[tuple[Any, int, bool, str]]: """每列的（列定义、所在列号、是不是主键、位置说明）：定下表头之后算一次，逐行复用。""" plan = self.__dict__.get("_plan_cache") @@ -2073,6 +2212,8 @@ class _ListRun: v = None values.append(v) lineage.append((col.name, r, c)) + if self.synth: + values += [_PERIOD_START, _PERIOD_END] ignored = self.ignored claims = {c: ("ignored_column" if c in ignored else "value") for c in inb} self.sh.layout(r, self.layout_data) @@ -2084,6 +2225,12 @@ class _ListRun: if self.rowid_first is None: self.rowid_first = rowid self.rowid_last = rowid + if self.sub_first is None: + self.sub_first = rowid + if self.group_idx is not None: + group = values[self.group_idx] + if self.run_value is _NO_GROUP or group != self.run_value: + self.run_first, self.run_value = rowid, group self.emit(r, cells, claims, True) def _total_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], @@ -2142,7 +2289,67 @@ class _ListRun: self.stop = ("total", r) self.emit(r, cells, claims, True) - def _ref_lookup(self, rc: tuple[int, int]) -> str | int: + def _subtotal_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], + label: GridCell) -> None: + """期 5：穿插在明细中间的一行小计。不进数据：标签格和数字格的去向是 total_label / total_value（同合计行）， + 每个数字格一条合计格（DerivedItem，kind=column_sum，分段键是块 id + SUBTOTAL_KEY），核对模块按它所管的那组 + 明细（rowid 闭区间）用 SQL 逐列求和比对，执行器不做加法（H1）；整行记进排除的行（reason=subtotal）。配了 + keep_as 的另存一行（小计项、按组时的分组取值、各数字列）。 + + 它管哪几行：since_previous 是上一行小计（或表头）之后的全部明细；group_column 是其中紧挨在它上面、分组列 + 取值相同的那一段。一行明细都没有时区间为空，核对判「无法核对」（合计行之上没有明细行）。数字格全空的小计 + 没有一列能核对，报 subtotal_blank（结构类）：不报的话这一行就悄悄没了核对。""" + eng, sh, block, sub = self.eng, self.sh, self.block, self.sub + assert sub is not None + claims = {c: ("ignored_column" if c in self.ignored else "total_label") for c in inb} + label_raw = _text(label.value) + first = self.sub_first + if sub.scope == "group_column" and first is not None and self.run_first is not None: + first = max(first, self.run_first) + last = self.rowid_last if first is not None else None + numeric = [(i, col) for i, col in enumerate(self.cols) if col.type in ("INTEGER", "REAL")] + if numeric and not any(self.colmap[i] in inb for i, _ in numeric): + coord = sh.coord(r, self.colmap[self.sub_idx]) # type: ignore[index] + where = f"{self.where}：小计行（{coord.split('!')[-1]}）的数字格全部为空" + eng.problems.add( + "subtotal_blank", + f"{where}，无法按这一组明细核对。请在文件中补上小计后重新上传；原表不再有小计时，在配方面板的" + "「小计行」中取消「有小计行」", + f"{where}，无法核对", [coord], sheet=sh.name) + key = f"{block.id}{SUBTOTAL_KEY}" + keep_values: list[Any] = [] + for i, col in numeric: + c = self.colmap[i] + cell = inb.get(c) + if cell is None: + keep_values.append(None) + continue + claims[c] = "total_value" + got = eng.convert_number(cell, col.type, block.values, sh.name, f"{self.where}的小计行", r, c, + data_area=False) + keep_values.append(got) + item = DerivedItem( + kind="column_sum", segment=key, sheet=sh.name, cell=sh.coord(r, c), label_raw=label_raw, + label=canon(label_raw), value=None if got is _FAIL else got, is_formula=cell.formula is not None, + formula=cell.formula, base_table=block.table, base_value=col.name, rowid_first=first, + rowid_last=last, hidden_in_range=self.sub_hidden, keep_table=sub.keep_as) + if item.is_formula: + eng.resolve_refs(item, cell.formula or "", self._ref_lookup) + eng.ex.derived.append(item) + if sub.keep_as and sub.keep_as in eng.sinks: + ks = eng.sinks[sub.keep_as] + ks.sources.append(f"{sh.sr.id}/{block.id}") + group = [None if self.run_value is _NO_GROUP else self.run_value] if self.group_idx is not None else [] + lin = [(col.name, r, self.colmap[i]) for i, col in numeric] + ks.add([canon(label_raw), *group, *keep_values], sheet=sh.name, + at=(r, self.colmap[self.sub_idx]), cells=lin, fails=True) # type: ignore[index] + n = sum(1 for role in claims.values() if role in ("total_label", "total_value")) + sh.exclude("subtotal", r, r, n, block=block.id, anchor=sub.pick) + self.sub_rows[r] = list(range(first, last + 1)) if first is not None and last is not None else [] + self.sub_first, self.run_first, self.run_value, self.sub_hidden = None, None, _NO_GROUP, False + self.emit(r, cells, claims, True) + + def _ref_lookup(self, rc: tuple[int, int]) -> str | int | list[int]: r, c = rc if self.c1 <= c <= self.c2: rowid = self.row_rowid.get(r) @@ -2150,6 +2357,9 @@ class _ListRun: return rowid if r in self.excluded_rows: return "hidden" + if r in self.sub_rows: + # 合计行的公式引用了小计格（=C5+C9）：小计格代表它管的那组明细 + return self.sub_rows[r] return "outside" @@ -2343,7 +2553,8 @@ class _Engine: self.canon_log[key] = CanonEntry(table, col.name, text, stored, 1, _coord(sheet, r, c)) return stored - def resolve_refs(self, item: DerivedItem, formula: str, lookup: Callable[[tuple[int, int]], str | int]) -> None: + def resolve_refs(self, item: DerivedItem, formula: str, + lookup: Callable[[tuple[int, int]], str | int | list[int]]) -> None: """合计公式的引用经溯源：全在基表 → ref_rowids；有隐藏行或分类汇总函数 → hidden； 有不在基表的 → outside；形状认不出 → unrecognized。执行器不做加法（H1）。""" shape = formula_refs(formula) @@ -2357,7 +2568,9 @@ class _Engine: outside = not cells for rc in cells: hit = lookup(rc) - if isinstance(hit, int): + if isinstance(hit, list): + rowids.extend(hit) # 期 5：引用了小计格，换算成它管的明细 + elif isinstance(hit, int): rowids.append(hit) elif hit == "hidden": hidden = True @@ -2607,8 +2820,9 @@ class _Engine: self.ex.problems = self.problems.finish(set()) return self.ex self.cols = cols + synth = {t: (spec[1], spec[2]) for t, spec in list_period_columns(self.recipe).items() if spec[0] == "synthesize"} for name, cs in cols.items(): - self.sinks[name] = _Sink(self, name, cs, self.specs.get(name)) + self.sinks[name] = _Sink(self, name, cs, self.specs.get(name), synth.get(name)) if self.db_path is not None: _remove_db(self.db_path) self.conn = _open_sqlite(self.db_path, readonly=False) @@ -2622,6 +2836,11 @@ class _Engine: crosstabs.extend(self._phase1(sh)) t2 = time.perf_counter() period, period_reported, _ = self._period() + if self.ex.period is not None: + # 期 5：列表加的统计期起止两列现在才有值（列表在统计期定下来之前就抽完了） + for sink in self.sinks.values(): + if sink.synth_idx is not None: + sink.fill_period(self.ex.period.start, self.ex.period.end) for ct in crosstabs: ct.complete_axis(period, period_reported) ct.extract() @@ -2650,7 +2869,7 @@ class _Engine: _remove_db(self.db_path) # type: ignore[arg-type] else: for sink in self.sinks.values(): - sink.flush() + sink.flush(final=True) self.conn.execute("COMMIT") self.conn.close() self.conn = None @@ -2748,7 +2967,9 @@ class _Engine: sh.failed = True return [] b = s.bounds - if b is not None and not self.partial and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): + # 稀疏判定挡的是 Excel 里误输入、残留格式撑大的已用区域；CSV 的每一格都是写出来的，空字段多只是数据稀疏 + if b is not None and not self.partial and self.scan.format != "csv" \ + and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): far = "、".join(list(dict.fromkeys(s.far_cells))[:3]) msg = (f"工作表「{sh.name}」的已用区域 {b.a1()} 共 {b.area:,} 个单元格，但只有 {s.nonempty:,} 个单元格有内容；" f"远处的单元格（如 {far}）可能是误输入或残留格式。请在 Excel 中删除这些单元格后重新上传") diff --git a/backend/app/data/recipe_fixes.py b/backend/app/data/recipe_fixes.py index 54f6be61..5921ebfb 100644 --- a/backend/app/data/recipe_fixes.py +++ b/backend/app/data/recipe_fixes.py @@ -253,6 +253,8 @@ def _key(kind: str, target: dict[str, Any]) -> str: return f"declare_hidden:{t['sheet']}:{t['axis']}" if kind == "rename_sheet": return f"rename_sheet:{t['sheet_id']}" + if kind == "declare_subtotals": + return f"declare_subtotals:{t['block']}" raise KeyError(kind) @@ -750,6 +752,62 @@ def _o_declare_total(recipe: dict, p: FixProposal, opt: FixOption, reason: str | return ops, summary +# ========================================================================== +# 期 5：列表中间像小计的行改作小计核对 +# ========================================================================== + + +def _subtotal_keep_name(recipe: dict, block: dict) -> str: + used = {collide_key(t.get("name") or "") for t in recipe.get("tables") or []} + return unique_name(f"{block.get('table')}_小计", used) + + +def _p_declare_subtotals(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + """subtotal_suspect → 列表块加 rows.subtotals（标签列、开头词取自执行器给的 fix_args：文件里的合计字样）。 + 块已经配了小计、标签列不在块里、开头词含数字时不提议。""" + found = find_block(recipe, args.get("block")) + if found is None: + return None + _si, _bi, block = found + label, pick = args.get("label_column"), args.get("pick") + if block.get("layout") != "list" or (block.get("rows") or {}).get("subtotals"): + return None + if label not in [c.get("name") for c in block.get("columns") or []] or not isinstance(pick, str) \ + or not pick or has_digit(pick): + return None + target = {"block": block["id"], "label_column": label, "pick": pick} + keep = _subtotal_keep_name(recipe, block) + return _proposal("declare_subtotals", _key("declare_subtotals", target), code=_get(prob, "code"), + title=f"列表「{block['id']}」中间「{pick}」开头的行是小计", + cells=list(args.get("cells") or _get(prob, "cells") or []), target=target, + options=[FixOption("check_only", "改作小计核对，不另存", + detail="每行小计按上一行小计之后的明细逐列求和核对，小计行不进数据，记进回执的" + "「排除的行」"), + FixOption("keep", f"改作小计核对，原值另存表「{keep}」", + detail="同上，原表的小计另存成一张表；另存的小计表不能按期累积")], + anchor=anchor) + + +def _o_declare_subtotals(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + found = find_block(recipe, p.target.get("block")) + if found is None: + raise _Stale + si, bi, block = found + if (block.get("rows") or {}).get("subtotals"): + raise _Stale + keep = _subtotal_keep_name(recipe, block) if opt.value == "keep" else None + value = {"label_column": p.target["label_column"], "pick": p.target["pick"], "scope": "since_previous", + "group_column": None, "keep_as": keep} + ops: list[dict[str, Any]] = [{"op": "add", "path": ptr("sheets", si, "blocks", bi, "rows", "subtotals"), + "value": value}] + if keep: + ops.append({"op": "add", "path": "/tables/-", "value": { + "name": keep, "grain": [], "kind": "reported_total", "units": {}, "note": ""}}) + summary = [f"列表「{block['id']}」中间「{p.target['pick']}」开头的行改作小计核对，不进数据" + + (f"，原值另存表「{keep}」" if keep else "，不另存")] + return ops, summary + + # ========================================================================== # ⑥ 忽略（按文字记） # ========================================================================== @@ -998,12 +1056,13 @@ _Builder = Callable[[dict, Any, dict, FixAnchor], "FixProposal | None"] _BUILDERS: dict[str, _Builder] = { "remove_label": _p_remove_label, "add_label": _p_add_label, "rename_title": _p_rename_title, "declare_total": _p_declare_total, "ignore_cells": _p_ignore_cells, "declare_placeholder": _p_declare_placeholder, - "declare_hidden": _p_declare_hidden, "rename_sheet": _p_rename_sheet, + "declare_hidden": _p_declare_hidden, "rename_sheet": _p_rename_sheet, "declare_subtotals": _p_declare_subtotals, } _OPS: dict[str, Callable[[dict, FixProposal, FixOption, str | None], tuple[list, list[str]]]] = { "remove_label": _o_remove_label, "add_label": _o_add_label, "edit_members": _o_edit_members, "rename_title": _o_rename_title, "declare_total": _o_declare_total, "ignore_cells": _o_ignore_cells, "declare_placeholder": _o_declare_placeholder, "declare_hidden": _o_declare_hidden, "rename_sheet": _o_rename_sheet, + "declare_subtotals": _o_declare_subtotals, } @@ -1064,6 +1123,8 @@ def _edit_title(p: FixProposal, opt: FixOption) -> str: return f"工作表「{t.get('sheet')}」：{opt.label}" if p.kind == "rename_sheet": return opt.label + if p.kind == "declare_subtotals": + return f"列表「{t.get('block')}」中间「{t.get('pick')}」开头的行{opt.label}" return p.title diff --git a/backend/app/data/recipe_imports.py b/backend/app/data/recipe_imports.py index 34acb9f0..ee8d338c 100644 --- a/backend/app/data/recipe_imports.py +++ b/backend/app/data/recipe_imports.py @@ -177,7 +177,8 @@ class Pipeline: recipe_sha256: Callable[[Recipe], str] canonical: Callable[[Recipe], dict[str, Any]] apply_patch: Callable[[dict[str, Any], list[dict[str, Any]]], dict[str, Any]] - scan: Callable[[bytes], WorkbookScan] + #: (原件, 文件名) → 扫描结果。文件名定按 CSV 读还是按工作簿读（csv_grid.scan_any） + scan: Callable[[bytes, str], WorkbookScan] scan_to_json: Callable[[WorkbookScan], dict[str, Any]] scan_from_json: Callable[[dict[str, Any]], WorkbookScan] read_grid: Callable[..., Grid] @@ -225,7 +226,7 @@ class Pipeline: def default_pipeline() -> Pipeline: """接上真模块。函数体里惰性导入：这些模块（openpyxl、执行器、起草器）导入不便宜，接口模块加载时不必付。""" from app.data import ( - recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, + csv_grid, recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, recipe_engine, recipe_fixes, recipe_notes, recipe_select, recipe_suggest, xlsx_cells, xlsx_scan, ) @@ -234,7 +235,8 @@ def default_pipeline() -> Pipeline: recipe_sha256=recipe.recipe_sha256, canonical=recipe.canonical_recipe, apply_patch=recipe.apply_patch, - scan=xlsx_scan.scan, + # CSV / TSV 也走配方：csv_grid 适配成一张「工作表」，其余照旧交给 xlsx_scan（Excel 期 5） + scan=csv_grid.scan_any, scan_to_json=xlsx_scan.scan_to_json, scan_from_json=xlsx_scan.scan_from_json, read_grid=xlsx_cells.read_grid, @@ -473,7 +475,7 @@ def _open_book(p: Pipeline, raw: bytes, scan: WorkbookScan) -> _Book: def _book_of(p: Pipeline, snap: _Snap) -> _Book: raw = _read_raw(snap.raw_sha256) - scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, snap.file_name) return _open_book(p, raw, scan) @@ -771,7 +773,7 @@ class _Staged: def _stage_work(p: Pipeline, raw: bytes, filename: str) -> _Staged: """首次导入（或切换）的暂存：扫描、网格预览、规则起草（带干跑）、对草稿做一次静态校验。""" - scan = p.scan(raw) + scan = p.scan(raw, filename) book = _open_book(p, raw, scan) dry = _Capture(_dry_fn(p, book, filename, {})) d = p.draft(scan, book.grids, filename, validate=lambda data, f, o: p.validate(data, facts=f, origin=o), @@ -859,7 +861,7 @@ async def stage_upload(session: AsyncSession, *, raw: bytes, filename: str, name def _replay_work(p: Pipeline, raw: bytes, current: dict[str, Any], filename: str, snap: _Snap, *, evaluate: bool) -> tuple[_Staged, _Eval | None]: """上传新一期 / 修改配方：扫描（或取存下的扫描）、按现行配方的写法取系统发现，修改配方时再评估一遍。""" - scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, filename) book = _open_book(p, raw, scan) facts = p.detect_facts(book.grids, scan, recipe=current) staged = _Staged(scan_json=p.scan_to_json(scan), previews=[p.preview(g) for g in book.grids.values()], @@ -1859,7 +1861,7 @@ def _trial_work(p: Pipeline, snap: _Snap, work: dict[str, Any], facts: DraftFact if parsed is None or problems: return _TrialRun(parsed=parsed, problems=list(problems)) raw = _read_raw(snap.raw_sha256) - scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, snap.file_name) path.parent.mkdir(parents=True, exist_ok=True) # 先写 .tmp- 再改名：半成品归启动清理（_sweep_tmp）管，发布、回收不会把半个库当成试运行库 tmp = path.parent / f"{path.name}{raw_store.TMP_MARK}{secrets.token_hex(6)}" diff --git a/backend/app/data/recipe_notes.py b/backend/app/data/recipe_notes.py index b4464cf7..48c8da20 100644 --- a/backend/app/data/recipe_notes.py +++ b/backend/app/data/recipe_notes.py @@ -27,6 +27,9 @@ from app.data.names import canon from app.data.recipe_checks import date_columns, plan_checks, split_coord from app.data.recipe_parsers import UNITS from app.data.recipe_types import ( + LIST_PERIOD_END, + LIST_PERIOD_START, + SUBTOTAL_KEY, Acceptance, CheckResult, ColumnOut, @@ -46,6 +49,7 @@ from app.data.recipe_types import ( TableNote, UnionNotePart, derive_tables, + list_period_columns, prose_problems, render_prose, ) @@ -134,6 +138,13 @@ NOTE_TEXT = { "ignored_rows_union": "部分期的原表另有按配方忽略的行，内容见证据面板", "ignored_cells_union": "部分期的原表另有按配方忽略的单元格，内容见证据面板", "dim_values_vary": "各期的 {dim} 取值不完全相同：跨期比较前请先确认两期都有的取值", + # ---- 期 5：列表按期累积、穿插的小计。列表可以累积以后，空行之后的文字行也要有并集说法 + "rows_after_stop_union": "部分期的原表在空行之后另有未导入的文字行，内容见证据面板", + "subtotal_kept": "原表穿插的小计在 {table}，不要与本表相加", + "subtotal_not_kept": "原表穿插的小计行未导入本表", + "list_subtotal_passed": "原表穿插的小计行的值，已按各组明细求和核对一致：不要与 {base} 相加，也不要彼此相加", + "list_subtotal_unverified": "原表穿插的小计行的值，按原表保存的值存储。本期部分小计未能核对：不要与 {base} 相加，" + "也不要彼此相加", } #: 列说明的片段模板 @@ -148,6 +159,9 @@ COLUMN_TEXT = { "col_partial_periods": "部分期没有这一列的数据，为空值", "col_absent": "当前版本的数据中没有这一列，为空值", "col_retired": "这一列自某一期起不再导入，此后各期为空值", + # ---- 期 5：列表按期累积时由系统加的统计期起止两列（ListPeriod.synthesize）。不含数字（说明不许报数） + "period_start": "本期统计期的第一天，取自表内的统计期或人工录入的统计期，格式 YYYY-MM-DD", + "period_end": "本期统计期的最后一天，取自表内的统计期或人工录入的统计期，格式 YYYY-MM-DD", } #: 「逐日 / 逐行」「个别日期 / 个别行」：粒度只有一列日期时按日说，否则按行说 @@ -161,7 +175,7 @@ _K_REASON = {r: f"total_k_{r}" for r in ("uncached", "blank", "null_detail", "ti _T_REASON = {r: f"list_total_{r}" for r in ("uncached", "blank", "null_detail", "tiling", "hidden")} #: 允许出现「一致」的模板：只在对应的 K / T 通过时才会选用 CONSISTENT_KEYS = frozenset({"total_k_passed", "total_k_passed_g_unverifiable", "list_total_passed", - "list_total_passed_partial"}) + "list_total_passed_partial", "list_subtotal_passed"}) def _tok(kind: str, name: str) -> str: @@ -202,6 +216,8 @@ class _Writers: lists: list[ListBlock] = field(default_factory=list) #: 把合计行另存进本表的列表块 list_totals: list[ListBlock] = field(default_factory=list) + #: 期 5：把穿插的小计另存进本表的列表块 + list_subtotals: list[ListBlock] = field(default_factory=list) def add_sheet(self, sheet: SheetRecipe) -> None: if all(s is not sheet for s in self.sheets): @@ -242,6 +258,9 @@ def _writers(recipe: Recipe) -> dict[str, _Writers]: total = block.rows.total_row if total is not None and total.keep_as: w(total.keep_as, sheet).list_totals.append(block) + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + w(sub.keep_as, sheet).list_subtotals.append(block) return out @@ -395,6 +414,15 @@ def _base_total_fragments(w: _Writers) -> list[NoteFragment]: b.rows.total_row is not None and not b.rows.total_row.keep_as for b in w.lists) if dropped: out.append(NoteFragment("total_not_kept", None, NOTE_TEXT["total_not_kept"])) + # 期 5：穿插的小计行不在本表里：另存在哪，或者没导入 + for b in w.lists: + sub = b.rows.subtotals + if sub is None: + continue + if sub.keep_as: + _add_unique(out, _frag("subtotal_kept", sub.keep_as, NOTE_TEXT["subtotal_kept"].format(table=_tab(sub.keep_as)))) + elif all(f.key != "subtotal_not_kept" for f in out): + out.append(NoteFragment("subtotal_not_kept", None, NOTE_TEXT["subtotal_not_kept"])) return out @@ -450,6 +478,13 @@ def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[NoteFragme else: frag = _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table))) _add_unique(out, frag) + for block in w.list_subtotals: + # 期 5：另存的小计：T 通过才说「核对一致」，没核对上的（无法核对）照实说部分未能核对；不一致的整份拒收 + t = ctx.check(table, ctx.plan.t.get(f"{block.id}{SUBTOTAL_KEY}"), f"块「{block.id}」的小计行") + if t is None or _blocking(t): + continue + key = "list_subtotal_passed" if t.status == "passed" else "list_subtotal_unverified" + _add_unique(out, _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table)))) return out @@ -653,10 +688,17 @@ def _date_key(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> str | None: return None +def _period_column_key(col: ColumnOut) -> str | None: + """期 5：系统加的统计期起止两列（role=period）用哪条列说明。""" + if col.role != "period": + return None + return "period_start" if col.name == LIST_PERIOD_START else "period_end" if col.name == LIST_PERIOD_END else None + + def _null_eligible(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> bool: """这一列要不要考虑空值说明：值列；主键不会是空值；表内合计表的空值已由表说明（未能核对）交代。""" value_col = col.role in ("measure", "value") and col.type in ("INTEGER", "REAL") - is_total = bool(w.kept_from or w.list_totals) + is_total = bool(w.kept_from or w.list_totals or w.list_subtotals) return value_col and not is_total and col.name not in set(ctx.grains.get(table, [])) @@ -668,6 +710,8 @@ def _column_templates(ctx: _Ctx, table: str, null_counts: Mapping[str, Mapping[s frags: list[str] = [] if (date := _date_key(ctx, w, table, col)) is not None: frags.append(COLUMN_TEXT[date]) + if (period := _period_column_key(col)) is not None: + frags.append(COLUMN_TEXT[period]) if col.unit: frags.append(COLUMN_TEXT["unit"].format(unit=_tok("单位", col.unit))) has_nulls = counts is None or counts.get(col.name, 0) > 0 @@ -793,22 +837,27 @@ _FAMILY: dict[str, str] = { "hidden_excluded": "hidden_excluded", "hidden_included": "hidden_included", "ignored_columns": "ignored_columns", "ignored_rows": "ignored_rows", "ignored_cells": "ignored_cells", "rows_after_stop": "rows_after_stop", + "subtotal_kept": "subtotal_kept", "subtotal_not_kept": "subtotal_not_kept", + "list_subtotal_passed": "list_total", "list_subtotal_unverified": "list_total", "period_conflict_accepted": "period_conflict", "period_conflict_pending": "period_conflict", "period_human": "period_human", "outside_digits": "outside_digits", "note": "note", } #: 合并后的拼接顺序（与 _table_fragments 一致；累积特有的几句排在用户写的表说明之前） -_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", "total_k", - "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", +_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", + "subtotal_kept", "subtotal_not_kept", "total_k", "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", "ignored_rows", "ignored_cells", "rows_after_stop", "period_conflict", "period_human", "outside_digits", "union", "note") #: 与核对结论无关、只描述表结构的片段族：用目标配方那一期的；只在早期出现的 subject（退役的表、另存表）用 #: 最后一个含它的那一期的 -_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note"}) +_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note", + "subtotal_kept", "subtotal_not_kept"}) #: 「任一期出现就写」的片段族 → 并集里的说法 _ANY_UNION = {"hidden_excluded": "hidden_excluded_union", "hidden_included": "hidden_included_union", "ignored_columns": "ignored_columns_union", "ignored_rows": "ignored_rows_union", - "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union"} + "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union", + # 期 5：列表可以按期累积了，空行之后的文字行任一期有就写 + "rows_after_stop": "rows_after_stop_union"} _K_PASSED = frozenset({"total_k_passed", "total_k_passed_g_unverifiable"}) @@ -943,20 +992,26 @@ def _merge_group(table: str, family: str, subject: str | None, group: dict[int, if family == "period_human": key = "period_human_all" if len(present) == len(having) else "period_human_union" return NoteFragment(key, subject, NOTE_TEXT[key]) - # list_total、rows_after_stop 只有列表才有，而列表不能按期累积（P3-SPEC 2.2）；不认识的片段族同样没有并集说法。 + # list_total 只在列表的合计行、小计行另存的表上，这种表不能按期累积（accumulate_blockers）；不认识的片段族同样 + # 没有并集说法。 # 记 problems（不能保存），绝不沿用某一期的单期说法。消息给人看，不露片段族的键名 problems.append(f"表「{table}」：{_UNMERGEABLE.get(family, '有一段说明')}不能按期合并，无法生成说明") return None #: 没有并集说法的片段族 → 给人看的叫法（problems 里不露键名） -_UNMERGEABLE = {"list_total": "列表合计行的核对结论", "rows_after_stop": "列表空行之后的文字行"} +_UNMERGEABLE = {"list_total": "列表合计行、小计行的核对结论"} def _axis_column(recipe: Recipe, table: str) -> str | None: + """各期按哪一列区分（说明里「请按 {col} 筛选」）：交叉表的日期轴；期 5 起列表按 period 绑定的日期列，或者 + 系统加的统计期起那一列。""" tables, _ = derive_tables(recipe) axes = [c.name for c in tables.get(table, []) if c.role == "axis"] - return axes[0] if len(axes) == 1 else None + if len(axes) == 1: + return axes[0] + spec = list_period_columns(recipe).get(table) + return spec[1] if spec else None def _union_table_fragments(table: str, having: list[int], parts: list[_UPart], ref: int, target: Recipe, @@ -1037,6 +1092,8 @@ def _union_column_templates(table: str, columns: list[ColumnOut], having: list[i frags.append(COLUMN_TEXT["date_year_from_period"]) elif "date" in dates or (not with_col and col.role == "axis"): frags.append(COLUMN_TEXT["date"]) + if (period := _period_column_key(col)) is not None: + frags.append(COLUMN_TEXT[period]) if target_cols is not None and col.name in target_cols: unit = target_cols[col.name].unit elif with_col: diff --git a/backend/app/data/recipe_select.py b/backend/app/data/recipe_select.py index daa7332c..34697950 100644 --- a/backend/app/data/recipe_select.py +++ b/backend/app/data/recipe_select.py @@ -427,6 +427,21 @@ def _bottom_box(ctx: _Ctx, plan: Any, first: int) -> bool: ctx.fail("selection_bottom_not_anchorable", f"框里只有表头和合计行（第 {last} 行），没有数据行：请把框扩大到数据所在的行") return False + # 期 5：框里夹在中间、第一列以合计字样开头、下面紧接着还有同一张表的明细的行（起草器 _scan_block 的同一条 + # 规则），照起草器提议成小计（_settle_subtotals）；提议不成的照旧报错 + mid = [r for r in data[1:] if S._total_word(S._label_of(grid.get(r, c1))) is not None] + rest = [r for r in data if r not in mid] + if mid and rest and not any(grid.get(r, c) is not None and S._total_word(grid.get(r, c).value) is not None + for r in data for c in range(c1 + 1, c2 + 1)): + trial = S._ListPlan(header_rows=plan.header_rows, cols=plan.cols, headers=plan.headers, rows=rest, + subtotal_rows=list(mid), + subtotal_words=[S._total_word(S._label_of(grid.get(r, c1))) or "" for r in mid]) + types = S._types_of(grid, rest, plan.cols) + if all(S._more_rows(grid, r, trial, S._bottom(grid), types) for r in mid): + S._settle_subtotals(grid, trial) + if trial.subtotal_word is not None: + plan.subtotal_rows, plan.subtotal_word = trial.subtotal_rows, trial.subtotal_word + data = [r for r in data if r not in mid] if not _no_stray_totals(ctx, data, c1, c2): return False bottom = S._bottom(grid) diff --git a/backend/app/data/recipe_suggest.py b/backend/app/data/recipe_suggest.py index 53314072..9a176511 100644 --- a/backend/app/data/recipe_suggest.py +++ b/backend/app/data/recipe_suggest.py @@ -12,6 +12,11 @@ validate）和执行器干跑（注入的 dry_run），再由人逐条确认之 哪些是区域外文字、哪些格没有去处）直接用执行器的结论：同一件事只有执行器一套判法，这里自己定位只是没有 干跑时的近似。 +**期 5（下一轮 03 细案 E）。** 有 Excel 表格对象（插入 → 表格）的工作表优先按表格对象出列表块（_lists_from_tables： +表头、列范围、汇总行都是表格对象给的，锚点仍按表头文字）；夹在明细中间、第一列以同一个合计字样开头的行提议成 +穿插的小计（_settle_subtotals），不再一律记起草失败；每个列表都问一道「每期怎么更新」（q_list_period），选按期 +累积时同时绑定每一行属于哪一期（按日期列核对，或加统计期起止两列）。 + **问题的 effects 怎么写。** 每个选项是一组 JSON Patch，路径按传进来的配方写；接口层从 answers_base 起、按 问题顺序重放全部回答（P2-SPEC 7.2）。因此： - 关系的两个选项都用整条 replace（「登记」写回完整的 sum_eq / not_comparable，「不登记」写 dismissed）， @@ -63,7 +68,7 @@ from app.data.recipe_types import ( derive_tables, ) from app.data.tabular import DATE_HEADER_MIN, EXCEL_ERRORS -from app.data.xlsx_scan import WorkbookScan, cell_ref, col_letter, parse_ref +from app.data.xlsx_scan import TableScan, WorkbookScan, cell_ref, col_letter, parse_ref logger = logging.getLogger(__name__) @@ -612,8 +617,15 @@ class _ListPlan: #: 按数据行收进来并记起草失败 adjacent: list[tuple[int, list[int]]] = field(default_factory=list) #: 不在末尾第一列的合计字样：(行, 列)。第一列在中间或最上面、或者落在别的列上，都无法按合计行核对， - #: 记起草失败；不记就会把合计当明细导入，求和翻倍（H2） + #: 记起草失败；不记就会把合计当明细导入，求和翻倍（H2）。期 5：第一列里同一个字样的穿插行先记进 subtotal_rows， + #: 收完整块再判能不能提议成小计（_settle_subtotals）；判不了的才回到这里 stray_totals: list[tuple[int, int]] = field(default_factory=list) + #: 期 5：穿插在明细中间、第一列以合计字样开头的行（不进 rows）和它们的字样；能提议成小计时 subtotal_word 非空 + subtotal_rows: list[int] = field(default_factory=list) + subtotal_words: list[str] = field(default_factory=list) + subtotal_word: str | None = None + #: 期 5：按 Excel 的表格对象起草的（xlsx_scan.TableScan 的名字）：表名优先用它，卡片里写明 + table_object: str | None = None # 命名阶段填 id: str = "" table: str = "" @@ -808,13 +820,44 @@ def _scan_block(grid: Grid, plan: _ListPlan, bottom: int) -> None: if word is not None and plan.rows and not _more_rows(grid, r, plan, bottom, types): plan.total_row, plan.total_word = r, word break + others = [(r, c) for c, cell in zip(cols[1:], cells[1:]) + if cell is not None and _total_word(cell.value) is not None] + plan.stray_totals.extend(others) + if word is not None and plan.rows and not others: + # 期 5：夹在明细中间的合计字样，先当小计候选：不进数据行，收完整块再判（_settle_subtotals） + plan.subtotal_rows.append(r) + plan.subtotal_words.append(word) + r += 1 + continue if word is not None: plan.stray_totals.append((r, cols[0])) - plan.stray_totals.extend((r, c) for c, cell in zip(cols[1:], cells[1:]) - if cell is not None and _total_word(cell.value) is not None) plan.rows.append(r) acc.add(r) r += 1 + _settle_subtotals(grid, plan) + + +def _settle_subtotals(grid: Grid, plan: _ListPlan) -> None: + """期 5：夹在明细中间的合计字样能不能提议成小计（ListRows.subtotals，规则起草不再把它们记成失败）。 + + 能的条件：都在第一列、是同一个字样，每一行至少有一格数（没有数的小计核对不了）。末尾那一行和它们字样相同时 + （表以最后一组的小计结束，没有总计），它也是小计，不是合计行。不能的（字样不同、有一行没有数）照旧记进 + stray_totals（起草失败，交给人或 AI），这些行回到数据行里——和期 4 的结果一样。""" + if not plan.subtotal_rows: + return + words = set(plan.subtotal_words) + numeric = [c for c in plan.cols[1:] if any(_value_num(grid.get(r, c)) is not None for r in plan.rows)] + ok = len(words) == 1 and all(any(_value_num(grid.get(r, c)) is not None for c in numeric) + for r in plan.subtotal_rows) + if ok: + plan.subtotal_word = next(iter(words)) + if plan.total_row is not None and plan.total_word == plan.subtotal_word: + plan.subtotal_rows.append(plan.total_row) + plan.total_row = plan.total_word = None + return + plan.stray_totals.extend((r, plan.cols[0]) for r in plan.subtotal_rows) + plan.rows = sorted(set(plan.rows) | set(plan.subtotal_rows)) + plan.subtotal_rows, plan.subtotal_words = [], [] def _list_failures(sheet: str, grid: Grid, plan: _ListPlan) -> list[_Failure]: @@ -898,7 +941,8 @@ def _analyze_lists(grid: Grid) -> _Lists: _finish_block(grid, plan) out.failures.extend(_list_failures(grid.sheet, grid, plan)) out.blocks.append(plan) - end = max((p.total_row or (p.rows[-1] if p.rows else p.header_rows[-1])) for p in made) + end = max(max([p.total_row or 0, *p.subtotal_rows, p.rows[-1] if p.rows else p.header_rows[-1]]) + for p in made) while r_idx < len(rows) and rows[r_idx] <= end: r_idx += 1 return out @@ -945,7 +989,7 @@ def _finish_block(grid: Grid, plan: _ListPlan) -> None: break if cells: break - last = plan.total_row or rows[-1] + last = max([plan.total_row or 0, rows[-1], *plan.subtotal_rows]) for r in range(plan.header_rows[0], last + 1): for c in cols: if grid.get(r, c) is not None or r in plan.header_rows: @@ -975,15 +1019,77 @@ class _Sheet: hidden_cols: list[int] = field(default_factory=list) -def _analyze_sheet(grid: Grid, sid: str) -> _Sheet: +def _lists_from_tables(grid: Grid, objects: list[TableScan]) -> _Lists | None: + """期 5：按 Excel 的表格对象（插入 → 表格）起草列表块。表格对象明确给出了表头行、列范围、数据行和汇总行，比按 + 「像不像表头」猜可靠：数字格空着的行不会被当成表头，紧挨着的两张表也分得开。 + + 出来的块和按规则猜的一样只记文字：锚点仍按表头文字（不记坐标，下一期表格挪了位置照样认得出），汇总行按第一列 + 的文字映射成合计行（total_row），夹在中间的合计字样按小计提议（_settle_subtotals）。表格对象的 ref 写坏了、 + 没有表头行时返回 None，退回按规则猜。""" + out = _Lists(sheet=grid.sheet) + spans = sorted(((t.span(), t) for t in objects), key=lambda x: (x[0] or (0, 0, 0, 0))[:2]) + for span, t in spans: + if span is None or t.header_row_count < 1: + return None + r1, c1, r2, c2 = span + hrows = list(range(r1, r1 + t.header_row_count)) + cols = list(range(c1, c2 + 1)) + texts = _header_texts(grid, hrows, cols) + if set(texts) != set(cols): + blank = [col_letter(c) for c in cols if c not in texts] + msg = (f"工作表「{grid.sheet}」：表格「{t.name}」的表头有空着的列（{'、'.join(blank)} 列），" + "按列表导入要每一列都有表头文字") + out.failures.append(_Failure(msg, msg)) + continue + plan = _ListPlan(header_rows=hrows, cols=cols, headers={c: texts[c] for c in cols}, table_object=t.name) + last_data = r2 - max(t.totals_row_count, 0) + rows = [r for r in grid.row_numbers() if hrows[-1] < r <= last_data and any(grid.get(r, c) is not None + for c in cols)] + prev = hrows[-1] + for r in rows: + if r > prev + 1: + plan.skip_blank = True + plan.skipped.extend(range(prev + 1, r)) + prev = r + word = _total_word(_label_of(grid.get(r, c1))) + if word is not None and plan.rows: + plan.subtotal_rows.append(r) + plan.subtotal_words.append(word) + continue + plan.rows.append(r) + if not plan.rows: + msg = f"工作表「{grid.sheet}」：表格「{t.name}」的表头下面没有数据行" + out.failures.append(_Failure(msg, msg)) + continue + if t.totals_row_count > 0: + tr = last_data + 1 + label = _label_of(grid.get(tr, c1)) + word = _total_word(label) or (label if label and len(label) <= 12 and not any( + ch.isdigit() for ch in label) else None) + if word is not None: + plan.total_row, plan.total_word = tr, word + else: + why = ("起草只读了表格前面的部分，没读到汇总行" if grid.truncated and tr > max(grid.row_numbers()) + else "汇总行的第一列没有可用作合计标签的文字（不能含数字、不超过 12 个字）") + msg = f"工作表「{grid.sheet}」：表格「{t.name}」的汇总行（第 {tr} 行）无法按合计行核对：{why}" + out.failures.append(_Failure(msg, msg)) + _settle_subtotals(grid, plan) + _finish_block(grid, plan) + out.failures.extend(_list_failures(grid.sheet, grid, plan)) + out.blocks.append(plan) + return out + + +def _analyze_sheet(grid: Grid, sid: str, objects: list[TableScan] | None = None) -> _Sheet: sh = _Sheet(sheet=grid.sheet, sid=sid) axis = _axis_row(grid) + from_objects = _lists_from_tables(grid, objects) if objects and axis is None else None if axis is not None: sh.cross = _analyze_crosstab(grid, *axis) sh.region = set(sh.cross.region) sh.failures.extend(sh.cross.failures) else: - sh.lists = _analyze_lists(grid) + sh.lists = from_objects if from_objects is not None else _analyze_lists(grid) sh.failures.extend(sh.lists.failures) for b in sh.lists.blocks: sh.region |= b.region @@ -1111,7 +1217,8 @@ def _measure_column(label: str, pos: int) -> tuple[str, str | None, str | None]: def _analyze(scan: WorkbookScan, grids: dict[str, Grid]) -> _Analysis: visible = _visible_grids(scan, grids) hidden = [s.name for s in scan.sheets if s.state != "visible"] - sheets = [_analyze_sheet(g, f"s{i + 1}") for i, g in enumerate(visible)] + sheets = [_analyze_sheet(g, f"s{i + 1}", [t for t in scan.tables if t.sheet == g.sheet]) + for i, g in enumerate(visible)] failures = [f for sh in sheets for f in sh.failures] tables_used: set[str] = set() @@ -1266,7 +1373,9 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] 合计表名撞了已用的名字时照起草器的老行为：不另建表定义，keep_as 仍写这个名字（交给静态校验报）。""" b.id = block_id if table is None: - title = b.title if b.title and not any(ch.isdigit() for ch in b.title) else None + # 期 5：按表格对象起草的，表名优先用表格对象的名字（不含数字时；Excel 默认的「表1」不用） + named = b.table_object if b.table_object and not any(ch.isdigit() for ch in b.table_object) else None + title = named or (b.title if b.title and not any(ch.isdigit() for ch in b.title) else None) want = _ENUM_PREFIX.sub("", P.split_unit_suffix(title)[0]) if title else sheet n_fallback = len(tables_used) + 1 b.table = _unique(to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}", tables_used) @@ -1293,6 +1402,11 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] total_spec = {"name": keep, "grain": ["合计项"], "kind": "reported_total", "units": num_units, "note": ""} b.keep_table = keep total = {"label_column": b.names[b.cols[0]], "pick": b.total_word, "keep_as": keep} + subtotals = None + if b.subtotal_word is not None: + # 期 5：穿插的小计只核对、不另存（另存的小计表不能按期累积，要另存由人在配方面板里打开） + subtotals = {"label_column": b.names[b.cols[0]], "pick": b.subtotal_word, "scope": "since_previous", + "group_column": None, "keep_as": None} grain = [b.names[b.grain_col]] if b.grain_col is not None else [] main = {"name": b.table, "grain": grain, "kind": "data", "units": dict(b.units), "note": ""} fill = bool(b.merged_fill) and all(r2 > r1 and c1 == c2 and b.types.get(c1) == "TEXT" @@ -1300,7 +1414,8 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] block = { "id": b.id, "layout": "list", "table": b.table, "header_rows": len(b.header_rows), "after_title": None, "columns": cols_out, "extra_columns": "reject", - "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total}, + "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total, + **({"subtotals": subtotals} if subtotals is not None else {})}, "values": {"placeholders": [{"text": e[0], "meaning": "无数据"} for e in b.placeholders.values()], "blank": "null", "text_number": "reject", "formula": "accept_cached"}, "merged_data": "fill" if fill else "reject", @@ -1797,6 +1912,8 @@ class _ListLoc: rows: list[int] total_row: int | None region: set[tuple[int, int]] = field(default_factory=set) + #: 期 5：穿插的小计行 + subtotal_rows: list[int] = field(default_factory=list) def _find_grid(name: str, grids: dict[str, Grid]) -> Grid | None: @@ -1943,19 +2060,26 @@ def _locate_list(block: ListBlock, grid: Grid, taken: set[int]) -> _ListLoc | No bottom = _bottom(grid) rr = r + 1 pick = P.match_key(block.rows.total_row.pick) if block.rows.total_row else None + sub_pick = P.match_key(block.rows.subtotals.pick) if block.rows.subtotals else None + subs: list[int] = [] while rr <= bottom: if all(grid.get(rr, c) is None for c in cols): if block.rows.blank_rows == "skip": rr += 1 continue break - if pick and P.match_key(_label_of(grid.get(rr, cols[0])) or "").startswith(pick): + label = P.match_key(_label_of(grid.get(rr, cols[0])) or "") + if pick and label.startswith(pick): total = rr break + if sub_pick and label.startswith(sub_pick): + subs.append(rr) + rr += 1 + continue rows.append(rr) rr += 1 - loc = _ListLoc(hrows, cols, rows, total) - for row in [*hrows, *rows, *([total] if total else [])]: + loc = _ListLoc(hrows, cols, rows, total, subtotal_rows=subs) + for row in [*hrows, *rows, *subs, *([total] if total else [])]: loc.region.update((row, c) for c in cols) return loc @@ -2130,6 +2254,9 @@ def questions_for(recipe: dict[str, Any], facts: DraftFacts, grids: dict[str, Gr if ext is not None and ext.sheets.skipped_hidden: for nm in _dedupe([item.get("sheet", "") for item in ext.sheets.skipped_hidden]): cards.append(_hidden_sheet_card(nm)) + # 期 5：列表每期怎么更新的问题排在其余问题之后、q_mode 之前：它只和按期累积有关，不该顶掉期 2 起排在前面的问题 + questions = ([q for q in questions if not q.id.startswith("q_list_period:")] + + [q for q in questions if q.id.startswith("q_list_period:")]) blockers = accumulate_blockers(parsed) cards.append(_mode_card(blockers)) if not blockers: @@ -2543,9 +2670,57 @@ def _list_cards(si: int, bi: int, sheet: Any, block: ListBlock, loc: _ListLoc | reason += f"，原值另存表「{total.keep_as}」" if total.keep_as else "，只核对不另存" cells = _coords(name, [(loc.total_row, loc.cols[0])]) if loc is not None and loc.total_row else [] cards.append(Card(f"total_row:{block.id}", f"{where}是合计行，改作核对", reason, cells)) + subt = block.rows.subtotals + if subt is not None: + rows = loc.subtotal_rows if loc is not None else [] + where = _rows_text(rows) if rows else f"「{subt.pick}」开头的行" + scope = ("上一行小计之后的全部明细" if subt.scope == "since_previous" + else f"紧挨在它上面、「{subt.group_column}」相同的那几行明细") + reason = f"每行小计按列求和与{scope}核对，小计行不进数据" + reason += f"，原值另存表「{subt.keep_as}」" if subt.keep_as else "，只核对不另存" + cells = _coords(name, [(r, loc.cols[0]) for r in rows]) if loc is not None else [] + cards.append(Card(f"subtotals:{block.id}", f"{where}是穿插的小计行，改作核对", reason, cells)) + q = _list_period_question(si, bi, sheet, block, recipe) + questions.append(q) + cards.append(Card(f"list_period:{block.id}", "这份明细每期怎么更新", + "每期替换：当前版本只含最新一期。按期累积：每期一份明细加进当前版本、可以跨期查询，要说明每一行" + "属于哪一期——选一个日期列，核对每行都在本期统计期内；明细里没有能代表统计期的日期列时，" + "选「每行加上本期统计期的起止两列」", [], q.id)) return cards, questions +def _list_period_question(si: int, bi: int, sheet: Any, block: ListBlock, recipe: dict[str, Any]) -> Question: + """期 5：列表每期怎么更新、按期累积时每一行属于哪一期（ListBlock.period），一道题定：每期替换（不绑定）；按期 + 累积并按某个日期列核对（每个日期列一个选项）；按期累积并给每行加统计期起止两列。后两种同时把导入模式改成按期 + 累积，所在工作表没配统计期的一并加上统计期（回执里统计期取自表内，没有就请人录入）。 + + 为什么不分成「先绑定、再问 q_mode」两道题：问题集合按起草的草稿生成、回答时不变（recipe_imports.answer）， + 列表的草稿还没绑定统计期，不符合按期累积的资格，q_mode 根本不会出现。几个列表各答各的：有的选累积、有的选替换 + 时，静态校验照实报出没绑定的那个列表不能累积。""" + base = ["sheets", si, "blocks", bi, "period"] + context = [] if sheet.context else _add_ops(recipe, ["sheets", si, "context"], [ + {"id": "统计期", "kind": "period", "parser": "cn_date_range", "prefer_prefix": None, "cross_check": "filename"}]) + mode = _add_ops(recipe, ["mode"], "accumulate") + # 「每期替换」排第一个：它就是草稿本来的样子（期 4 起各处取第一个选项的地方都不会因此改了配方） + options: list[dict[str, Any]] = [{"value": "none", "label": "每期替换", "needs_reason": False}] + effects: dict[str, list[dict[str, Any]]] = {"none": _add_ops(recipe, base, None)} + for c in block.columns: + if c.type != "DATE": + continue + value = f"column:{c.name}" + options.append({"value": value, "label": f"按期累积，按「{c.name}」核对每行都在本期统计期内", + "needs_reason": False}) + effects[value] = context + mode + _add_ops(recipe, base, {"column": c.name, "check": "within_context", + "synthesize": None}) + options.append({"value": "synthesize", "label": "按期累积，每行加上本期统计期的起止两列", "needs_reason": False}) + effects["synthesize"] = context + mode + _add_ops(recipe, base, {"column": None, "check": "within_context", + "synthesize": "统计期"}) + current = block.period + default = ("synthesize" if current is not None and current.synthesize else + f"column:{current.column}" if current is not None and current.column else "none") + return Question(f"q_list_period:{block.id}", f"列表「{block.id}」每期怎么更新", options, default, effects) + + # ========================================================================== # 期 3：按规则重新起草，并把名字对齐到现行配方（P3-SPEC 6.3） # ========================================================================== diff --git a/backend/app/data/recipe_types.py b/backend/app/data/recipe_types.py index 7643144f..215cd7f3 100644 --- a/backend/app/data/recipe_types.py +++ b/backend/app/data/recipe_types.py @@ -300,11 +300,32 @@ class TotalRow(_M): keep_as: str | None = None +class Subtotals(_M): + """穿插在明细中间的小计行：不作数据，按它所管的那一组明细逐列求和核对；keep_as 给了就另存一张表。""" + + #: 小计标签所在的列（列名） + label_column: str + #: 小计行的标签以这个词开头（match_key 比较）。取自实际格子文字的候选词，不能含数字 + pick: str = Field(min_length=1, max_length=12) + #: 一行小计管哪几行明细：since_previous 上一行小计（或表头）之后的全部明细；group_column 紧挨在它上面、 + #: group_column 这一列取值相同的那几行明细 + scope: Literal["since_previous", "group_column"] = "since_previous" + #: scope=group_column 时按哪一列分组（列名）；since_previous 时为 None + group_column: str | None = None + #: 原表的小计另存成哪张表（kind=reported_total）；None 只核对不另存 + keep_as: str | None = None + + class ListRows(_M): #: 数据中间的空行：stop 到此为止（默认）；skip 跳过继续（回执记个数） blank_rows: Literal["stop", "skip"] = "stop" #: 合计行：到合计行为止，合计行按列求和核对 total_row: TotalRow | None = None + #: 期 5：穿插在明细中间的小计行（Subtotals）。默认 None = 期 4 的行为（像小计的行照常是数据或起草失败） + subtotals: Subtotals | None = None + #: 期 5：人确认过这张列表没有小计行：以合计字样开头的行就是数据（「累计充值」这类键值清单），执行器不再报 + #: subtotal_suspect，每期也就不用再勾那一项确认。和 subtotals 二选一（静态校验 subtotals_conflict） + no_subtotals: bool = False class ListValues(_M): @@ -317,6 +338,18 @@ class ListValues(_M): formula: Literal["accept_cached", "reject"] = "accept_cached" +class ListPeriod(_M): + """列表按期累积时，每一行属于哪一期，二选一：column 按表里的一个日期列，每行的日期都要落在本期的统计期内； + synthesize 每行加上本期统计期的起、止两列（明细里没有日期列，或者日期不代表它属于哪一期时）。""" + + #: 按这个日期列（ListColumn.name，类型 DATE）核对；和 synthesize 二选一 + column: str | None = None + #: 怎么核对：within_context 每行的日期都要落在本期统计期内（空着的也不行） + check: Literal["within_context"] = "within_context" + #: 加统计期起、止两列（LIST_PERIOD_START、LIST_PERIOD_END）；和 column 二选一 + synthesize: Literal["统计期"] | None = None + + class ListBlock(_M): id: str = Field(min_length=1, max_length=32) layout: Literal["list"] @@ -333,6 +366,8 @@ class ListBlock(_M): merged_data: Literal["reject", "fill"] = "reject" #: 期 3：按表头忽略的列（IgnoreColumn），只针对这些列；其余多出的列照 extra_columns。默认空 = 期 2 的行为 ignore_columns: list[IgnoreColumn] = Field(default_factory=list, max_length=50) + #: 期 5：按期累积时每一行属于哪一期（ListPeriod）。默认 None = 期 4 的行为（列表只能每期替换） + period: ListPeriod | None = None Block = Annotated[Union[CrosstabBlock, ListBlock], Field(discriminator="layout")] @@ -441,9 +476,10 @@ class Problem: fix_args: dict[str, Any] | None = None -#: 修复按钮的九种（P3-SPEC 3.2 ①–⑨）。Problem.fix、FixProposal.kind、确认项 fix:<目标键> 的 kind 都取自这里 +#: 修复按钮的种类（P3-SPEC 3.2 ①–⑨；期 5 加 declare_subtotals：列表中间像小计的行改作小计核对）。Problem.fix、 +#: FixProposal.kind、确认项 fix:<目标键> 的 kind 都取自这里 FIX_KINDS = ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", "ignore_cells", - "declare_placeholder", "declare_hidden", "rename_sheet") + "declare_placeholder", "declare_hidden", "rename_sheet", "declare_subtotals") @dataclass @@ -462,13 +498,14 @@ CheckStatus = Literal["passed", "mismatch", "unverifiable", "info"] @dataclass class CheckResult: - #: 稳定 id：K1… 合计核对、G1… 公式引用、T1… 列表合计、R1… 关系（沿用配方里的 id）、C1 统计期、C2 文件名、 - #: N1… 行数、P1… 主键；期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期（物化时跑，都是结构类，P3-SPEC 2.7）。 + #: 稳定 id：K1… 合计核对、G1… 公式引用、T1… 列表合计（期 5 起也含列表中间的小计）、R1… 关系（沿用配方里的 + #: id）、C1 统计期、C2 文件名、N1… 行数、P1… 主键、D1… 列表的日期落在统计期内（期 5，ListPeriod.column）； + #: 期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期（物化时跑，都是结构类，P3-SPEC 2.7）。 #: F 编号留给系统发现的事实（Fact.id），两者不混用 id: str kind: Literal["derived_sum", "formula_refs", "column_sum", "relation_sum_eq", "relation_not_comparable", "context_agree", "filename_period", "row_count", "pk_unique", - "union_rows", "union_pk", "union_period"] + "union_rows", "union_pk", "union_period", "period_rows"] title: str status: CheckStatus #: 不通过时的类别：structure / data_quality / info @@ -759,9 +796,10 @@ class SheetsOut: #: hidden_excluded = hidden.rows=exclude 排除的隐藏行；blank_skipped = 列表 blank_rows=skip 跳过的空行； #: ignored_rows = 交叉表 ignore_rows 忽略的行；ignored_outside = ignore_outside 忽略了区域外数字格的行； #: after_stop = 列表停止之后没导入的文字行；total_not_kept = 参与核对、但不另存的合计行（交叉表 keep_as 为空的 -#: derived 段、列表 total_row.keep_as 为空的合计行） +#: derived 段、列表 total_row.keep_as 为空的合计行）；期 5：subtotal = 列表中间的小计行（参与核对、不进数据， +#: 另存与否都记在这里：它们都不在本表里） ExcludedReason = Literal["hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", - "total_not_kept"] + "total_not_kept", "subtotal"] @dataclass @@ -1031,6 +1069,15 @@ def prose_problems(template: str, *, known: dict[str, set[str]] | None = None) - #: 列表合计行另存成表时，合计项列的固定列名 LIST_TOTAL_DIM = "合计项" +#: 期 5：列表的小计行另存成表时，小计项列的固定列名 +LIST_SUBTOTAL_DIM = "小计项" +#: 期 5：列表按期累积、由系统给每行加统计期时（ListPeriod.synthesize）的两列：本期统计期的第一天、最后一天， +#: 写法 YYYY-MM-DD（同 DATE 列，存 TEXT） +LIST_PERIOD_START = "统计期起" +LIST_PERIOD_END = "统计期止" +#: 期 5：小计核对的键（核对编号 T、合计格 DerivedItem.segment）= 块 id 加这个后缀。块 id 在整份配方内唯一， +#: 同一块的合计行用块 id 本身，两者不相混 +SUBTOTAL_KEY = "·小计" _SQL_TYPE = {"INTEGER": "INTEGER", "REAL": "REAL", "TEXT": "TEXT", "DATE": "TEXT"} @@ -1107,16 +1154,42 @@ def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[Reci cols = [ColumnOut(c.name, _SQL_TYPE[c.type], header=c.header, role="measure" if c.type in ("INTEGER", "REAL") else "text") for c in block.columns] + if block.period is not None and block.period.synthesize: + # 期 5：每行加上本期统计期的起止（role=period：没有格子来历，取值来自统计期） + cols += [ColumnOut(LIST_PERIOD_START, "TEXT", role="period"), + ColumnOut(LIST_PERIOD_END, "TEXT", role="period")] put(block.table, cols, base) total = block.rows.total_row + numeric = [c for c in cols if c.type in ("INTEGER", "REAL")] if total is not None and total.keep_as: tcols = [ColumnOut(LIST_TOTAL_DIM, "TEXT", role="dim")] - tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") - for c in cols if c.type in ("INTEGER", "REAL")] + tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") for c in numeric] put(total.keep_as, tcols, f"{base}/rows/total_row") + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + # 期 5：小计另存：小计项（标签的规范写法）、按组小计时的分组列、各数字列 + scols = [ColumnOut(LIST_SUBTOTAL_DIM, "TEXT", role="dim")] + if sub.scope == "group_column" and sub.group_column: + scols.append(ColumnOut(sub.group_column, "TEXT", role="dim")) + scols += [ColumnOut(c.name, c.type, header=c.header, role="value") for c in numeric] + put(sub.keep_as, scols, f"{base}/rows/subtotals") return out, problems +def list_period_columns(recipe: Recipe) -> dict[str, tuple[str, ...]]: + """期 5：列表写入的表 → 它按什么分期：("column", 日期列) 或 ("synthesize", 统计期起列, 统计期止列)。只列配了 + ListBlock.period 的列表；累积计划的 U3、单期的 D 核对、说明里「按哪一列筛选」共用这一份。""" + out: dict[str, tuple[str, ...]] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock) and block.period is not None: + if block.period.synthesize: + out.setdefault(block.table, ("synthesize", LIST_PERIOD_START, LIST_PERIOD_END)) + elif block.period.column: + out.setdefault(block.table, ("column", block.period.column)) + return out + + # ========================================================================== # 期 3：按期累积的资格（静态校验、起草器、累积计划共用同一份，P3-SPEC 2.2） # ========================================================================== @@ -1130,10 +1203,13 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: 1. 至少一个工作表配置了统计期上下文（SheetRecipe.context 非空），否则报 accumulate_needs_period（path=/mode）； 2. derive_tables 推出的每张表都「按统计期分得开」，否则逐表报 accumulate_unkeyed（path=/tables/{ti}，表没在 tables 里声明时指向第一个写入它的块），消息写明哪一条不满足： - - 只由交叉表块写入（measures、dimension 段，或者 derived 段的 keep_as）——列表写入的表一律不满足； - - 写入它的每个交叉表块，axis.checks 都含 covers_context（日期恰好落在统计期内）； - - 这张表的 grain 含那个块的 axis.name。 - 两条都成立时，各期的日期集合互不相交（统计期不重叠由累积计划保证），主键里不需要另加统计期列。 + - 交叉表写入的（measures、dimension 段，或者 derived 段的 keep_as）：写入它的每个交叉表块，axis.checks 都含 + covers_context（日期恰好落在统计期内），并且这张表的 grain 含那个块的 axis.name； + - 期 5：列表写入的明细表：写入它的每个列表块都配了 period（按日期列核对，或加统计期起止两列），并且所在的 + 工作表配了统计期。列表的合计行、小计行另存的表不能累积：各期的合计项、小计项相同，主键分不开各期； + - 同一张表不能既由交叉表又由列表写入。 + 都成立时，各期的行按统计期分得开（统计期不重叠由累积计划保证；列表明细跨期重复的同一条记录由并集的主键核对 + U2 拒收）。 """ problems: list[RecipeProblem] = [] if not any(sheet.context for sheet in recipe.sheets): @@ -1144,6 +1220,11 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: writers: dict[str, list[CrosstabBlock | ListBlock]] = {} first_path: dict[str, str] = {} + #: 列表的合计行、小计行另存的表 → 写入它的块 + totals: dict[str, list[ListBlock]] = {} + #: 列表块 → 所在工作表有没有配统计期 + ctx_of: dict[int, bool] = {} + def _add(table: str, block: CrosstabBlock | ListBlock, path: str) -> None: got = writers.setdefault(table, []) if all(b is not block for b in got): @@ -1161,16 +1242,32 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: _add(seg.keep_as.table, block, path) else: _add(block.table, block, path) - total = block.rows.total_row - if total is not None and total.keep_as: - _add(total.keep_as, block, path) + ctx_of[id(block)] = bool(sheet.context) + for kept in (block.rows.total_row.keep_as if block.rows.total_row else None, + block.rows.subtotals.keep_as if block.rows.subtotals else None): + if kept: + _add(kept, block, path) + totals.setdefault(kept, []).append(block) index = {t.name: ti for ti, t in enumerate(recipe.tables)} grains = {t.name: list(t.grain) for t in recipe.tables} for name in tables: blocks = writers.get(name, []) - if any(isinstance(b, ListBlock) for b in blocks): - reasons = ["列表形态的表目前只支持每期替换"] + lists = [b for b in blocks if isinstance(b, ListBlock)] + if lists and len(lists) != len(blocks): + reasons = ["同一张表既由交叉表又由列表写入，各期的行分不开"] + elif name in totals: + reasons = ["列表的合计行、小计行另存的表不能按期累积：各期的合计项、小计项相同，主键分不开各期" + "（可以改为只核对、不另存）"] + elif lists: + reasons = [] + unbound = [b.id for b in lists if b.period is None] + if unbound: + reasons.append("、".join(f"列表「{x}」" for x in unbound) + + "没有指定每一行属于哪一期（按日期列核对，或加统计期起止两列）") + no_ctx = [b.id for b in lists if b.period is not None and not ctx_of.get(id(b))] + if no_ctx: + reasons.append("、".join(f"列表「{x}」" for x in no_ctx) + "所在的工作表没有配置统计期") else: reasons = [] uncovered = [b.id for b in blocks if isinstance(b, CrosstabBlock) and "covers_context" not in b.axis.checks] diff --git a/backend/app/data/xlsx_cells.py b/backend/app/data/xlsx_cells.py index 28f137e0..69f93fad 100644 --- a/backend/app/data/xlsx_cells.py +++ b/backend/app/data/xlsx_cells.py @@ -24,7 +24,7 @@ from collections.abc import Iterator from typing import Any from xml.etree import ElementTree as ET -from app.data import tabular +from app.data import csv_grid, tabular from app.data.recipe_types import Grid, GridCell from app.data.xlsx_scan import SheetScan, UnsupportedTable, WorkbookScan, _broken, cell_ref, parse_ref @@ -70,6 +70,14 @@ class Workbooks: self.raw = raw self._values: Any = None self._formulas: Any = None + self._csv: Any = None + + def csv(self, delimiter: str | None = None) -> Any: + """CSV 原件解析好的记录（csv_grid.CsvTable），第一次要读时才解析，同一次执行里复用。delimiter 是扫描时定下的 + 分隔符（csv_grid.sheet_delimiter），不给时再嗅一次。""" + if self._csv is None: + self._csv = csv_grid.parse(self.raw, delimiter) + return self._csv def values(self) -> Any: if self._values is None: @@ -126,6 +134,11 @@ def sheet_rows(books: Workbooks, sheet: SheetScan, *, min_row: int, max_row: int 行数和每行的宽度都一样。文件内容写坏了（见 _READ_ERRORS）抛 UnsupportedTable，日志只记异常类型。 """ try: + if csv_grid.is_csv_sheet(sheet): + # CSV 适配出来的「工作表」：从解析好的记录里取，口径同下（只给非空格、整行为空的不给）。分隔符用扫描时定下的 + yield from csv_grid.rows(books.csv(csv_grid.sheet_delimiter(sheet)), min_row=min_row, max_row=max_row, + min_col=min_col, max_col=max_col) + return yield from _sheet_rows(books, sheet, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col) except UnsupportedTable: raise diff --git a/backend/app/data/xlsx_scan.py b/backend/app/data/xlsx_scan.py index 4910c6ef..5acd20fd 100644 --- a/backend/app/data/xlsx_scan.py +++ b/backend/app/data/xlsx_scan.py @@ -216,6 +216,9 @@ class WorkbookScan: tables: list[TableScan] = field(default_factory=list) #: 定义名称 → 原文。工作表级的名称记作「工作表名!名称」，免得同名互相覆盖 defined_names: dict[str, str] = field(default_factory=dict) + #: 原件的格式：xlsx，或者 csv（csv_grid.scan 适配出来的一张「工作表」，读取层据 SheetScan.path 从 CSV 里读）。 + #: 老暂存区的扫描结果里没有这个键，按 xlsx 读 + format: str = "xlsx" def sheet(self, name: str) -> SheetScan | None: return next((s for s in self.sheets if s.name == name), None) @@ -1090,4 +1093,5 @@ def scan_from_json(data: dict[str, Any]) -> WorkbookScan: for t in tables: t.columns = list(t.columns) return WorkbookScan(sheets=sheets, full_calc_on_load=bool(data.get("full_calc_on_load")), tables=tables, - defined_names=dict(data.get("defined_names") or {})) + defined_names=dict(data.get("defined_names") or {}), + format=str(data.get("format") or "xlsx")) diff --git a/backend/tests/fixtures/dialect/mariadb.sql b/backend/tests/fixtures/dialect/mariadb.sql new file mode 100644 index 00000000..0130c4d9 --- /dev/null +++ b/backend/tests/fixtures/dialect/mariadb.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：mariadb（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'sales' AND TABLE_NAME = 'orders'; +-- stats_quoted +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT(`level`) AS n, COUNT(DISTINCT `level`) AS d FROM `visits`; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY DATE(paid_at) ORDER BY d; +-- sample +-- （这种方言没有这类查询） diff --git a/backend/tests/fixtures/dialect/mysql.sql b/backend/tests/fixtures/dialect/mysql.sql new file mode 100644 index 00000000..1b046e36 --- /dev/null +++ b/backend/tests/fixtures/dialect/mysql.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：mysql（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'sales' AND TABLE_NAME = 'orders'; +-- stats_quoted +SELECT TABLE_ROWS FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT(`level`) AS n, COUNT(DISTINCT `level`) AS d FROM `visits`; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT DATE(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY DATE(paid_at) ORDER BY d; +-- sample +-- （这种方言没有这类查询） diff --git a/backend/tests/fixtures/dialect/oracle.sql b/backend/tests/fixtures/dialect/oracle.sql new file mode 100644 index 00000000..c338ead9 --- /dev/null +++ b/backend/tests/fixtures/dialect/oracle.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：oracle（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = 'SALES' AND TABLE_NAME = 'ORDERS'; +-- stats_quoted +SELECT NUM_ROWS FROM ALL_TABLES WHERE OWNER = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') AND TABLE_NAME = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT * FROM (SELECT 1 AS one FROM sales.orders) WHERE ROWNUM <= 100001) s; +-- distinct +SELECT * FROM (SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL) WHERE ROWNUM <= 2000; +-- distinct_capped +SELECT * FROM (SELECT DISTINCT store_code FROM (SELECT * FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL) WHERE ROWNUM <= 100000) s) WHERE ROWNUM <= 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- value_counts_capped +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM (SELECT * FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL) WHERE ROWNUM <= 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; +-- day_volume +SELECT TRUNC(paid_at) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY TRUNC(paid_at) ORDER BY d; +-- sample +SELECT * FROM (SELECT status, COUNT(*) AS cnt FROM sales.orders SAMPLE BLOCK (1) WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status) WHERE ROWNUM <= 21; diff --git a/backend/tests/fixtures/dialect/postgresql.sql b/backend/tests/fixtures/dialect/postgresql.sql new file mode 100644 index 00000000..1416da24 --- /dev/null +++ b/backend/tests/fixtures/dialect/postgresql.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：postgresql（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +SELECT c.reltuples FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'sales' AND c.relname = 'orders'; +-- stats_quoted +SELECT c.reltuples FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = current_schema() AND c.relname = 'visits'; +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= DATE '0001-01-02' AND paid_at < DATE '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < DATE '0001-01-02' OR paid_at >= DATE '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT CAST(paid_at AS DATE) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= DATE '2026-08-27' AND paid_at < DATE '9999-12-31' GROUP BY CAST(paid_at AS DATE) ORDER BY d; +-- sample +SELECT status, COUNT(*) AS cnt FROM sales.orders TABLESAMPLE SYSTEM (1) WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; diff --git a/backend/tests/fixtures/dialect/sqlite.sql b/backend/tests/fixtures/dialect/sqlite.sql new file mode 100644 index 00000000..8b10405f --- /dev/null +++ b/backend/tests/fixtures/dialect/sqlite.sql @@ -0,0 +1,27 @@ +-- 剖析与取值观察的 SQL 金样：sqlite（tests/test_profile_dialect_golden.py 生成，改动要人审） +-- stats +-- （这种方言没有这类查询） +-- stats_quoted +-- （这种方言没有这类查询） +-- bounded +SELECT COUNT(*) AS n FROM (SELECT 1 AS one FROM sales.orders LIMIT 100001) s; +-- distinct +SELECT DISTINCT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 2000; +-- distinct_capped +SELECT DISTINCT store_code FROM (SELECT store_code FROM sales.orders WHERE store_code IS NOT NULL LIMIT 100000) s LIMIT 2000; +-- match +SELECT COUNT(DISTINCT code) AS n FROM stores WHERE code IN ('12', '7'); +-- unique +SELECT COUNT("level") AS n, COUNT(DISTINCT "level") AS d FROM "visits"; +-- value_counts +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- value_counts_capped +SELECT status, COUNT(*) AS cnt FROM (SELECT status FROM sales.orders WHERE status IS NOT NULL LIMIT 100000) s WHERE status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- min_max +SELECT MIN(CASE WHEN paid_at >= '0001-01-02' AND paid_at < '9999-12-31' THEN paid_at END), MAX(CASE WHEN paid_at >= '0001-01-02' AND paid_at < '9999-12-31' THEN paid_at END), SUM(CASE WHEN paid_at < '0001-01-02' OR paid_at >= '9999-12-31' THEN 1 ELSE 0 END), MIN(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), MAX(CASE WHEN biz_day >= '0001-01-02' AND biz_day < '9999-12-31' THEN biz_day END), SUM(CASE WHEN biz_day < '0001-01-02' OR biz_day >= '9999-12-31' THEN 1 ELSE 0 END) FROM sales.orders; +-- date_window +SELECT status, COUNT(*) AS cnt FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '9999-12-31' AND status IS NOT NULL GROUP BY status ORDER BY cnt DESC, status LIMIT 21; +-- day_volume +SELECT substr(paid_at, 1, 10) AS d, COUNT(*) AS n FROM sales.orders WHERE paid_at >= '2026-08-27' AND paid_at < '9999-12-31' GROUP BY substr(paid_at, 1, 10) ORDER BY d; +-- sample +-- （这种方言没有这类查询） diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py index 37f3705b..c65eff02 100644 --- a/backend/tests/test_catalog_draft_facts.py +++ b/backend/tests/test_catalog_draft_facts.py @@ -402,7 +402,7 @@ async def test_a_big_table_is_checked_on_its_first_rows(scenic): found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], "status = 1") assert found.sampled and found.rows == 100 - assert found.note.startswith("剖析核对：按前 100 行统计，") + assert found.note.startswith("剖析核对：按存储顺序取前 100 行统计，") async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index fee64c5f..0a1ef009 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -23,7 +23,7 @@ from httpx import ASGITransport, AsyncClient from sqlalchemy import event from sqlalchemy.engine import Engine -from app.data import catalog_profile, catalog_profile_jobs +from app.data import catalog, catalog_profile, catalog_profile_jobs from app.data import engine as data_engine from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings from app.db.base import SessionLocal @@ -240,7 +240,7 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert rel["to_table"] == parent assert (rel["source"], rel["status"]) == ("profile", "verified") assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" - assert "数据剖析" in rel["note"] and "抽样" in rel["note"] + assert "数据剖析" in rel["note"] and "子表取了" in rel["note"] and "抽样" not in rel["note"] # 子表一侧数过 COUNT 和 COUNT(DISTINCT)、确有重复：基数是用数据核实的（A6，一对多关联的检查据此给 error） assert rel["cardinality_checked"] is True # A7：说明里不写日期，时间看这一项的 updated_at（界面按本地时间显示） @@ -318,7 +318,7 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") assert rel["status"] == "proposed" and rel["source"] == "profile" assert rel["coverage"] == 0.15 - assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] + assert "覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] finding = _table(body, "parking_records")["findings"][0] assert finding["status"] == "proposed" and finding["coverage"] == 0.15 # 两个日期类列：不提议业务日期，取值范围只在剖析结果里 @@ -327,6 +327,39 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"] +async def test_text_keys_that_cannot_be_integers_count_as_unmatched(client, make_source, tmp_path): + """模型起草的关系：orders.customer_ref（文字）指向 customers.id（整数主键），可取值大多是散客代号。不是整数写法的 + 文字永远对不上整数主键，算作没对上、留在分母里——不能从分母里拿掉，把 1 个对上的键说成覆盖率 100% 升为有确证。""" + path = str(tmp_path / "walkin.db") + db = sqlite3.connect(path) + try: + db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT NOT NULL);" + "CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_ref TEXT, amount INTEGER);") + db.executemany("INSERT INTO customers VALUES (?, ?)", [(i, f"会员{i}") for i in (1, 2, 3)]) + refs = ["WALKIN"] * 90 + ["1"] * 10 + [f"GUEST{i}" for i in range(30)] + db.executemany("INSERT INTO orders VALUES (?, ?, ?)", [(i, ref, 10) for i, ref in enumerate(refs, 1)]) + db.commit() + finally: + db.close() + sid = await make_source(path, options=_on()) + rel_id = catalog.relation_id("orders", ["customer_ref"], "customers", ["id"]) + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", {"relations": [ + {"id": rel_id, "columns": ["customer_ref"], "to_table": "customers", "to_columns": ["id"], + "cardinality": "many_to_one", "coverage": None, "source": "llm", "status": "proposed"}]}, + if_version=0, actor=None) + + body = await _profile(client, sid, ["orders"]) + rel = _relation((await _notes(client, sid, "orders"))["notes"], "customer_ref") + assert (rel["source"], rel["status"]) == ("profile", "proposed") + assert rel["coverage"] == round(1 / 32, 4) + assert "覆盖率 3.1%" in rel["note"] and "可能不是这条关系" in rel["note"] + assert "子表取了 32 个不同键值，父表对上 1 个" in rel["note"] + assert "31 个不是整数写法" in rel["note"] and "未计入" not in rel["note"] + finding = _table(body, "orders")["findings"][0] + assert (finding["status"], finding["sample"], finding["matched"]) == ("proposed", 32, 1) + + async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, make_source, scenic_db): sid = await make_source(scenic_db, options=_on(), draft=["visits"]) detail = await _notes(client, sid, "visits") diff --git a/backend/tests/test_catalog_profile_units.py b/backend/tests/test_catalog_profile_units.py index 4fa1472f..22933754 100644 --- a/backend/tests/test_catalog_profile_units.py +++ b/backend/tests/test_catalog_profile_units.py @@ -40,7 +40,7 @@ def test_settings_default_off_and_parse_leniently(): ({"table_max_s": 9}, "每张表时长上限"), ({"table_max_s": 601}, "每张表时长上限"), ({"query_timeout_s": 61}, "单条查询时限"), - ({"sample_size": 5}, "抽样键值数"), + ({"sample_size": 5}, "核对键值数"), ({"max_scan_rows": -1}, "整表统计行数上限"), ({"enabled": "true"}, "开启数据剖析"), ({"table_max_queries": True}, "每张表最多查询数"), @@ -143,8 +143,16 @@ def test_profile_sql_passes_readonly_guard_on_every_dialect(kind): d.unique_check_sql(tbl, col), d.value_counts_sql(tbl, col, 21, scan_cap=None), d.value_counts_sql(tbl, col, 21, scan_cap=100_000), - d.min_max_sql(tbl, [col]), + d.min_max_sql(tbl, [(col, False)]), + d.min_max_sql(tbl, [(col, True)]), + d.day_volume_sql(tbl, col, "2026-08-01"), + d.window_value_counts_sql(tbl, col, col, "2026-08-01", 21), ] + sampled = d.sample_value_counts_sql(tbl, col, 1, 21) + if kind in ("postgresql", "oracle"): + sqls.append(sampled) + else: + assert sampled is None # MySQL、SQLite 没有块采样 for sql in sqls: assert guard.check(sql, readonly=True, dialect=kind) == sql, sql diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index 25eb9aad..d15e0332 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -14,6 +14,7 @@ """ from __future__ import annotations +import re import sqlite3 import pytest @@ -365,7 +366,7 @@ async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(c capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"]) await _profile(client, capped, ["visits"]) codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"] - assert "按前 10 行统计" in codes["note"] and "complete" not in codes + assert "按存储顺序取前 10 行统计" in codes["note"] and "complete" not in codes # ========================================================================== @@ -408,6 +409,95 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak assert codes["note"].startswith("数据剖析：") and "2026-10-03" not in codes["note"] +#: 升级前剖析写的说法 → 现在的说法（下一轮 C1：大表看的是按存储顺序的前 N 行，不是抽样）。测试里反过来用：把新写法 +#: 改回旧写法，模拟升级前写进目录的备注 +_WORDING = (("覆盖率 ", "抽样覆盖率 ", "，可能不是这条关系"), ("子表取了 ", "子表抽样 ", None), + ("（按存储顺序只看前 ", "（只看前 ", None), ("按存储顺序取前 ", "按前 ", None)) + + +def _legacy(note: str) -> str: + out = note + for new, old, tail in _WORDING: + if tail: + out = re.sub(re.escape(new) + r"(\S+?)" + re.escape(tail), lambda m: old + m.group(1) + tail, out) + else: + out = out.replace(new, old) + return out + + +def _profile_notes(notes: dict) -> list[dict]: + """目录里剖析写了备注的项（关系、码值、业务日期）。""" + found = [r for r in notes.get("relations") or [] if catalog.is_profile_note(r.get("note"))] + for column in (notes.get("columns") or {}).values(): + found += [item for item in column.values() if isinstance(item, dict) + and catalog.is_profile_note(item.get("note"))] + bdate = notes.get("business_date") + return found + ([bdate] if isinstance(bdate, dict) and catalog.is_profile_note(bdate.get("note")) else []) + + +def test_profile_notes_in_the_old_wording_are_the_same_note(): + """改过的四处说法：旧写法和新写法一字之外都一样才算同一句；数变了、或者不是剖析写的备注，照旧算不同。""" + old = "数据剖析：抽样覆盖率 15%，可能不是这条关系；子表抽样 20 个不同键值（只看前 100 行），父表对上 3 个；被指向列是主键。" + new = "数据剖析：覆盖率 15%，可能不是这条关系；子表取了 20 个不同键值（按存储顺序只看前 100 行），父表对上 3 个；被指向列是主键。" + assert catalog.same_profile_note(old, new) and catalog.same_profile_note(new, old) + assert catalog.same_profile_note(old.replace("数据剖析：", "数据剖析（2026-10-01）："), new) + assert not catalog.same_profile_note(old, new.replace("对上 3 个", "对上 4 个")) + codes_old = "数据剖析（2026-10-01）：按前 10 行统计，10 行非空值共 2 个取值：1（90%）、0（10%）。" + codes_new = "数据剖析：按存储顺序取前 10 行统计，10 行非空值共 2 个取值：1（90%）、0（10%）。" + assert catalog.same_profile_note(codes_old, codes_new) + assert not catalog.same_profile_note(codes_old, codes_new.replace("前 10 行", "前 20 行")) + # 起草写的有效行过滤备注：只认接在后面的剖析核对那句 + filter_old = "依据：status 为 1 才是有效记录。剖析核对：按前 100,000 行统计，排除 3% 的行" + filter_new = "依据：status 为 1 才是有效记录。剖析核对：按存储顺序取前 100,000 行统计，排除 3% 的行" + assert catalog.same_profile_note(filter_old, filter_new) + assert not catalog.same_profile_note(filter_old, filter_new.replace("3%", "4%")) + # 不是剖析写的备注：「抽样覆盖率」是人写的原话，不换 + assert not catalog.same_profile_note("人工备注：抽样覆盖率 15%，可能不是这条关系", + "人工备注：覆盖率 15%，可能不是这条关系") + assert not catalog.same_profile_note(new, "覆盖率 15%，可能不是这条关系") + + +async def test_rerun_after_upgrade_keeps_the_version_when_only_the_wording_changed(client, make_source, tmp_path): + """升级前写进目录的剖析备注用的是旧说法（「抽样覆盖率」「子表抽样」「（只看前 N 行）」「按前 N 行统计」）。数据 + 没变、升级后第一次重新剖析：结论一字不差、只是措辞改了，不算变化——否则每张有剖析结论的表都升一个版本，发布时 + 记下的目录版本对不上，之后每次正式运行都提醒「目录有变化」。""" + path = scenic.build(tmp_path / "wording.db") + db = sqlite3.connect(path) + try: + db.execute("CREATE TABLE lots (id INTEGER PRIMARY KEY, name TEXT NOT NULL)") + db.executemany("INSERT INTO lots VALUES (?, ?)", [(i, f"{i} 号停车区") for i in (1, 2, 3)]) + db.executemany("INSERT INTO parking_records VALUES (?, ?, ?, ?, ?, ?)", + [(i, 1 + i % 20, None, "2026-08-01 09:00:00", None, 10.0) for i in range(1, 101)]) + db.commit() + finally: + db.close() + tables = ["visits", "parking_records"] + whole = await make_source(path, options=_on(), draft=tables) + capped = await make_source(path, options=_on(max_scan_rows=0, sample_size=10), draft=tables) + seen: set[str] = set() + for sid in (whole, capped): + await _profile(client, sid, tables) + for table in tables: + detail = await _notes(client, sid, table) + notes = detail["notes"] + for item in _profile_notes(notes): + old = _legacy(item["note"]) + seen |= {o for _, o, _ in _WORDING if o in old} + item["note"] = old + async with SessionLocal() as session: + await catalog.write_entry(session, sid, table, notes, if_version=detail["version"], actor=None) + assert seen == {o for _, o, _ in _WORDING}, "四处旧说法都写进了目录" + + for sid in (whole, capped): + before = {t: await _notes(client, sid, t) for t in tables} + body = await _profile(client, sid, tables) + for table in tables: + done = _table(body, table) + assert (done["added"], done["updated"], done["removed"]) == (0, 0, 0), (table, done) + assert done["version"] == before[table]["version"] + assert (await _notes(client, sid, table))["notes"] == before[table]["notes"] + + def test_merge_ignores_a_profile_note_that_only_changed_its_date(): old = {"columns": {"status": {"codes": catalog.make_item( {"1": "", "0": ""}, "profile", "proposed", note="数据剖析（2026-10-01）：统计全表，1500 行非空值共 2 个取值。", diff --git a/backend/tests/test_csv_grid.py b/backend/tests/test_csv_grid.py new file mode 100644 index 00000000..dca02df7 --- /dev/null +++ b/backend/tests/test_csv_grid.py @@ -0,0 +1,149 @@ +"""CSV 走配方（Excel 期 5 第 4 项）：CSV 适配成网格，后面的执行器、核对、溯源全部复用。 + +格子坐标沿用「工作表!A1」（见 csv_grid 的模块说明）：CSV 没有工作表，名字固定是「CSV」，每月的文件名变了也认得出。 +""" +from __future__ import annotations + +import pytest + +from app.data import csv_grid, recipe_engine, xlsx_cells +from app.data.recipe_types import Recipe +from app.data.xlsx_scan import SheetScan, UnsupportedTable, scan_from_json, scan_to_json + +CSV = ("订单号,下单日期,门店,金额,备注\n" + "A001,2026-08-01,一号店,120,\n" + "A002,2026-08-02,二号店,80.5,\"带,逗号\"\n" + "\n" + "A003,2026-08-03,一号店,1,234,多出的一格\n").encode("utf-8") + + +def test_text_is_sniffed_as_csv_and_binary_is_not(): + assert csv_grid.looks_like_csv(CSV) + assert csv_grid.looks_like_csv("a;b\n1;2\n".encode("gb18030")) + assert csv_grid.looks_like_csv(b"\xff\xfe" + "a\tb\n".encode("utf-16-le")) + assert not csv_grid.looks_like_csv(b"PK\x03\x04rest-of-zip") + assert not csv_grid.looks_like_csv(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100) # 旧版 .xls + assert not csv_grid.looks_like_csv(b"\x89PNG\r\n\x1a\n\x00\x00") + assert not csv_grid.looks_like_csv(b"") + + +def test_scan_gives_one_visible_sheet_with_bounds_and_row_counts(): + scan = csv_grid.scan(CSV) + assert scan.format == "csv" and len(scan.sheets) == 1 + sh = scan.sheets[0] + assert sh.name == csv_grid.CSV_SHEET and sh.state == "visible" and sh.path == csv_grid.CSV_PATH + "," + assert (sh.bounds.min_row, sh.bounds.min_col, sh.bounds.max_row, sh.bounds.max_col) == (1, 1, 5, 6) + # 空字段不算非空格；第 4 行是空行 + assert sh.row_runs == [(1, 1, 5), (2, 2, 4), (3, 3, 5), (5, 5, 6)] + assert sh.nonempty == 20 + again = scan_from_json(scan_to_json(scan)) + assert again.format == "csv" and again.sheets[0].path == csv_grid.CSV_PATH + "," + + +def test_cell_values_are_typed_only_when_unambiguous(): + """整数、小数写成数；千分位、前导零、正号、科学计数法、日期都留作文字，交给配方的规则（text_number、类型）判。""" + v = csv_grid.cell_value + assert v("120") == 120 and v(" 80.5 ") == 80.5 and v("-3") == -3 and v("0") == 0 + for text in ("1,234", "007", "+5", "1e5", "2026-08-01", "12%", "TRUE", "=SUM(A1:A2)", "1234567890123456"): + assert v(text) == text, text + assert v("") is None and v(" ") is None + + +def test_grid_and_rows_read_the_same_cells(): + scan = csv_grid.scan(CSV) + grid = xlsx_cells.read_grid(CSV, scan, csv_grid.CSV_SHEET) + assert grid.get(2, 4).value == 120 and grid.get(3, 4).value == 80.5 and grid.get(3, 5).value == "带,逗号" + assert grid.get(2, 5) is None and grid.get(1, 1).value == "订单号" + assert grid.get(5, 4).value == 1 and grid.get(5, 5).value == 234 # 没加引号的千分位被逗号拆开了：照实读 + rows = list(xlsx_cells.iter_rows(CSV, scan, csv_grid.CSV_SHEET, min_row=1, max_row=5, min_col=1, max_col=6)) + assert [r for r, _ in rows] == [1, 2, 3, 5] + assert sum(len(cells) for _, cells in rows) == len(grid.cells) == scan.sheets[0].nonempty + small = xlsx_cells.read_grid(CSV, scan, csv_grid.CSV_SHEET, max_rows=2) + assert small.truncated and set(r for r, _ in small.cells) == {1, 2} + + +def test_semicolon_and_tab_delimiters_and_gbk(): + scan = csv_grid.scan("名称;数量\n甲;3\n".encode("gb18030")) + grid = xlsx_cells.read_grid("名称;数量\n甲;3\n".encode("gb18030"), scan, csv_grid.CSV_SHEET) + assert grid.get(2, 1).value == "甲" and grid.get(2, 2).value == 3 + raw = "a\tb\n1\t2\n".encode("utf-8") + assert xlsx_cells.read_grid(raw, csv_grid.scan(raw), csv_grid.CSV_SHEET).get(2, 2).value == 2 + + +def test_scan_any_dispatches_and_keeps_the_xlsx_refusal(): + assert csv_grid.scan_any(CSV, "orders.csv").format == "csv" + assert csv_grid.scan_any(CSV, "ORDERS.CSV").format == "csv" + with pytest.raises(UnsupportedTable): + csv_grid.scan_any(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100, "old.xls") + + +def test_only_csv_and_tsv_names_are_read_as_csv(): + """复核反例：以前只看内容，不是 zip / OLE、前 64KB 没有 NUL 就当 CSV 读。系统导出的「HTML 表格冒充 .xls」、 + 误传的文本文件，都被拆成一张叫「CSV」的工作表去起草。现在和简单导入（tabular）一样按扩展名定：只有 .csv / .tsv + 按 CSV 读；别的照旧交给 xlsx_scan 拒收。""" + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan_any(CSV, "orders.txt") + assert "不是有效的 .xlsx" in str(e.value) + html = "
订单号金额
".encode("utf-8") + for raw in (html, b"\xef\xbb\xbf \r\n" + html, b"\xff\xfe" + "
a
".encode("utf-16-le"), + b''): + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan_any(raw, "report.xls") + assert "网页或 XML" in str(e.value) and "另存为 .xlsx 或 .csv" in str(e.value) + # 扩展名是 .csv、内容却不是文字（图片之类改了扩展名）：按 CSV 读不了，照实说 + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan_any(b"\x89PNG\r\n\x1a\n\x00\x00", "orders.csv") + assert "不是文字" in str(e.value) + + +def test_tsv_is_always_tab_separated_and_the_delimiter_is_kept_for_reading(): + """.tsv 一律按制表符分隔（同 tabular）：每行逗号个数一样的 TSV，嗅探会挑逗号（csv.Sniffer 优先逗号）。 + 扫描时定下的分隔符记在工作表的 path 里，之后读格子（重放、试运行）用同一个，不再各嗅各的。""" + raw = "名称,规格\t金额\n甲,1kg\t10\n乙,2kg\t20\n".encode("utf-8") # 每行一个逗号、一个制表符 + scan = csv_grid.scan_any(raw, "goods.tsv") + assert scan.sheets[0].path == csv_grid.CSV_PATH + "\t" + grid = xlsx_cells.read_grid(raw, scan_from_json(scan_to_json(scan)), csv_grid.CSV_SHEET) + assert grid.get(2, 1).value == "甲,1kg" and grid.get(2, 2).value == 10 + # 这一版之前的暂存区：path 只有「csv:」，读的时候再嗅 + assert csv_grid.is_csv_sheet(SheetScan(name="CSV", state="visible", path=csv_grid.CSV_PATH)) + assert csv_grid.sheet_delimiter(SheetScan(name="CSV", state="visible", path=csv_grid.CSV_PATH)) is None + assert not csv_grid.is_csv_sheet(SheetScan(name="明细", state="visible", path="xl/worksheets/sheet1.xml")) + sniffed = csv_grid.scan_any(raw, "goods.csv") + assert sniffed.sheets[0].path == csv_grid.CSV_PATH + "," # 同样的内容叫 .csv：照嗅探的来（逗号） + grid = xlsx_cells.read_grid(raw, sniffed, csv_grid.CSV_SHEET) + assert grid.get(2, 1).value == "甲" and grid.get(2, 2).value == "1kg\t10" + + +def test_too_many_columns_is_refused(): + raw = (",".join(["x"] * (csv_grid.MAX_COLS + 1)) + "\n").encode() + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan(raw) + assert "列" in str(e.value) + + +LIST_RECIPE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "CSV"}, + "blocks": [{"id": "列表1", "layout": "list", "table": "订单", + "columns": [{"header": "订单号", "name": "订单号", "type": "TEXT"}, + {"header": "下单日期", "name": "下单日期", "type": "DATE"}, + {"header": "门店", "name": "门店", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "REAL"}], + "rows": {"blank_rows": "stop"}}]}], + "tables": [{"name": "订单", "grain": ["订单号"]}], +} + + +def test_engine_runs_a_list_recipe_on_csv(tmp_path): + raw = "订单号,下单日期,门店,金额\nA001,2026-08-01,一号店,120\nA002,2026-08-02,二号店,80.5\n".encode() + scan = csv_grid.scan(raw) + db = tmp_path / "t.sqlite" + ex = recipe_engine.execute(Recipe.model_validate(LIST_RECIPE), raw, "orders_2026-08.csv", scan, str(db)) + assert ex.ok, [p.message for p in ex.problems] + assert ex.tables[0].rows == 2 and ex.tables[0].sheet == "CSV" + # 溯源照常：订单号这一列从 CSV!A2 起往下 2 格 + run = ex.lineage["订单"]["订单号"][0] + assert run[1:] == ["CSV", "A2", 2, "down"] + import sqlite3 + rows = sqlite3.connect(db).execute('SELECT "订单号", "下单日期", "金额" FROM "订单" ORDER BY rowid').fetchall() + assert rows == [("A001", "2026-08-01", 120.0), ("A002", "2026-08-02", 80.5)] diff --git a/backend/tests/test_profile_dialect_golden.py b/backend/tests/test_profile_dialect_golden.py new file mode 100644 index 00000000..7e7c6321 --- /dev/null +++ b/backend/tests/test_profile_dialect_golden.py @@ -0,0 +1,178 @@ +"""剖析 SQL 的方言金样（下一轮 03 细案 C2）：没有真库时挡住方言写法的回退。 + +三层，各挡各的： + +1. **金样**：每种方言（mysql、mariadb、postgresql、oracle、sqlite）× 每类查询（统计信息、有上限的计数、取不同键值、 + 对键值、唯一性、取值分布、日期最小最大、时间窗口里的取值分布、按日行数、块采样），完整 SQL 钉在 + tests/fixtures/dialect/<方言>.sql，人审过一次；以后拼法有任何改动都会出 diff。要更新金样： + ``UPDATE_DIALECT_GOLDEN=1 pytest tests/test_profile_dialect_golden.py``，再逐行看 git diff。 +2. **sqlglot 逐条解析**：按方言读一遍（mariadb 按 mysql 读），挡住语法写坏了（少括号、关键字拼错）。 +3. **IN 两边类型族一致**：用 sqlglot 的 qualify + annotate_types，配上表结构里的类型，核对 IN 列表的字面量和被指向 + 列是同一类型族（PostgreSQL 上 varchar 父列配整数子列时，字面量必须带引号）。 + +挡不住的照实写：sqlglot 不知道真库的统计信息是不是过期、零日期驱动怎么返回、Oracle 小写对象是否存在、排序规则 +怎么比——这些要靠 scripts/dialect-it.sh 在本机容器里跑真库（不进 CI）。 +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +import sqlglot +from sqlglot import exp +from sqlglot.optimizer.annotate_types import annotate_types +from sqlglot.optimizer.qualify import qualify + +from app.data import catalog, guard +from app.data.catalog_profile import IN_CHUNK, SqlDialect, key_literals + +GOLDEN = Path(__file__).parent / "fixtures" / "dialect" +DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite") +#: sqlglot 的方言名 +READ = {"mysql": "mysql", "mariadb": "mysql", "postgresql": "postgres", "oracle": "oracle", "sqlite": "sqlite"} + +#: 夹具表结构：一张带 schema 的订单表、一张门店表（编码是文字），Oracle 的一张小写带引号的表和一个小写带引号的列 +META = { + "orders": {"schema": "sales", "columns": [ + {"name": "id", "type": "INTEGER"}, {"name": "store_code", "type": "INTEGER"}, + {"name": "status", "type": "CHAR(2)"}, {"name": "paid_at", "type": "TIMESTAMP"}, + {"name": "biz_day", "type": "VARCHAR(10)"}, {"name": "level", "type": "INTEGER"}], + "primary_key": ["id"]}, + "stores": {"schema": None, "columns": [{"name": "code", "type": "VARCHAR(10)"}], "primary_key": ["code"]}, + "visits": {"schema": None, "quote_name": True, "columns": [ + {"name": "id", "type": "INTEGER"}, {"name": "level", "type": "INTEGER", "quote_name": True}], + "primary_key": ["id"]}, +} + + +def _queries(kind: str) -> list[tuple[str, str | None]]: + """一种方言的全部剖析、观察查询：(名字, SQL)。没有这种查询（SQLite 没有统计信息、MySQL 没有块采样）的 SQL 为 None。""" + d = SqlDialect(kind) + orders, stores, visits = d.table(META["orders"], "orders"), d.table(META["stores"], "stores"), \ + d.table(META["visits"], "visits") + status, code = d.column(META["orders"], "status"), d.column(META["stores"], "code") + level = d.column(META["visits"], "level") + keys = key_literals([12, 7], "number", parent="text", child_type="INTEGER").literals + out: list[tuple[str, str | None]] = [ + ("stats", d.stats_sql(META["orders"], "orders")), + ("stats_quoted", d.stats_sql(META["visits"], "visits")), + ("bounded", d.bounded_count_sql(orders, 100_001)), + ("distinct", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, scan_cap=None)), + ("distinct_capped", d.distinct_sample_sql(orders, d.column(META["orders"], "store_code"), 2000, + scan_cap=100_000)), + ("match", d.match_count_sql(stores, code, keys)), + ("unique", d.unique_check_sql(visits, level)), + ("value_counts", d.value_counts_sql(orders, status, 21, scan_cap=None)), + ("value_counts_capped", d.value_counts_sql(orders, status, 21, scan_cap=100_000)), + ("min_max", d.min_max_sql(orders, [(d.column(META["orders"], "paid_at"), False), + (d.column(META["orders"], "biz_day"), True)])), + ("date_window", d.window_value_counts_sql(orders, status, d.column(META["orders"], "paid_at"), + "2026-08-27", 21)), + ("day_volume", d.day_volume_sql(orders, d.column(META["orders"], "paid_at"), "2026-08-27")), + ("sample", d.sample_value_counts_sql(orders, status, 1, 21)), + ] + return out + + +def _render(kind: str) -> str: + lines = [f"-- 剖析与取值观察的 SQL 金样：{kind}（tests/test_profile_dialect_golden.py 生成，改动要人审）"] + for name, sql in _queries(kind): + lines.append(f"-- {name}") + lines.append(f"{sql};" if sql is not None else "-- （这种方言没有这类查询）") + return "\n".join(lines) + "\n" + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_profile_sql_matches_the_golden_file(kind): + path = GOLDEN / f"{kind}.sql" + text = _render(kind) + if os.environ.get("UPDATE_DIALECT_GOLDEN") == "1": + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + assert path.read_text(encoding="utf-8") == text, f"{kind} 的剖析 SQL 和金样不一致：看 diff，确认无误后重新生成金样" + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_every_profile_query_parses_and_passes_the_guard(kind): + for name, sql in _queries(kind): + if sql is None: + continue + parsed = sqlglot.parse(sql, read=READ[kind], error_level=sqlglot.ErrorLevel.RAISE) + assert len(parsed) == 1 and parsed[0] is not None, (kind, name) + assert guard.check(sql, readonly=True, dialect=kind) == sql, (kind, name) + + +@pytest.mark.parametrize("kind", ["postgresql", "oracle", "mysql"]) +def test_observation_queries_use_the_right_day_and_sample_functions(kind): + """按天截断和块采样各家写法不同，解析成语法树再看用的是哪个函数，不只比字符串。""" + sql = dict(_queries(kind)) + day = sqlglot.parse_one(sql["day_volume"], read=READ[kind]) + if kind == "oracle": + assert any(isinstance(n, (exp.DateTrunc, exp.TimestampTrunc, exp.Anonymous)) or n.key == "trunc" + for n in day.walk()), sql["day_volume"] + assert "TRUNC(paid_at)" in sql["day_volume"] + elif kind == "postgresql": + assert any(isinstance(n, exp.Cast) for n in day.walk()) + else: + assert "DATE(paid_at)" in sql["day_volume"] + if kind == "mysql": + assert sql["sample"] is None + else: + sampled = sqlglot.parse_one(sql["sample"], read=READ[kind]) + assert any(isinstance(n, exp.TableSample) for n in sampled.walk()), sql["sample"] + + +# ---------------------------------------------------------------- IN 两边的类型族 + + +_FAMILY_OF = [(exp.DataType.TEXT_TYPES, "text"), (exp.DataType.NUMERIC_TYPES, "number"), + (exp.DataType.TEMPORAL_TYPES, "time")] + + +def _sg_family(t: exp.DataType | None) -> str | None: + if t is None: + return None + for group, name in _FAMILY_OF: + if t.this in group: + return name + return None + + +def _in_families(sql: str, kind: str, schema: dict[str, dict[str, str]]) -> list[tuple[str | None, set[str | None]]]: + """sql 里每个 IN：(被比较的列的类型族, 字面量的类型族集合)，按 annotate_types 配上表结构的类型推出来。""" + tree = qualify(sqlglot.parse_one(sql, read=READ[kind]), schema=schema, dialect=READ[kind], + validate_qualify_columns=False) + tree = annotate_types(tree, schema=schema, dialect=READ[kind]) + return [(_sg_family(node.this.type), {_sg_family(x.type) for x in node.expressions}) + for node in tree.find_all(exp.In)] + + +@pytest.mark.parametrize("kind", DIALECTS) +@pytest.mark.parametrize("parent_type, child_values, child_type", [ + ("VARCHAR(10)", [12, 7, 30], "INTEGER"), # 父列文字、子列数：字面量要带引号 + ("INTEGER", ["12", "0007", "A1"], "VARCHAR(10)"), # 父列数、子列文字：只收整数写法，写成数字 + ("VARCHAR(10)", ["A ", "BC"], "CHAR(2)"), # 定长子列：去掉补齐的空格 + ("INTEGER", [1, 2], "INTEGER"), +]) +def test_in_list_literals_have_the_parent_columns_family(kind, parent_type, child_values, child_type): + d = SqlDialect(kind) + parent = {"name": "code", "type": parent_type} + literals = key_literals(child_values, None, parent=catalog.column_family(parent), child_type=child_type).literals + assert literals + sql = d.match_count_sql("stores", d.quote("code"), literals[:IN_CHUNK]) + found = _in_families(sql, kind, {"stores": {"code": parent_type}}) + assert found, sql + for column_family, literal_families in found: + assert literal_families == {column_family}, (kind, sql, column_family, literal_families) + + +def test_the_old_way_would_mismatch_on_postgres(): + """对照：以前按子表取回的值写字面量，varchar 父列配整数子列在 sqlglot 推出的类型上就两边不同——PostgreSQL 真库上 + 这是「operator does not exist: character varying = integer」。""" + from app.data.catalog_profile import sql_literal + + old = [lit for v in (12, 7) if (lit := sql_literal(v, "number"))] + sql = SqlDialect("postgresql").match_count_sql("stores", "code", old) + [(column_family, literal_families)] = _in_families(sql, "postgresql", {"stores": {"code": "VARCHAR(10)"}}) + assert column_family == "text" and literal_families == {"number"} diff --git a/backend/tests/test_profile_dialect_units.py b/backend/tests/test_profile_dialect_units.py new file mode 100644 index 00000000..d2bc226d --- /dev/null +++ b/backend/tests/test_profile_dialect_units.py @@ -0,0 +1,428 @@ +"""剖析与探查的方言修正（下一轮 03 细案 C1）：不连库的部分。 + +- 列类型按方言编译的完整写法：类型族、日期列、码值候选都认得 ENUM / SET、VARCHAR2(20 CHAR)、INTERVAL 这些写法； +- IN 列表的字面量按被指向列的类型族生成（PG 上文字列配整数子列会直接报错，Oracle 会把父列转成数报 ORA-01722）； +- Oracle 小写带引号的对象（quote_name）强制加引号、不转大写； +- 统计信息估出的行数不超过整表统计上限时再数一次确认； +- 日期列的最小值、最大值排除零日期和哨兵日期，照实写排除了几个； +- 取值观察要用的三个方言方法（date_literal / day_trunc / sample_from）； +- 说明文字不再说「抽样」：大表看的是按存储顺序的前 N 行。 +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +from typing import Any + +import pytest +from sqlalchemy import VARCHAR, Date, Numeric +from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite +from sqlalchemy.sql.elements import quoted_name + +from app.data import catalog, catalog_profile, introspect +from app.data.catalog_profile import ProfileSettings, SqlDialect, TableSize +from app.data.engine import QueryResult + +DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite") + + +# ---------------------------------------------------------------- 类型的写法 + + +@pytest.mark.parametrize("type_name, family", [ + ("ENUM('paid','refund')", "text"), + ("ENUM('int','date')", "text"), # 取值里的词不能让它变成数字或时间 + ("SET('a','b')", "text"), + ("VARCHAR2(20 CHAR)", "text"), + ("NVARCHAR2(10)", "text"), + ("VARCHAR(20) COLLATE utf8mb4_bin", "text"), + ("NUMBER(10, 0)", "number"), + ("INTEGER(11)", "number"), + ("BIGINT UNSIGNED", "number"), + ("DOUBLE PRECISION", "number"), + ("TIMESTAMP WITH TIME ZONE", "time"), + ("TIMESTAMP(3)", "time"), + ("INTERVAL DAY(2) TO SECOND(6)", None), # 时长不是数也不是时刻；以前按「INT」认成了数字 + ("order_status", None), + ("", None), +]) +def test_type_family_reads_dialect_full_types(type_name, family): + assert catalog._type_family(type_name) == family + + +def test_type_family_of_a_column_uses_its_enum_values(): + """PostgreSQL 的枚举类型编译出来是类型名（order_status），只有取值列表能说明它是文字。""" + assert catalog.column_family({"name": "status", "type": "order_status", "enum_values": ["a", "b"]}) == "text" + assert catalog.column_family({"name": "n", "type": "INTEGER"}) == "number" + assert catalog.column_family({"name": "x"}) is None + + +def test_enum_values_never_make_a_date_column(): + assert not catalog.is_date_column("status", "ENUM('date','time')") + assert catalog.is_date_column("paid_at", "TIMESTAMP WITH TIME ZONE") + assert catalog.is_date_column("biz_dt", "VARCHAR2(10 CHAR)") # 文字存日期、名字像日期 + assert not catalog.is_date_column("gap", "INTERVAL DAY(2) TO SECOND(6)") + + +# PostgreSQL 的枚举、域编译出来是类型名本身：名字以 set、enum 开头不能被当成 MySQL 的 SET / ENUM，PG 的枚举一律靠 +# 探查记下的 enum_values 认 +@pytest.mark.parametrize("type_name, family", [ + ("setup_status", None), + ("settlement_amount", None), + ("enum_kind", None), + ("SET ('a','b')", "text"), + ("enum('paid')", "text"), +]) +def test_only_mysql_enum_and_set_spellings_are_enum(type_name, family): + assert catalog._type_family(type_name) == family + assert catalog_profile._is_enum(type_name) is (family == "text") + + +def test_postgres_type_names_starting_with_set_or_enum(): + # 带取值列表的 PG 枚举：是码值候选、是文字，但不是 MySQL 的 SET + assert catalog_profile._is_enum("setup_status", ["draft", "done"]) + assert catalog.column_family({"name": "stage", "type": "setup_status", "enum_values": ["draft"]}) == "text" + # 名字以 set 开头的数值域：不是文字、不是码值候选 + assert not catalog_profile._code_type_ok("settlement_amount") + # 名字以 set 开头的日期域：照样是日期列 + assert catalog.is_date_column("settled_on", "settlement_date") + # 枚举的类型名里碰巧有 DATE（UPDATE 里的）：有取值列表就不是日期列 + assert not catalog.is_date_column("reason", "update_reason", enum_values=["manual", "sync"]) + assert not catalog_profile._is_date_column({"name": "reason", "type": "update_reason", + "enum_values": ["manual", "sync"]}) + + +def test_postgres_enum_named_like_set_keeps_its_unseen_values(): + """PG 枚举 setup_status 不是 MySQL 的 SET：列定义里这次没出现的取值也写进码值（含义待填）。""" + settings = ProfileSettings(enabled=True) + source = type("S", (), {"id": "s1", "name": "shop", "kind": "postgresql", "options": {}})() + tables = {"jobs": {"columns": [{"name": "stage", "type": "setup_status", + "enum_values": ["draft", "running", "done"]}]}} + job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect("postgresql"), + runner=None, tables=tables, masked=set()) + job.sizes["jobs"] = TableSize(10, "count") + result = catalog_profile.TableResult(table="jobs") + cx = catalog_profile._TableCx(job, "jobs", result, catalog_profile.TableHooks()) + + async def run(sql: str, **_kw: Any) -> QueryResult: + return QueryResult(columns=["v", "n"], rows=[["draft", 6], ["done", 4]], row_count=2, truncated=False, + elapsed_ms=1, sql=sql) + cx.run = run # type: ignore[method-assign] + out = catalog_profile._TableOutcome() + asyncio.run(catalog_profile._check_codes(cx, "jobs", "stage", out, result)) + [codes] = out.codes + assert codes.declared == ["running"] and "列定义里另有 1 个取值" in codes.note + + +@pytest.mark.parametrize("col, ok", [ + ({"type": "VARCHAR2(20 CHAR)"}, True), + ({"type": "VARCHAR2(200 CHAR)"}, False), + ({"type": "CHAR(2)"}, True), + ({"type": "ENUM('paid','refund')"}, True), + ({"type": "SET('a','b')"}, True), + ({"type": "order_status", "enum_values": ["a", "b"]}, True), + ({"type": "NUMBER(10, 0)"}, True), + ({"type": "NUMBER(10, 2)"}, False), + ({"type": "TIMESTAMP(3)"}, False), + ({"type": "INTERVAL DAY(2) TO SECOND(6)"}, False), +]) +def test_code_type_accepts_enum_and_char_semantics(col, ok): + assert catalog_profile._code_type_ok(col.get("type"), col.get("enum_values")) is ok + + +def test_enum_columns_are_code_candidates_whatever_their_name(): + """ENUM 的取值列表有限，直接算码值候选：pay_way 这种名字不像码值的列以前被漏掉。""" + meta = {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "pay_way", "type": "ENUM('cash','card')", + "enum_values": ["cash", "card"]}, + {"name": "pay_note", "type": "VARCHAR(20)"}], + "primary_key": ["id"]} + assert catalog_profile._code_columns(meta, {}) == ["pay_way"] + + +# ---------------------------------------------------------------- 探查：类型写法、取值列表、引号、索引首列 + + +@pytest.mark.parametrize("dialect, col_type, text", [ + (mysql.dialect(), mysql.ENUM("paid", "refund"), "ENUM('paid','refund')"), + (mysql.dialect(), mysql.TINYINT(display_width=1), "TINYINT(1)"), + (oracle.dialect(), oracle.VARCHAR2(20), "VARCHAR2(20 CHAR)"), + # Oracle 的 VARCHAR2 反射回来是通用的 VARCHAR（容器集成测试实测）：照数据字典写成 VARCHAR2 + (oracle.dialect(), VARCHAR(10), "VARCHAR2(10 CHAR)"), + (oracle.dialect(), oracle.NUMBER(10, 0), "NUMBER(10, 0)"), + (postgresql.dialect(), postgresql.ENUM("a", "b", name="order_status"), "order_status"), + # PG 的域编译出来只是域名，名字里的词（set、UPDATE 里的 DATE）会让它被认错：记它底下的类型 + (postgresql.dialect(), postgresql.DOMAIN("settlement_amount", Numeric(12, 2)), "NUMERIC(12, 2)"), + (postgresql.dialect(), postgresql.DOMAIN("update_day", postgresql.DOMAIN("biz_day", Date())), "DATE"), + (sqlite.dialect(), sqlite.DATETIME(), "DATETIME"), +]) +def test_reflected_type_is_compiled_for_the_dialect(dialect, col_type, text): + assert introspect.type_text(col_type, dialect) == text + + +def test_type_that_cannot_compile_falls_back_to_str(): + from sqlalchemy.types import NullType + + assert introspect.type_text(NullType(), sqlite.dialect()) == "NULL" + assert introspect.type_text(None, sqlite.dialect()) == "" + + +def test_column_entry_records_enum_values_and_oracle_quoted_names(): + ora = oracle.dialect() + entry = introspect.column_entry({"name": quoted_name("level", quote=True), "type": oracle.VARCHAR2(10), + "nullable": True, "comment": None}, ora) + assert entry == {"name": "level", "type": "VARCHAR2(10 CHAR)", "nullable": True, "comment": None, + "quote_name": True} + plain = introspect.column_entry({"name": "status", "type": mysql.ENUM("a", "b"), "nullable": False}, mysql.dialect()) + assert plain["enum_values"] == ["a", "b"] and "quote_name" not in plain + members = introspect.column_entry({"name": "tags", "type": mysql.SET("x", "y")}, mysql.dialect()) + assert members["enum_values"] == ["x", "y"] and members["type"] == "SET('x','y')" + + +def test_domain_over_an_enum_keeps_the_enum_values(): + status = postgresql.ENUM("draft", "done", name="setup_status") + entry = introspect.column_entry({"name": "stage", "type": postgresql.DOMAIN("stage_kind", status)}, + postgresql.dialect()) + assert entry["type"] == "setup_status" and entry["enum_values"] == ["draft", "done"] + + +def test_index_first_columns_are_recorded(): + indexes = [{"column_names": ["paid_at", "id"]}, {"column_names": [None]}, {"column_names": ["store_id"]}, + {"column_names": ["paid_at"]}] + assert introspect.index_first_columns(indexes, ["id"]) == ["id", "paid_at", "store_id"] + + +# ---------------------------------------------------------------- IN 列表的字面量按被指向列的类型族 + + +def test_key_literals_follow_the_parent_family(): + assert catalog_profile.key_literals([12, 7], "number", parent="text", child_type="INTEGER") == ( + ["'12'", "'7'"], 0, 0) + # 父列是数、子列是文字：只拼整数写法；'A1' 按类型就对不上整数父键，记进 unmatchable（调用方算作没对上） + assert catalog_profile.key_literals(["12", "A1", "-3", "A1 "], "text", parent="number", + child_type="VARCHAR(10)") == (["12", "-3"], 1, 0) + # 父列是时间的关系整条不核对（_check_relation 先跳过），这里不收 + with pytest.raises(ValueError): + catalog_profile.key_literals([1, 2], "number", parent="time", child_type="INTEGER") + # 父列类型不知道：照子表取回的值的类型写（以前的做法） + assert catalog_profile.key_literals([3, "x"], None, parent=None, child_type=None) == (["3", "'x'"], 0, 0) + + +def test_key_literals_count_unsafe_values_but_not_duplicate_spellings(): + """不能安全拼进 SQL 的值记进 unsafe（备注写「类型不支持核对，未计入」）；'0012' 和 '12' 写成同一个字面量，只算一个， + 不能被当成「不支持核对」。""" + found = catalog_profile.key_literals(["0012", "12", "a\\b", True], "text", parent="text", child_type="TEXT") + assert found == (["'0012'", "'12'"], 0, 2) + found = catalog_profile.key_literals(["0012", "12"], "text", parent="number", child_type="TEXT") + assert found == (["12"], 0, 0) + + +_REL = {"id": "r1", "columns": ["customer_ref"], "to_table": "customers", "to_columns": ["id"], "source": "llm", + "status": "proposed"} + + +def _relation_cx(values: list[Any], parent_type: str, matched: int): + """orders.customer_ref（文字）→ customers.id（parent_type）的关系核对用的执行状态：取值查询回 values，父表的核对 + 查询回 matched，唯一性计数回「有重复」。seen 记下发出的每条 SQL。""" + settings = ProfileSettings(enabled=True) + source = type("S", (), {"id": "s1", "name": "shop", "kind": "postgresql", "options": {}})() + tables = {"orders": {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "customer_ref", "type": "TEXT"}], + "primary_key": ["id"]}, + "customers": {"columns": [{"name": "id", "type": parent_type}], "primary_key": ["id"]}} + job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect("postgresql"), + runner=None, tables=tables, masked=set()) + job.sizes.update({"orders": TableSize(130, "count"), "customers": TableSize(3, "count")}) + cx = catalog_profile._TableCx(job, "orders", catalog_profile.TableResult(table="orders"), + catalog_profile.TableHooks()) + seen: list[str] = [] + + async def run(sql: str, **_kw: Any) -> QueryResult: + seen.append(sql) + if "SELECT DISTINCT" in sql: + rows = [[v] for v in values] + return QueryResult(columns=["k"], rows=rows, row_count=len(rows), truncated=False, elapsed_ms=1, sql=sql, + column_types={"k": "text"}) + if " IN (" in sql: + return _result(matched) + return QueryResult(columns=["a", "b"], rows=[[130, len(values)]], row_count=1, truncated=False, elapsed_ms=1, + sql=sql) + cx.run = run # type: ignore[method-assign] + return cx, seen + + +@pytest.mark.parametrize("values, parent_type, coverage, sample, matched", [ + # 九成是散客代号、只有 1 个整数写法对得上：以前分母里拿掉了对不上类型的，测成 100% + (["WALKIN", "1"] + [f"GUEST{i}" for i in range(30)], "INTEGER", round(1 / 32, 4), 32, 1), + # 全都不是整数写法：一条 IN 都不用发，覆盖率 0%，照样是「可能不是这条关系」，不是「无法核对」 + (["WALKIN", "GUEST1"], "INTEGER", 0.0, 2, 0), + (["1", "2"], "INTEGER", 1.0, 2, 2), +]) +def test_relation_coverage_counts_type_mismatches_as_unmatched(values, parent_type, coverage, sample, matched): + cx, seen = _relation_cx(values, parent_type, matched) + out, result = catalog_profile._TableOutcome(), catalog_profile.TableResult(table="orders") + asyncio.run(catalog_profile._check_relation(cx, "orders", _REL, out, result)) + [rel] = out.relations + assert (rel.coverage, rel.sample, rel.matched, rel.holds) == (coverage, sample, matched, coverage == 1.0) + if coverage < 1: + assert "可能不是这条关系" in rel.note and "未计入" not in rel.note + if sample > matched and matched == 0: + assert not any("IN (" in sql for sql in seen), "没有整数写法的键值：不发父表的核对查询" + + +def test_time_keys_are_not_checked_and_send_no_query(): + cx, seen = _relation_cx(["2026-09-01"], "DATE", 0) + out, result = catalog_profile._TableOutcome(), catalog_profile.TableResult(table="orders") + with pytest.raises(catalog_profile._Skip) as skipped: + asyncio.run(catalog_profile._check_relation(cx, "orders", _REL, out, result)) + assert skipped.value.reason == "unsupported" and "日期时间" in str(skipped.value) + assert "两端类型不同" not in str(skipped.value) + assert seen == [], "时间键不核对，一条查询都不发" + + +def test_char_keys_lose_their_padding(): + """PG、Oracle 的 CHAR(n) 取回来带尾随空格：拼成 'A ' 去和 VARCHAR 父列比就对不上。""" + lits = catalog_profile.key_literals(["A ", "BC"], "text", parent="text", child_type="CHAR(2)").literals + assert lits == ["'A'", "'BC'"] + lits = catalog_profile.key_literals(["A "], "text", parent="text", child_type="VARCHAR(2)").literals + assert lits == ["'A '"] + + +# ---------------------------------------------------------------- Oracle 小写带引号的对象 + + +def test_oracle_quote_name_forces_quotes_and_keeps_case(): + ora = SqlDialect("oracle") + meta = {"schema": "sales", "quote_name": True, + "columns": [{"name": "level", "type": "VARCHAR2(10 CHAR)", "quote_name": True}, + {"name": "amount", "type": "NUMBER(10, 2)"}]} + assert ora.table(meta, "visits") == 'sales."visits"' + assert ora.column(meta, "level") == '"level"' and ora.column(meta, "amount") == "amount" + stats = ora.stats_sql(meta, "visits") + assert "TABLE_NAME = 'visits'" in stats and "OWNER = 'SALES'" in stats + # 没有 quote_name 的照旧：不加引号，Oracle 自己转大写 + assert ora.table({"schema": None}, "visits") == "visits" + assert "TABLE_NAME = 'VISITS'" in ora.stats_sql({}, "visits") + + +# ---------------------------------------------------------------- 统计信息过期：再数一次确认 + + +def _result(value: Any) -> QueryResult: + return QueryResult(columns=["n"], rows=[[value]], row_count=1, truncated=False, elapsed_ms=1, sql="") + + +def _runner(answers: list[Any], seen: list[str]): + async def run(sql: str, **_kw: Any) -> QueryResult: + seen.append(sql) + return _result(answers.pop(0)) + return run + + +@pytest.mark.parametrize("stats, counted, expected", [ + (500, 500, TableSize(500, "count")), # 统计信息说小、数过也小：准数 + (500, 100_001, TableSize(None, "count", at_least=100_001)), # 统计信息过期：其实是大表 + (5_000_000, None, TableSize(5_000_000, "stats")), # 统计信息说大：不必再数 +]) +def test_small_stats_are_confirmed_with_a_bounded_count(stats, counted, expected): + seen: list[str] = [] + answers = [stats] + ([counted] if counted is not None else []) + size, skips = asyncio.run(catalog_profile.estimate_size(_runner(answers, seen), SqlDialect("postgresql"), + {"schema": None}, "visits", max_scan_rows=100_000)) + assert size == expected and skips == [] + assert len(seen) == (2 if counted is not None else 1) + if counted is not None: + assert "LIMIT 100001" in seen[1] + + +# ---------------------------------------------------------------- 日期列：排除零日期和哨兵日期 + + +@pytest.mark.parametrize("kind", DIALECTS) +def test_min_max_excludes_invalid_dates_and_counts_them(kind): + d = SqlDialect(kind) + sql = d.min_max_sql("t", [("paid_at", False), ("biz_day", True)]) + # 日期类型的列按日期字面量比，文字存的日期按字符串比（PG 上 varchar 和 date 比较直接报错） + lo, hi = d.date_literal("0001-01-02"), d.date_literal("9999-12-31") + assert f"paid_at >= {lo}" in sql and f"paid_at < {hi}" in sql + assert "biz_day >= '0001-01-02'" in sql and "biz_day < '9999-12-31'" in sql + assert sql.count("SUM(CASE WHEN") == 2 + + +def _dates_cx(row: list[Any], kind: str = "mysql"): + settings = ProfileSettings(enabled=True) + source = type("S", (), {"id": "s1", "name": "shop", "kind": kind, "options": {}})() + job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect(kind), + runner=None, tables={"orders": {"columns": [ + {"name": "paid_at", "type": "DATETIME"}]}}, masked=set()) + job.sizes["orders"] = TableSize(10, "count") + result = catalog_profile.TableResult(table="orders") + cx = catalog_profile._TableCx(job, "orders", result, catalog_profile.TableHooks()) + + async def run(sql: str, **_kw: Any) -> QueryResult: + return QueryResult(columns=["a", "b", "c"], rows=[row], row_count=1, truncated=False, elapsed_ms=1, sql=sql) + cx.run = run # type: ignore[method-assign] + return cx, result + + +def test_dates_note_says_how_many_invalid_values_were_left_out(): + cx, result = _dates_cx(["2026-01-03 08:00:00", "2026-09-30 21:00:00", 4]) + out = catalog_profile._TableOutcome() + asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, out, result)) + assert result.date_ranges == [{"column": "paid_at", "min": "2026-01-03 08:00:00", "max": "2026-09-30 21:00:00", + "invalid": 4}] + assert "另有 4 个无效日期" in out.business_date.note and "0000-00-00" in out.business_date.note + + +def test_only_invalid_dates_is_no_data(): + cx, result = _dates_cx([None, None, 9]) + with pytest.raises(catalog_profile._Skip) as e: + asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, catalog_profile._TableOutcome(), + result)) + assert e.value.reason == "no_data" and "9 个无效日期" in e.value.detail + + +# ---------------------------------------------------------------- 取值观察要用的方言方法 + + +@pytest.mark.parametrize("kind, literal, day, sample", [ + ("mysql", "DATE '2026-09-01'", "DATE(paid_at)", None), + ("mariadb", "DATE '2026-09-01'", "DATE(paid_at)", None), + ("postgresql", "DATE '2026-09-01'", "CAST(paid_at AS DATE)", "orders TABLESAMPLE SYSTEM (1.5)"), + ("oracle", "DATE '2026-09-01'", "TRUNC(paid_at)", "orders SAMPLE BLOCK (1.5)"), + ("sqlite", "'2026-09-01'", "substr(paid_at, 1, 10)", None), +]) +def test_observation_dialect_methods(kind, literal, day, sample): + d = SqlDialect(kind) + assert d.date_literal(_dt.date(2026, 9, 1)) == literal == d.date_literal("2026-09-01") + assert d.date_literal("2026-09-01", text=True) == "'2026-09-01'" + assert d.day_trunc("paid_at") == day + assert d.sample_from("orders", 1.5) == sample + + +@pytest.mark.parametrize("bad", ["2026-9-1", "2026-09-01'; --", "0000-00-00", 20260901, None]) +def test_date_literal_refuses_anything_but_a_real_date(bad): + with pytest.raises(ValueError): + SqlDialect("postgresql").date_literal(bad) + + +@pytest.mark.parametrize("bad", [0, -1, 100, 101, float("nan")]) +def test_sample_percent_must_be_between_0_and_100(bad): + with pytest.raises(ValueError): + SqlDialect("postgresql").sample_from("orders", bad) + + +# ---------------------------------------------------------------- 说明文字 + + +def test_notes_say_storage_order_not_sampling(): + note = catalog_profile._relation_note(sample=2000, matched=2000, coverage=1.0, scan_cap=100_000, skipped_values=0, + unmatchable=0, parent=(True, "主键"), child=(False, "count"), holds=True, + collation=None) + assert "按存储顺序只看前 100000 行" in note and "抽样" not in note + assert catalog_profile.FILTER_SAMPLED.format(rows="100", share="1%").startswith("剖析核对：按存储顺序取前 100 行统计") + + +def test_mysql_notes_mention_the_collation(): + note = catalog_profile._relation_note(sample=10, matched=10, coverage=1.0, scan_cap=None, skipped_values=0, + unmatchable=0, parent=(True, "主键"), child=(False, "count"), holds=True, + collation=catalog_profile.MYSQL_COLLATION_NOTE) + assert "排序规则" in note and "不分大小写" in note diff --git a/backend/tests/test_recipe_contract.py b/backend/tests/test_recipe_contract.py index 34800e71..03827e7e 100644 --- a/backend/tests/test_recipe_contract.py +++ b/backend/tests/test_recipe_contract.py @@ -370,10 +370,12 @@ def test_accumulate_blockers_reference_recipe_is_eligible(): def test_accumulate_blockers_list_shape_is_unkeyed(): data = json.loads(json.dumps(C01_LIKE)) - data["sheets"][0]["context"] = [{}] # 即使配了统计期，列表写入的表也不能累积 + data["sheets"][0]["context"] = [{}] # 配了统计期，列表却没说每一行属于哪一期：不能累积 got = T.accumulate_blockers(T.Recipe.model_validate(data)) assert [(p.code, p.path) for p in got] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] - assert all("列表形态的表目前只支持每期替换" in p.message for p in got) + assert "没有指定每一行属于哪一期" in got[0].message + # 合计行另存的表：各期的合计项相同，主键分不开各期（期 5 列表可以累积以后仍然不行） + assert "合计行、小计行另存的表不能按期累积" in got[1].message assert "「月报」" in got[0].message and "「月报_表内合计」" in got[1].message # c01 的草稿本来就没有统计期：两条都报，统计期那条在前（起草卡片取第一条的人话） got = T.accumulate_blockers(T.Recipe.model_validate(C01_LIKE)) @@ -479,13 +481,15 @@ def test_closed_value_sets(): assert {"union_rows", "union_pk", "union_period"} <= set(_literal(T.CheckResult, "kind")) assert "ignored" in typing.get_args(T.Role) and "ignored_column" in typing.get_args(T.Role) assert typing.get_args(T.ExcludedReason) == ( - "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept") + "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept", + "subtotal") assert _literal(T.FixAnchor, "kind") == ("problem", "recipe_problem", "sheet_renamed") assert _literal(T.Anchor, "kind") == ( "header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text") assert _literal(T.EditResult, "kind") == ("fix", "selection") assert _literal(T.ChangeClass, "change") == ("same", "compatible", "retire", "semantic") - assert T.FIX_KINDS == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", + assert T.FIX_KINDS[-1] == "declare_subtotals" # 期 5 加在末尾 + assert T.FIX_KINDS[:-1] == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", "ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet") assert T.SELECTION_AS == ("list", "crosstab", "segment", "derived", "section_title", "ignore_rows", "ignore_columns", "ignore_outside") diff --git a/backend/tests/test_recipe_engine_messages.py b/backend/tests/test_recipe_engine_messages.py index 26dd3d84..c2013ab1 100644 --- a/backend/tests/test_recipe_engine_messages.py +++ b/backend/tests/test_recipe_engine_messages.py @@ -318,6 +318,30 @@ def c_total_row_blank(mp): return _run(sales_book(edit=edit), LIST), [N] +def c_subtotal_blank(mp): + """期 5：穿插的小计行数字格全空：没有一列能核对，结构类。""" + data = list_recipe() + data["sheets"][0]["blocks"][0]["rows"]["subtotals"] = {"label_column": "地区", "pick": "小计"} + + def edit(ws): + ws["D4"] = N + ws.insert_rows(6) + ws["A6"] = "小计" # 第 6 行是小计：标签在，数字格全空 + return _run(sales_book(edit=edit), data), [N] + + +def c_subtotal_suspect(mp): + """期 5：没配小计时，首列以合计字样开头、其余文字列空着的行像夹在中间的小计：照常导入，要人确认。""" + data = list_recipe() + + def edit(ws): + ws["D4"] = N + ws.insert_rows(6) + ws["A6"] = "小计" + ws["D6"] = N + return _run(sales_book(edit=edit), data), [N] + + def c_value_not_integer(mp): return _run(_flow(_set(C11=9876.54)), FLOW), [9876.54] @@ -423,6 +447,7 @@ NEXT_STEPS = { "hidden_cols": "在配方面板的「隐藏列」中选择「照常导入」", "total_row_missing": "在配方面板的「合计行」中取消「有合计行」", "total_row_blank": "在配方面板的「合计行」中取消「有合计行」", + "subtotal_blank": "在配方面板的「小计行」中取消「有小计行」", } diff --git a/backend/tests/test_recipe_engine_p3.py b/backend/tests/test_recipe_engine_p3.py index a4c6ad63..f827a6bf 100644 --- a/backend/tests/test_recipe_engine_p3.py +++ b/backend/tests/test_recipe_engine_p3.py @@ -119,7 +119,9 @@ def test_problem_fix_table_follows_spec_3_1(): "label_unparsed": "declare_total", "value_not_number": "declare_placeholder", "axis_extra_cells": "ignore_cells", "column_extra": "ignore_cells", "row_unclaimed": "ignore_cells", "outside_number": "ignore_cells", "hidden_rows": "declare_hidden", "hidden_cols": "declare_hidden", - "sheet_missing": "rename_sheet"} + "sheet_missing": "rename_sheet", + # 期 5：没配小计的列表里像小计的行 + "subtotal_suspect": "declare_subtotals"} assert set(recipe_engine.PROBLEM_FIX.values()) <= set(FIX_KINDS) # edit_members 只来自静态校验；cell_unclaimed、rows_after_stop 没有修复按钮 assert "edit_members" not in recipe_engine.PROBLEM_FIX.values() @@ -367,9 +369,17 @@ def _column_extra(): return sales_book(edit=edit), LIST +def _subtotal_suspect(): + def edit(ws): + ws.insert_rows(6) + ws["A6"], ws["D6"], ws["E6"] = "小计", N, N + 1 + return sales_book(edit=edit), LIST + + @pytest.mark.parametrize("code", sorted(recipe_engine.PROBLEM_FIX)) def test_fix_args_never_carry_cell_values(code): - book, recipe = _column_extra() if code == "column_extra" else _crosstab_cases()[code] + book, recipe = (_column_extra() if code == "column_extra" else _subtotal_suspect() if code == "subtotal_suspect" + else _crosstab_cases()[code]) ex = run(book, recipe) hits = [p for p in ex.problems if p.code == code and p.fix_args is not None] assert hits, f"{code}：{[(p.code, p.fix, p.message) for p in ex.problems]}" diff --git a/backend/tests/test_recipe_imports.py b/backend/tests/test_recipe_imports.py index cdfed879..74b54535 100644 --- a/backend/tests/test_recipe_imports.py +++ b/backend/tests/test_recipe_imports.py @@ -260,7 +260,7 @@ def fake_pipeline(state: FakeState) -> Pipeline: return Pipeline( validate=validate, recipe_sha256=recipe.recipe_sha256, canonical=recipe.canonical_recipe, - apply_patch=recipe.apply_patch, scan=lambda raw: fake_scan(raw, state), + apply_patch=recipe.apply_patch, scan=lambda raw, _filename: fake_scan(raw, state), scan_to_json=xlsx_scan.scan_to_json, scan_from_json=xlsx_scan.scan_from_json, read_grid=fake_grid, preview=xlsx_cells.preview, execute=execute, run_checks=run_checks, column_null_counts=null_counts, build_notes=build_notes, apply_notes=recipe_notes.apply_notes, detect_facts=detect_facts, draft=draft, diff --git a/backend/tests/test_recipe_notes_p3.py b/backend/tests/test_recipe_notes_p3.py index ba454c55..cfc9a6e8 100644 --- a/backend/tests/test_recipe_notes_p3.py +++ b/backend/tests/test_recipe_notes_p3.py @@ -532,7 +532,8 @@ def test_problems_for_inconsistent_inputs(): def test_list_tables_cannot_be_merged_and_are_problems(): - """列表不能按期累积（P3-SPEC 2.2）：真把两期列表并起来，列表合计没有并集说法，记 problems，绝不沿用单期说法。""" + """列表合计行另存的表不能按期累积（accumulate_blockers）：真把两期这样的列表并起来，列表合计没有并集说法，记 + problems，绝不沿用单期说法。""" data = deepcopy(LIST_RECIPE) def lpart(span): @@ -544,7 +545,7 @@ def test_list_tables_cannot_be_merged_and_are_problems(): tgt = Recipe.model_validate(data) notes = build_union_notes(tgt, [lpart(AUG), lpart(SEP)], union_tables=derive_tables(tgt)[0], null_counts=None, part_null_counts=None, added=[], retired=[], label_sets=[], gaps=False) - assert any("列表合计行的核对结论不能按期合并" in p for p in notes.problems) + assert any("列表合计行、小计行的核对结论不能按期合并" in p for p in notes.problems) assert not any("list_total" in p for p in notes.problems) # 不露键名 assert "一致" not in notes.tables["销售_表内合计"].comment diff --git a/backend/tests/test_recipe_p5.py b/backend/tests/test_recipe_p5.py new file mode 100644 index 00000000..163a7bf7 --- /dev/null +++ b/backend/tests/test_recipe_p5.py @@ -0,0 +1,537 @@ +"""Excel 期 5（下一轮 03 细案 E）：列表按期累积、穿插的小计、表格对象起草、CSV 走配方。 + +不连接口的部分：执行器、核对、静态校验、累积资格、并集物化、起草。夹具全是合成的通用名字（门店一、分区甲、 +订单 A001……），数字是手写的小数，手算得出。接口的端到端在 test_recipe_p5_e2e.py。 +""" +from __future__ import annotations + +import copy +import io +import os +import sqlite3 +import tempfile +from pathlib import Path +from typing import Any + +import pytest +from openpyxl import Workbook +from openpyxl.worksheet.table import Table + +from app.data import csv_grid, raw_store, recipe_checks, recipe_engine, recipe_suggest, table_versions, xlsx_cells +from app.data import xlsx_scan +from app.data.recipe import validate_recipe +from app.data.recipe_accumulate import materialize_union +from app.data.recipe_types import ( + LIST_PERIOD_END, LIST_PERIOD_START, SUBTOTAL_KEY, Recipe, UnionPart, accumulate_blockers, derive_tables, +) + + +def _book(rows: list[tuple[Any, ...]], *, sheet: str = "明细", top: list[tuple[str, Any]] | None = None, + start: int = 1) -> bytes: + wb = Workbook() + ws = wb.active + ws.title = sheet + for ref, value in top or []: + ws[ref] = value + for r, row in enumerate(rows, start=start): + for c, v in enumerate(row, start=1): + if v is not None: + ws.cell(r, c, v) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def _run(raw: bytes, recipe: dict[str, Any], db: str | None = None, filename: str = "明细.xlsx", **kw: Any): + scan = csv_grid.scan_any(raw, filename) + return recipe_engine.execute(Recipe.model_validate(recipe), raw, filename, scan, db, **kw) + + +def _rows(db: str, sql: str) -> list[tuple[Any, ...]]: + conn = sqlite3.connect(db) + try: + return conn.execute(sql).fetchall() + finally: + conn.close() + + +def _checks(db: str, ex: Any, recipe: dict[str, Any]) -> dict[str, Any]: + return {c.id: c for c in recipe_checks.run_checks(db, ex, Recipe.model_validate(recipe))} + + +def _list(columns: list[tuple[str, str]], *, table: str = "明细", sheet: str = "明细", context: bool = False, + **block: Any) -> dict[str, Any]: + out = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": sheet}, + "context": [{"id": "统计期"}] if context else [], + "blocks": [{"id": "列表1", "layout": "list", "table": table, + "columns": [{"header": h, "name": h, "type": t} for h, t in columns], **block}]}], + "tables": [{"name": table, "grain": []}], + } + return out + + +# ========================================================================== +# 穿插的小计 +# ========================================================================== + +STORES = [("地区", "门店", "金额", "数量"), + ("分区甲", "门店一", 100, 1), ("分区甲", "门店二", 200, 2), ("小计", None, 300, 3), + ("分区乙", "门店三", 150, 4), ("分区乙", "门店四", 150, 5), ("小计", None, 300, 9), + ("合计", None, 600, 12)] +STORE_COLS = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER"), ("数量", "INTEGER")] + + +def _stores_recipe(**sub: Any) -> dict[str, Any]: + return _list(STORE_COLS, rows={"total_row": {"label_column": "地区", "pick": "合计"}, + "subtotals": {"label_column": "地区", "pick": "小计", **sub}}) + + +def test_subtotal_rows_are_checked_against_their_group_and_kept_out_of_the_data(tmp_path): + db = str(tmp_path / "s.db") + recipe = _stores_recipe() + ex = _run(_book(STORES), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + # 只有 4 行明细进了数据；两行小计、一行合计都不在 + assert _rows(db, 'SELECT "门店", "金额" FROM "明细" ORDER BY rowid') == [ + ("门店一", 100), ("门店二", 200), ("门店三", 150), ("门店四", 150)] + subs = [d for d in ex.derived if d.segment == f"列表1{SUBTOTAL_KEY}"] + assert [(d.cell, d.rowid_first, d.rowid_last) for d in subs] == [ + ("明细!C4", 1, 2), ("明细!D4", 1, 2), ("明细!C7", 3, 4), ("明细!D7", 3, 4)] + [excluded] = [x for x in ex.rows_excluded if x.reason == "subtotal"] + assert excluded.rows == [[4, 4], [7, 7]] and excluded.anchor == "小计" and excluded.block == "列表1" + checks = _checks(db, ex, recipe) + assert checks["T1"].title == "合计行按列求和（块「列表1」）" and checks["T1"].status == "passed" + assert checks["T2"].title == "小计行按各组明细求和（块「列表1」）" and checks["T2"].status == "passed" + assert checks["T2"].checked == 4 + + +def test_a_wrong_subtotal_is_a_mismatch(tmp_path): + rows = list(STORES) + rows[6] = ("小计", None, 300, 10) # 4 + 5 ≠ 10 + db = str(tmp_path / "s.db") + recipe = _stores_recipe() + ex = _run(_book(rows), recipe, db) + checks = _checks(db, ex, recipe) + t2 = checks["T2"] + assert t2.status == "mismatch" and t2.failed == 1 and t2.cells == ["明细!D7"] + assert "按明细求和为 9" in t2.details[0] + + +def test_group_column_scope_only_covers_the_rows_of_the_same_group(tmp_path): + """按组小计：紧挨在小计上面、分组列取值相同的那几行。小计行的标签写在「门店」列，分组按「地区」。 + 同一份数据按「上一行小计之后的全部明细」核对就对不上（350 ≠ 50）。""" + rows = [("地区", "门店", "金额"), ("分区甲", "门店一", 100), ("分区甲", "门店二", 200), + ("分区乙", "门店三", 50), ("分区乙", "小计", 50)] + cols = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER")] + recipe = _list(cols, rows={"subtotals": {"label_column": "门店", "pick": "小计", "scope": "group_column", + "group_column": "地区", "keep_as": "明细_小计"}}) + recipe["tables"].append({"name": "明细_小计", "kind": "reported_total"}) + db = str(tmp_path / "g.db") + ex = _run(_book(rows), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + assert _checks(db, ex, recipe)["T1"].status == "passed" + assert _rows(db, 'SELECT * FROM "明细_小计"') == [("小计", "分区乙", 50)] + assert [c.name for c in derive_tables(Recipe.model_validate(recipe))[0]["明细_小计"]] == ["小计项", "地区", "金额"] + since = copy.deepcopy(recipe) + since["sheets"][0]["blocks"][0]["rows"]["subtotals"].update(scope="since_previous", group_column=None) + since["tables"][1]["name"] = "明细_小计" + db2 = str(tmp_path / "g2.db") + ex2 = _run(_book(rows), since, db2) + assert _checks(db2, ex2, since)["T1"].status == "mismatch" + + +def test_subtotal_validation(): + def problems(**sub: Any) -> list[str]: + data = _stores_recipe(**sub) + _, probs = validate_recipe(data, facts=None, origin="rules") + return [p.code for p in probs] + + assert problems() == [] + assert "subtotal_group_missing" in problems(scope="group_column") + assert "column_unknown" in problems(scope="group_column", group_column="城市") + assert "subtotal_group_missing" in problems(group_column="地区") + assert "period_literal" in problems(pick="2026小计") + bad = _stores_recipe() + bad["sheets"][0]["blocks"][0]["rows"]["subtotals"]["pick"] = "合计" + assert "subtotal_pick_conflict" in [p.code for p in validate_recipe(bad, facts=None, origin="rules")[1]] + + +# ========================================================================== +# 列表按期累积：每一行属于哪一期 +# ========================================================================== + +PERIOD_CELL = ("A1", "统计时间：2026年8月1日至2026年8月31日") +ORDERS = [("订单号", "下单日期", "金额"), ("A001", "2026-08-01", 120), ("A002", "2026-08-15", 80), + ("A003", "2026-08-31", 40)] +ORDER_COLS = [("订单号", "TEXT"), ("下单日期", "DATE"), ("金额", "INTEGER")] + + +def _orders(period: dict[str, Any], *, context: bool = True, grain: list[str] | None = None) -> dict[str, Any]: + data = _list(ORDER_COLS, table="订单", context=context, period=period) + data["tables"][0]["grain"] = grain if grain is not None else ["订单号"] + return data + + +def test_synthesized_period_columns_carry_this_periods_bounds(tmp_path): + recipe = _orders({"synthesize": "统计期"}, grain=["订单号", LIST_PERIOD_START]) + cols = derive_tables(Recipe.model_validate(recipe))[0]["订单"] + assert [(c.name, c.role) for c in cols][-2:] == [(LIST_PERIOD_START, "period"), (LIST_PERIOD_END, "period")] + db = str(tmp_path / "p.db") + ex = _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + assert _rows(db, f'SELECT "订单号", "{LIST_PERIOD_START}", "{LIST_PERIOD_END}" FROM "订单" ORDER BY rowid') == [ + ("A001", "2026-08-01", "2026-08-31"), ("A002", "2026-08-01", "2026-08-31"), + ("A003", "2026-08-01", "2026-08-31")] + assert ex.table_hashes["订单"] + # 统计期起止列没有格子来历：溯源只有表头推出来的三列 + assert set(ex.lineage["订单"]) == {"订单号", "下单日期", "金额"} + # 干跑（不建库）同样走得通 + assert _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe).ok + + +def test_synthesized_period_with_a_human_period(tmp_path): + import datetime as dt + + from app.data.recipe_types import PeriodInput + + recipe = _orders({"synthesize": "统计期"}) + raw = _book(ORDERS) + assert {p.code for p in _run(raw, recipe).problems} == {"period_missing"} + db = str(tmp_path / "h.db") + ex = _run(raw, recipe, db, context_inputs={"统计期": PeriodInput(dt.date(2026, 8, 1), dt.date(2026, 8, 31))}) + assert ex.ok and _rows(db, f'SELECT DISTINCT "{LIST_PERIOD_START}" FROM "订单"') == [("2026-08-01",)] + + +def test_rows_must_fall_inside_the_period_when_bound_to_a_date_column(tmp_path): + recipe = _orders({"column": "下单日期"}) + db = str(tmp_path / "w.db") + ex = _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, db) + assert ex.ok + d1 = _checks(db, ex, recipe)["D1"] + assert d1.kind == "period_rows" and d1.status == "passed" and d1.checked == 3 + late = list(ORDERS) + [("A004", "2026-09-02", 10)] + db2 = str(tmp_path / "w2.db") + ex2 = _run(_book(late, top=[PERIOD_CELL], start=3), recipe, db2) + d1 = _checks(db2, ex2, recipe)["D1"] + assert d1.status == "mismatch" and d1.category == "structure" and d1.failed == 1 + assert d1.cells == ["明细!B7"] and "2026-08-01 至 2026-08-31" in d1.details[0] + + +def test_list_period_validation(): + def codes(data: dict[str, Any]) -> list[str]: + return [p.code for p in validate_recipe(data, facts=None, origin="rules")[1]] + + assert codes(_orders({"column": "下单日期"})) == [] + assert "list_period_invalid" in codes(_orders({"column": "金额"})) + assert "column_unknown" in codes(_orders({"column": "付款日期"})) + assert "list_period_invalid" in codes(_orders({"column": "下单日期", "synthesize": "统计期"})) + assert "list_period_invalid" in codes(_orders({})) + assert "covers_without_context" in codes(_orders({"synthesize": "统计期"}, context=False)) + clash = _list(ORDER_COLS + [(LIST_PERIOD_START, "TEXT")], table="订单", context=True, + period={"synthesize": "统计期"}) + assert "name_collision" in codes(clash) + + +def test_lists_become_eligible_for_accumulation_once_bound(): + unbound = Recipe.model_validate({**_list(ORDER_COLS, table="订单", context=True), "mode": "accumulate"}) + [p] = accumulate_blockers(unbound) + assert "没有指定每一行属于哪一期" in p.message + for period in ({"column": "下单日期"}, {"synthesize": "统计期"}): + data = {**_orders(period), "mode": "accumulate"} + assert accumulate_blockers(Recipe.model_validate(data)) == [] + assert validate_recipe(data, facts=None, origin="rules")[1] == [] + kept = {**_orders({"column": "下单日期"}), "mode": "accumulate"} + kept["sheets"][0]["blocks"][0]["rows"] = {"total_row": {"label_column": "订单号", "pick": "合计", + "keep_as": "订单_表内合计"}} + kept["tables"].append({"name": "订单_表内合计", "grain": ["合计项"], "kind": "reported_total"}) + blockers = accumulate_blockers(Recipe.model_validate(kept)) + assert [p.path for p in blockers] == ["/tables/1"] and "合计行、小计行另存的表" in blockers[0].message + + +def _built(raw: bytes, recipe: dict[str, Any], tag: str) -> tuple[str, Any]: + folder = Path(tempfile.mkdtemp(prefix="p5-built-")) + db = folder / f"{tag}.db" + ex = _run(raw, recipe, str(db)) + assert ex.ok, [p.message for p in ex.problems] + table_versions.settle_journal(db) + return str(db), ex + + +def _part(db: str, ex: Any, recipe: dict[str, Any], **kw: Any) -> UnionPart: + return UnionPart(import_id=None, start=kw.get("start", ex.period.start), end=kw.get("end", ex.period.end), + db_path=db, recipe=Recipe.model_validate(recipe), db_sha256=raw_store.sha256_file(db), + table_hashes=dict(ex.table_hashes)) + + +@pytest.mark.parametrize("period", [{"column": "下单日期"}, {"synthesize": "统计期"}]) +def test_two_months_of_a_list_materialize_into_one_union(tmp_path, period): + recipe = {**_orders(period), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, "aug") + sep_rows = [ORDERS[0], ("B001", "2026-09-03", 70), ("B002", "2026-09-30", 30)] + sep = _built(_book(sep_rows, top=[("A1", "统计时间：2026年9月1日至2026年9月30日")], start=3), recipe, "sep") + out = tmp_path / "u.db" + report = materialize_union(out, target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe), _part(*sep, recipe)]) + assert report.ok, [c.details for c in report.checks] + assert report.rows == {"订单": 5} + assert {c.id: c.status for c in report.checks} == {"U1": "passed", "U2": "passed", "U3": "passed"} + assert report.checks[2].checked == 2 + # 登记的统计期和库里的对不上（把 8 月的库说成 9 月）：U3 拒收 + bad = materialize_union(tmp_path / "bad.db", target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe, start="2026-09-01", end="2026-09-30")]) + u3 = next(c for c in bad.checks if c.id == "U3") + assert not bad.ok and u3.status == "mismatch" and "第 1 期" in u3.details[0] + + +def test_the_same_order_in_two_periods_is_a_union_key_conflict(tmp_path): + """明细跨期重复（同一订单两期都出现）按日期列分期时主键冲突，U2 拒收（细案风险条：语义要在配方里显式声明）。""" + recipe = {**_orders({"synthesize": "统计期"}), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, "aug2") + sep = _built(_book(ORDERS, top=[("A1", "统计时间：2026年9月1日至2026年9月30日")], start=3), recipe, "sep2") + report = materialize_union(tmp_path / "u.db", target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe), _part(*sep, recipe)]) + u2 = next(c for c in report.checks if c.id == "U2") + assert not report.ok and u2.status == "mismatch" + # 主键带上统计期起就分得开：每月一份的快照式明细 + keyed = {**_orders({"synthesize": "统计期"}, grain=["订单号", LIST_PERIOD_START]), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), keyed, "aug3") + sep = _built(_book(ORDERS, top=[("A1", "统计时间：2026年9月1日至2026年9月30日")], start=3), keyed, "sep3") + assert materialize_union(tmp_path / "k.db", target=Recipe.model_validate(keyed), + parts=[_part(*aug, keyed), _part(*sep, keyed)]).rows == {"订单": 6} + + +# ========================================================================== +# 起草：表格对象、穿插的小计、每一行属于哪一期的问题 +# ========================================================================== + + +def _ok_validate(data, facts, origin): + from app.data.recipe import validate_recipe as v + + return v(data, facts=facts, origin=origin) + + +def _draft(raw: bytes, filename: str = "明细.xlsx"): + scan = csv_grid.scan_any(raw, filename) + grids = {s.name: xlsx_cells.read_grid(raw, scan, s.name, max_rows=recipe_suggest.DRAFT_MAX_ROWS) + for s in scan.sheets if s.state == "visible" and s.nonempty} + + def dry(recipe: Recipe): + return recipe_engine.execute(recipe, raw, filename, scan, None) + return recipe_suggest.draft(scan, grids, filename, validate=_ok_validate, dry_run=dry), scan, grids + + +def _table_book(*, totals: bool = True, name: str = "门店销售") -> bytes: + """一个表格对象：B3:D7（表头、3 行明细、汇总行），表格上面有一行说明、右边有一个空格隔开的备注列。""" + wb = Workbook() + ws = wb.active + ws.title = "销售" + ws["B1"] = "门店销售明细" + rows = [("门店", "日期", "金额"), ("门店一", "2026-08-01", 10), ("门店二", "2026-08-02", 20), + (None, None, None), ("门店三", "2026-08-03", 30)] + if totals: + rows.append(("汇总", None, 60)) + for r, row in enumerate(rows, start=3): + for c, v in enumerate(row, start=2): + if v is not None: + ws.cell(r, c, v) + t = Table(displayName=name, ref=f"B3:D{2 + len(rows)}") + if totals: + t.totalsRowCount = 1 + ws.add_table(t) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def test_a_table_object_is_drafted_as_a_list_without_ai(): + raw = _table_book() + scan = xlsx_scan.scan(raw) + assert [(t.name, t.ref, t.totals_row_count) for t in scan.tables] == [("门店销售", "B3:D8", 1)] + draft, _, _ = _draft(raw) + assert draft.complete and draft.origin == "rules", draft.failures + [block] = draft.recipe["sheets"][0]["blocks"] + assert block["layout"] == "list" and block["table"] == "门店销售" + assert [c["header"] for c in block["columns"]] == ["门店", "日期", "金额"] + assert block["rows"]["total_row"] == {"label_column": "门店", "pick": "汇总", "keep_as": "门店销售_表内合计"} + assert block["rows"]["blank_rows"] == "skip" + # 锚点按表头文字，不记坐标 + assert "B3" not in str(block) + + +def test_a_table_object_with_digits_in_its_name_falls_back_to_the_title(): + draft, _, _ = _draft(_table_book(name="表1")) + assert draft.complete and draft.recipe["sheets"][0]["blocks"][0]["table"] == "门店销售明细" + + +def test_drafted_subtotals_run_and_verify(tmp_path): + raw = _book(STORES) + draft, scan, _ = _draft(raw) + assert draft.complete, draft.failures + block = draft.recipe["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"]["pick"] == "小计" and block["rows"]["total_row"]["pick"] == "合计" + db = str(tmp_path / "d.db") + recipe = Recipe.model_validate(draft.recipe) + ex = recipe_engine.execute(recipe, raw, "明细.xlsx", scan, db) + assert ex.ok and ex.tables[0].rows == 4 + assert all(c.status == "passed" for c in recipe_checks.run_checks(db, ex, recipe) if c.kind == "column_sum") + + +def test_period_question_binds_the_list_and_switches_to_accumulation(): + raw = _book(ORDERS, top=[PERIOD_CELL], start=3) + draft, _, grids = _draft(raw) + assert draft.complete, draft.failures + from app.data import recipe as R + + recipe = draft.recipe + cards, questions = recipe_suggest.questions_for(recipe, draft.facts, grids) + q = next(q for q in questions if q.id == "q_list_period:列表1") + assert [o["value"] for o in q.options] == ["none", "column:下单日期", "synthesize"] and q.default == "none" + assert not any(q.id == "q_mode" for q in questions) + bound = R.apply_patch(recipe, q.effects["column:下单日期"]) + assert bound["sheets"][0]["blocks"][0]["period"]["column"] == "下单日期" and bound["mode"] == "accumulate" + assert validate_recipe(bound, facts=None, origin="rules")[1] == [] + assert R.apply_patch(recipe, q.effects["none"])["mode"] == "replace" + # 工作表没配统计期的，选了以后一并加上 + plain = _book(ORDERS) + draft2, _, grids2 = _draft(plain) + assert draft2.recipe["sheets"][0]["context"] == [] + _, qs = recipe_suggest.questions_for(draft2.recipe, draft2.facts, grids2) + q2 = next(q for q in qs if q.id == "q_list_period:列表1") + bound2 = R.apply_patch(draft2.recipe, q2.effects["synthesize"]) + assert bound2["sheets"][0]["context"][0]["id"] == "统计期" + assert bound2["sheets"][0]["blocks"][0]["period"]["synthesize"] == "统计期" + assert accumulate_blockers(Recipe.model_validate(bound2)) == [] + + +# ========================================================================== +# CSV +# ========================================================================== + + +def test_csv_drafts_a_list_and_the_engine_runs_it(tmp_path): + raw = ("订单号,下单日期,门店,金额\n" + "A001,2026-08-01,一号店,120\n" + "A002,2026-08-02,二号店,80.5\n" + "小计,,,200.5\n" + "A003,2026-08-03,一号店,1\n" + "合计,,,201.5\n").encode("gb18030") + draft, scan, _ = _draft(raw, "orders_2026-08.csv") + assert draft.complete and draft.origin == "rules", draft.failures + sheet = draft.recipe["sheets"][0] + assert sheet["match"]["name"] == csv_grid.CSV_SHEET + block = sheet["blocks"][0] + assert [c["type"] for c in block["columns"]] == ["TEXT", "DATE", "TEXT", "REAL"] + assert block["rows"]["subtotals"]["pick"] == "小计" + db = str(tmp_path / "c.db") + recipe = Recipe.model_validate(draft.recipe) + ex = recipe_engine.execute(recipe, raw, "orders_2026-08.csv", scan, db) + assert ex.ok and ex.tables[0].rows == 3 + assert all(c.status == "passed" for c in recipe_checks.run_checks(db, ex, recipe) if c.kind == "column_sum") + # 回执的坐标照 Excel 打开这个 CSV 时的 A1 写 + [excluded] = [x for x in ex.rows_excluded if x.reason == "subtotal"] + assert excluded.sheet == "CSV" and excluded.rows == [[4, 4]] + + +# ========================================================================== +# 没配小计时，像小计的行：照常导入、要人确认，带「改作小计核对」的修复按钮 +# ========================================================================== + + +def test_a_subtotal_like_row_without_subtotals_is_flagged_and_fixable(tmp_path): + from app.data import recipe_confirm, recipe_fixes + from app.data.recipe import apply_patch + + plain = _list(STORE_COLS, rows={"total_row": {"label_column": "地区", "pick": "合计"}}) + ex = _run(_book(STORES), plain) + [p] = [p for p in ex.problems if p.code == "subtotal_suspect"] + assert p.category == "confirm" and p.fix == "declare_subtotals" + assert p.cells == ["明细!A4", "明细!A7"] and p.fix_args["pick"] == "小计" and p.fix_args["label_column"] == "地区" + assert ex.ok # 需确认类：照常导入，提交时勾确认项 + full = Recipe.model_validate(plain).model_dump(mode="json") + [prop] = recipe_fixes.propose_fixes(full, [recipe_fixes.dataclasses.asdict(p)], [], None) + assert prop.kind == "declare_subtotals" and [o.value for o in prop.options] == ["check_only", "keep"] + res = recipe_fixes.fix_ops(full, prop, "check_only", None) + assert res.ok, res.problems + fixed = apply_patch(full, res.ops) + assert fixed["sheets"][0]["blocks"][0]["rows"]["subtotals"]["pick"] == "小计" + db = str(tmp_path / "f.db") + ex2 = _run(_book(STORES), fixed, db) + assert ex2.ok and not [p for p in ex2.problems if p.code == "subtotal_suspect"] + assert _checks(db, ex2, fixed)["T2"].status == "passed" + keep = apply_patch(full, recipe_fixes.fix_ops(full, prop, "keep", None).ops) + assert keep["tables"][-1] == {"name": "明细_小计", "grain": [], "kind": "reported_total", "units": {}, "note": ""} + assert validate_recipe(keep, facts=None, origin="rules")[1] == [] + # 确认项：没改作小计核对就提交的，要勾「确实是数据」 + items = recipe_confirm.confirm_items(recipe_confirm.ConfirmContext(kind="first", recipe=plain, extraction=ex, + checks=[])) + [item] = [i for i in items if i.id == "subtotal_suspect:列表1"] + assert item.required and "改作小计核对" in item.label + + +def test_ordinary_rows_starting_with_a_total_word_are_not_flagged(): + """首列以「累计」开头、别的文字列有字的是数据（「累计积分卡」这种商品名），不当成像小计的行。""" + rows = [("商品", "门店", "金额"), ("累计积分卡", "门店一", 10), ("普通卡", "门店二", 20)] + ex = _run(_book(rows), _list([("商品", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER")])) + assert ex.ok and not [p for p in ex.problems if p.code == "subtotal_suspect"] + + +def _suspects(data: list[tuple[Any, ...]], columns: list[tuple[str, str]], **block: Any) -> list[str]: + ex = _run(_book(data), _list(columns, **block)) + assert ex.ok, [p.message for p in ex.problems] + return [c for p in ex.problems if p.code == "subtotal_suspect" for c in p.cells] + + +def test_a_two_column_key_value_list_is_never_flagged(): + """复核反例：只有「标签文字列 + 数字列」的键值清单，「其余文字列为空」自然成立，不能据此说它像小计—— + 「累计收入」「汇总调整」是普通的数据行。最后一行以合计字样开头也一样。""" + cols = [("项目", "TEXT"), ("金额", "INTEGER")] + assert _suspects([("项目", "金额"), ("门票收入", 100), ("累计收入", 300), ("汇总调整", 5), ("餐饮收入", 50)], + cols) == [] + assert _suspects([("项目", "金额"), ("门票收入", 100), ("餐饮收入", 50), ("合计调整项", 5)], cols) == [] + + +def test_only_rows_between_details_are_flagged(): + """像小计的行要夹在明细中间：上下相邻的明细在某个文字列上有值、这一行空着；后面还得有明细（或者是合计行）。 + 放在最后、后面没有明细的不报（那不是「夹在中间」）；上下的明细那一列本来就空着的也不报。""" + cols = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER")] + head = ("地区", "门店", "金额") + # 最后一行：后面没有明细，也没有合计行 + assert _suspects([head, ("分区甲", "门店一", 100), ("分区甲", "门店二", 200), ("小计", None, 300)], cols) == [] + # 上面的明细「门店」也空着：不是因为小计才空的 + assert _suspects([head, ("分区甲", None, 100), ("累计调整", None, 5), ("分区乙", "门店三", 50)], cols) == [] + # 下面的明细「门店」空着：同样不报 + assert _suspects([head, ("分区甲", "门店一", 100), ("累计调整", None, 5), ("分区乙", None, 50)], cols) == [] + # 夹在明细中间：报 + assert _suspects([head, ("分区甲", "门店一", 100), ("小计", None, 100), ("分区乙", "门店三", 50)], + cols) == ["明细!A3"] + # 最后一组的小计后面紧跟合计行：也在列表里面，报 + total = {"rows": {"total_row": {"label_column": "地区", "pick": "合计"}}} + assert _suspects([head, ("分区甲", "门店一", 100), ("小计", None, 100), ("合计", None, 100)], cols, + **total) == ["明细!A3"] + + +def test_declaring_no_subtotals_keeps_such_rows_as_data_without_asking(tmp_path): + """人确认过「这张列表没有小计行」（rows.no_subtotals）：以合计字样开头的行照常是数据，不再每期报 subtotal_suspect、 + 不再每期要勾确认；这个声明本身是放宽，配上的那一次要确认一次。和 subtotals 只能二选一。""" + from app.data import recipe_confirm + + recipe = _list(STORE_COLS, rows={"total_row": {"label_column": "地区", "pick": "合计"}, "no_subtotals": True}) + assert validate_recipe(recipe, facts=None, origin="rules")[1] == [] + db = str(tmp_path / "n.db") + ex = _run(_book(STORES), recipe, db) + assert ex.ok and not [p for p in ex.problems if p.code == "subtotal_suspect"] + assert _rows(db, 'SELECT COUNT(*) FROM "明细"') == [(6,)] # 两行「小计」照常是数据 + items = recipe_confirm.confirm_items(recipe_confirm.ConfirmContext(kind="first", recipe=recipe, extraction=ex, + checks=[])) + ids = [i.id for i in items] + assert "no_subtotals:列表1" in ids and not [i for i in ids if i.startswith("subtotal_suspect:")] + sigs = recipe_confirm.recipe_item_signatures(Recipe.model_validate(recipe)) + assert sigs["no_subtotals:列表1"] != recipe_confirm.recipe_item_signatures( + Recipe.model_validate(_list(STORE_COLS)))["no_subtotals:列表1"] + both = _stores_recipe() + both["sheets"][0]["blocks"][0]["rows"]["no_subtotals"] = True + probs = validate_recipe(both, facts=None, origin="rules")[1] + assert [p.code for p in probs] == ["subtotals_conflict"] and probs[0].path.endswith("/rows/no_subtotals") diff --git a/backend/tests/test_recipe_p5_e2e.py b/backend/tests/test_recipe_p5_e2e.py new file mode 100644 index 00000000..d5538bee --- /dev/null +++ b/backend/tests/test_recipe_p5_e2e.py @@ -0,0 +1,191 @@ +"""Excel 期 5 的端到端（真管线，经接口）：每月一份的 CSV 明细按期累积成一张表，重叠的那期被拒，同一期再传要确认 +替换；带穿插小计的 Excel 规则起草成功、小计核对通过、小计行不进数据。 + +夹具全部合成（订单号 A001…、门店一…），不调用任何模型（规则起草不完整时的 AI 可用性查询换成「不可用」的桩）。 +""" +from __future__ import annotations + +import io +import json +import uuid +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from openpyxl import Workbook + +from app.core.config import settings +from app.main import app + +API = "/api/datasources" +CSV_MIME = "text/csv" + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture(autouse=True) +def no_model(monkeypatch): + from app.data import recipe_ai + from app.data.recipe_types import AiAvailability + + async def unavailable(session): + return None, "", AiAvailability(False, reason="验收测试不接模型") + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", unavailable) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def _name() -> str: + return f"p5_{uuid.uuid4().hex[:10]}" + + +def month_csv(month: int, orders: list[tuple[str, int, int]]) -> tuple[bytes, str]: + """一个月的订单导出：订单号、下单日期、门店、金额。文件名带统计期（只作旁证，CSV 里没有统计期格）。""" + lines = ["订单号,下单日期,门店,金额"] + for oid, day, amount in orders: + lines.append(f"{oid},2026-{month:02d}-{day:02d},门店{'一二'[day % 2]},{amount}") + last = 31 if month in (8, 10) else 30 + return ("\n".join(lines) + "\n").encode("utf-8"), f"orders_2026-{month:02d}-01_2026-{month:02d}-{last}.csv" + + +AUG = [("A001", 1, 120), ("A002", 15, 80), ("A003", 31, 40)] +SEP = [("B001", 3, 70), ("B002", 30, 30)] + + +async def stage(client, raw: bytes, filename: str, name: str) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/stage", files={"file": (filename, raw, CSV_MIME)}, data={"name": name}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def answer(client, sid: str, answers: dict[str, Any]) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/answers", json={"answers": answers}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def trial(client, sid: str, **body: Any) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/trial", json=body) + assert resp.status_code == 200, resp.text + return resp.json() + + +def _period(month: int) -> dict[str, Any]: + last = 31 if month in (8, 10) else 30 + return {"统计期": {"start": f"2026-{month:02d}-01", "end": f"2026-{month:02d}-{last}"}} + + +async def commit(client, st: dict[str, Any], confirmations: list[str] | None = None): + ids = [i["id"] for i in st["trial"]["confirm_items"]] if confirmations is None else confirmations + return await client.post(f"{API}/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ids}) + + +async def rows(client, name: str, sql: str) -> list[list[Any]]: + resp = await client.post(f"/api/tools/db_query__{name}/run", json={"args": {"sql": sql}}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["ok"], out + return json.loads(out["result"])["rows"] + + +async def first_csv(client, name: str) -> dict[str, Any]: + raw, fn = month_csv(8, AUG) + st = await stage(client, raw, fn, name) + assert st["recipe"]["sheets"][0]["match"]["name"] == "CSV" + assert not st["recipe_problems"], st["recipe_problems"] + q = next(q for q in st["questions"] if q["id"].startswith("q_list_period:")) + # 列表的草稿还不能按期累积，没有 q_mode；「每期怎么更新」这一道题同时定下导入模式和每一行属于哪一期 + assert not any(x["id"] == "q_mode" for x in st["questions"]) + st = await answer(client, st["id"], {q["id"]: {"value": "synthesize"}}) + assert st["recipe"]["mode"] == "accumulate" and not st["recipe_problems"], st["recipe_problems"] + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + st = await trial(client, st["id"], context_inputs=_period(8)) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = [i["id"] for i in st["trial"]["confirm_items"]] + assert "list_period:列表1" in ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + return {**resp.json(), "table": st["recipe"]["sheets"][0]["blocks"][0]["table"]} + + +async def reupload(client, source_id: str, month: int, orders: list[tuple[str, int, int]], + period: dict[str, Any] | None = None) -> dict[str, Any]: + raw, fn = month_csv(month, orders) + resp = await client.post(f"{API}/{source_id}/reupload", files={"file": (fn, raw, CSV_MIME)}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + return await trial(client, st["id"], context_inputs=period or _period(month)) + + +async def test_monthly_csv_exports_accumulate_into_one_table(client): + name = _name() + first = await first_csv(client, name) + sid, table = first["source"]["id"], first["table"] + assert await rows(client, name, f'SELECT "订单号", "统计期起", "统计期止" FROM "{table}" ORDER BY rowid') == [ + ["A001", "2026-08-01", "2026-08-31"], ["A002", "2026-08-01", "2026-08-31"], ["A003", "2026-08-01", "2026-08-31"]] + st = await reupload(client, sid, 9, SEP) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert st["trial"]["accumulate"]["action"] == "append" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + got = await rows(client, name, f'SELECT "统计期起", COUNT(*) FROM "{table}" GROUP BY 1 ORDER BY 1') + assert got == [["2026-08-01", 3], ["2026-09-01", 2]] + # 部分重叠的一期被拒收 + st = await reupload(client, sid, 9, SEP, period={"统计期": {"start": "2026-08-15", "end": "2026-09-14"}}) + assert st["trial"]["status"] == "rejected" and any(p["code"] == "period_overlap" for p in st["trial"]["problems"]) + # 同一期再传（数不同）：要确认替换 + st = await reupload(client, sid, 9, [("B001", 3, 75)]) + assert st["trial"]["status"] == "passed" and st["trial"]["accumulate"]["action"] == "replace_period" + ids = [i["id"] for i in st["trial"]["confirm_items"]] + replace = "period_replace:2026-09-01~2026-09-30" + assert replace in ids + resp = await commit(client, st, [i for i in ids if i != replace]) + assert resp.status_code == 422 + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await rows(client, name, f'SELECT COUNT(*), SUM("金额") FROM "{table}"') == [[4, 315]] + + +async def test_excel_with_subtotals_drafts_checks_and_keeps_them_out(client): + wb = Workbook() + ws = wb.active + ws.title = "门店" + for row in [("门店", "金额"), ("门店一", 100), ("门店二", 200), ("小计", 300), ("门店三", 150), ("门店四", 150), + ("小计", 300), ("合计", 600)]: + ws.append(row) + buf = io.BytesIO() + wb.save(buf) + name = _name() + resp = await client.post(f"{API}/imports/stage", files={"file": ("门店.xlsx", buf.getvalue(), "application/octet-stream")}, + data={"name": name}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert not st["draft_problems"] and not st["recipe_problems"], (st["draft_problems"], st["recipe_problems"]) + block = st["recipe"]["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"]["pick"] == "小计" + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + checks = {c["id"]: c for c in st["trial"]["checks"]} + assert checks["T2"]["status"] == "passed" and "小计" in checks["T2"]["title"] + excluded = st["trial"]["receipt"]["rows_excluded"] + assert any(x["reason"] == "subtotal" and x["rows"] == [[4, 4], [7, 7]] for x in excluded), excluded + ids = [i["id"] for i in st["trial"]["confirm_items"]] + assert "subtotals:列表1" in ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + table = st["recipe"]["sheets"][0]["blocks"][0]["table"] + assert await rows(client, name, f'SELECT COUNT(*), SUM("金额") FROM "{table}"') == [[4, 600]] diff --git a/backend/tests/test_recipe_select.py b/backend/tests/test_recipe_select.py index 6824450b..9e408f16 100644 --- a/backend/tests/test_recipe_select.py +++ b/backend/tests/test_recipe_select.py @@ -210,10 +210,8 @@ def _subtotals() -> Book: @pytest.mark.parametrize("ref,bottom,where", [ - ("A1:C7", "box", "第 4 行像合计行"), # 小计夹在框中间 ("A1:C4", "box", "第 4 行像合计行"), # 框到小计为止：它下面紧接着还有明细，是夹在中间的小计 ("A1:C3", "box", "第 4 行像合计行"), # 框到小计上一行：下一行是小计，同上 - ("A1:C3", "auto", "第 4 行像合计行"), # 按规则推断：起草器在这里记失败，框选同样不收 ]) def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): b = _subtotals() @@ -222,6 +220,22 @@ def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): assert where in res.problems[0].message and "明细!A4" in res.problems[0].cells +@pytest.mark.parametrize("ref,bottom", [ + ("A1:C7", "box"), # 小计夹在框中间、合计在框底 + ("A1:C3", "auto"), # 按规则推断：起草器把夹在中间的小计提议成小计（期 5），框选同样 +]) +def test_subtotal_in_the_middle_becomes_subtotals_not_data(ref, bottom): + """期 5：夹在中间、第一列以合计字样开头的行提议成小计（ListRows.subtotals）：不进数据，按各组明细核对。""" + b = _subtotals() + res = selection_edit(None, b.grid, sel(ref, "list", "明细", bottom=bottom), extraction=None) + assert res.ok, res.problems + [op] = res.ops # 配方为空：一条根替换，新建只有这一个块的配方 + block = op["value"]["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"] == {"label_column": "地区", "pick": "小计", "scope": "since_previous", + "group_column": None, "keep_as": None} + assert block["rows"]["total_row"]["pick"] == "合计" + + def test_total_word_in_another_column_is_not_imported_as_data(): """合计字样写在第二列（第一列空着）：起草器记失败，框选同样报错，不当明细导入。""" wb = Workbook() diff --git a/backend/tests/test_recipe_suggest.py b/backend/tests/test_recipe_suggest.py index 40c62af1..44bd2164 100644 --- a/backend/tests/test_recipe_suggest.py +++ b/backend/tests/test_recipe_suggest.py @@ -951,7 +951,7 @@ def test_list_subtotal_rows_in_middle_fail(): assert [f for f in d.failures if "像合计行" in f] == [ "工作表「门店」：第 4 行像合计行（B 列「小计」），但合计字样不在第一列，无法按合计行核对", "工作表「门店」：第 7 行像合计行（B 列「小计」），但合计字样不在第一列，无法按合计行核对"] - # 小计写在第一列、但下面还有明细：不是末尾的合计行，整块照样收进来，不另报「数据区外的数字」 + # 期 5：小计写在第一列、下面还有明细：提议成穿插的小计（不进数据，按上一行小计之后的明细核对），不再记失败 wb2 = Workbook() ws2 = wb2.active ws2.title = "门店" @@ -960,10 +960,21 @@ def test_list_subtotal_rows_in_middle_fail(): ws2.append(row) sc2, grids2 = build(wb2) d2 = S.draft(sc2, grids2, "x.xlsx", validate=ok_validate()) - assert not d2.complete - assert d2.failures == ["工作表「门店」：第 4 行像合计行（A 列「小计」），但不在列表末尾，无法确定它汇总的是哪几行"] + assert d2.complete and d2.failures == [] block = d2.recipe["sheets"][0]["blocks"][0] assert block["rows"]["total_row"]["pick"] == "合计" + assert block["rows"]["subtotals"] == {"label_column": "门店", "pick": "小计", "scope": "since_previous", + "group_column": None, "keep_as": None} + # 字样不一样（「小计」「合计」都夹在中间）：分不清，照旧记失败 + wb3 = Workbook() + ws3 = wb3.active + ws3.title = "门店" + for row in [["门店", "金额"], ["门店一", 100], ["小计", 100], ["门店二", 200], ["合计", 200], ["门店三", 150], + ["总计", 450]]: + ws3.append(row) + sc3, grids3 = build(wb3) + d3 = S.draft(sc3, grids3, "x.xlsx", validate=ok_validate()) + assert not d3.complete and any("不在列表末尾" in f for f in d3.failures) # 末尾第一列的合计行照旧改作核对，不算失败（test_list_total_row_is_verified_and_kept） diff --git a/backend/tests/test_recipe_validate.py b/backend/tests/test_recipe_validate.py index 0d2f945e..3e9fe110 100644 --- a/backend/tests/test_recipe_validate.py +++ b/backend/tests/test_recipe_validate.py @@ -315,7 +315,7 @@ def test_accumulate_mode_follows_eligibility(): probs = run(lr, facts=None) assert [(p.code, p.path) for p in probs] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] - assert "列表形态的表目前只支持每期替换" in probs[0].message + assert "没有指定每一行属于哪一期" in probs[0].message # 同一份配方每期替换时不报 lr["mode"] = "replace" assert run(lr, facts=None) == [] diff --git a/backend/tests/test_recipe_validate_p3.py b/backend/tests/test_recipe_validate_p3.py index a2e2f775..b4f0acca 100644 --- a/backend/tests/test_recipe_validate_p3.py +++ b/backend/tests/test_recipe_validate_p3.py @@ -79,7 +79,7 @@ def test_accumulate_list_table_message_does_not_promise_future_versions(): r["sheets"][0]["context"] = [{}] _rec, problems = validate_recipe(r) assert [p.code for p in problems] == ["accumulate_unkeyed"] - assert "列表形态的表目前只支持每期替换" in problems[0].message and "后续版本" not in problems[0].message + assert "没有指定每一行属于哪一期" in problems[0].message and "后续版本" not in problems[0].message # ========================================================================== diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 17c24466..248b23f0 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1246,8 +1246,6 @@ export const UPLOAD_TEXT = { shapeTitle: '该表格不是一行一条记录的明细表', shapeLead: '检测到以下结构，默认不导入：', shapeRecipe: '这类表格可以按配方导入：指定表头、分段和合计行的处理方式，试运行核对、逐条确认后再启用。', - /** CSV 没有配方这条路（配方只认 Excel 的工作表结构） */ - shapeRecipeCsv: '按配方导入目前只支持 Excel（.xlsx）文件：可另存为 .xlsx 后再按配方导入。', shapeHeader: (row: number) => `如果表头不在第 ${row} 行，请取消后修改表头行号，再重新上传。`, /** 结构问题之外，预告的混合列：选按原样导入以后还要再选一次 */ shapeMixedLead: '另外，以下各列以数字为主，但混有非数字的值。选择按原样导入后，还需要选择这些值的处理方式：', @@ -1515,9 +1513,9 @@ export const RECIPE_TEXT = { descriptionLabel: '说明（供助手参考）', pickDrop: '拖入文件，或点击选择', /** 与选择框的 accept 一致 */ - pickFormats: '支持 Excel（.xlsx、.xlsm）', + pickFormats: '支持 Excel（.xlsx、.xlsm）和 CSV / TSV', pickAria: '选择表格文件', - pickWrongType: (name: string) => `「${name}」不是 Excel 文件：按配方导入只支持 .xlsx、.xlsm`, + pickWrongType: (name: string) => `「${name}」不是 Excel 或 CSV 文件：按配方导入支持 .xlsx、.xlsm、.csv、.tsv`, cancelUpload: '取消上传', cancel: '取消', close: '关闭', @@ -1652,6 +1650,20 @@ export const RECIPE_TEXT = { totalLabelColumn: '合计标签所在的列', totalPick: '合计行以此开头', totalKeep: '原表合计另存为表（留空则只核对）', + // 期 5：穿插的小计行（ListRows.subtotals）、列表每期怎么更新（ListBlock.period） + subtotals: '小计行', + subtotalsOn: '有小计行', + subtotalsHint: '夹在明细中间的小计不进数据，按它所管的那组明细逐列求和核对', + subtotalLabelColumn: '小计标签所在的列', + subtotalPick: '小计行以此开头', + subtotalScope: '一行小计管哪些明细', + subtotalGroupColumn: '按哪一列分组', + subtotalKeep: '原表小计另存为表（留空则只核对）', + /** 人确认过列表里没有小计（ListRows.no_subtotals）：像小计的行照常是数据，不再每期报「像小计的行」要人确认 */ + noSubtotals: '没有小计行（以合计字样开头的行也是数据，不再提醒）', + listPeriod: '按期累积时每一行属于哪一期', + listPeriodColumn: '按哪个日期列核对', + listPeriodHint: '列表要按期累积，每一行都要说得出属于哪一期；所在工作表要配置统计期（表内没有时，每期试运行时录入）。导入模式另在「导入模式」中选', hiddenRows: '隐藏行', hiddenCols: '隐藏列', fallback: '工作表改名时', @@ -1864,6 +1876,7 @@ export const FIX_KIND_LABEL: Record = { edit_members: '更新关系成员', rename_title: '改分段标题', declare_total: '改作合计核对', + declare_subtotals: '改作小计核对', ignore_cells: '按文字忽略', declare_placeholder: '声明占位符', declare_hidden: '设置隐藏行处理', @@ -2009,6 +2022,7 @@ export const RECEIPT_BLOCK_TEXT = { ignored_outside: '按配方忽略了区域外数字的行', after_stop: '列表结束之后的文字行', total_not_kept: '只核对、不另存的合计行', + subtotal: '穿插的小计行（核对后不进数据）', } as Record, excludedOther: '其他原因', excludedTitle: '排除的行', @@ -2122,6 +2136,8 @@ export const RECIPE_CHOICE_LABEL = { colType: { TEXT: '文字', INTEGER: '整数', REAL: '小数', DATE: '日期' } as Record, colStore: { '': '默认', canonical: '规范写法', raw: '原文' } as Record, axisChecks: { contiguous: '逐日连续', covers_context: '恰好覆盖统计期' } as Record, + subtotalScope: { since_previous: '上一行小计之后的全部明细', group_column: '紧挨在上面、分组列取值相同的明细' } as Record, + listPeriod: { none: '不区分（只能每期替换）', column: '按日期列核对每行都在本期统计期内', synthesize: '每行加上本期统计期的起止两列' } as Record, } // =========================================================================== @@ -2786,7 +2802,7 @@ export const SCHEMA_PARTIAL_TEXT = { // - 剖析是后台任务：逐张表进行，每张表剖析完就写进目录；查询一条一条地发，可以随时停止。关掉对话框照常进行。 // 目录只在整张表结束时写一次：停止、关停服务时正在做的那张来得及收尾，做完的项照常写进去；进程被杀时那张表没来得及收尾， // 这次做完的项一项都没写进去。说停下以后写进了什么，按逐表的状态照实说（cancelledNote / interruptedNote），不一律说「已写进」。 -// - 预算按表算：「每张表最多查询数」「每张表时长上限」用完只停这张表，下一张表重新计数；另有单条查询时限、抽样键值数、 +// - 预算按表算：「每张表最多查询数」「每张表时长上限」用完只停这张表，下一张表重新计数；另有单条查询时限、核对键值数、 // 整表统计行数上限。标签和服务端校验报错里的叫法一致（catalog_profile._NUMBER_FIELDS），保存被拒时按机读码认出是哪一项。 // - 没有整次总量上限：任务自己在连续失败、多数超时时停下（止损），写明原因。 // - 「没查成」≠ 结论错了：因为次数、时长、超时、出错没查成的项保留剖析以前的结论，可以「重试没做完的表」。 @@ -2798,7 +2814,7 @@ export const PROFILE_FIELD_LABEL: Record<'enabled' | CatalogProfileNumberKey, st table_max_queries: '每张表最多查询数', table_max_s: '每张表时长上限', query_timeout_s: '单条查询时限', - sample_size: '抽样键值数', + sample_size: '核对键值数', max_scan_rows: '整表统计行数上限', } @@ -2814,7 +2830,7 @@ export const PROFILE_FIELD_HINT: Record = { table_max_queries: '剖析一张表最多发出的查询条数，用完只停这张表，下一张表重新计数', table_max_s: '剖析一张表最多用的时长，到时只停这张表，下一张表重新计时', query_timeout_s: '超过即中断这条查询；不超过数据源自身的查询时限，也不超过这张表剩下的时长', - sample_size: '核对一条关联关系时，从本表抽取的不同键值个数', + sample_size: '核对一条关联关系时，从本表取的不同键值个数：取的是数据库先给出的那些，不是随机抽样', max_scan_rows: '不超过此行数的表才做去重计数、最小值和最大值等整表统计', } @@ -2878,7 +2894,7 @@ export const PROFILE_TEXT = { budgetPerTable: (queries: number, secs: number) => `每张表最多 ${formatNumber(queries)} 条查询、${formatNumber(secs)} 秒，用完只停这张表，下一张表重新计数`, budgetQuery: (secs: number) => `单条查询不超过 ${formatNumber(secs)} 秒`, - budgetSample: (n: number) => `每条关联关系抽样 ${formatNumber(n)} 个键值`, + budgetSample: (n: number) => `每条关联关系取 ${formatNumber(n)} 个不同键值核对`, budgetScan: (n: number) => (n > 0 ? `行数超过 ${formatNumber(n)} 的表不做整表统计` : '不做整表统计'), /** 上限的估算：表数 × 每张表的上限 */ budgetTotal: (tables: number, queries: number, span: string) => @@ -3017,7 +3033,7 @@ export const PROFILE_TEXT = { relationLowCoverage: '覆盖率不足，可能不是这条关系', relationNotUnique: '被指向列未核实唯一', coverage: (pct: string) => `覆盖率 ${pct}`, - sampled: (sample: number, matched: number) => `抽样 ${formatNumber(sample)} 个键值，对上 ${formatNumber(matched)} 个`, + sampled: (sample: number, matched: number) => `取了 ${formatNumber(sample)} 个不同键值，对上 ${formatNumber(matched)} 个`, cardinalityUnknown: '基数未核实', codeRows: (n: number) => `${formatNumber(n)} 行`, codesSummary: (values: number, rows: number) => `${formatNumber(values)} 个取值，共 ${formatNumber(rows)} 行`, @@ -3027,10 +3043,12 @@ export const PROFILE_TEXT = { dateProposal: (col: string) => `提议按 ${col} 作为业务日期`, dateSpan: (min: string, max: string) => `取值从 ${min} 到 ${max}`, dateRanges: '日期列的取值范围', + /** 日期列的最小值、最大值不计入的值（catalog_profile.VALID_DATE_FROM）：零日期、0001-01-01、9999-12-31、无穷大 */ + dateInvalid: (n: number) => `另有 ${formatNumber(n)} 个无效日期未计入（零日期或占位日期）`, skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`, moreSkipped: (n: number) => `…另有 ${formatNumber(n)} 项`, queriesTitle: (n: number) => `看发过的查询（${formatNumber(n)} 条）`, - queriesHint: 'SQL 里的取值都换成了「?」：抽样到的键值不保存在 AgentLab 中；遮罩的列不出现在任何查询里', + queriesHint: 'SQL 里的取值都换成了「?」：取到的键值不保存在 AgentLab 中；遮罩的列不出现在任何查询里', queryKind: { row_estimate: '行数估算', relation: '关系', codes: '码值', date: '日期' } as Record, queryOutcome: { ok: '成功', timeout: '超时', error: '失败', rejected: '未通过只读守卫' } as Record, queryRows: (n: number) => `${formatNumber(n)} 行`, diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index a7585768..cfd0c97e 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -1773,7 +1773,7 @@ function UploadDecisionBody({ decision, choices, headerRow, sheetNoun, onRecipe )}
-

{onRecipe ? UPLOAD_TEXT.shapeRecipe : UPLOAD_TEXT.shapeRecipeCsv}

+

{UPLOAD_TEXT.shapeRecipe}

{onRecipe && (