diff --git a/backend/app/data/csv_grid.py b/backend/app/data/csv_grid.py new file mode 100644 index 00000000..83e6177c --- /dev/null +++ b/backend/app/data/csv_grid.py @@ -0,0 +1,154 @@ +"""CSV 走配方(Excel 期 5 第 4 项):把 CSV 适配成配方读取层认的样子——一张可见工作表的扫描结果(WorkbookScan) +和网格(Grid / 逐行流),之后的执行器、核对、说明、溯源、按期累积全部复用,不另写一套 CSV 的解析(否则会和配方 +语言分叉,违背「一种说法」)。 + +**怎么认出是 CSV。** 看内容不看扩展名(配方的几个入口只拿得到原件字节):不是 zip(xlsx)、不是旧版 .xls 的 OLE +文件头、前 64 KB 里没有 NUL 字节(图片、PDF 这类二进制文件都有),就按 CSV 读。编码识别、分隔符嗅探复用 tabular +读 CSV 的那一段(UTF-8 / GB18030 / 带 BOM 的 UTF-16,逗号、分号、制表符、竖线)。 + +**坐标沿用「工作表!A1」。** 细案原写「格子坐标记成 R{行}C{列}」,这里没这么做(见交付说明的偏离项):坐标在执行器、 +核对、修复按钮、框选、溯源、裁判摘录、前端网格里一共有十来处解析和拼写,引入第二种写法就要处处认两种,漏一处就是 +着色错位、下钻找不到格子;期 1 读 CSV 时(tabular._shape_reasons)用的也是 A1。CSV 没有工作表,名字固定为 +CSV_SHEET:每月文件名不同(orders_2026-08.csv、orders_2026-09.csv),配方按工作表名认,名字必须稳定。行号是第几条 +记录(引号里的换行不另起一行),列号是第几个字段,和用 Excel 打开这个 CSV 看到的 A1 坐标一致。 + +**格子的值。** CSV 里只有文字。只有写法毫无歧义的才转成数(-12、80.5,不超过 15 位有效数字),其余一律原样留作文字: +「1,234」(千分位,交给配方的 text_number 决定)、「007」(前导零是编码的一部分)、「+5」「1e5」「12%」、日期写法 +(执行器按 DATE 列解析文字日期,起草按文字认日期)、「=SUM(…)」(CSV 里不是公式,Excel 打开时才会当公式算)。 +空字段、只有空白的字段是空格(和 xlsx_scan 的口径一致)。没有公式、合并单元格、隐藏行列。 +""" +from __future__ import annotations + +import csv +import io +import re +from dataclasses import dataclass +from typing import Any, Iterator + +from app.data.recipe_types import GridCell +from app.data.xlsx_scan import Bounds, SheetScan, UnsupportedTable, WorkbookScan + +__all__ = ["CSV_PATH", "CSV_SHEET", "MAX_COLS", "MAX_ROWS", "CsvTable", "cell_value", "looks_like_csv", "parse", + "rows", "scan", "scan_any"] + +#: CSV 的「工作表名」:配方按它认(SheetMatch.name),每月的文件名不同也认得出 +CSV_SHEET = "CSV" +#: SheetScan.path 的取值:读取层据此知道这张「工作表」要从 CSV 里读,不去压缩包里找 XML +CSV_PATH = "csv:" +#: 和 Excel 的上限一致:超过的 CSV 用 Excel 打开也装不下,交给模型的格子坐标也写不出来 +MAX_ROWS = 1_048_576 +MAX_COLS = 16_384 +#: 嗅探看多少字节 +_SNIFF = 64 * 1024 +#: 写法毫无歧义的整数、小数(不带正号、不带前导零、不用科学计数法) +_INT = re.compile(r"-?(?:0|[1-9][0-9]*)") +_DEC = re.compile(r"-?(?:0|[1-9][0-9]*)\.[0-9]+") +#: 超过这么多位有效数字的不转成数:转成浮点会丢位(19 位卡号撞值),留作文字 +_DIGITS_MAX = 15 + + +def looks_like_csv(raw: bytes) -> bool: + """这份原件按 CSV 读:不是 zip、不是 OLE(旧版 .xls)、开头没有 NUL 字节、不是空文件。带 BOM 的 UTF-16 例外 + (每个字符都带 NUL,Excel「Unicode 文本」另存出来的就是它)。""" + if not raw or raw[:4] == b"PK\x03\x04" or raw[:8] == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1": + return False + if raw[:2] in (b"\xff\xfe", b"\xfe\xff"): + return True + return b"\x00" not in raw[:_SNIFF] + + +def cell_value(text: str) -> Any: + """CSV 的一个字段 → 格子的值:空 / 只有空白是 None(空格),写法毫无歧义的整数、小数转成数,其余原样(文字)。""" + t = text.strip() + if not t: + return None + digits = len(t.lstrip("-").replace(".", "").lstrip("0")) or 1 + if digits <= _DIGITS_MAX: + if _INT.fullmatch(t): + return int(t) + if _DEC.fullmatch(t): + return float(t) + return text + + +@dataclass +class CsvTable: + """解析好的 CSV:records[i] 是第 i + 1 条记录的字段(原文)。""" + + records: list[list[str]] + delimiter: str + + +def parse(raw: bytes) -> CsvTable: + """原件 → 记录。编码、分隔符照 tabular 读 CSV 的做法;字段太长、引号不配对读不了时抛 UnsupportedTable。""" + from app.data import tabular + + text = tabular._decode_csv(raw) + try: + delimiter = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|").delimiter + except csv.Error: + delimiter = "," # 嗅不出来就按最常见的来 + records: list[list[str]] = [] + try: + for record in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter): + if len(records) >= MAX_ROWS: + raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行,超出可以导入的上限") + if len(record) > MAX_COLS: + raise UnsupportedTable(f"CSV 文件第 {len(records) + 1} 行有 {len(record):,} 列,超过 {MAX_COLS:,} 列的上限") + records.append(record) + except csv.Error as e: + raise UnsupportedTable(f"无法按 CSV 读取这个文件:第 {len(records) + 1} 行的写法不正确({e})。" + "请在 Excel 中重新另存为 CSV 后上传") from e + return CsvTable(records=records, delimiter=delimiter) + + +def _cells(table: CsvTable) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + for r, record in enumerate(table.records, 1): + out = [(c, GridCell(v)) for c, field in enumerate(record, 1) if (v := cell_value(field)) is not None] + if out: + yield r, out + + +def scan(raw: bytes) -> WorkbookScan: + """CSV 原件 → 扫描结果:一张可见工作表(名字 CSV_SHEET、路径 CSV_PATH),边界、非空格数、逐行非空格数的游程。 + 没有合并区、隐藏行列、公式、错误值、离群格(CSV 里每一格都是写出来的,没有残留格式)。""" + table = parse(raw) + sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH) + lo_r = lo_c = hi_r = hi_c = 0 + runs: list[tuple[int, int, int]] = [] + for r, cells in _cells(table): + n = len(cells) + sheet.nonempty += n + first, last = cells[0][0], cells[-1][0] + if not lo_r: + lo_r, lo_c, hi_c = r, first, last + lo_c, hi_c, hi_r = min(lo_c, first), max(hi_c, last), r + if runs and runs[-1][1] == r - 1 and runs[-1][2] == n: + runs[-1] = (runs[-1][0], r, n) + else: + runs.append((r, r, n)) + if lo_r: + sheet.bounds = Bounds(lo_r, lo_c, hi_r, hi_c) + sheet.dimension = sheet.bounds.a1() + sheet.row_runs = runs + return WorkbookScan(sheets=[sheet], format="csv") + + +def scan_any(raw: bytes) -> WorkbookScan: + """配方导入的扫描入口:CSV 走这里的适配,其余照旧交给 xlsx_scan(读不了照旧拒收,文案不变)。""" + from app.data import xlsx_scan + + if looks_like_csv(raw): + return scan(raw) + return xlsx_scan.scan(raw) + + +def rows(table: CsvTable, *, min_row: int, max_row: int, min_col: int, + max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + """按行流式给一个矩形里的非空格:(行号, [(列号, 格)…]),口径同 xlsx_cells.sheet_rows(整行为空的不给)。""" + for r in range(max(min_row, 1), min(max_row, len(table.records)) + 1): + record = table.records[r - 1] + out = [(c, GridCell(v)) for c in range(max(min_col, 1), min(max_col, len(record)) + 1) + if (v := cell_value(record[c - 1])) is not None] + if out: + yield r, out diff --git a/backend/app/data/recipe.py b/backend/app/data/recipe.py index 6e4d4a6d..a4835456 100644 --- a/backend/app/data/recipe.py +++ b/backend/app/data/recipe.py @@ -43,6 +43,9 @@ from app.data.recipe_parsers import ( text_label, ) from app.data.recipe_types import ( + LIST_PERIOD_END, + LIST_PERIOD_START, + LIST_SUBTOTAL_DIM, LIST_TOTAL_DIM, ColumnOut, CrosstabBlock, @@ -271,6 +274,8 @@ _FIELD_LABEL = { "parts": "组成列", "claims": "认领", "a": "一方", "b": "另一方", "reason": "理由", "ignore_rows": "按行标签忽略的行", "ignore_columns": "按表头忽略的列", "ignore_outside": "按同一行文字忽略的数字", "label": "标签", "anchor": "同一行的文字", + "subtotals": "小计行", "scope": "小计管哪些明细", "group_column": "分组的列", "period": "每行属于哪一期", + "column": "日期列", "check": "核对方式", "synthesize": "加统计期列", } @@ -608,15 +613,25 @@ class _Checker: for si, bi, block in self.lists: base = ("sheets", si, "blocks", bi) cols = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns)] + if block.period is not None and block.period.synthesize: + # 期 5:加上的统计期起止两列也是这张表的列,和表头推出的列名不能撞 + pp = base + ("period", "synthesize") + cols += [(LIST_PERIOD_START, pp), (LIST_PERIOD_END, pp)] out.append(_Writer(block.table, base + ("table",), "data", cols, origin=_pointer(base))) + numeric = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns) if c.type in _NUMERIC] total = block.rows.total_row if total is not None and total.keep_as: tp = base + ("rows", "total_row", "keep_as") - tcols = [(LIST_TOTAL_DIM, tp)] - tcols += [(c.name, base + ("columns", n, "name")) - for n, c in enumerate(block.columns) if c.type in _NUMERIC] - out.append(_Writer(total.keep_as, tp, "reported_total", tcols, + out.append(_Writer(total.keep_as, tp, "reported_total", [(LIST_TOTAL_DIM, tp)] + numeric, origin=_pointer(base + ("rows", "total_row")))) + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + sp = base + ("rows", "subtotals", "keep_as") + scols = [(LIST_SUBTOTAL_DIM, sp)] + if sub.scope == "group_column" and sub.group_column: + scols.append((sub.group_column, base + ("rows", "subtotals", "group_column"))) + out.append(_Writer(sub.keep_as, sp, "reported_total", scols + numeric, + origin=_pointer(base + ("rows", "subtotals")))) return out # ---- 8. 名字 ---- @@ -680,6 +695,10 @@ class _Checker: if sheet.context: continue for bi, block in enumerate(sheet.blocks): + if isinstance(block, ListBlock) and block.period is not None: + # 期 5:每行属于哪一期要拿统计期来核对(按日期列)或填进去(加起止两列) + self.add(("sheets", si, "blocks", bi, "period"), "covers_without_context", + "要按统计期核对或填写每一行的统计期,但这个工作表没有配置统计期") if not isinstance(block, CrosstabBlock): continue ap = ("sheets", si, "blocks", bi, "axis") @@ -873,6 +892,10 @@ class _Checker: if total is not None and any(ch.isdigit() for ch in total.pick): self.add(("sheets", si, "blocks", bi, "rows", "total_row", "pick"), "period_literal", f"合计行的开头词「{total.pick}」不能含数字(配方里不能写死年份或期次)") + sub = block.rows.subtotals + if sub is not None and any(ch.isdigit() for ch in sub.pick): + self.add(("sheets", si, "blocks", bi, "rows", "subtotals", "pick"), "period_literal", + f"小计行的开头词「{sub.pick}」不能含数字(配方里不能写死年份或期次)") # ---- 4、9、13. 单位:键是本表列名,值在词表里,与原表标签、表头里的单位一致 ---- @@ -902,10 +925,13 @@ class _Checker: self.unit_against(seg.keep_as.table, seg.keep_as.value, target.locate.title, "分段标题") for _si, _bi, block in self.lists: total = block.rows.total_row + sub = block.rows.subtotals for c in block.columns: self.unit_against(block.table, c.name, c.header, "表头") if total is not None and total.keep_as and c.type in _NUMERIC: self.unit_against(total.keep_as, c.name, c.header, "表头") + if sub is not None and sub.keep_as and c.type in _NUMERIC: + self.unit_against(sub.keep_as, c.name, c.header, "表头") def after_target(self, si: int, bi: int, seg: DerivedSegment) -> DimensionSegment | None: """合计分段紧跟的那个 dimension 分段(同一块里按 id 找);找不到或不是 dimension 段时已报 derived_after_invalid。""" @@ -998,6 +1024,8 @@ class _Checker: if total is not None and total.label_column not in names: self.add(base + ("rows", "total_row", "label_column"), "column_unknown", f"合计标签所在的列「{total.label_column}」不在这个列表的列里{self._near(total.label_column, names)}") + self.check_subtotals(base, block, names) + self.check_list_period(base, block, names) seen: dict[str, str] = {} for n, c in enumerate(block.columns): key = match_key(c.header) @@ -1006,6 +1034,50 @@ class _Checker: f"表头「{c.header}」与「{seen[key]}」是同一个表头") seen.setdefault(key, c.header) + def check_subtotals(self, base: tuple[Any, ...], block: ListBlock, names: list[str]) -> None: + """期 5:小计行的标签列、分组列要是这个列表的列;按组小计要给分组列;小计和合计的开头词不能互为前缀 + (「合计」「合计小计」:一行就分不清是小计还是合计)。""" + sub = block.rows.subtotals + if sub is None: + return + sp = base + ("rows", "subtotals") + if sub.label_column not in names: + self.add(sp + ("label_column",), "column_unknown", + f"小计标签所在的列「{sub.label_column}」不在这个列表的列里{self._near(sub.label_column, names)}") + if sub.scope == "group_column": + if not sub.group_column: + self.add(sp + ("group_column",), "subtotal_group_missing", "按组小计时要指明按哪一列分组") + elif sub.group_column not in names: + self.add(sp + ("group_column",), "column_unknown", + f"小计分组的列「{sub.group_column}」不在这个列表的列里{self._near(sub.group_column, names)}") + elif sub.group_column: + self.add(sp + ("group_column",), "subtotal_group_missing", + "小计管「上一行小计之后的全部明细」时不按列分组,请去掉分组的列,或改为按组小计") + total = block.rows.total_row + if total is not None: + a, b = match_key(sub.pick), match_key(total.pick) + if a.startswith(b) or b.startswith(a): + self.add(sp + ("pick",), "subtotal_pick_conflict", + f"小计行的开头词「{sub.pick}」和合计行的「{total.pick}」一个是另一个的开头,分不清是小计还是合计") + + def check_list_period(self, base: tuple[Any, ...], block: ListBlock, names: list[str]) -> None: + """期 5:每行属于哪一期,按日期列核对和加起止两列只能二选一;按日期列时那一列要是这个列表的日期列。""" + period = block.period + if period is None: + return + pp = base + ("period",) + if (period.column is None) == (period.synthesize is None): + self.add(pp, "list_period_invalid", "每一行属于哪一期只能二选一:按一个日期列核对,或加统计期起止两列") + return + if period.column is not None: + col = next((c for c in block.columns if c.name == period.column), None) + if col is None: + self.add(pp + ("column",), "column_unknown", + f"核对统计期的列「{period.column}」不在这个列表的列里{self._near(period.column, names)}") + elif col.type != "DATE": + self.add(pp + ("column",), "list_period_invalid", + f"核对统计期的列「{period.column}」的类型要是日期") + # ---- 期 3:忽略规则、表头上方的标题(P3-SPEC 9.2) ---- def check_ignores(self) -> None: diff --git a/backend/app/data/recipe_accumulate.py b/backend/app/data/recipe_accumulate.py index 5f6a8b54..ac689387 100644 --- a/backend/app/data/recipe_accumulate.py +++ b/backend/app/data/recipe_accumulate.py @@ -30,6 +30,7 @@ from app.data.recipe_parsers import hour_range, hour_range_total, match_key, tex from app.data.recipe_types import ( UNION_VER, AccumulatePlan, ChangeClass, CheckResult, ColumnOut, CrosstabBlock, DerivedSegment, DimensionSegment, PartInfo, Recipe, UnionPart, UnionReport, accumulate_blockers, derive_tables, + list_period_columns, ) from app.data.table_versions import settle_journal, sha_json @@ -508,14 +509,19 @@ def union_options(parts: list[tuple[str, str, str]], target_recipe_sha: str, *, class _Table: - """并集里的一张表:列(目标配方的在前、退役的按给定顺序追加在后)、主键、日期轴列。""" + """并集里的一张表:列(目标配方的在前、退役的按给定顺序追加在后)、主键、日期轴列。期 5:列表写入的表按 period + 分期(list_period_columns):按日期列的,U3 核对那一列落在该期统计期内;加统计期起止两列的,核对两列就是该期。""" - def __init__(self, name: str, cols: list[ColumnOut], grain: list[str]) -> None: + def __init__(self, name: str, cols: list[ColumnOut], grain: list[str], + period: tuple[str, ...] | None = None) -> None: self.name = name self.cols = cols self.grain = grain self.axis = next((c.name for c in cols if c.role == "axis"), None) self.names = [c.name for c in cols] + self.period = period if period is None or all(c in self.names for c in period[1:]) else None + if self.axis is None and self.period is not None and self.period[0] == "column": + self.axis = self.period[1] def _copy_col(col: ColumnOut) -> ColumnOut: @@ -551,6 +557,7 @@ def _layout(target: Recipe, parts: list[UnionPart], out: dict[str, _Table] = {} retired = retired or {} + periods = list_period_columns(target) for name, cols in tables.items(): mine = [_copy_col(c) for c in cols] known = {c.name for c in mine} @@ -561,7 +568,7 @@ def _layout(target: Recipe, parts: list[UnionPart], mine.append(_copy_col(found) if found is not None else _copy_col(col)) known.add(col.name) spec = specs.get(name) - out[name] = _Table(name, mine, list(spec.grain) if spec is not None else []) + out[name] = _Table(name, mine, list(spec.grain) if spec is not None else [], periods.get(name)) for name, cols in retired.items(): if name in out: continue @@ -831,20 +838,36 @@ def _finish(conn: sqlite3.Connection, layout: dict[str, _Table], parts: list[Uni u2.append(f"表「{table.name}」有 {dup} 组主键重复") checks.append(_check("U2", "union_pk", "累积后主键不重复", checked=len(keyed), failed=u2, sql=sql2)) - # U3 统计期:第 k 期的 rowid 区间里,日期都在 [start_k, end_k] 之内 + # U3 统计期:第 k 期的 rowid 区间里,日期都在 [start_k, end_k] 之内。期 5:按日期列分期的列表,日期为空也算不在; + # 加统计期起止两列的列表,两列必须正好是这一期 u3: list[str] = [] sql3, params3, checked3 = None, [], 0 for k, (part, ranges) in enumerate(zip(parts, part_rows), 1): for name, span in ranges.items(): - axis = layout[name].axis - if axis is None or axis not in part_cols[k - 1].get(name, []): - continue + table = layout[name] + have = part_cols[k - 1].get(name, []) a, b = span["union"] + if table.period is not None and table.period[0] == "synthesize": + start_col, end_col = table.period[1], table.period[2] + if start_col not in have or end_col not in have or b < a: + continue + checked3 += 1 + sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " + f"AND ({_q(start_col)} IS NOT ? OR {_q(end_col)} IS NOT ?)") + params3 = [a, b, part.start, part.end] + bad = conn.execute(sql3, params3).fetchone()[0] + if bad: + u3.append(f"第 {k} 期({_period_text(part.start, part.end)})的「{name}」有 {bad} 行的统计期不是这一期") + continue + axis = table.axis + if axis is None or axis not in have: + continue if b < a: continue checked3 += 1 + nulls = f"{_q(axis)} IS NULL OR " if table.period is not None else "" sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " - f"AND ({_q(axis)} < ? OR {_q(axis)} > ?)") + f"AND ({nulls}{_q(axis)} < ? OR {_q(axis)} > ?)") params3 = [a, b, part.start, part.end] bad = conn.execute(sql3, params3).fetchone()[0] if bad: diff --git a/backend/app/data/recipe_checks.py b/backend/app/data/recipe_checks.py index e095ada7..79a21b96 100644 --- a/backend/app/data/recipe_checks.py +++ b/backend/app/data/recipe_checks.py @@ -23,6 +23,7 @@ from typing import Any, Callable, Iterable from app.data.engine import open_checked_sqlite from app.data.recipe_types import ( + SUBTOTAL_KEY, CheckResult, ColumnOut, CrosstabBlock, @@ -34,6 +35,7 @@ from app.data.recipe_types import ( Recipe, SumEq, derive_tables, + list_period_columns, ) #: 无法核对的原因(契约 CheckResult.reasons 的键)的固定顺序:细节、说明都按这个顺序列 @@ -71,24 +73,33 @@ class CheckPlan: k: dict[str, str] = field(default_factory=dict) #: derived 分段键 → G 编号(只给有公式格的 derived 段) g: dict[str, str] = field(default_factory=dict) - #: 列表块键 → T 编号(每个有合计行的列表块一条) + #: 列表块键 → T 编号(每个有合计行的列表块一条);期 5:块键 + SUBTOTAL_KEY → 这个块的小计核对(一条) t: dict[str, str] = field(default_factory=dict) items: dict[str, list[DerivedItem]] = field(default_factory=dict) def _recipe_keys(recipe: Recipe) -> tuple[list[str], list[str]]: - """配方里 derived 分段键、带合计行的列表块键,按配方里的先后。""" + """配方里 derived 分段键、带合计行的列表块键,按配方里的先后。期 5:带小计的列表块另有一个键(块键 + + SUBTOTAL_KEY),排在这个块的合计行后面:同一块的小计和合计各编一个 T,结论分开写。""" derived: list[str] = [] totals: list[str] = [] for sheet in recipe.sheets: for block in sheet.blocks: if isinstance(block, CrosstabBlock): derived += [s.id for s in block.segments if isinstance(s, DerivedSegment)] - elif block.rows.total_row is not None: + continue + if block.rows.total_row is not None: totals.append(block.id) + if block.rows.subtotals is not None: + totals.append(f"{block.id}{SUBTOTAL_KEY}") return derived, totals +def is_subtotal_key(key: str) -> bool: + """T 编号的键是不是一个块的小计核对(期 5)。""" + return key.endswith(SUBTOTAL_KEY) + + def _ordered(recipe_keys: list[str], items: list[DerivedItem]) -> list[str]: """先按配方里的先后;配方里找不到的分段(不该出现)按执行结果里第一次出现的先后排在后面,不丢。""" present: list[str] = [] @@ -570,6 +581,25 @@ def _check_pk(sql: _Sql, cid: str, title: str, table: str, grain: list[str]) -> # ---------------------------------------------------------------- 入口 +def _check_period_rows(sql: _Sql, cid: str, title: str, table: str, column: str, period: Any, + lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult: + """期 5:列表按日期列分期(ListPeriod.column、check=within_context)时,每行的这一列都落在本期统计期内。 + 日期为空的行也算不在:看不出它属于哪一期。不在的行按期累积会混进别的期(统计期互不重叠的前提就破了), + 所以是结构类、不能写理由接受。统计期没定下来(要人录入)时不会走到这里:那时不建库。""" + if period is None or not period.start or not period.end: + raise _Broken("本期没有统计期,无法核对每行的日期") + q = (f"SELECT rowid FROM {_q(table)} WHERE {_q(column)} IS NULL OR {_q(column)} < ? OR {_q(column)} > ? " + "ORDER BY rowid") + params = [period.start, period.end] + bad = [r[0] for r in sql.all(q, params)] + total = sql.one(f"SELECT COUNT(*) FROM {_q(table)}")[0] + cells = [c for rowid in bad[:_DETAIL_MAX] if (c := lineage_cell(lineage, table, column, rowid))] + details = [f"有 {len(bad)} 行的「{column}」为空或不在 {period.start} 至 {period.end} 之内"] if bad else [] + return CheckResult(id=cid, kind="period_rows", title=title, status="mismatch" if bad else "passed", + category="structure", checked=int(total), failed=len(bad), sql=q, params=params, + details=details, cells=cells) + + def _axis_names(recipe: Recipe) -> dict[str, str]: """derived 分段键 → 所在交叉表块的轴列名(K 的铺满判定不按日期分)。""" out: dict[str, str] = {} @@ -629,7 +659,9 @@ def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[Che jobs.append((cid, "formula_refs", f"合计公式引用(分段「{seg}」)", lambda t, seg=seg, cid=cid: _check_g(sql, cid, t, plan.items[seg]))) for block, cid in plan.t.items(): - jobs.append((cid, "column_sum", f"合计行按列求和(块「{block}」)", + title = (f"小计行按各组明细求和(块「{block.removesuffix(SUBTOTAL_KEY)}」)" if is_subtotal_key(block) + else f"合计行按列求和(块「{block}」)") + jobs.append((cid, "column_sum", title, lambda t, block=block, cid=cid: _check_t(sql, cid, t, plan.items[block], full_calc))) for rel in recipe.relations: if isinstance(rel, SumEq): @@ -646,6 +678,12 @@ def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[Che for i, table in enumerate([t for t in tables if grains.get(t)], 1): jobs.append((f"P{i}", "pk_unique", f"主键唯一:「{table}」", lambda t, table=table, i=i: _check_pk(sql, f"P{i}", t, table, grains[table]))) + period = extraction.period + dated = [(t, spec[1]) for t, spec in list_period_columns(recipe).items() if spec[0] == "column" and t in tables] + for i, (table, column) in enumerate(dated, 1): + jobs.append((f"D{i}", "period_rows", f"「{table}」每行的「{column}」都在本期统计期内", + lambda t, table=table, column=column, i=i: _check_period_rows( + sql, f"D{i}", t, table, column, period, extraction.lineage))) try: return list(extraction.context_checks) + [_guarded(sql, *job) for job in jobs] finally: diff --git a/backend/app/data/recipe_confirm.py b/backend/app/data/recipe_confirm.py index f2a9419c..3271eed9 100644 --- a/backend/app/data/recipe_confirm.py +++ b/backend/app/data/recipe_confirm.py @@ -43,7 +43,7 @@ from app.data.recipe_diff import ( ) from app.data.recipe_parsers import UNITS, match_key, period_residue, split_unit_suffix from app.data.recipe_types import ( - ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, + LIST_PERIOD_END, LIST_PERIOD_START, SUBTOTAL_KEY, ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, SheetRecipe, SumEq, derive_tables, ) @@ -523,6 +523,12 @@ def _signatures(recipe: Recipe, sheet_name: Callable[[SheetRecipe], str]) -> dic total = block.rows.total_row out[f"total_row:{block.id}"] = _sig( None if total is None else (total.label_column, total.pick, total.keep_as)) + # 期 5:穿插的小计、每行属于哪一期 + subt = block.rows.subtotals + out[f"subtotals:{block.id}"] = _sig( + None if subt is None else (subt.label_column, subt.pick, subt.scope, subt.group_column, subt.keep_as)) + period = block.period + out[f"list_period:{block.id}"] = _sig(None if period is None else (period.column, period.synthesize)) for r in recipe.relations: if isinstance(r, SumEq): out[f"relation:{r.id}"] = _sig("sum_eq", r.table, r.total, sorted(r.parts)) @@ -1058,6 +1064,25 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *, detail = (f"按明细核对的列:{'、'.join(nums)};其余列合计行里的内容不核对" if nums else "这个块没有数字列,合计行里的内容不核对") bag.add(f"total_row:{b.id}", f"{prefix}改作核对{keep}", detail) + for b in lists: + # 期 5:穿插在明细中间的小计行:不进数据,按各组明细核对。要人确认:这几行从此不在本表里 + subt = b.rows.subtotals + if subt is not None: + rows = rows_text(derived_rows.get(f"{b.id}{SUBTOTAL_KEY}")) + prefix = f"「{b.id}」中间的小计行" + (f"({rows})" if rows else "") + keep = f",原值另存表「{subt.keep_as}」" if subt.keep_as else ",原值不另存" + scope = ("每行小计核对上一行小计之后的全部明细" if subt.scope == "since_previous" + else f"每行小计核对紧挨在它上面、「{subt.group_column}」相同的那几行明细") + nums = [c.name for c in b.columns if c.type in ("INTEGER", "REAL")] + detail = f"{scope};按明细核对的列:{'、'.join(nums)}" if nums else f"{scope};这个块没有数字列,不核对" + bag.add(f"subtotals:{b.id}", f"{prefix}改作核对、不进数据{keep}", detail) + period = b.period + if period is not None and period.column: + bag.add(f"list_period:{b.id}", f"「{b.id}」每一行的「{period.column}」都要在本期统计期内", + "按期累积时各期的行按这一列区分;有一行不在本期统计期内(或为空),这一期整份拒收") + elif period is not None and period.synthesize: + bag.add(f"list_period:{b.id}", f"「{b.id}」每一行加上本期统计期的起、止两列", + f"列名「{LIST_PERIOD_START}」「{LIST_PERIOD_END}」,取自表内的统计期或人工录入的统计期") # 年份取自统计期:表头是日期格时年份来自格子本身,不算 forms: dict[str, set[str]] = {} diff --git a/backend/app/data/recipe_engine.py b/backend/app/data/recipe_engine.py index 7c33e010..f44803da 100644 --- a/backend/app/data/recipe_engine.py +++ b/backend/app/data/recipe_engine.py @@ -39,6 +39,14 @@ pk_duplicate 同时出现)。块级的定位失败(找不到日期表头、 个矩形。 - 回执里的「排除的行」rows_excluded(H2:排除的行要写进回执):隐藏行、列表跳过的空行、按规则忽略的行、 停止之后没导入的文字行、只核对不另存的合计行。 + +**期 5(下一轮 03 细案 E)又加了两件事,同样没用新字段的配方写出的库逐字节不变:** +- 列表中间穿插的小计(ListRows.subtotals):小计行不进数据,每个数字格一条合计格,按它所管的那组明细(上一行小计 + 之后的全部明细,或紧挨在上面、分组列相同的那几行)交给核对模块用 SQL 求和比对;整行记进排除的行(subtotal)。 +- 列表按期累积时由系统加的统计期起止两列(ListPeriod.synthesize):列表在统计期定下来之前就抽完了,这两格先放 + 占位,统计期定下来以后写库时换成真值(_Sink.fill_period);按日期列分期的(ListPeriod.column)由核对模块的 D + 核对查每行都在本期统计期内。 +CSV 原件经 csv_grid 适配成一张「工作表」,这里照常按网格或逐行流读(xlsx_cells 按 SheetScan.path 分派)。 """ from __future__ import annotations @@ -64,10 +72,10 @@ from app.data.recipe_parsers import ( period_residue, text_label, thousands_number, ) from app.data.recipe_types import ( - AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, DimensionSegment, - ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, + SUBTOTAL_KEY, AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, + DimensionSegment, ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, PeriodInput, PeriodOut, Problem, Recipe, RegionMark, SegmentLabels, SheetRecipe, TableOut, TableSpec, - derive_tables, + derive_tables, list_period_columns, ) from app.data.xlsx_scan import MAX_MERGED, SheetScan, WorkbookScan, cell_ref, col_letter @@ -101,7 +109,7 @@ PROBLEM_CODES: dict[str, str] = { # 列表定位 "header_not_found": "structure", "header_ambiguous": "structure", "column_missing": "structure", "column_extra": "structure", "rows_after_stop": "confirm", "total_row_missing": "structure", - "total_row_blank": "structure", "list_empty": "confirm", + "total_row_blank": "structure", "list_empty": "confirm", "subtotal_blank": "structure", # 值 "value_not_integer": "structure", "value_text_number": "structure", "value_blank": "structure", "value_error": "structure", "value_not_number": "structure", "value_not_date": "structure", @@ -143,7 +151,7 @@ PLACEHOLDER_TEXTS_MAX = 8 PLACEHOLDER_TEXT_LEN = 8 #: 排除的行在回执里的顺序(同一工作表内按原因、块、锚点、首行排) _EXCLUDED_ORDER = ("hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", - "total_not_kept") + "total_not_kept", "subtotal") #: 公式区域展开的上限:更大的区域不可能全落在基表里,直接判「引用了基表之外的格」 REF_CELLS_MAX = 200_000 @@ -613,11 +621,18 @@ class _Problems: class _Sink: """一张表的行:主键去重、计数、溯源、批量写库(有库时)。rowid 就是插入序号(新库、1 起)。""" - def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None) -> None: + def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None, + synth: tuple[str, str] | None = None) -> None: self.eng = eng self.name = name self.cols = cols names = [c.name for c in cols] + #: 期 5:由系统加的统计期起止两列(ListPeriod.synthesize)在一行里的位置。列表是在统计期定下来之前抽完的, + #: 这两格先放占位(_PERIOD_START / _PERIOD_END),统计期定下来以后(fill_period)写库时换成真值;在那之前 + #: 不分批写库(flush 等着),整张表留在内存里 + self.synth_idx = (names.index(synth[0]), names.index(synth[1])) \ + if synth and synth[0] in names and synth[1] in names else None + self.period: tuple[str | None, str | None] | None = None self.grain = [g for g in (spec.grain if spec else []) if g in names] self.grain_idx = [names.index(g) for g in self.grain] self.check_pk = bool(self.grain_idx) and not eng.partial @@ -682,9 +697,22 @@ class _Sink: self.eng.problems.issue("value_blank", sheet, f"工作表「{sheet}」的表「{self.name}」的主键列{_quoted(names)}", coords, text=_PK_NULL, group="pk_null") - def flush(self) -> None: + def fill_period(self, start: str | None, end: str | None) -> None: + """统计期定下来了:加的统计期起止两列从此写真值(start、end 是 YYYY-MM-DD)。""" + self.period = (start, end) + + def flush(self, *, final: bool = False) -> None: + if self.synth_idx is not None and self.period is None and not final: + return # 统计期还没定:加的两列还是占位,先不写 if self.buffer and self.eng.conn is not None: - self.eng.conn.executemany(self.insert_sql, self.buffer) + rows = self.buffer + if self.synth_idx is not None: + a, b = self.synth_idx + start, end = self.period or (None, None) + fill = {a: start, b: end} + rows = [tuple(fill.get(i, v) if v is _PERIOD_START or v is _PERIOD_END else v + for i, v in enumerate(row)) for row in rows] + self.eng.conn.executemany(self.insert_sql, rows) self.buffer.clear() @@ -695,6 +723,12 @@ class _Sink: _FAIL = object() #: 列表数据区里不能填充的合并单元格覆盖的空格(已经报了 merged_in_values) _MERGED = object() +#: 期 5:加的统计期起止两列在统计期定下来之前的占位(_Sink.fill_period)。主键判重时它们和自己相等: +#: 同一期的行统计期都一样,判重只看其余的主键列,结论不变 +_PERIOD_START = object() +_PERIOD_END = object() +#: 小计按组时「还没有组」 +_NO_GROUP = object() # -------------------------------------------------------------------------- @@ -1625,6 +1659,22 @@ class _ListRun: self.ignore_keys = {k for rule in block.ignore_columns if (k := match_key(rule.header))} total = block.rows.total_row self.total_idx = next((i for i, c in enumerate(self.cols) if total and c.name == total.label_column), None) + # 期 5:穿插的小计行(ListRows.subtotals)。sub_first:上一行小计(或表头)之后第一行明细的 rowid;按组小计 + # 时 run_first / run_value 是紧挨在上面、分组列取值相同的那一段明细的首个 rowid 和取值。sub_rows:小计所在的 + # 行 → 它管的明细的 rowid(合计行的公式引用了小计格时,按它换算回明细) + self.sub = block.rows.subtotals + sub = self.sub + self.sub_idx = next((i for i, c in enumerate(self.cols) if sub and c.name == sub.label_column), None) + self.sub_key = match_key(sub.pick) if sub else "" + self.group_idx = next((i for i, c in enumerate(self.cols) + if sub and sub.scope == "group_column" and c.name == sub.group_column), None) + self.sub_first: int | None = None + self.run_first: int | None = None + self.run_value: Any = _NO_GROUP + self.sub_hidden = False + self.sub_rows: dict[int, list[int]] = {} + #: 期 5:加统计期起止两列(ListPeriod.synthesize):每行末尾两格占位,统计期定下来以后由 _Sink 换成真值 + self.synth = block.period is not None and block.period.synthesize is not None # ---------------- 喂入 ---------------- @@ -2001,6 +2051,7 @@ class _ListRun: fills = self._merges_at(r, inb) if self.sh.hidden_row(r): self.hidden_seen = True + self.sub_hidden = True if self.sh.sr.hidden.rows == "exclude": self.excluded_rows.add(r) self.sh.layout(r, self.layout_hidden) @@ -2012,6 +2063,11 @@ class _ListRun: if label is not None and match_key(_text(label.value)).startswith(pick): self._total_row(r, inb, cells, label) return + if self.sub_idx is not None: + label = inb.get(self.colmap.get(self.sub_idx, -1)) + if label is not None and match_key(_text(label.value)).startswith(self.sub_key): + self._subtotal_row(r, inb, cells, label) + return self._record(r, inb, cells, fills) def _plan(self) -> list[tuple[Any, int, bool, str]]: @@ -2073,6 +2129,8 @@ class _ListRun: v = None values.append(v) lineage.append((col.name, r, c)) + if self.synth: + values += [_PERIOD_START, _PERIOD_END] ignored = self.ignored claims = {c: ("ignored_column" if c in ignored else "value") for c in inb} self.sh.layout(r, self.layout_data) @@ -2084,6 +2142,12 @@ class _ListRun: if self.rowid_first is None: self.rowid_first = rowid self.rowid_last = rowid + if self.sub_first is None: + self.sub_first = rowid + if self.group_idx is not None: + group = values[self.group_idx] + if self.run_value is _NO_GROUP or group != self.run_value: + self.run_first, self.run_value = rowid, group self.emit(r, cells, claims, True) def _total_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], @@ -2142,7 +2206,67 @@ class _ListRun: self.stop = ("total", r) self.emit(r, cells, claims, True) - def _ref_lookup(self, rc: tuple[int, int]) -> str | int: + def _subtotal_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], + label: GridCell) -> None: + """期 5:穿插在明细中间的一行小计。不进数据:标签格和数字格的去向是 total_label / total_value(同合计行), + 每个数字格一条合计格(DerivedItem,kind=column_sum,分段键是块 id + SUBTOTAL_KEY),核对模块按它所管的那组 + 明细(rowid 闭区间)用 SQL 逐列求和比对,执行器不做加法(H1);整行记进排除的行(reason=subtotal)。配了 + keep_as 的另存一行(小计项、按组时的分组取值、各数字列)。 + + 它管哪几行:since_previous 是上一行小计(或表头)之后的全部明细;group_column 是其中紧挨在它上面、分组列 + 取值相同的那一段。一行明细都没有时区间为空,核对判「无法核对」(合计行之上没有明细行)。数字格全空的小计 + 没有一列能核对,报 subtotal_blank(结构类):不报的话这一行就悄悄没了核对。""" + eng, sh, block, sub = self.eng, self.sh, self.block, self.sub + assert sub is not None + claims = {c: ("ignored_column" if c in self.ignored else "total_label") for c in inb} + label_raw = _text(label.value) + first = self.sub_first + if sub.scope == "group_column" and first is not None and self.run_first is not None: + first = max(first, self.run_first) + last = self.rowid_last if first is not None else None + numeric = [(i, col) for i, col in enumerate(self.cols) if col.type in ("INTEGER", "REAL")] + if numeric and not any(self.colmap[i] in inb for i, _ in numeric): + coord = sh.coord(r, self.colmap[self.sub_idx]) # type: ignore[index] + where = f"{self.where}:小计行({coord.split('!')[-1]})的数字格全部为空" + eng.problems.add( + "subtotal_blank", + f"{where},无法按这一组明细核对。请在文件中补上小计后重新上传;原表不再有小计时,在配方面板的" + "「小计行」中取消「有小计行」", + f"{where},无法核对", [coord], sheet=sh.name) + key = f"{block.id}{SUBTOTAL_KEY}" + keep_values: list[Any] = [] + for i, col in numeric: + c = self.colmap[i] + cell = inb.get(c) + if cell is None: + keep_values.append(None) + continue + claims[c] = "total_value" + got = eng.convert_number(cell, col.type, block.values, sh.name, f"{self.where}的小计行", r, c, + data_area=False) + keep_values.append(got) + item = DerivedItem( + kind="column_sum", segment=key, sheet=sh.name, cell=sh.coord(r, c), label_raw=label_raw, + label=canon(label_raw), value=None if got is _FAIL else got, is_formula=cell.formula is not None, + formula=cell.formula, base_table=block.table, base_value=col.name, rowid_first=first, + rowid_last=last, hidden_in_range=self.sub_hidden, keep_table=sub.keep_as) + if item.is_formula: + eng.resolve_refs(item, cell.formula or "", self._ref_lookup) + eng.ex.derived.append(item) + if sub.keep_as and sub.keep_as in eng.sinks: + ks = eng.sinks[sub.keep_as] + ks.sources.append(f"{sh.sr.id}/{block.id}") + group = [None if self.run_value is _NO_GROUP else self.run_value] if self.group_idx is not None else [] + lin = [(col.name, r, self.colmap[i]) for i, col in numeric] + ks.add([canon(label_raw), *group, *keep_values], sheet=sh.name, + at=(r, self.colmap[self.sub_idx]), cells=lin, fails=True) # type: ignore[index] + n = sum(1 for role in claims.values() if role in ("total_label", "total_value")) + sh.exclude("subtotal", r, r, n, block=block.id, anchor=sub.pick) + self.sub_rows[r] = list(range(first, last + 1)) if first is not None and last is not None else [] + self.sub_first, self.run_first, self.run_value, self.sub_hidden = None, None, _NO_GROUP, False + self.emit(r, cells, claims, True) + + def _ref_lookup(self, rc: tuple[int, int]) -> str | int | list[int]: r, c = rc if self.c1 <= c <= self.c2: rowid = self.row_rowid.get(r) @@ -2150,6 +2274,9 @@ class _ListRun: return rowid if r in self.excluded_rows: return "hidden" + if r in self.sub_rows: + # 合计行的公式引用了小计格(=C5+C9):小计格代表它管的那组明细 + return self.sub_rows[r] return "outside" @@ -2343,7 +2470,8 @@ class _Engine: self.canon_log[key] = CanonEntry(table, col.name, text, stored, 1, _coord(sheet, r, c)) return stored - def resolve_refs(self, item: DerivedItem, formula: str, lookup: Callable[[tuple[int, int]], str | int]) -> None: + def resolve_refs(self, item: DerivedItem, formula: str, + lookup: Callable[[tuple[int, int]], str | int | list[int]]) -> None: """合计公式的引用经溯源:全在基表 → ref_rowids;有隐藏行或分类汇总函数 → hidden; 有不在基表的 → outside;形状认不出 → unrecognized。执行器不做加法(H1)。""" shape = formula_refs(formula) @@ -2357,7 +2485,9 @@ class _Engine: outside = not cells for rc in cells: hit = lookup(rc) - if isinstance(hit, int): + if isinstance(hit, list): + rowids.extend(hit) # 期 5:引用了小计格,换算成它管的明细 + elif isinstance(hit, int): rowids.append(hit) elif hit == "hidden": hidden = True @@ -2607,8 +2737,9 @@ class _Engine: self.ex.problems = self.problems.finish(set()) return self.ex self.cols = cols + synth = {t: (spec[1], spec[2]) for t, spec in list_period_columns(self.recipe).items() if spec[0] == "synthesize"} for name, cs in cols.items(): - self.sinks[name] = _Sink(self, name, cs, self.specs.get(name)) + self.sinks[name] = _Sink(self, name, cs, self.specs.get(name), synth.get(name)) if self.db_path is not None: _remove_db(self.db_path) self.conn = _open_sqlite(self.db_path, readonly=False) @@ -2622,6 +2753,11 @@ class _Engine: crosstabs.extend(self._phase1(sh)) t2 = time.perf_counter() period, period_reported, _ = self._period() + if self.ex.period is not None: + # 期 5:列表加的统计期起止两列现在才有值(列表在统计期定下来之前就抽完了) + for sink in self.sinks.values(): + if sink.synth_idx is not None: + sink.fill_period(self.ex.period.start, self.ex.period.end) for ct in crosstabs: ct.complete_axis(period, period_reported) ct.extract() @@ -2650,7 +2786,7 @@ class _Engine: _remove_db(self.db_path) # type: ignore[arg-type] else: for sink in self.sinks.values(): - sink.flush() + sink.flush(final=True) self.conn.execute("COMMIT") self.conn.close() self.conn = None @@ -2748,7 +2884,9 @@ class _Engine: sh.failed = True return [] b = s.bounds - if b is not None and not self.partial and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): + # 稀疏判定挡的是 Excel 里误输入、残留格式撑大的已用区域;CSV 的每一格都是写出来的,空字段多只是数据稀疏 + if b is not None and not self.partial and self.scan.format != "csv" \ + and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): far = "、".join(list(dict.fromkeys(s.far_cells))[:3]) msg = (f"工作表「{sh.name}」的已用区域 {b.a1()} 共 {b.area:,} 个单元格,但只有 {s.nonempty:,} 个单元格有内容;" f"远处的单元格(如 {far})可能是误输入或残留格式。请在 Excel 中删除这些单元格后重新上传") diff --git a/backend/app/data/recipe_imports.py b/backend/app/data/recipe_imports.py index 40dba2b1..e715a08c 100644 --- a/backend/app/data/recipe_imports.py +++ b/backend/app/data/recipe_imports.py @@ -225,7 +225,7 @@ class Pipeline: def default_pipeline() -> Pipeline: """接上真模块。函数体里惰性导入:这些模块(openpyxl、执行器、起草器)导入不便宜,接口模块加载时不必付。""" from app.data import ( - recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, + csv_grid, recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, recipe_engine, recipe_fixes, recipe_notes, recipe_select, recipe_suggest, xlsx_cells, xlsx_scan, ) @@ -234,7 +234,8 @@ def default_pipeline() -> Pipeline: recipe_sha256=recipe.recipe_sha256, canonical=recipe.canonical_recipe, apply_patch=recipe.apply_patch, - scan=xlsx_scan.scan, + # CSV / TSV 也走配方:csv_grid 适配成一张「工作表」,其余照旧交给 xlsx_scan(Excel 期 5) + scan=csv_grid.scan_any, scan_to_json=xlsx_scan.scan_to_json, scan_from_json=xlsx_scan.scan_from_json, read_grid=xlsx_cells.read_grid, diff --git a/backend/app/data/recipe_notes.py b/backend/app/data/recipe_notes.py index b4464cf7..48c8da20 100644 --- a/backend/app/data/recipe_notes.py +++ b/backend/app/data/recipe_notes.py @@ -27,6 +27,9 @@ from app.data.names import canon from app.data.recipe_checks import date_columns, plan_checks, split_coord from app.data.recipe_parsers import UNITS from app.data.recipe_types import ( + LIST_PERIOD_END, + LIST_PERIOD_START, + SUBTOTAL_KEY, Acceptance, CheckResult, ColumnOut, @@ -46,6 +49,7 @@ from app.data.recipe_types import ( TableNote, UnionNotePart, derive_tables, + list_period_columns, prose_problems, render_prose, ) @@ -134,6 +138,13 @@ NOTE_TEXT = { "ignored_rows_union": "部分期的原表另有按配方忽略的行,内容见证据面板", "ignored_cells_union": "部分期的原表另有按配方忽略的单元格,内容见证据面板", "dim_values_vary": "各期的 {dim} 取值不完全相同:跨期比较前请先确认两期都有的取值", + # ---- 期 5:列表按期累积、穿插的小计。列表可以累积以后,空行之后的文字行也要有并集说法 + "rows_after_stop_union": "部分期的原表在空行之后另有未导入的文字行,内容见证据面板", + "subtotal_kept": "原表穿插的小计在 {table},不要与本表相加", + "subtotal_not_kept": "原表穿插的小计行未导入本表", + "list_subtotal_passed": "原表穿插的小计行的值,已按各组明细求和核对一致:不要与 {base} 相加,也不要彼此相加", + "list_subtotal_unverified": "原表穿插的小计行的值,按原表保存的值存储。本期部分小计未能核对:不要与 {base} 相加," + "也不要彼此相加", } #: 列说明的片段模板 @@ -148,6 +159,9 @@ COLUMN_TEXT = { "col_partial_periods": "部分期没有这一列的数据,为空值", "col_absent": "当前版本的数据中没有这一列,为空值", "col_retired": "这一列自某一期起不再导入,此后各期为空值", + # ---- 期 5:列表按期累积时由系统加的统计期起止两列(ListPeriod.synthesize)。不含数字(说明不许报数) + "period_start": "本期统计期的第一天,取自表内的统计期或人工录入的统计期,格式 YYYY-MM-DD", + "period_end": "本期统计期的最后一天,取自表内的统计期或人工录入的统计期,格式 YYYY-MM-DD", } #: 「逐日 / 逐行」「个别日期 / 个别行」:粒度只有一列日期时按日说,否则按行说 @@ -161,7 +175,7 @@ _K_REASON = {r: f"total_k_{r}" for r in ("uncached", "blank", "null_detail", "ti _T_REASON = {r: f"list_total_{r}" for r in ("uncached", "blank", "null_detail", "tiling", "hidden")} #: 允许出现「一致」的模板:只在对应的 K / T 通过时才会选用 CONSISTENT_KEYS = frozenset({"total_k_passed", "total_k_passed_g_unverifiable", "list_total_passed", - "list_total_passed_partial"}) + "list_total_passed_partial", "list_subtotal_passed"}) def _tok(kind: str, name: str) -> str: @@ -202,6 +216,8 @@ class _Writers: lists: list[ListBlock] = field(default_factory=list) #: 把合计行另存进本表的列表块 list_totals: list[ListBlock] = field(default_factory=list) + #: 期 5:把穿插的小计另存进本表的列表块 + list_subtotals: list[ListBlock] = field(default_factory=list) def add_sheet(self, sheet: SheetRecipe) -> None: if all(s is not sheet for s in self.sheets): @@ -242,6 +258,9 @@ def _writers(recipe: Recipe) -> dict[str, _Writers]: total = block.rows.total_row if total is not None and total.keep_as: w(total.keep_as, sheet).list_totals.append(block) + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + w(sub.keep_as, sheet).list_subtotals.append(block) return out @@ -395,6 +414,15 @@ def _base_total_fragments(w: _Writers) -> list[NoteFragment]: b.rows.total_row is not None and not b.rows.total_row.keep_as for b in w.lists) if dropped: out.append(NoteFragment("total_not_kept", None, NOTE_TEXT["total_not_kept"])) + # 期 5:穿插的小计行不在本表里:另存在哪,或者没导入 + for b in w.lists: + sub = b.rows.subtotals + if sub is None: + continue + if sub.keep_as: + _add_unique(out, _frag("subtotal_kept", sub.keep_as, NOTE_TEXT["subtotal_kept"].format(table=_tab(sub.keep_as)))) + elif all(f.key != "subtotal_not_kept" for f in out): + out.append(NoteFragment("subtotal_not_kept", None, NOTE_TEXT["subtotal_not_kept"])) return out @@ -450,6 +478,13 @@ def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[NoteFragme else: frag = _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table))) _add_unique(out, frag) + for block in w.list_subtotals: + # 期 5:另存的小计:T 通过才说「核对一致」,没核对上的(无法核对)照实说部分未能核对;不一致的整份拒收 + t = ctx.check(table, ctx.plan.t.get(f"{block.id}{SUBTOTAL_KEY}"), f"块「{block.id}」的小计行") + if t is None or _blocking(t): + continue + key = "list_subtotal_passed" if t.status == "passed" else "list_subtotal_unverified" + _add_unique(out, _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table)))) return out @@ -653,10 +688,17 @@ def _date_key(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> str | None: return None +def _period_column_key(col: ColumnOut) -> str | None: + """期 5:系统加的统计期起止两列(role=period)用哪条列说明。""" + if col.role != "period": + return None + return "period_start" if col.name == LIST_PERIOD_START else "period_end" if col.name == LIST_PERIOD_END else None + + def _null_eligible(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> bool: """这一列要不要考虑空值说明:值列;主键不会是空值;表内合计表的空值已由表说明(未能核对)交代。""" value_col = col.role in ("measure", "value") and col.type in ("INTEGER", "REAL") - is_total = bool(w.kept_from or w.list_totals) + is_total = bool(w.kept_from or w.list_totals or w.list_subtotals) return value_col and not is_total and col.name not in set(ctx.grains.get(table, [])) @@ -668,6 +710,8 @@ def _column_templates(ctx: _Ctx, table: str, null_counts: Mapping[str, Mapping[s frags: list[str] = [] if (date := _date_key(ctx, w, table, col)) is not None: frags.append(COLUMN_TEXT[date]) + if (period := _period_column_key(col)) is not None: + frags.append(COLUMN_TEXT[period]) if col.unit: frags.append(COLUMN_TEXT["unit"].format(unit=_tok("单位", col.unit))) has_nulls = counts is None or counts.get(col.name, 0) > 0 @@ -793,22 +837,27 @@ _FAMILY: dict[str, str] = { "hidden_excluded": "hidden_excluded", "hidden_included": "hidden_included", "ignored_columns": "ignored_columns", "ignored_rows": "ignored_rows", "ignored_cells": "ignored_cells", "rows_after_stop": "rows_after_stop", + "subtotal_kept": "subtotal_kept", "subtotal_not_kept": "subtotal_not_kept", + "list_subtotal_passed": "list_total", "list_subtotal_unverified": "list_total", "period_conflict_accepted": "period_conflict", "period_conflict_pending": "period_conflict", "period_human": "period_human", "outside_digits": "outside_digits", "note": "note", } #: 合并后的拼接顺序(与 _table_fragments 一致;累积特有的几句排在用户写的表说明之前) -_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", "total_k", - "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", +_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", + "subtotal_kept", "subtotal_not_kept", "total_k", "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", "ignored_rows", "ignored_cells", "rows_after_stop", "period_conflict", "period_human", "outside_digits", "union", "note") #: 与核对结论无关、只描述表结构的片段族:用目标配方那一期的;只在早期出现的 subject(退役的表、另存表)用 #: 最后一个含它的那一期的 -_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note"}) +_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note", + "subtotal_kept", "subtotal_not_kept"}) #: 「任一期出现就写」的片段族 → 并集里的说法 _ANY_UNION = {"hidden_excluded": "hidden_excluded_union", "hidden_included": "hidden_included_union", "ignored_columns": "ignored_columns_union", "ignored_rows": "ignored_rows_union", - "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union"} + "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union", + # 期 5:列表可以按期累积了,空行之后的文字行任一期有就写 + "rows_after_stop": "rows_after_stop_union"} _K_PASSED = frozenset({"total_k_passed", "total_k_passed_g_unverifiable"}) @@ -943,20 +992,26 @@ def _merge_group(table: str, family: str, subject: str | None, group: dict[int, if family == "period_human": key = "period_human_all" if len(present) == len(having) else "period_human_union" return NoteFragment(key, subject, NOTE_TEXT[key]) - # list_total、rows_after_stop 只有列表才有,而列表不能按期累积(P3-SPEC 2.2);不认识的片段族同样没有并集说法。 + # list_total 只在列表的合计行、小计行另存的表上,这种表不能按期累积(accumulate_blockers);不认识的片段族同样 + # 没有并集说法。 # 记 problems(不能保存),绝不沿用某一期的单期说法。消息给人看,不露片段族的键名 problems.append(f"表「{table}」:{_UNMERGEABLE.get(family, '有一段说明')}不能按期合并,无法生成说明") return None #: 没有并集说法的片段族 → 给人看的叫法(problems 里不露键名) -_UNMERGEABLE = {"list_total": "列表合计行的核对结论", "rows_after_stop": "列表空行之后的文字行"} +_UNMERGEABLE = {"list_total": "列表合计行、小计行的核对结论"} def _axis_column(recipe: Recipe, table: str) -> str | None: + """各期按哪一列区分(说明里「请按 {col} 筛选」):交叉表的日期轴;期 5 起列表按 period 绑定的日期列,或者 + 系统加的统计期起那一列。""" tables, _ = derive_tables(recipe) axes = [c.name for c in tables.get(table, []) if c.role == "axis"] - return axes[0] if len(axes) == 1 else None + if len(axes) == 1: + return axes[0] + spec = list_period_columns(recipe).get(table) + return spec[1] if spec else None def _union_table_fragments(table: str, having: list[int], parts: list[_UPart], ref: int, target: Recipe, @@ -1037,6 +1092,8 @@ def _union_column_templates(table: str, columns: list[ColumnOut], having: list[i frags.append(COLUMN_TEXT["date_year_from_period"]) elif "date" in dates or (not with_col and col.role == "axis"): frags.append(COLUMN_TEXT["date"]) + if (period := _period_column_key(col)) is not None: + frags.append(COLUMN_TEXT[period]) if target_cols is not None and col.name in target_cols: unit = target_cols[col.name].unit elif with_col: diff --git a/backend/app/data/recipe_select.py b/backend/app/data/recipe_select.py index daa7332c..34697950 100644 --- a/backend/app/data/recipe_select.py +++ b/backend/app/data/recipe_select.py @@ -427,6 +427,21 @@ def _bottom_box(ctx: _Ctx, plan: Any, first: int) -> bool: ctx.fail("selection_bottom_not_anchorable", f"框里只有表头和合计行(第 {last} 行),没有数据行:请把框扩大到数据所在的行") return False + # 期 5:框里夹在中间、第一列以合计字样开头、下面紧接着还有同一张表的明细的行(起草器 _scan_block 的同一条 + # 规则),照起草器提议成小计(_settle_subtotals);提议不成的照旧报错 + mid = [r for r in data[1:] if S._total_word(S._label_of(grid.get(r, c1))) is not None] + rest = [r for r in data if r not in mid] + if mid and rest and not any(grid.get(r, c) is not None and S._total_word(grid.get(r, c).value) is not None + for r in data for c in range(c1 + 1, c2 + 1)): + trial = S._ListPlan(header_rows=plan.header_rows, cols=plan.cols, headers=plan.headers, rows=rest, + subtotal_rows=list(mid), + subtotal_words=[S._total_word(S._label_of(grid.get(r, c1))) or "" for r in mid]) + types = S._types_of(grid, rest, plan.cols) + if all(S._more_rows(grid, r, trial, S._bottom(grid), types) for r in mid): + S._settle_subtotals(grid, trial) + if trial.subtotal_word is not None: + plan.subtotal_rows, plan.subtotal_word = trial.subtotal_rows, trial.subtotal_word + data = [r for r in data if r not in mid] if not _no_stray_totals(ctx, data, c1, c2): return False bottom = S._bottom(grid) diff --git a/backend/app/data/recipe_suggest.py b/backend/app/data/recipe_suggest.py index 53314072..9a176511 100644 --- a/backend/app/data/recipe_suggest.py +++ b/backend/app/data/recipe_suggest.py @@ -12,6 +12,11 @@ validate)和执行器干跑(注入的 dry_run),再由人逐条确认之 哪些是区域外文字、哪些格没有去处)直接用执行器的结论:同一件事只有执行器一套判法,这里自己定位只是没有 干跑时的近似。 +**期 5(下一轮 03 细案 E)。** 有 Excel 表格对象(插入 → 表格)的工作表优先按表格对象出列表块(_lists_from_tables: +表头、列范围、汇总行都是表格对象给的,锚点仍按表头文字);夹在明细中间、第一列以同一个合计字样开头的行提议成 +穿插的小计(_settle_subtotals),不再一律记起草失败;每个列表都问一道「每期怎么更新」(q_list_period),选按期 +累积时同时绑定每一行属于哪一期(按日期列核对,或加统计期起止两列)。 + **问题的 effects 怎么写。** 每个选项是一组 JSON Patch,路径按传进来的配方写;接口层从 answers_base 起、按 问题顺序重放全部回答(P2-SPEC 7.2)。因此: - 关系的两个选项都用整条 replace(「登记」写回完整的 sum_eq / not_comparable,「不登记」写 dismissed), @@ -63,7 +68,7 @@ from app.data.recipe_types import ( derive_tables, ) from app.data.tabular import DATE_HEADER_MIN, EXCEL_ERRORS -from app.data.xlsx_scan import WorkbookScan, cell_ref, col_letter, parse_ref +from app.data.xlsx_scan import TableScan, WorkbookScan, cell_ref, col_letter, parse_ref logger = logging.getLogger(__name__) @@ -612,8 +617,15 @@ class _ListPlan: #: 按数据行收进来并记起草失败 adjacent: list[tuple[int, list[int]]] = field(default_factory=list) #: 不在末尾第一列的合计字样:(行, 列)。第一列在中间或最上面、或者落在别的列上,都无法按合计行核对, - #: 记起草失败;不记就会把合计当明细导入,求和翻倍(H2) + #: 记起草失败;不记就会把合计当明细导入,求和翻倍(H2)。期 5:第一列里同一个字样的穿插行先记进 subtotal_rows, + #: 收完整块再判能不能提议成小计(_settle_subtotals);判不了的才回到这里 stray_totals: list[tuple[int, int]] = field(default_factory=list) + #: 期 5:穿插在明细中间、第一列以合计字样开头的行(不进 rows)和它们的字样;能提议成小计时 subtotal_word 非空 + subtotal_rows: list[int] = field(default_factory=list) + subtotal_words: list[str] = field(default_factory=list) + subtotal_word: str | None = None + #: 期 5:按 Excel 的表格对象起草的(xlsx_scan.TableScan 的名字):表名优先用它,卡片里写明 + table_object: str | None = None # 命名阶段填 id: str = "" table: str = "" @@ -808,13 +820,44 @@ def _scan_block(grid: Grid, plan: _ListPlan, bottom: int) -> None: if word is not None and plan.rows and not _more_rows(grid, r, plan, bottom, types): plan.total_row, plan.total_word = r, word break + others = [(r, c) for c, cell in zip(cols[1:], cells[1:]) + if cell is not None and _total_word(cell.value) is not None] + plan.stray_totals.extend(others) + if word is not None and plan.rows and not others: + # 期 5:夹在明细中间的合计字样,先当小计候选:不进数据行,收完整块再判(_settle_subtotals) + plan.subtotal_rows.append(r) + plan.subtotal_words.append(word) + r += 1 + continue if word is not None: plan.stray_totals.append((r, cols[0])) - plan.stray_totals.extend((r, c) for c, cell in zip(cols[1:], cells[1:]) - if cell is not None and _total_word(cell.value) is not None) plan.rows.append(r) acc.add(r) r += 1 + _settle_subtotals(grid, plan) + + +def _settle_subtotals(grid: Grid, plan: _ListPlan) -> None: + """期 5:夹在明细中间的合计字样能不能提议成小计(ListRows.subtotals,规则起草不再把它们记成失败)。 + + 能的条件:都在第一列、是同一个字样,每一行至少有一格数(没有数的小计核对不了)。末尾那一行和它们字样相同时 + (表以最后一组的小计结束,没有总计),它也是小计,不是合计行。不能的(字样不同、有一行没有数)照旧记进 + stray_totals(起草失败,交给人或 AI),这些行回到数据行里——和期 4 的结果一样。""" + if not plan.subtotal_rows: + return + words = set(plan.subtotal_words) + numeric = [c for c in plan.cols[1:] if any(_value_num(grid.get(r, c)) is not None for r in plan.rows)] + ok = len(words) == 1 and all(any(_value_num(grid.get(r, c)) is not None for c in numeric) + for r in plan.subtotal_rows) + if ok: + plan.subtotal_word = next(iter(words)) + if plan.total_row is not None and plan.total_word == plan.subtotal_word: + plan.subtotal_rows.append(plan.total_row) + plan.total_row = plan.total_word = None + return + plan.stray_totals.extend((r, plan.cols[0]) for r in plan.subtotal_rows) + plan.rows = sorted(set(plan.rows) | set(plan.subtotal_rows)) + plan.subtotal_rows, plan.subtotal_words = [], [] def _list_failures(sheet: str, grid: Grid, plan: _ListPlan) -> list[_Failure]: @@ -898,7 +941,8 @@ def _analyze_lists(grid: Grid) -> _Lists: _finish_block(grid, plan) out.failures.extend(_list_failures(grid.sheet, grid, plan)) out.blocks.append(plan) - end = max((p.total_row or (p.rows[-1] if p.rows else p.header_rows[-1])) for p in made) + end = max(max([p.total_row or 0, *p.subtotal_rows, p.rows[-1] if p.rows else p.header_rows[-1]]) + for p in made) while r_idx < len(rows) and rows[r_idx] <= end: r_idx += 1 return out @@ -945,7 +989,7 @@ def _finish_block(grid: Grid, plan: _ListPlan) -> None: break if cells: break - last = plan.total_row or rows[-1] + last = max([plan.total_row or 0, rows[-1], *plan.subtotal_rows]) for r in range(plan.header_rows[0], last + 1): for c in cols: if grid.get(r, c) is not None or r in plan.header_rows: @@ -975,15 +1019,77 @@ class _Sheet: hidden_cols: list[int] = field(default_factory=list) -def _analyze_sheet(grid: Grid, sid: str) -> _Sheet: +def _lists_from_tables(grid: Grid, objects: list[TableScan]) -> _Lists | None: + """期 5:按 Excel 的表格对象(插入 → 表格)起草列表块。表格对象明确给出了表头行、列范围、数据行和汇总行,比按 + 「像不像表头」猜可靠:数字格空着的行不会被当成表头,紧挨着的两张表也分得开。 + + 出来的块和按规则猜的一样只记文字:锚点仍按表头文字(不记坐标,下一期表格挪了位置照样认得出),汇总行按第一列 + 的文字映射成合计行(total_row),夹在中间的合计字样按小计提议(_settle_subtotals)。表格对象的 ref 写坏了、 + 没有表头行时返回 None,退回按规则猜。""" + out = _Lists(sheet=grid.sheet) + spans = sorted(((t.span(), t) for t in objects), key=lambda x: (x[0] or (0, 0, 0, 0))[:2]) + for span, t in spans: + if span is None or t.header_row_count < 1: + return None + r1, c1, r2, c2 = span + hrows = list(range(r1, r1 + t.header_row_count)) + cols = list(range(c1, c2 + 1)) + texts = _header_texts(grid, hrows, cols) + if set(texts) != set(cols): + blank = [col_letter(c) for c in cols if c not in texts] + msg = (f"工作表「{grid.sheet}」:表格「{t.name}」的表头有空着的列({'、'.join(blank)} 列)," + "按列表导入要每一列都有表头文字") + out.failures.append(_Failure(msg, msg)) + continue + plan = _ListPlan(header_rows=hrows, cols=cols, headers={c: texts[c] for c in cols}, table_object=t.name) + last_data = r2 - max(t.totals_row_count, 0) + rows = [r for r in grid.row_numbers() if hrows[-1] < r <= last_data and any(grid.get(r, c) is not None + for c in cols)] + prev = hrows[-1] + for r in rows: + if r > prev + 1: + plan.skip_blank = True + plan.skipped.extend(range(prev + 1, r)) + prev = r + word = _total_word(_label_of(grid.get(r, c1))) + if word is not None and plan.rows: + plan.subtotal_rows.append(r) + plan.subtotal_words.append(word) + continue + plan.rows.append(r) + if not plan.rows: + msg = f"工作表「{grid.sheet}」:表格「{t.name}」的表头下面没有数据行" + out.failures.append(_Failure(msg, msg)) + continue + if t.totals_row_count > 0: + tr = last_data + 1 + label = _label_of(grid.get(tr, c1)) + word = _total_word(label) or (label if label and len(label) <= 12 and not any( + ch.isdigit() for ch in label) else None) + if word is not None: + plan.total_row, plan.total_word = tr, word + else: + why = ("起草只读了表格前面的部分,没读到汇总行" if grid.truncated and tr > max(grid.row_numbers()) + else "汇总行的第一列没有可用作合计标签的文字(不能含数字、不超过 12 个字)") + msg = f"工作表「{grid.sheet}」:表格「{t.name}」的汇总行(第 {tr} 行)无法按合计行核对:{why}" + out.failures.append(_Failure(msg, msg)) + _settle_subtotals(grid, plan) + _finish_block(grid, plan) + out.failures.extend(_list_failures(grid.sheet, grid, plan)) + out.blocks.append(plan) + return out + + +def _analyze_sheet(grid: Grid, sid: str, objects: list[TableScan] | None = None) -> _Sheet: sh = _Sheet(sheet=grid.sheet, sid=sid) axis = _axis_row(grid) + from_objects = _lists_from_tables(grid, objects) if objects and axis is None else None if axis is not None: sh.cross = _analyze_crosstab(grid, *axis) sh.region = set(sh.cross.region) sh.failures.extend(sh.cross.failures) else: - sh.lists = _analyze_lists(grid) + sh.lists = from_objects if from_objects is not None else _analyze_lists(grid) sh.failures.extend(sh.lists.failures) for b in sh.lists.blocks: sh.region |= b.region @@ -1111,7 +1217,8 @@ def _measure_column(label: str, pos: int) -> tuple[str, str | None, str | None]: def _analyze(scan: WorkbookScan, grids: dict[str, Grid]) -> _Analysis: visible = _visible_grids(scan, grids) hidden = [s.name for s in scan.sheets if s.state != "visible"] - sheets = [_analyze_sheet(g, f"s{i + 1}") for i, g in enumerate(visible)] + sheets = [_analyze_sheet(g, f"s{i + 1}", [t for t in scan.tables if t.sheet == g.sheet]) + for i, g in enumerate(visible)] failures = [f for sh in sheets for f in sh.failures] tables_used: set[str] = set() @@ -1266,7 +1373,9 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] 合计表名撞了已用的名字时照起草器的老行为:不另建表定义,keep_as 仍写这个名字(交给静态校验报)。""" b.id = block_id if table is None: - title = b.title if b.title and not any(ch.isdigit() for ch in b.title) else None + # 期 5:按表格对象起草的,表名优先用表格对象的名字(不含数字时;Excel 默认的「表1」不用) + named = b.table_object if b.table_object and not any(ch.isdigit() for ch in b.table_object) else None + title = named or (b.title if b.title and not any(ch.isdigit() for ch in b.title) else None) want = _ENUM_PREFIX.sub("", P.split_unit_suffix(title)[0]) if title else sheet n_fallback = len(tables_used) + 1 b.table = _unique(to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}", tables_used) @@ -1293,6 +1402,11 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] total_spec = {"name": keep, "grain": ["合计项"], "kind": "reported_total", "units": num_units, "note": ""} b.keep_table = keep total = {"label_column": b.names[b.cols[0]], "pick": b.total_word, "keep_as": keep} + subtotals = None + if b.subtotal_word is not None: + # 期 5:穿插的小计只核对、不另存(另存的小计表不能按期累积,要另存由人在配方面板里打开) + subtotals = {"label_column": b.names[b.cols[0]], "pick": b.subtotal_word, "scope": "since_previous", + "group_column": None, "keep_as": None} grain = [b.names[b.grain_col]] if b.grain_col is not None else [] main = {"name": b.table, "grain": grain, "kind": "data", "units": dict(b.units), "note": ""} fill = bool(b.merged_fill) and all(r2 > r1 and c1 == c2 and b.types.get(c1) == "TEXT" @@ -1300,7 +1414,8 @@ def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str] block = { "id": b.id, "layout": "list", "table": b.table, "header_rows": len(b.header_rows), "after_title": None, "columns": cols_out, "extra_columns": "reject", - "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total}, + "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total, + **({"subtotals": subtotals} if subtotals is not None else {})}, "values": {"placeholders": [{"text": e[0], "meaning": "无数据"} for e in b.placeholders.values()], "blank": "null", "text_number": "reject", "formula": "accept_cached"}, "merged_data": "fill" if fill else "reject", @@ -1797,6 +1912,8 @@ class _ListLoc: rows: list[int] total_row: int | None region: set[tuple[int, int]] = field(default_factory=set) + #: 期 5:穿插的小计行 + subtotal_rows: list[int] = field(default_factory=list) def _find_grid(name: str, grids: dict[str, Grid]) -> Grid | None: @@ -1943,19 +2060,26 @@ def _locate_list(block: ListBlock, grid: Grid, taken: set[int]) -> _ListLoc | No bottom = _bottom(grid) rr = r + 1 pick = P.match_key(block.rows.total_row.pick) if block.rows.total_row else None + sub_pick = P.match_key(block.rows.subtotals.pick) if block.rows.subtotals else None + subs: list[int] = [] while rr <= bottom: if all(grid.get(rr, c) is None for c in cols): if block.rows.blank_rows == "skip": rr += 1 continue break - if pick and P.match_key(_label_of(grid.get(rr, cols[0])) or "").startswith(pick): + label = P.match_key(_label_of(grid.get(rr, cols[0])) or "") + if pick and label.startswith(pick): total = rr break + if sub_pick and label.startswith(sub_pick): + subs.append(rr) + rr += 1 + continue rows.append(rr) rr += 1 - loc = _ListLoc(hrows, cols, rows, total) - for row in [*hrows, *rows, *([total] if total else [])]: + loc = _ListLoc(hrows, cols, rows, total, subtotal_rows=subs) + for row in [*hrows, *rows, *subs, *([total] if total else [])]: loc.region.update((row, c) for c in cols) return loc @@ -2130,6 +2254,9 @@ def questions_for(recipe: dict[str, Any], facts: DraftFacts, grids: dict[str, Gr if ext is not None and ext.sheets.skipped_hidden: for nm in _dedupe([item.get("sheet", "") for item in ext.sheets.skipped_hidden]): cards.append(_hidden_sheet_card(nm)) + # 期 5:列表每期怎么更新的问题排在其余问题之后、q_mode 之前:它只和按期累积有关,不该顶掉期 2 起排在前面的问题 + questions = ([q for q in questions if not q.id.startswith("q_list_period:")] + + [q for q in questions if q.id.startswith("q_list_period:")]) blockers = accumulate_blockers(parsed) cards.append(_mode_card(blockers)) if not blockers: @@ -2543,9 +2670,57 @@ def _list_cards(si: int, bi: int, sheet: Any, block: ListBlock, loc: _ListLoc | reason += f",原值另存表「{total.keep_as}」" if total.keep_as else ",只核对不另存" cells = _coords(name, [(loc.total_row, loc.cols[0])]) if loc is not None and loc.total_row else [] cards.append(Card(f"total_row:{block.id}", f"{where}是合计行,改作核对", reason, cells)) + subt = block.rows.subtotals + if subt is not None: + rows = loc.subtotal_rows if loc is not None else [] + where = _rows_text(rows) if rows else f"「{subt.pick}」开头的行" + scope = ("上一行小计之后的全部明细" if subt.scope == "since_previous" + else f"紧挨在它上面、「{subt.group_column}」相同的那几行明细") + reason = f"每行小计按列求和与{scope}核对,小计行不进数据" + reason += f",原值另存表「{subt.keep_as}」" if subt.keep_as else ",只核对不另存" + cells = _coords(name, [(r, loc.cols[0]) for r in rows]) if loc is not None else [] + cards.append(Card(f"subtotals:{block.id}", f"{where}是穿插的小计行,改作核对", reason, cells)) + q = _list_period_question(si, bi, sheet, block, recipe) + questions.append(q) + cards.append(Card(f"list_period:{block.id}", "这份明细每期怎么更新", + "每期替换:当前版本只含最新一期。按期累积:每期一份明细加进当前版本、可以跨期查询,要说明每一行" + "属于哪一期——选一个日期列,核对每行都在本期统计期内;明细里没有能代表统计期的日期列时," + "选「每行加上本期统计期的起止两列」", [], q.id)) return cards, questions +def _list_period_question(si: int, bi: int, sheet: Any, block: ListBlock, recipe: dict[str, Any]) -> Question: + """期 5:列表每期怎么更新、按期累积时每一行属于哪一期(ListBlock.period),一道题定:每期替换(不绑定);按期 + 累积并按某个日期列核对(每个日期列一个选项);按期累积并给每行加统计期起止两列。后两种同时把导入模式改成按期 + 累积,所在工作表没配统计期的一并加上统计期(回执里统计期取自表内,没有就请人录入)。 + + 为什么不分成「先绑定、再问 q_mode」两道题:问题集合按起草的草稿生成、回答时不变(recipe_imports.answer), + 列表的草稿还没绑定统计期,不符合按期累积的资格,q_mode 根本不会出现。几个列表各答各的:有的选累积、有的选替换 + 时,静态校验照实报出没绑定的那个列表不能累积。""" + base = ["sheets", si, "blocks", bi, "period"] + context = [] if sheet.context else _add_ops(recipe, ["sheets", si, "context"], [ + {"id": "统计期", "kind": "period", "parser": "cn_date_range", "prefer_prefix": None, "cross_check": "filename"}]) + mode = _add_ops(recipe, ["mode"], "accumulate") + # 「每期替换」排第一个:它就是草稿本来的样子(期 4 起各处取第一个选项的地方都不会因此改了配方) + options: list[dict[str, Any]] = [{"value": "none", "label": "每期替换", "needs_reason": False}] + effects: dict[str, list[dict[str, Any]]] = {"none": _add_ops(recipe, base, None)} + for c in block.columns: + if c.type != "DATE": + continue + value = f"column:{c.name}" + options.append({"value": value, "label": f"按期累积,按「{c.name}」核对每行都在本期统计期内", + "needs_reason": False}) + effects[value] = context + mode + _add_ops(recipe, base, {"column": c.name, "check": "within_context", + "synthesize": None}) + options.append({"value": "synthesize", "label": "按期累积,每行加上本期统计期的起止两列", "needs_reason": False}) + effects["synthesize"] = context + mode + _add_ops(recipe, base, {"column": None, "check": "within_context", + "synthesize": "统计期"}) + current = block.period + default = ("synthesize" if current is not None and current.synthesize else + f"column:{current.column}" if current is not None and current.column else "none") + return Question(f"q_list_period:{block.id}", f"列表「{block.id}」每期怎么更新", options, default, effects) + + # ========================================================================== # 期 3:按规则重新起草,并把名字对齐到现行配方(P3-SPEC 6.3) # ========================================================================== diff --git a/backend/app/data/recipe_types.py b/backend/app/data/recipe_types.py index 7643144f..c4b8d62c 100644 --- a/backend/app/data/recipe_types.py +++ b/backend/app/data/recipe_types.py @@ -300,11 +300,29 @@ class TotalRow(_M): keep_as: str | None = None +class Subtotals(_M): + """穿插在明细中间的小计行:不作数据,按它所管的那一组明细逐列求和核对;keep_as 给了就另存一张表。""" + + #: 小计标签所在的列(列名) + label_column: str + #: 小计行的标签以这个词开头(match_key 比较)。取自实际格子文字的候选词,不能含数字 + pick: str = Field(min_length=1, max_length=12) + #: 一行小计管哪几行明细:since_previous 上一行小计(或表头)之后的全部明细;group_column 紧挨在它上面、 + #: group_column 这一列取值相同的那几行明细 + scope: Literal["since_previous", "group_column"] = "since_previous" + #: scope=group_column 时按哪一列分组(列名);since_previous 时为 None + group_column: str | None = None + #: 原表的小计另存成哪张表(kind=reported_total);None 只核对不另存 + keep_as: str | None = None + + class ListRows(_M): #: 数据中间的空行:stop 到此为止(默认);skip 跳过继续(回执记个数) blank_rows: Literal["stop", "skip"] = "stop" #: 合计行:到合计行为止,合计行按列求和核对 total_row: TotalRow | None = None + #: 期 5:穿插在明细中间的小计行(Subtotals)。默认 None = 期 4 的行为(像小计的行照常是数据或起草失败) + subtotals: Subtotals | None = None class ListValues(_M): @@ -317,6 +335,18 @@ class ListValues(_M): formula: Literal["accept_cached", "reject"] = "accept_cached" +class ListPeriod(_M): + """列表按期累积时,每一行属于哪一期,二选一:column 按表里的一个日期列,每行的日期都要落在本期的统计期内; + synthesize 每行加上本期统计期的起、止两列(明细里没有日期列,或者日期不代表它属于哪一期时)。""" + + #: 按这个日期列(ListColumn.name,类型 DATE)核对;和 synthesize 二选一 + column: str | None = None + #: 怎么核对:within_context 每行的日期都要落在本期统计期内(空着的也不行) + check: Literal["within_context"] = "within_context" + #: 加统计期起、止两列(LIST_PERIOD_START、LIST_PERIOD_END);和 column 二选一 + synthesize: Literal["统计期"] | None = None + + class ListBlock(_M): id: str = Field(min_length=1, max_length=32) layout: Literal["list"] @@ -333,6 +363,8 @@ class ListBlock(_M): merged_data: Literal["reject", "fill"] = "reject" #: 期 3:按表头忽略的列(IgnoreColumn),只针对这些列;其余多出的列照 extra_columns。默认空 = 期 2 的行为 ignore_columns: list[IgnoreColumn] = Field(default_factory=list, max_length=50) + #: 期 5:按期累积时每一行属于哪一期(ListPeriod)。默认 None = 期 4 的行为(列表只能每期替换) + period: ListPeriod | None = None Block = Annotated[Union[CrosstabBlock, ListBlock], Field(discriminator="layout")] @@ -462,13 +494,14 @@ CheckStatus = Literal["passed", "mismatch", "unverifiable", "info"] @dataclass class CheckResult: - #: 稳定 id:K1… 合计核对、G1… 公式引用、T1… 列表合计、R1… 关系(沿用配方里的 id)、C1 统计期、C2 文件名、 - #: N1… 行数、P1… 主键;期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期(物化时跑,都是结构类,P3-SPEC 2.7)。 + #: 稳定 id:K1… 合计核对、G1… 公式引用、T1… 列表合计(期 5 起也含列表中间的小计)、R1… 关系(沿用配方里的 + #: id)、C1 统计期、C2 文件名、N1… 行数、P1… 主键、D1… 列表的日期落在统计期内(期 5,ListPeriod.column); + #: 期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期(物化时跑,都是结构类,P3-SPEC 2.7)。 #: F 编号留给系统发现的事实(Fact.id),两者不混用 id: str kind: Literal["derived_sum", "formula_refs", "column_sum", "relation_sum_eq", "relation_not_comparable", "context_agree", "filename_period", "row_count", "pk_unique", - "union_rows", "union_pk", "union_period"] + "union_rows", "union_pk", "union_period", "period_rows"] title: str status: CheckStatus #: 不通过时的类别:structure / data_quality / info @@ -759,9 +792,10 @@ class SheetsOut: #: hidden_excluded = hidden.rows=exclude 排除的隐藏行;blank_skipped = 列表 blank_rows=skip 跳过的空行; #: ignored_rows = 交叉表 ignore_rows 忽略的行;ignored_outside = ignore_outside 忽略了区域外数字格的行; #: after_stop = 列表停止之后没导入的文字行;total_not_kept = 参与核对、但不另存的合计行(交叉表 keep_as 为空的 -#: derived 段、列表 total_row.keep_as 为空的合计行) +#: derived 段、列表 total_row.keep_as 为空的合计行);期 5:subtotal = 列表中间的小计行(参与核对、不进数据, +#: 另存与否都记在这里:它们都不在本表里) ExcludedReason = Literal["hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", - "total_not_kept"] + "total_not_kept", "subtotal"] @dataclass @@ -1031,6 +1065,15 @@ def prose_problems(template: str, *, known: dict[str, set[str]] | None = None) - #: 列表合计行另存成表时,合计项列的固定列名 LIST_TOTAL_DIM = "合计项" +#: 期 5:列表的小计行另存成表时,小计项列的固定列名 +LIST_SUBTOTAL_DIM = "小计项" +#: 期 5:列表按期累积、由系统给每行加统计期时(ListPeriod.synthesize)的两列:本期统计期的第一天、最后一天, +#: 写法 YYYY-MM-DD(同 DATE 列,存 TEXT) +LIST_PERIOD_START = "统计期起" +LIST_PERIOD_END = "统计期止" +#: 期 5:小计核对的键(核对编号 T、合计格 DerivedItem.segment)= 块 id 加这个后缀。块 id 在整份配方内唯一, +#: 同一块的合计行用块 id 本身,两者不相混 +SUBTOTAL_KEY = "·小计" _SQL_TYPE = {"INTEGER": "INTEGER", "REAL": "REAL", "TEXT": "TEXT", "DATE": "TEXT"} @@ -1107,16 +1150,42 @@ def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[Reci cols = [ColumnOut(c.name, _SQL_TYPE[c.type], header=c.header, role="measure" if c.type in ("INTEGER", "REAL") else "text") for c in block.columns] + if block.period is not None and block.period.synthesize: + # 期 5:每行加上本期统计期的起止(role=period:没有格子来历,取值来自统计期) + cols += [ColumnOut(LIST_PERIOD_START, "TEXT", role="period"), + ColumnOut(LIST_PERIOD_END, "TEXT", role="period")] put(block.table, cols, base) total = block.rows.total_row + numeric = [c for c in cols if c.type in ("INTEGER", "REAL")] if total is not None and total.keep_as: tcols = [ColumnOut(LIST_TOTAL_DIM, "TEXT", role="dim")] - tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") - for c in cols if c.type in ("INTEGER", "REAL")] + tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") for c in numeric] put(total.keep_as, tcols, f"{base}/rows/total_row") + sub = block.rows.subtotals + if sub is not None and sub.keep_as: + # 期 5:小计另存:小计项(标签的规范写法)、按组小计时的分组列、各数字列 + scols = [ColumnOut(LIST_SUBTOTAL_DIM, "TEXT", role="dim")] + if sub.scope == "group_column" and sub.group_column: + scols.append(ColumnOut(sub.group_column, "TEXT", role="dim")) + scols += [ColumnOut(c.name, c.type, header=c.header, role="value") for c in numeric] + put(sub.keep_as, scols, f"{base}/rows/subtotals") return out, problems +def list_period_columns(recipe: Recipe) -> dict[str, tuple[str, ...]]: + """期 5:列表写入的表 → 它按什么分期:("column", 日期列) 或 ("synthesize", 统计期起列, 统计期止列)。只列配了 + ListBlock.period 的列表;累积计划的 U3、单期的 D 核对、说明里「按哪一列筛选」共用这一份。""" + out: dict[str, tuple[str, ...]] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock) and block.period is not None: + if block.period.synthesize: + out.setdefault(block.table, ("synthesize", LIST_PERIOD_START, LIST_PERIOD_END)) + elif block.period.column: + out.setdefault(block.table, ("column", block.period.column)) + return out + + # ========================================================================== # 期 3:按期累积的资格(静态校验、起草器、累积计划共用同一份,P3-SPEC 2.2) # ========================================================================== @@ -1130,10 +1199,13 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: 1. 至少一个工作表配置了统计期上下文(SheetRecipe.context 非空),否则报 accumulate_needs_period(path=/mode); 2. derive_tables 推出的每张表都「按统计期分得开」,否则逐表报 accumulate_unkeyed(path=/tables/{ti},表没在 tables 里声明时指向第一个写入它的块),消息写明哪一条不满足: - - 只由交叉表块写入(measures、dimension 段,或者 derived 段的 keep_as)——列表写入的表一律不满足; - - 写入它的每个交叉表块,axis.checks 都含 covers_context(日期恰好落在统计期内); - - 这张表的 grain 含那个块的 axis.name。 - 两条都成立时,各期的日期集合互不相交(统计期不重叠由累积计划保证),主键里不需要另加统计期列。 + - 交叉表写入的(measures、dimension 段,或者 derived 段的 keep_as):写入它的每个交叉表块,axis.checks 都含 + covers_context(日期恰好落在统计期内),并且这张表的 grain 含那个块的 axis.name; + - 期 5:列表写入的明细表:写入它的每个列表块都配了 period(按日期列核对,或加统计期起止两列),并且所在的 + 工作表配了统计期。列表的合计行、小计行另存的表不能累积:各期的合计项、小计项相同,主键分不开各期; + - 同一张表不能既由交叉表又由列表写入。 + 都成立时,各期的行按统计期分得开(统计期不重叠由累积计划保证;列表明细跨期重复的同一条记录由并集的主键核对 + U2 拒收)。 """ problems: list[RecipeProblem] = [] if not any(sheet.context for sheet in recipe.sheets): @@ -1144,6 +1216,11 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: writers: dict[str, list[CrosstabBlock | ListBlock]] = {} first_path: dict[str, str] = {} + #: 列表的合计行、小计行另存的表 → 写入它的块 + totals: dict[str, list[ListBlock]] = {} + #: 列表块 → 所在工作表有没有配统计期 + ctx_of: dict[int, bool] = {} + def _add(table: str, block: CrosstabBlock | ListBlock, path: str) -> None: got = writers.setdefault(table, []) if all(b is not block for b in got): @@ -1161,16 +1238,32 @@ def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: _add(seg.keep_as.table, block, path) else: _add(block.table, block, path) - total = block.rows.total_row - if total is not None and total.keep_as: - _add(total.keep_as, block, path) + ctx_of[id(block)] = bool(sheet.context) + for kept in (block.rows.total_row.keep_as if block.rows.total_row else None, + block.rows.subtotals.keep_as if block.rows.subtotals else None): + if kept: + _add(kept, block, path) + totals.setdefault(kept, []).append(block) index = {t.name: ti for ti, t in enumerate(recipe.tables)} grains = {t.name: list(t.grain) for t in recipe.tables} for name in tables: blocks = writers.get(name, []) - if any(isinstance(b, ListBlock) for b in blocks): - reasons = ["列表形态的表目前只支持每期替换"] + lists = [b for b in blocks if isinstance(b, ListBlock)] + if lists and len(lists) != len(blocks): + reasons = ["同一张表既由交叉表又由列表写入,各期的行分不开"] + elif name in totals: + reasons = ["列表的合计行、小计行另存的表不能按期累积:各期的合计项、小计项相同,主键分不开各期" + "(可以改为只核对、不另存)"] + elif lists: + reasons = [] + unbound = [b.id for b in lists if b.period is None] + if unbound: + reasons.append("、".join(f"列表「{x}」" for x in unbound) + + "没有指定每一行属于哪一期(按日期列核对,或加统计期起止两列)") + no_ctx = [b.id for b in lists if b.period is not None and not ctx_of.get(id(b))] + if no_ctx: + reasons.append("、".join(f"列表「{x}」" for x in no_ctx) + "所在的工作表没有配置统计期") else: reasons = [] uncovered = [b.id for b in blocks if isinstance(b, CrosstabBlock) and "covers_context" not in b.axis.checks] diff --git a/backend/app/data/xlsx_cells.py b/backend/app/data/xlsx_cells.py index 28f137e0..c270f8bb 100644 --- a/backend/app/data/xlsx_cells.py +++ b/backend/app/data/xlsx_cells.py @@ -24,7 +24,7 @@ from collections.abc import Iterator from typing import Any from xml.etree import ElementTree as ET -from app.data import tabular +from app.data import csv_grid, tabular from app.data.recipe_types import Grid, GridCell from app.data.xlsx_scan import SheetScan, UnsupportedTable, WorkbookScan, _broken, cell_ref, parse_ref @@ -70,6 +70,13 @@ class Workbooks: self.raw = raw self._values: Any = None self._formulas: Any = None + self._csv: Any = None + + def csv(self) -> Any: + """CSV 原件解析好的记录(csv_grid.CsvTable),第一次要读时才解析,同一次执行里复用。""" + if self._csv is None: + self._csv = csv_grid.parse(self.raw) + return self._csv def values(self) -> Any: if self._values is None: @@ -126,6 +133,10 @@ def sheet_rows(books: Workbooks, sheet: SheetScan, *, min_row: int, max_row: int 行数和每行的宽度都一样。文件内容写坏了(见 _READ_ERRORS)抛 UnsupportedTable,日志只记异常类型。 """ try: + if sheet.path == csv_grid.CSV_PATH: + # CSV 适配出来的「工作表」:从解析好的记录里取,口径同下(只给非空格、整行为空的不给) + yield from csv_grid.rows(books.csv(), min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col) + return yield from _sheet_rows(books, sheet, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col) except UnsupportedTable: raise diff --git a/backend/app/data/xlsx_scan.py b/backend/app/data/xlsx_scan.py index 4910c6ef..5acd20fd 100644 --- a/backend/app/data/xlsx_scan.py +++ b/backend/app/data/xlsx_scan.py @@ -216,6 +216,9 @@ class WorkbookScan: tables: list[TableScan] = field(default_factory=list) #: 定义名称 → 原文。工作表级的名称记作「工作表名!名称」,免得同名互相覆盖 defined_names: dict[str, str] = field(default_factory=dict) + #: 原件的格式:xlsx,或者 csv(csv_grid.scan 适配出来的一张「工作表」,读取层据 SheetScan.path 从 CSV 里读)。 + #: 老暂存区的扫描结果里没有这个键,按 xlsx 读 + format: str = "xlsx" def sheet(self, name: str) -> SheetScan | None: return next((s for s in self.sheets if s.name == name), None) @@ -1090,4 +1093,5 @@ def scan_from_json(data: dict[str, Any]) -> WorkbookScan: for t in tables: t.columns = list(t.columns) return WorkbookScan(sheets=sheets, full_calc_on_load=bool(data.get("full_calc_on_load")), tables=tables, - defined_names=dict(data.get("defined_names") or {})) + defined_names=dict(data.get("defined_names") or {}), + format=str(data.get("format") or "xlsx")) diff --git a/backend/tests/test_csv_grid.py b/backend/tests/test_csv_grid.py new file mode 100644 index 00000000..30a5f78a --- /dev/null +++ b/backend/tests/test_csv_grid.py @@ -0,0 +1,111 @@ +"""CSV 走配方(Excel 期 5 第 4 项):CSV 适配成网格,后面的执行器、核对、溯源全部复用。 + +格子坐标沿用「工作表!A1」(见 csv_grid 的模块说明):CSV 没有工作表,名字固定是「CSV」,每月的文件名变了也认得出。 +""" +from __future__ import annotations + +import pytest + +from app.data import csv_grid, recipe_engine, xlsx_cells +from app.data.recipe_types import Recipe +from app.data.xlsx_scan import UnsupportedTable, scan_from_json, scan_to_json + +CSV = ("订单号,下单日期,门店,金额,备注\n" + "A001,2026-08-01,一号店,120,\n" + "A002,2026-08-02,二号店,80.5,\"带,逗号\"\n" + "\n" + "A003,2026-08-03,一号店,1,234,多出的一格\n").encode("utf-8") + + +def test_text_is_sniffed_as_csv_and_binary_is_not(): + assert csv_grid.looks_like_csv(CSV) + assert csv_grid.looks_like_csv("a;b\n1;2\n".encode("gb18030")) + assert csv_grid.looks_like_csv(b"\xff\xfe" + "a\tb\n".encode("utf-16-le")) + assert not csv_grid.looks_like_csv(b"PK\x03\x04rest-of-zip") + assert not csv_grid.looks_like_csv(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100) # 旧版 .xls + assert not csv_grid.looks_like_csv(b"\x89PNG\r\n\x1a\n\x00\x00") + assert not csv_grid.looks_like_csv(b"") + + +def test_scan_gives_one_visible_sheet_with_bounds_and_row_counts(): + scan = csv_grid.scan(CSV) + assert scan.format == "csv" and len(scan.sheets) == 1 + sh = scan.sheets[0] + assert sh.name == csv_grid.CSV_SHEET and sh.state == "visible" and sh.path == csv_grid.CSV_PATH + assert (sh.bounds.min_row, sh.bounds.min_col, sh.bounds.max_row, sh.bounds.max_col) == (1, 1, 5, 6) + # 空字段不算非空格;第 4 行是空行 + assert sh.row_runs == [(1, 1, 5), (2, 2, 4), (3, 3, 5), (5, 5, 6)] + assert sh.nonempty == 20 + again = scan_from_json(scan_to_json(scan)) + assert again.format == "csv" and again.sheets[0].path == csv_grid.CSV_PATH + + +def test_cell_values_are_typed_only_when_unambiguous(): + """整数、小数写成数;千分位、前导零、正号、科学计数法、日期都留作文字,交给配方的规则(text_number、类型)判。""" + v = csv_grid.cell_value + assert v("120") == 120 and v(" 80.5 ") == 80.5 and v("-3") == -3 and v("0") == 0 + for text in ("1,234", "007", "+5", "1e5", "2026-08-01", "12%", "TRUE", "=SUM(A1:A2)", "1234567890123456"): + assert v(text) == text, text + assert v("") is None and v(" ") is None + + +def test_grid_and_rows_read_the_same_cells(): + scan = csv_grid.scan(CSV) + grid = xlsx_cells.read_grid(CSV, scan, csv_grid.CSV_SHEET) + assert grid.get(2, 4).value == 120 and grid.get(3, 4).value == 80.5 and grid.get(3, 5).value == "带,逗号" + assert grid.get(2, 5) is None and grid.get(1, 1).value == "订单号" + assert grid.get(5, 4).value == 1 and grid.get(5, 5).value == 234 # 没加引号的千分位被逗号拆开了:照实读 + rows = list(xlsx_cells.iter_rows(CSV, scan, csv_grid.CSV_SHEET, min_row=1, max_row=5, min_col=1, max_col=6)) + assert [r for r, _ in rows] == [1, 2, 3, 5] + assert sum(len(cells) for _, cells in rows) == len(grid.cells) == scan.sheets[0].nonempty + small = xlsx_cells.read_grid(CSV, scan, csv_grid.CSV_SHEET, max_rows=2) + assert small.truncated and set(r for r, _ in small.cells) == {1, 2} + + +def test_semicolon_and_tab_delimiters_and_gbk(): + scan = csv_grid.scan("名称;数量\n甲;3\n".encode("gb18030")) + grid = xlsx_cells.read_grid("名称;数量\n甲;3\n".encode("gb18030"), scan, csv_grid.CSV_SHEET) + assert grid.get(2, 1).value == "甲" and grid.get(2, 2).value == 3 + raw = "a\tb\n1\t2\n".encode("utf-8") + assert xlsx_cells.read_grid(raw, csv_grid.scan(raw), csv_grid.CSV_SHEET).get(2, 2).value == 2 + + +def test_scan_any_dispatches_and_keeps_the_xlsx_refusal(): + assert csv_grid.scan_any(CSV).format == "csv" + with pytest.raises(UnsupportedTable): + csv_grid.scan_any(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100) + + +def test_too_many_columns_is_refused(): + raw = (",".join(["x"] * (csv_grid.MAX_COLS + 1)) + "\n").encode() + with pytest.raises(UnsupportedTable) as e: + csv_grid.scan(raw) + assert "列" in str(e.value) + + +LIST_RECIPE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "CSV"}, + "blocks": [{"id": "列表1", "layout": "list", "table": "订单", + "columns": [{"header": "订单号", "name": "订单号", "type": "TEXT"}, + {"header": "下单日期", "name": "下单日期", "type": "DATE"}, + {"header": "门店", "name": "门店", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "REAL"}], + "rows": {"blank_rows": "stop"}}]}], + "tables": [{"name": "订单", "grain": ["订单号"]}], +} + + +def test_engine_runs_a_list_recipe_on_csv(tmp_path): + raw = "订单号,下单日期,门店,金额\nA001,2026-08-01,一号店,120\nA002,2026-08-02,二号店,80.5\n".encode() + scan = csv_grid.scan(raw) + db = tmp_path / "t.sqlite" + ex = recipe_engine.execute(Recipe.model_validate(LIST_RECIPE), raw, "orders_2026-08.csv", scan, str(db)) + assert ex.ok, [p.message for p in ex.problems] + assert ex.tables[0].rows == 2 and ex.tables[0].sheet == "CSV" + # 溯源照常:订单号这一列从 CSV!A2 起往下 2 格 + run = ex.lineage["订单"]["订单号"][0] + assert run[1:] == ["CSV", "A2", 2, "down"] + import sqlite3 + rows = sqlite3.connect(db).execute('SELECT "订单号", "下单日期", "金额" FROM "订单" ORDER BY rowid').fetchall() + assert rows == [("A001", "2026-08-01", 120.0), ("A002", "2026-08-02", 80.5)] diff --git a/backend/tests/test_recipe_contract.py b/backend/tests/test_recipe_contract.py index 34800e71..50b1f0b9 100644 --- a/backend/tests/test_recipe_contract.py +++ b/backend/tests/test_recipe_contract.py @@ -370,10 +370,12 @@ def test_accumulate_blockers_reference_recipe_is_eligible(): def test_accumulate_blockers_list_shape_is_unkeyed(): data = json.loads(json.dumps(C01_LIKE)) - data["sheets"][0]["context"] = [{}] # 即使配了统计期,列表写入的表也不能累积 + data["sheets"][0]["context"] = [{}] # 配了统计期,列表却没说每一行属于哪一期:不能累积 got = T.accumulate_blockers(T.Recipe.model_validate(data)) assert [(p.code, p.path) for p in got] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] - assert all("列表形态的表目前只支持每期替换" in p.message for p in got) + assert "没有指定每一行属于哪一期" in got[0].message + # 合计行另存的表:各期的合计项相同,主键分不开各期(期 5 列表可以累积以后仍然不行) + assert "合计行、小计行另存的表不能按期累积" in got[1].message assert "「月报」" in got[0].message and "「月报_表内合计」" in got[1].message # c01 的草稿本来就没有统计期:两条都报,统计期那条在前(起草卡片取第一条的人话) got = T.accumulate_blockers(T.Recipe.model_validate(C01_LIKE)) @@ -479,7 +481,8 @@ def test_closed_value_sets(): assert {"union_rows", "union_pk", "union_period"} <= set(_literal(T.CheckResult, "kind")) assert "ignored" in typing.get_args(T.Role) and "ignored_column" in typing.get_args(T.Role) assert typing.get_args(T.ExcludedReason) == ( - "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept") + "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept", + "subtotal") assert _literal(T.FixAnchor, "kind") == ("problem", "recipe_problem", "sheet_renamed") assert _literal(T.Anchor, "kind") == ( "header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text") diff --git a/backend/tests/test_recipe_engine_messages.py b/backend/tests/test_recipe_engine_messages.py index 26dd3d84..fef58344 100644 --- a/backend/tests/test_recipe_engine_messages.py +++ b/backend/tests/test_recipe_engine_messages.py @@ -318,6 +318,18 @@ def c_total_row_blank(mp): return _run(sales_book(edit=edit), LIST), [N] +def c_subtotal_blank(mp): + """期 5:穿插的小计行数字格全空:没有一列能核对,结构类。""" + data = list_recipe() + data["sheets"][0]["blocks"][0]["rows"]["subtotals"] = {"label_column": "地区", "pick": "小计"} + + def edit(ws): + ws["D4"] = N + ws.insert_rows(6) + ws["A6"] = "小计" # 第 6 行是小计:标签在,数字格全空 + return _run(sales_book(edit=edit), data), [N] + + def c_value_not_integer(mp): return _run(_flow(_set(C11=9876.54)), FLOW), [9876.54] @@ -423,6 +435,7 @@ NEXT_STEPS = { "hidden_cols": "在配方面板的「隐藏列」中选择「照常导入」", "total_row_missing": "在配方面板的「合计行」中取消「有合计行」", "total_row_blank": "在配方面板的「合计行」中取消「有合计行」", + "subtotal_blank": "在配方面板的「小计行」中取消「有小计行」", } diff --git a/backend/tests/test_recipe_notes_p3.py b/backend/tests/test_recipe_notes_p3.py index ba454c55..cfc9a6e8 100644 --- a/backend/tests/test_recipe_notes_p3.py +++ b/backend/tests/test_recipe_notes_p3.py @@ -532,7 +532,8 @@ def test_problems_for_inconsistent_inputs(): def test_list_tables_cannot_be_merged_and_are_problems(): - """列表不能按期累积(P3-SPEC 2.2):真把两期列表并起来,列表合计没有并集说法,记 problems,绝不沿用单期说法。""" + """列表合计行另存的表不能按期累积(accumulate_blockers):真把两期这样的列表并起来,列表合计没有并集说法,记 + problems,绝不沿用单期说法。""" data = deepcopy(LIST_RECIPE) def lpart(span): @@ -544,7 +545,7 @@ def test_list_tables_cannot_be_merged_and_are_problems(): tgt = Recipe.model_validate(data) notes = build_union_notes(tgt, [lpart(AUG), lpart(SEP)], union_tables=derive_tables(tgt)[0], null_counts=None, part_null_counts=None, added=[], retired=[], label_sets=[], gaps=False) - assert any("列表合计行的核对结论不能按期合并" in p for p in notes.problems) + assert any("列表合计行、小计行的核对结论不能按期合并" in p for p in notes.problems) assert not any("list_total" in p for p in notes.problems) # 不露键名 assert "一致" not in notes.tables["销售_表内合计"].comment diff --git a/backend/tests/test_recipe_p5.py b/backend/tests/test_recipe_p5.py new file mode 100644 index 00000000..b635a014 --- /dev/null +++ b/backend/tests/test_recipe_p5.py @@ -0,0 +1,435 @@ +"""Excel 期 5(下一轮 03 细案 E):列表按期累积、穿插的小计、表格对象起草、CSV 走配方。 + +不连接口的部分:执行器、核对、静态校验、累积资格、并集物化、起草。夹具全是合成的通用名字(门店一、分区甲、 +订单 A001……),数字是手写的小数,手算得出。接口的端到端在 test_recipe_p5_e2e.py。 +""" +from __future__ import annotations + +import copy +import io +import os +import sqlite3 +import tempfile +from pathlib import Path +from typing import Any + +import pytest +from openpyxl import Workbook +from openpyxl.worksheet.table import Table + +from app.data import csv_grid, raw_store, recipe_checks, recipe_engine, recipe_suggest, table_versions, xlsx_cells +from app.data import xlsx_scan +from app.data.recipe import validate_recipe +from app.data.recipe_accumulate import materialize_union +from app.data.recipe_types import ( + LIST_PERIOD_END, LIST_PERIOD_START, SUBTOTAL_KEY, Recipe, UnionPart, accumulate_blockers, derive_tables, +) + + +def _book(rows: list[tuple[Any, ...]], *, sheet: str = "明细", top: list[tuple[str, Any]] | None = None, + start: int = 1) -> bytes: + wb = Workbook() + ws = wb.active + ws.title = sheet + for ref, value in top or []: + ws[ref] = value + for r, row in enumerate(rows, start=start): + for c, v in enumerate(row, start=1): + if v is not None: + ws.cell(r, c, v) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def _run(raw: bytes, recipe: dict[str, Any], db: str | None = None, filename: str = "明细.xlsx", **kw: Any): + scan = csv_grid.scan_any(raw) + return recipe_engine.execute(Recipe.model_validate(recipe), raw, filename, scan, db, **kw) + + +def _rows(db: str, sql: str) -> list[tuple[Any, ...]]: + conn = sqlite3.connect(db) + try: + return conn.execute(sql).fetchall() + finally: + conn.close() + + +def _checks(db: str, ex: Any, recipe: dict[str, Any]) -> dict[str, Any]: + return {c.id: c for c in recipe_checks.run_checks(db, ex, Recipe.model_validate(recipe))} + + +def _list(columns: list[tuple[str, str]], *, table: str = "明细", sheet: str = "明细", context: bool = False, + **block: Any) -> dict[str, Any]: + out = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": sheet}, + "context": [{"id": "统计期"}] if context else [], + "blocks": [{"id": "列表1", "layout": "list", "table": table, + "columns": [{"header": h, "name": h, "type": t} for h, t in columns], **block}]}], + "tables": [{"name": table, "grain": []}], + } + return out + + +# ========================================================================== +# 穿插的小计 +# ========================================================================== + +STORES = [("地区", "门店", "金额", "数量"), + ("分区甲", "门店一", 100, 1), ("分区甲", "门店二", 200, 2), ("小计", None, 300, 3), + ("分区乙", "门店三", 150, 4), ("分区乙", "门店四", 150, 5), ("小计", None, 300, 9), + ("合计", None, 600, 12)] +STORE_COLS = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER"), ("数量", "INTEGER")] + + +def _stores_recipe(**sub: Any) -> dict[str, Any]: + return _list(STORE_COLS, rows={"total_row": {"label_column": "地区", "pick": "合计"}, + "subtotals": {"label_column": "地区", "pick": "小计", **sub}}) + + +def test_subtotal_rows_are_checked_against_their_group_and_kept_out_of_the_data(tmp_path): + db = str(tmp_path / "s.db") + recipe = _stores_recipe() + ex = _run(_book(STORES), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + # 只有 4 行明细进了数据;两行小计、一行合计都不在 + assert _rows(db, 'SELECT "门店", "金额" FROM "明细" ORDER BY rowid') == [ + ("门店一", 100), ("门店二", 200), ("门店三", 150), ("门店四", 150)] + subs = [d for d in ex.derived if d.segment == f"列表1{SUBTOTAL_KEY}"] + assert [(d.cell, d.rowid_first, d.rowid_last) for d in subs] == [ + ("明细!C4", 1, 2), ("明细!D4", 1, 2), ("明细!C7", 3, 4), ("明细!D7", 3, 4)] + [excluded] = [x for x in ex.rows_excluded if x.reason == "subtotal"] + assert excluded.rows == [[4, 4], [7, 7]] and excluded.anchor == "小计" and excluded.block == "列表1" + checks = _checks(db, ex, recipe) + assert checks["T1"].title == "合计行按列求和(块「列表1」)" and checks["T1"].status == "passed" + assert checks["T2"].title == "小计行按各组明细求和(块「列表1」)" and checks["T2"].status == "passed" + assert checks["T2"].checked == 4 + + +def test_a_wrong_subtotal_is_a_mismatch(tmp_path): + rows = list(STORES) + rows[6] = ("小计", None, 300, 10) # 4 + 5 ≠ 10 + db = str(tmp_path / "s.db") + recipe = _stores_recipe() + ex = _run(_book(rows), recipe, db) + checks = _checks(db, ex, recipe) + t2 = checks["T2"] + assert t2.status == "mismatch" and t2.failed == 1 and t2.cells == ["明细!D7"] + assert "按明细求和为 9" in t2.details[0] + + +def test_group_column_scope_only_covers_the_rows_of_the_same_group(tmp_path): + """按组小计:紧挨在小计上面、分组列取值相同的那几行。小计行的标签写在「门店」列,分组按「地区」。 + 同一份数据按「上一行小计之后的全部明细」核对就对不上(350 ≠ 50)。""" + rows = [("地区", "门店", "金额"), ("分区甲", "门店一", 100), ("分区甲", "门店二", 200), + ("分区乙", "门店三", 50), ("分区乙", "小计", 50)] + cols = [("地区", "TEXT"), ("门店", "TEXT"), ("金额", "INTEGER")] + recipe = _list(cols, rows={"subtotals": {"label_column": "门店", "pick": "小计", "scope": "group_column", + "group_column": "地区", "keep_as": "明细_小计"}}) + recipe["tables"].append({"name": "明细_小计", "kind": "reported_total"}) + db = str(tmp_path / "g.db") + ex = _run(_book(rows), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + assert _checks(db, ex, recipe)["T1"].status == "passed" + assert _rows(db, 'SELECT * FROM "明细_小计"') == [("小计", "分区乙", 50)] + assert [c.name for c in derive_tables(Recipe.model_validate(recipe))[0]["明细_小计"]] == ["小计项", "地区", "金额"] + since = copy.deepcopy(recipe) + since["sheets"][0]["blocks"][0]["rows"]["subtotals"].update(scope="since_previous", group_column=None) + since["tables"][1]["name"] = "明细_小计" + db2 = str(tmp_path / "g2.db") + ex2 = _run(_book(rows), since, db2) + assert _checks(db2, ex2, since)["T1"].status == "mismatch" + + +def test_subtotal_validation(): + def problems(**sub: Any) -> list[str]: + data = _stores_recipe(**sub) + _, probs = validate_recipe(data, facts=None, origin="rules") + return [p.code for p in probs] + + assert problems() == [] + assert "subtotal_group_missing" in problems(scope="group_column") + assert "column_unknown" in problems(scope="group_column", group_column="城市") + assert "subtotal_group_missing" in problems(group_column="地区") + assert "period_literal" in problems(pick="2026小计") + bad = _stores_recipe() + bad["sheets"][0]["blocks"][0]["rows"]["subtotals"]["pick"] = "合计" + assert "subtotal_pick_conflict" in [p.code for p in validate_recipe(bad, facts=None, origin="rules")[1]] + + +# ========================================================================== +# 列表按期累积:每一行属于哪一期 +# ========================================================================== + +PERIOD_CELL = ("A1", "统计时间:2026年8月1日至2026年8月31日") +ORDERS = [("订单号", "下单日期", "金额"), ("A001", "2026-08-01", 120), ("A002", "2026-08-15", 80), + ("A003", "2026-08-31", 40)] +ORDER_COLS = [("订单号", "TEXT"), ("下单日期", "DATE"), ("金额", "INTEGER")] + + +def _orders(period: dict[str, Any], *, context: bool = True, grain: list[str] | None = None) -> dict[str, Any]: + data = _list(ORDER_COLS, table="订单", context=context, period=period) + data["tables"][0]["grain"] = grain if grain is not None else ["订单号"] + return data + + +def test_synthesized_period_columns_carry_this_periods_bounds(tmp_path): + recipe = _orders({"synthesize": "统计期"}, grain=["订单号", LIST_PERIOD_START]) + cols = derive_tables(Recipe.model_validate(recipe))[0]["订单"] + assert [(c.name, c.role) for c in cols][-2:] == [(LIST_PERIOD_START, "period"), (LIST_PERIOD_END, "period")] + db = str(tmp_path / "p.db") + ex = _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, db) + assert ex.ok, [p.message for p in ex.problems] + assert _rows(db, f'SELECT "订单号", "{LIST_PERIOD_START}", "{LIST_PERIOD_END}" FROM "订单" ORDER BY rowid') == [ + ("A001", "2026-08-01", "2026-08-31"), ("A002", "2026-08-01", "2026-08-31"), + ("A003", "2026-08-01", "2026-08-31")] + assert ex.table_hashes["订单"] + # 统计期起止列没有格子来历:溯源只有表头推出来的三列 + assert set(ex.lineage["订单"]) == {"订单号", "下单日期", "金额"} + # 干跑(不建库)同样走得通 + assert _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe).ok + + +def test_synthesized_period_with_a_human_period(tmp_path): + import datetime as dt + + from app.data.recipe_types import PeriodInput + + recipe = _orders({"synthesize": "统计期"}) + raw = _book(ORDERS) + assert {p.code for p in _run(raw, recipe).problems} == {"period_missing"} + db = str(tmp_path / "h.db") + ex = _run(raw, recipe, db, context_inputs={"统计期": PeriodInput(dt.date(2026, 8, 1), dt.date(2026, 8, 31))}) + assert ex.ok and _rows(db, f'SELECT DISTINCT "{LIST_PERIOD_START}" FROM "订单"') == [("2026-08-01",)] + + +def test_rows_must_fall_inside_the_period_when_bound_to_a_date_column(tmp_path): + recipe = _orders({"column": "下单日期"}) + db = str(tmp_path / "w.db") + ex = _run(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, db) + assert ex.ok + d1 = _checks(db, ex, recipe)["D1"] + assert d1.kind == "period_rows" and d1.status == "passed" and d1.checked == 3 + late = list(ORDERS) + [("A004", "2026-09-02", 10)] + db2 = str(tmp_path / "w2.db") + ex2 = _run(_book(late, top=[PERIOD_CELL], start=3), recipe, db2) + d1 = _checks(db2, ex2, recipe)["D1"] + assert d1.status == "mismatch" and d1.category == "structure" and d1.failed == 1 + assert d1.cells == ["明细!B7"] and "2026-08-01 至 2026-08-31" in d1.details[0] + + +def test_list_period_validation(): + def codes(data: dict[str, Any]) -> list[str]: + return [p.code for p in validate_recipe(data, facts=None, origin="rules")[1]] + + assert codes(_orders({"column": "下单日期"})) == [] + assert "list_period_invalid" in codes(_orders({"column": "金额"})) + assert "column_unknown" in codes(_orders({"column": "付款日期"})) + assert "list_period_invalid" in codes(_orders({"column": "下单日期", "synthesize": "统计期"})) + assert "list_period_invalid" in codes(_orders({})) + assert "covers_without_context" in codes(_orders({"synthesize": "统计期"}, context=False)) + clash = _list(ORDER_COLS + [(LIST_PERIOD_START, "TEXT")], table="订单", context=True, + period={"synthesize": "统计期"}) + assert "name_collision" in codes(clash) + + +def test_lists_become_eligible_for_accumulation_once_bound(): + unbound = Recipe.model_validate({**_list(ORDER_COLS, table="订单", context=True), "mode": "accumulate"}) + [p] = accumulate_blockers(unbound) + assert "没有指定每一行属于哪一期" in p.message + for period in ({"column": "下单日期"}, {"synthesize": "统计期"}): + data = {**_orders(period), "mode": "accumulate"} + assert accumulate_blockers(Recipe.model_validate(data)) == [] + assert validate_recipe(data, facts=None, origin="rules")[1] == [] + kept = {**_orders({"column": "下单日期"}), "mode": "accumulate"} + kept["sheets"][0]["blocks"][0]["rows"] = {"total_row": {"label_column": "订单号", "pick": "合计", + "keep_as": "订单_表内合计"}} + kept["tables"].append({"name": "订单_表内合计", "grain": ["合计项"], "kind": "reported_total"}) + blockers = accumulate_blockers(Recipe.model_validate(kept)) + assert [p.path for p in blockers] == ["/tables/1"] and "合计行、小计行另存的表" in blockers[0].message + + +def _built(raw: bytes, recipe: dict[str, Any], tag: str) -> tuple[str, Any]: + folder = Path(tempfile.mkdtemp(prefix="p5-built-")) + db = folder / f"{tag}.db" + ex = _run(raw, recipe, str(db)) + assert ex.ok, [p.message for p in ex.problems] + table_versions.settle_journal(db) + return str(db), ex + + +def _part(db: str, ex: Any, recipe: dict[str, Any], **kw: Any) -> UnionPart: + return UnionPart(import_id=None, start=kw.get("start", ex.period.start), end=kw.get("end", ex.period.end), + db_path=db, recipe=Recipe.model_validate(recipe), db_sha256=raw_store.sha256_file(db), + table_hashes=dict(ex.table_hashes)) + + +@pytest.mark.parametrize("period", [{"column": "下单日期"}, {"synthesize": "统计期"}]) +def test_two_months_of_a_list_materialize_into_one_union(tmp_path, period): + recipe = {**_orders(period), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, "aug") + sep_rows = [ORDERS[0], ("B001", "2026-09-03", 70), ("B002", "2026-09-30", 30)] + sep = _built(_book(sep_rows, top=[("A1", "统计时间:2026年9月1日至2026年9月30日")], start=3), recipe, "sep") + out = tmp_path / "u.db" + report = materialize_union(out, target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe), _part(*sep, recipe)]) + assert report.ok, [c.details for c in report.checks] + assert report.rows == {"订单": 5} + assert {c.id: c.status for c in report.checks} == {"U1": "passed", "U2": "passed", "U3": "passed"} + assert report.checks[2].checked == 2 + # 登记的统计期和库里的对不上(把 8 月的库说成 9 月):U3 拒收 + bad = materialize_union(tmp_path / "bad.db", target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe, start="2026-09-01", end="2026-09-30")]) + u3 = next(c for c in bad.checks if c.id == "U3") + assert not bad.ok and u3.status == "mismatch" and "第 1 期" in u3.details[0] + + +def test_the_same_order_in_two_periods_is_a_union_key_conflict(tmp_path): + """明细跨期重复(同一订单两期都出现)按日期列分期时主键冲突,U2 拒收(细案风险条:语义要在配方里显式声明)。""" + recipe = {**_orders({"synthesize": "统计期"}), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), recipe, "aug2") + sep = _built(_book(ORDERS, top=[("A1", "统计时间:2026年9月1日至2026年9月30日")], start=3), recipe, "sep2") + report = materialize_union(tmp_path / "u.db", target=Recipe.model_validate(recipe), + parts=[_part(*aug, recipe), _part(*sep, recipe)]) + u2 = next(c for c in report.checks if c.id == "U2") + assert not report.ok and u2.status == "mismatch" + # 主键带上统计期起就分得开:每月一份的快照式明细 + keyed = {**_orders({"synthesize": "统计期"}, grain=["订单号", LIST_PERIOD_START]), "mode": "accumulate"} + aug = _built(_book(ORDERS, top=[PERIOD_CELL], start=3), keyed, "aug3") + sep = _built(_book(ORDERS, top=[("A1", "统计时间:2026年9月1日至2026年9月30日")], start=3), keyed, "sep3") + assert materialize_union(tmp_path / "k.db", target=Recipe.model_validate(keyed), + parts=[_part(*aug, keyed), _part(*sep, keyed)]).rows == {"订单": 6} + + +# ========================================================================== +# 起草:表格对象、穿插的小计、每一行属于哪一期的问题 +# ========================================================================== + + +def _ok_validate(data, facts, origin): + from app.data.recipe import validate_recipe as v + + return v(data, facts=facts, origin=origin) + + +def _draft(raw: bytes, filename: str = "明细.xlsx"): + scan = csv_grid.scan_any(raw) + grids = {s.name: xlsx_cells.read_grid(raw, scan, s.name, max_rows=recipe_suggest.DRAFT_MAX_ROWS) + for s in scan.sheets if s.state == "visible" and s.nonempty} + + def dry(recipe: Recipe): + return recipe_engine.execute(recipe, raw, filename, scan, None) + return recipe_suggest.draft(scan, grids, filename, validate=_ok_validate, dry_run=dry), scan, grids + + +def _table_book(*, totals: bool = True, name: str = "门店销售") -> bytes: + """一个表格对象:B3:D7(表头、3 行明细、汇总行),表格上面有一行说明、右边有一个空格隔开的备注列。""" + wb = Workbook() + ws = wb.active + ws.title = "销售" + ws["B1"] = "门店销售明细" + rows = [("门店", "日期", "金额"), ("门店一", "2026-08-01", 10), ("门店二", "2026-08-02", 20), + (None, None, None), ("门店三", "2026-08-03", 30)] + if totals: + rows.append(("汇总", None, 60)) + for r, row in enumerate(rows, start=3): + for c, v in enumerate(row, start=2): + if v is not None: + ws.cell(r, c, v) + t = Table(displayName=name, ref=f"B3:D{2 + len(rows)}") + if totals: + t.totalsRowCount = 1 + ws.add_table(t) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def test_a_table_object_is_drafted_as_a_list_without_ai(): + raw = _table_book() + scan = xlsx_scan.scan(raw) + assert [(t.name, t.ref, t.totals_row_count) for t in scan.tables] == [("门店销售", "B3:D8", 1)] + draft, _, _ = _draft(raw) + assert draft.complete and draft.origin == "rules", draft.failures + [block] = draft.recipe["sheets"][0]["blocks"] + assert block["layout"] == "list" and block["table"] == "门店销售" + assert [c["header"] for c in block["columns"]] == ["门店", "日期", "金额"] + assert block["rows"]["total_row"] == {"label_column": "门店", "pick": "汇总", "keep_as": "门店销售_表内合计"} + assert block["rows"]["blank_rows"] == "skip" + # 锚点按表头文字,不记坐标 + assert "B3" not in str(block) + + +def test_a_table_object_with_digits_in_its_name_falls_back_to_the_title(): + draft, _, _ = _draft(_table_book(name="表1")) + assert draft.complete and draft.recipe["sheets"][0]["blocks"][0]["table"] == "门店销售明细" + + +def test_drafted_subtotals_run_and_verify(tmp_path): + raw = _book(STORES) + draft, scan, _ = _draft(raw) + assert draft.complete, draft.failures + block = draft.recipe["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"]["pick"] == "小计" and block["rows"]["total_row"]["pick"] == "合计" + db = str(tmp_path / "d.db") + recipe = Recipe.model_validate(draft.recipe) + ex = recipe_engine.execute(recipe, raw, "明细.xlsx", scan, db) + assert ex.ok and ex.tables[0].rows == 4 + assert all(c.status == "passed" for c in recipe_checks.run_checks(db, ex, recipe) if c.kind == "column_sum") + + +def test_period_question_binds_the_list_and_switches_to_accumulation(): + raw = _book(ORDERS, top=[PERIOD_CELL], start=3) + draft, _, grids = _draft(raw) + assert draft.complete, draft.failures + from app.data import recipe as R + + recipe = draft.recipe + cards, questions = recipe_suggest.questions_for(recipe, draft.facts, grids) + q = next(q for q in questions if q.id == "q_list_period:列表1") + assert [o["value"] for o in q.options] == ["none", "column:下单日期", "synthesize"] and q.default == "none" + assert not any(q.id == "q_mode" for q in questions) + bound = R.apply_patch(recipe, q.effects["column:下单日期"]) + assert bound["sheets"][0]["blocks"][0]["period"]["column"] == "下单日期" and bound["mode"] == "accumulate" + assert validate_recipe(bound, facts=None, origin="rules")[1] == [] + assert R.apply_patch(recipe, q.effects["none"])["mode"] == "replace" + # 工作表没配统计期的,选了以后一并加上 + plain = _book(ORDERS) + draft2, _, grids2 = _draft(plain) + assert draft2.recipe["sheets"][0]["context"] == [] + _, qs = recipe_suggest.questions_for(draft2.recipe, draft2.facts, grids2) + q2 = next(q for q in qs if q.id == "q_list_period:列表1") + bound2 = R.apply_patch(draft2.recipe, q2.effects["synthesize"]) + assert bound2["sheets"][0]["context"][0]["id"] == "统计期" + assert bound2["sheets"][0]["blocks"][0]["period"]["synthesize"] == "统计期" + assert accumulate_blockers(Recipe.model_validate(bound2)) == [] + + +# ========================================================================== +# CSV +# ========================================================================== + + +def test_csv_drafts_a_list_and_the_engine_runs_it(tmp_path): + raw = ("订单号,下单日期,门店,金额\n" + "A001,2026-08-01,一号店,120\n" + "A002,2026-08-02,二号店,80.5\n" + "小计,,,200.5\n" + "A003,2026-08-03,一号店,1\n" + "合计,,,201.5\n").encode("gb18030") + draft, scan, _ = _draft(raw, "orders_2026-08.csv") + assert draft.complete and draft.origin == "rules", draft.failures + sheet = draft.recipe["sheets"][0] + assert sheet["match"]["name"] == csv_grid.CSV_SHEET + block = sheet["blocks"][0] + assert [c["type"] for c in block["columns"]] == ["TEXT", "DATE", "TEXT", "REAL"] + assert block["rows"]["subtotals"]["pick"] == "小计" + db = str(tmp_path / "c.db") + recipe = Recipe.model_validate(draft.recipe) + ex = recipe_engine.execute(recipe, raw, "orders_2026-08.csv", scan, db) + assert ex.ok and ex.tables[0].rows == 3 + assert all(c.status == "passed" for c in recipe_checks.run_checks(db, ex, recipe) if c.kind == "column_sum") + # 回执的坐标照 Excel 打开这个 CSV 时的 A1 写 + [excluded] = [x for x in ex.rows_excluded if x.reason == "subtotal"] + assert excluded.sheet == "CSV" and excluded.rows == [[4, 4]] diff --git a/backend/tests/test_recipe_p5_e2e.py b/backend/tests/test_recipe_p5_e2e.py new file mode 100644 index 00000000..d5538bee --- /dev/null +++ b/backend/tests/test_recipe_p5_e2e.py @@ -0,0 +1,191 @@ +"""Excel 期 5 的端到端(真管线,经接口):每月一份的 CSV 明细按期累积成一张表,重叠的那期被拒,同一期再传要确认 +替换;带穿插小计的 Excel 规则起草成功、小计核对通过、小计行不进数据。 + +夹具全部合成(订单号 A001…、门店一…),不调用任何模型(规则起草不完整时的 AI 可用性查询换成「不可用」的桩)。 +""" +from __future__ import annotations + +import io +import json +import uuid +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from openpyxl import Workbook + +from app.core.config import settings +from app.main import app + +API = "/api/datasources" +CSV_MIME = "text/csv" + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture(autouse=True) +def no_model(monkeypatch): + from app.data import recipe_ai + from app.data.recipe_types import AiAvailability + + async def unavailable(session): + return None, "", AiAvailability(False, reason="验收测试不接模型") + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", unavailable) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def _name() -> str: + return f"p5_{uuid.uuid4().hex[:10]}" + + +def month_csv(month: int, orders: list[tuple[str, int, int]]) -> tuple[bytes, str]: + """一个月的订单导出:订单号、下单日期、门店、金额。文件名带统计期(只作旁证,CSV 里没有统计期格)。""" + lines = ["订单号,下单日期,门店,金额"] + for oid, day, amount in orders: + lines.append(f"{oid},2026-{month:02d}-{day:02d},门店{'一二'[day % 2]},{amount}") + last = 31 if month in (8, 10) else 30 + return ("\n".join(lines) + "\n").encode("utf-8"), f"orders_2026-{month:02d}-01_2026-{month:02d}-{last}.csv" + + +AUG = [("A001", 1, 120), ("A002", 15, 80), ("A003", 31, 40)] +SEP = [("B001", 3, 70), ("B002", 30, 30)] + + +async def stage(client, raw: bytes, filename: str, name: str) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/stage", files={"file": (filename, raw, CSV_MIME)}, data={"name": name}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def answer(client, sid: str, answers: dict[str, Any]) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/answers", json={"answers": answers}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def trial(client, sid: str, **body: Any) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/trial", json=body) + assert resp.status_code == 200, resp.text + return resp.json() + + +def _period(month: int) -> dict[str, Any]: + last = 31 if month in (8, 10) else 30 + return {"统计期": {"start": f"2026-{month:02d}-01", "end": f"2026-{month:02d}-{last}"}} + + +async def commit(client, st: dict[str, Any], confirmations: list[str] | None = None): + ids = [i["id"] for i in st["trial"]["confirm_items"]] if confirmations is None else confirmations + return await client.post(f"{API}/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ids}) + + +async def rows(client, name: str, sql: str) -> list[list[Any]]: + resp = await client.post(f"/api/tools/db_query__{name}/run", json={"args": {"sql": sql}}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["ok"], out + return json.loads(out["result"])["rows"] + + +async def first_csv(client, name: str) -> dict[str, Any]: + raw, fn = month_csv(8, AUG) + st = await stage(client, raw, fn, name) + assert st["recipe"]["sheets"][0]["match"]["name"] == "CSV" + assert not st["recipe_problems"], st["recipe_problems"] + q = next(q for q in st["questions"] if q["id"].startswith("q_list_period:")) + # 列表的草稿还不能按期累积,没有 q_mode;「每期怎么更新」这一道题同时定下导入模式和每一行属于哪一期 + assert not any(x["id"] == "q_mode" for x in st["questions"]) + st = await answer(client, st["id"], {q["id"]: {"value": "synthesize"}}) + assert st["recipe"]["mode"] == "accumulate" and not st["recipe_problems"], st["recipe_problems"] + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + st = await trial(client, st["id"], context_inputs=_period(8)) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = [i["id"] for i in st["trial"]["confirm_items"]] + assert "list_period:列表1" in ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + return {**resp.json(), "table": st["recipe"]["sheets"][0]["blocks"][0]["table"]} + + +async def reupload(client, source_id: str, month: int, orders: list[tuple[str, int, int]], + period: dict[str, Any] | None = None) -> dict[str, Any]: + raw, fn = month_csv(month, orders) + resp = await client.post(f"{API}/{source_id}/reupload", files={"file": (fn, raw, CSV_MIME)}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + return await trial(client, st["id"], context_inputs=period or _period(month)) + + +async def test_monthly_csv_exports_accumulate_into_one_table(client): + name = _name() + first = await first_csv(client, name) + sid, table = first["source"]["id"], first["table"] + assert await rows(client, name, f'SELECT "订单号", "统计期起", "统计期止" FROM "{table}" ORDER BY rowid') == [ + ["A001", "2026-08-01", "2026-08-31"], ["A002", "2026-08-01", "2026-08-31"], ["A003", "2026-08-01", "2026-08-31"]] + st = await reupload(client, sid, 9, SEP) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert st["trial"]["accumulate"]["action"] == "append" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + got = await rows(client, name, f'SELECT "统计期起", COUNT(*) FROM "{table}" GROUP BY 1 ORDER BY 1') + assert got == [["2026-08-01", 3], ["2026-09-01", 2]] + # 部分重叠的一期被拒收 + st = await reupload(client, sid, 9, SEP, period={"统计期": {"start": "2026-08-15", "end": "2026-09-14"}}) + assert st["trial"]["status"] == "rejected" and any(p["code"] == "period_overlap" for p in st["trial"]["problems"]) + # 同一期再传(数不同):要确认替换 + st = await reupload(client, sid, 9, [("B001", 3, 75)]) + assert st["trial"]["status"] == "passed" and st["trial"]["accumulate"]["action"] == "replace_period" + ids = [i["id"] for i in st["trial"]["confirm_items"]] + replace = "period_replace:2026-09-01~2026-09-30" + assert replace in ids + resp = await commit(client, st, [i for i in ids if i != replace]) + assert resp.status_code == 422 + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await rows(client, name, f'SELECT COUNT(*), SUM("金额") FROM "{table}"') == [[4, 315]] + + +async def test_excel_with_subtotals_drafts_checks_and_keeps_them_out(client): + wb = Workbook() + ws = wb.active + ws.title = "门店" + for row in [("门店", "金额"), ("门店一", 100), ("门店二", 200), ("小计", 300), ("门店三", 150), ("门店四", 150), + ("小计", 300), ("合计", 600)]: + ws.append(row) + buf = io.BytesIO() + wb.save(buf) + name = _name() + resp = await client.post(f"{API}/imports/stage", files={"file": ("门店.xlsx", buf.getvalue(), "application/octet-stream")}, + data={"name": name}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert not st["draft_problems"] and not st["recipe_problems"], (st["draft_problems"], st["recipe_problems"]) + block = st["recipe"]["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"]["pick"] == "小计" + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + checks = {c["id"]: c for c in st["trial"]["checks"]} + assert checks["T2"]["status"] == "passed" and "小计" in checks["T2"]["title"] + excluded = st["trial"]["receipt"]["rows_excluded"] + assert any(x["reason"] == "subtotal" and x["rows"] == [[4, 4], [7, 7]] for x in excluded), excluded + ids = [i["id"] for i in st["trial"]["confirm_items"]] + assert "subtotals:列表1" in ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + table = st["recipe"]["sheets"][0]["blocks"][0]["table"] + assert await rows(client, name, f'SELECT COUNT(*), SUM("金额") FROM "{table}"') == [[4, 600]] diff --git a/backend/tests/test_recipe_select.py b/backend/tests/test_recipe_select.py index 6824450b..9e408f16 100644 --- a/backend/tests/test_recipe_select.py +++ b/backend/tests/test_recipe_select.py @@ -210,10 +210,8 @@ def _subtotals() -> Book: @pytest.mark.parametrize("ref,bottom,where", [ - ("A1:C7", "box", "第 4 行像合计行"), # 小计夹在框中间 ("A1:C4", "box", "第 4 行像合计行"), # 框到小计为止:它下面紧接着还有明细,是夹在中间的小计 ("A1:C3", "box", "第 4 行像合计行"), # 框到小计上一行:下一行是小计,同上 - ("A1:C3", "auto", "第 4 行像合计行"), # 按规则推断:起草器在这里记失败,框选同样不收 ]) def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): b = _subtotals() @@ -222,6 +220,22 @@ def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): assert where in res.problems[0].message and "明细!A4" in res.problems[0].cells +@pytest.mark.parametrize("ref,bottom", [ + ("A1:C7", "box"), # 小计夹在框中间、合计在框底 + ("A1:C3", "auto"), # 按规则推断:起草器把夹在中间的小计提议成小计(期 5),框选同样 +]) +def test_subtotal_in_the_middle_becomes_subtotals_not_data(ref, bottom): + """期 5:夹在中间、第一列以合计字样开头的行提议成小计(ListRows.subtotals):不进数据,按各组明细核对。""" + b = _subtotals() + res = selection_edit(None, b.grid, sel(ref, "list", "明细", bottom=bottom), extraction=None) + assert res.ok, res.problems + [op] = res.ops # 配方为空:一条根替换,新建只有这一个块的配方 + block = op["value"]["sheets"][0]["blocks"][0] + assert block["rows"]["subtotals"] == {"label_column": "地区", "pick": "小计", "scope": "since_previous", + "group_column": None, "keep_as": None} + assert block["rows"]["total_row"]["pick"] == "合计" + + def test_total_word_in_another_column_is_not_imported_as_data(): """合计字样写在第二列(第一列空着):起草器记失败,框选同样报错,不当明细导入。""" wb = Workbook() diff --git a/backend/tests/test_recipe_suggest.py b/backend/tests/test_recipe_suggest.py index 40c62af1..44bd2164 100644 --- a/backend/tests/test_recipe_suggest.py +++ b/backend/tests/test_recipe_suggest.py @@ -951,7 +951,7 @@ def test_list_subtotal_rows_in_middle_fail(): assert [f for f in d.failures if "像合计行" in f] == [ "工作表「门店」:第 4 行像合计行(B 列「小计」),但合计字样不在第一列,无法按合计行核对", "工作表「门店」:第 7 行像合计行(B 列「小计」),但合计字样不在第一列,无法按合计行核对"] - # 小计写在第一列、但下面还有明细:不是末尾的合计行,整块照样收进来,不另报「数据区外的数字」 + # 期 5:小计写在第一列、下面还有明细:提议成穿插的小计(不进数据,按上一行小计之后的明细核对),不再记失败 wb2 = Workbook() ws2 = wb2.active ws2.title = "门店" @@ -960,10 +960,21 @@ def test_list_subtotal_rows_in_middle_fail(): ws2.append(row) sc2, grids2 = build(wb2) d2 = S.draft(sc2, grids2, "x.xlsx", validate=ok_validate()) - assert not d2.complete - assert d2.failures == ["工作表「门店」:第 4 行像合计行(A 列「小计」),但不在列表末尾,无法确定它汇总的是哪几行"] + assert d2.complete and d2.failures == [] block = d2.recipe["sheets"][0]["blocks"][0] assert block["rows"]["total_row"]["pick"] == "合计" + assert block["rows"]["subtotals"] == {"label_column": "门店", "pick": "小计", "scope": "since_previous", + "group_column": None, "keep_as": None} + # 字样不一样(「小计」「合计」都夹在中间):分不清,照旧记失败 + wb3 = Workbook() + ws3 = wb3.active + ws3.title = "门店" + for row in [["门店", "金额"], ["门店一", 100], ["小计", 100], ["门店二", 200], ["合计", 200], ["门店三", 150], + ["总计", 450]]: + ws3.append(row) + sc3, grids3 = build(wb3) + d3 = S.draft(sc3, grids3, "x.xlsx", validate=ok_validate()) + assert not d3.complete and any("不在列表末尾" in f for f in d3.failures) # 末尾第一列的合计行照旧改作核对,不算失败(test_list_total_row_is_verified_and_kept) diff --git a/backend/tests/test_recipe_validate.py b/backend/tests/test_recipe_validate.py index 0d2f945e..3e9fe110 100644 --- a/backend/tests/test_recipe_validate.py +++ b/backend/tests/test_recipe_validate.py @@ -315,7 +315,7 @@ def test_accumulate_mode_follows_eligibility(): probs = run(lr, facts=None) assert [(p.code, p.path) for p in probs] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] - assert "列表形态的表目前只支持每期替换" in probs[0].message + assert "没有指定每一行属于哪一期" in probs[0].message # 同一份配方每期替换时不报 lr["mode"] = "replace" assert run(lr, facts=None) == [] diff --git a/backend/tests/test_recipe_validate_p3.py b/backend/tests/test_recipe_validate_p3.py index a2e2f775..b4f0acca 100644 --- a/backend/tests/test_recipe_validate_p3.py +++ b/backend/tests/test_recipe_validate_p3.py @@ -79,7 +79,7 @@ def test_accumulate_list_table_message_does_not_promise_future_versions(): r["sheets"][0]["context"] = [{}] _rec, problems = validate_recipe(r) assert [p.code for p in problems] == ["accumulate_unkeyed"] - assert "列表形态的表目前只支持每期替换" in problems[0].message and "后续版本" not in problems[0].message + assert "没有指定每一行属于哪一期" in problems[0].message and "后续版本" not in problems[0].message # ========================================================================== diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index ed0661cf..8d80043d 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -1231,8 +1231,6 @@ export const UPLOAD_TEXT = { shapeTitle: '该表格不是一行一条记录的明细表', shapeLead: '检测到以下结构,默认不导入:', shapeRecipe: '这类表格可以按配方导入:指定表头、分段和合计行的处理方式,试运行核对、逐条确认后再启用。', - /** CSV 没有配方这条路(配方只认 Excel 的工作表结构) */ - shapeRecipeCsv: '按配方导入目前只支持 Excel(.xlsx)文件:可另存为 .xlsx 后再按配方导入。', shapeHeader: (row: number) => `如果表头不在第 ${row} 行,请取消后修改表头行号,再重新上传。`, /** 结构问题之外,预告的混合列:选按原样导入以后还要再选一次 */ shapeMixedLead: '另外,以下各列以数字为主,但混有非数字的值。选择按原样导入后,还需要选择这些值的处理方式:', @@ -1500,9 +1498,9 @@ export const RECIPE_TEXT = { descriptionLabel: '说明(供助手参考)', pickDrop: '拖入文件,或点击选择', /** 与选择框的 accept 一致 */ - pickFormats: '支持 Excel(.xlsx、.xlsm)', + pickFormats: '支持 Excel(.xlsx、.xlsm)和 CSV / TSV', pickAria: '选择表格文件', - pickWrongType: (name: string) => `「${name}」不是 Excel 文件:按配方导入只支持 .xlsx、.xlsm`, + pickWrongType: (name: string) => `「${name}」不是 Excel 或 CSV 文件:按配方导入支持 .xlsx、.xlsm、.csv、.tsv`, cancelUpload: '取消上传', cancel: '取消', close: '关闭', @@ -1637,6 +1635,18 @@ export const RECIPE_TEXT = { totalLabelColumn: '合计标签所在的列', totalPick: '合计行以此开头', totalKeep: '原表合计另存为表(留空则只核对)', + // 期 5:穿插的小计行(ListRows.subtotals)、列表每期怎么更新(ListBlock.period) + subtotals: '小计行', + subtotalsOn: '有小计行', + subtotalsHint: '夹在明细中间的小计不进数据,按它所管的那组明细逐列求和核对', + subtotalLabelColumn: '小计标签所在的列', + subtotalPick: '小计行以此开头', + subtotalScope: '一行小计管哪些明细', + subtotalGroupColumn: '按哪一列分组', + subtotalKeep: '原表小计另存为表(留空则只核对)', + listPeriod: '按期累积时每一行属于哪一期', + listPeriodColumn: '按哪个日期列核对', + listPeriodHint: '列表要按期累积,每一行都要说得出属于哪一期;所在工作表要配置统计期(表内没有时,每期试运行时录入)。导入模式另在「导入模式」中选', hiddenRows: '隐藏行', hiddenCols: '隐藏列', fallback: '工作表改名时', @@ -1994,6 +2004,7 @@ export const RECEIPT_BLOCK_TEXT = { ignored_outside: '按配方忽略了区域外数字的行', after_stop: '列表结束之后的文字行', total_not_kept: '只核对、不另存的合计行', + subtotal: '穿插的小计行(核对后不进数据)', } as Record, excludedOther: '其他原因', excludedTitle: '排除的行', @@ -2107,6 +2118,8 @@ export const RECIPE_CHOICE_LABEL = { colType: { TEXT: '文字', INTEGER: '整数', REAL: '小数', DATE: '日期' } as Record, colStore: { '': '默认', canonical: '规范写法', raw: '原文' } as Record, axisChecks: { contiguous: '逐日连续', covers_context: '恰好覆盖统计期' } as Record, + subtotalScope: { since_previous: '上一行小计之后的全部明细', group_column: '紧挨在上面、分组列取值相同的明细' } as Record, + listPeriod: { none: '不区分(只能每期替换)', column: '按日期列核对每行都在本期统计期内', synthesize: '每行加上本期统计期的起止两列' } as Record, } // =========================================================================== diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index a7585768..cfd0c97e 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -1773,7 +1773,7 @@ function UploadDecisionBody({ decision, choices, headerRow, sheetNoun, onRecipe )}
-

{onRecipe ? UPLOAD_TEXT.shapeRecipe : UPLOAD_TEXT.shapeRecipeCsv}

+

{UPLOAD_TEXT.shapeRecipe}

{onRecipe && (