"""配方导入的读取层:一张可见工作表读成网格(小表)或逐行流(大列表),外加给界面的网格预览。
**两遍读取,口径一致。** 第一遍是 xlsx_scan(标准库流式扫描:边界、合并区、隐藏行列、逐行非空格数),
这里是第二遍(openpyxl read_only)。两遍各数一次非空格,配方执行器逐行对账:对不上说明两个解析器看到的
不是同一张表(XML 行号乱序、重复的行 openpyxl 会静默跳过),整份拒收,不猜。所以「非空」的口径必须和
xlsx_scan 一字不差:值不是 None、也不是只含空白的文本,或者是公式格(哪怕没有缓存值);0 和 False 不算空。
**值和公式各读一份。** 值来自 data_only 的工作簿(用户要的是数),公式原文只有 data_only=False 才拿得到。
工作表有公式(SheetScan.formulas > 0)时再开一份 data_only=False 的只读工作簿,两份按行同步迭代:两个生成器
解析的是同一段 XML,行与行一一对应。openpyxl 只读模式会把共享公式展开成各格自己的写法(B5 的
`` 读成 `=SUM(B1:B4)`),合计公式的引用核对靠的就是这一点。
**边界用扫描算出来的。** `` 是写文件的程序自己填的,可能陈旧或者干脆没有:先 reset_dimensions,
再按 SheetScan.bounds 传 min/max 行列(与期 1 的 tabular.load_into 相同)。
隐藏工作表一律不读(H10):worksheet_for 只认可见的工作表,按部件路径配对,名字、路径、状态对不上就拒收。
"""
from __future__ import annotations
import datetime as _dt
import logging
import zipfile
from collections.abc import Iterator
from typing import Any
from xml.etree import ElementTree as ET
from app.data import csv_grid, tabular
from app.data.recipe_types import Grid, GridCell
from app.data.xlsx_scan import SheetScan, UnsupportedTable, WorkbookScan, _broken, cell_ref, parse_ref
logger = logging.getLogger(__name__)
#: 读格子时 openpyxl、zipfile、XML 解析会抛的异常(数字格的 写坏了是 ValueError):一律转成 UnsupportedTable,
#: 固定中文文案。不转的话会经全局 ValueError 处理器把英文原文连同格子内容回成不带 code 的 400(SE-7)
_READ_ERRORS = (ValueError, KeyError, ET.ParseError, zipfile.BadZipFile, EOFError)
__all__ = [
"GRID_MAX_CELLS", "PREVIEW_MAX_COLS", "PREVIEW_MAX_ROWS", "GridTooLarge", "Workbooks", "display", "grid_from",
"iter_rows", "parse_range", "preview", "read_grid", "sheet_rows", "visible_sheet",
]
#: 物化网格的非空格上限。更大的工作表走流式(iter_rows),配方执行器在那条路上只支持一个列表块
GRID_MAX_CELLS = 200_000
#: 网格预览最多给多少行、多少列(界面只读展示,再多也看不过来)
PREVIEW_MAX_ROWS, PREVIEW_MAX_COLS = 300, 60
#: 预览里一格文字最多多少字(长备注截断,原文在证据面板)
PREVIEW_TEXT_MAX = 200
class GridTooLarge(UnsupportedTable):
"""非空格超过 GRID_MAX_CELLS、又没有给 max_rows:不物化,调用方改走 iter_rows。"""
# --------------------------------------------------------------------------
# 工作簿
# --------------------------------------------------------------------------
def _open_formula_book(raw: bytes) -> Any:
return tabular.open_book(raw, data_only=False)
class Workbooks:
"""一次执行里复用的两份只读工作簿:值一份(data_only),公式一份(第一次要读有公式的工作表时才开)。
打开一次工作簿要解析样式表和共享字符串表,多张工作表、多次读取各开一遍很浪费。用完 close():
只读工作簿一直开着压缩包。
"""
def __init__(self, raw: bytes) -> None:
self.raw = raw
self._values: Any = None
self._formulas: Any = None
self._csv: Any = None
def csv(self, delimiter: str | None = None) -> Any:
"""CSV 原件解析好的记录(csv_grid.CsvTable),第一次要读时才解析,同一次执行里复用。delimiter 是扫描时定下的
分隔符(csv_grid.sheet_delimiter),不给时再嗅一次。"""
if self._csv is None:
self._csv = csv_grid.parse(self.raw, delimiter)
return self._csv
def values(self) -> Any:
if self._values is None:
self._values = tabular.open_book(self.raw)
return self._values
def formulas(self) -> Any:
if self._formulas is None:
self._formulas = _open_formula_book(self.raw)
return self._formulas
def close(self) -> None:
for book in (self._values, self._formulas):
if book is not None:
try:
book.close()
except Exception: # noqa: BLE001 关闭失败不影响已经读出来的结果
pass
self._values = self._formulas = None
def __enter__(self) -> Workbooks:
return self
def __exit__(self, *_exc: object) -> None:
self.close()
def visible_sheet(scan: WorkbookScan, sheet: str) -> SheetScan:
"""扫描结果里的一张可见工作表;没有这张表、或者它是隐藏的,抛 UnsupportedTable。"""
found = scan.sheet(sheet)
if found is None:
raise UnsupportedTable(f"工作簿中没有名为「{sheet}」的工作表")
if found.state != "visible":
raise UnsupportedTable(f"工作表「{sheet}」是隐藏的工作表,不读取")
return found
# --------------------------------------------------------------------------
# 逐行读取
# --------------------------------------------------------------------------
def _formula_text(value: Any) -> str:
"""openpyxl 给的公式:普通公式是「=…」字符串,数组公式是 ArrayFormula(.text 是原文)。"""
text = value if isinstance(value, str) else getattr(value, "text", None) or str(value)
return text if text.startswith("=") else f"={text}"
def sheet_rows(books: Workbooks, sheet: SheetScan, *, min_row: int, max_row: int, min_col: int,
max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]:
"""按行流式读一个矩形:每行给 (行号, [(列号, 格)…]),只给非空格、按列升序;整行为空的不给。
值和公式两份工作簿按行同步迭代(zip):openpyxl 只读模式按行号补齐缺的行,两边解析同一段 XML,
行数和每行的宽度都一样。文件内容写坏了(见 _READ_ERRORS)抛 UnsupportedTable,日志只记异常类型。
"""
try:
if csv_grid.is_csv_sheet(sheet):
# CSV 适配出来的「工作表」:从解析好的记录里取,口径同下(只给非空格、整行为空的不给)。分隔符用扫描时定下的
yield from csv_grid.rows(books.csv(csv_grid.sheet_delimiter(sheet)), min_row=min_row, max_row=max_row,
min_col=min_col, max_col=max_col)
return
yield from _sheet_rows(books, sheet, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col)
except UnsupportedTable:
raise
except _READ_ERRORS as e:
logger.info("reading sheet failed: %s", type(e).__name__)
raise UnsupportedTable(_broken()) from e
def _sheet_rows(books: Workbooks, sheet: SheetScan, *, min_row: int, max_row: int, min_col: int,
max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]:
if min_row > max_row or min_col > max_col:
return
ws = tabular.worksheet_for(books.values(), sheet)
ws.reset_dimensions() # 不信 :边界用扫描算出来的
values = ws.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True)
formulas: Iterator[Any] | None = None
if sheet.formulas > 0:
fws = tabular.worksheet_for(books.formulas(), sheet)
fws.reset_dimensions()
formulas = iter(fws.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col,
values_only=False))
row = min_row
for vrow in values:
frow = next(formulas, None) if formulas is not None else None
out: list[tuple[int, GridCell]] = []
col = min_col
for i, v in enumerate(vrow):
f = None
if frow is not None and i < len(frow):
fc = frow[i]
if getattr(fc, "data_type", None) == "f" and fc.value is not None:
f = _formula_text(fc.value)
if f is None and (v is None or (type(v) is str and not v.strip())):
col += 1
continue
out.append((col, GridCell(v, f)))
col += 1
if out:
yield row, out
row += 1
def iter_rows(raw: bytes, scan: WorkbookScan, sheet: str, *, min_row: int, max_row: int,
min_col: int, max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]:
"""流式读一张可见工作表的一个矩形(大列表用):每行只给非空格(按列排序),整行为空的行不给。"""
found = visible_sheet(scan, sheet)
books = Workbooks(raw)
try:
yield from sheet_rows(books, found, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col)
finally:
books.close()
# --------------------------------------------------------------------------
# 网格
# --------------------------------------------------------------------------
def parse_range(ref: str) -> tuple[int, int, int, int] | None:
"""「B2:AG2」→ (2, 2, 2, 33);单格「B2」→ (2, 2, 2, 2);写坏了 None。"""
a, _, b = ref.replace("$", "").partition(":")
try:
r1, c1 = parse_ref(a)
r2, c2 = parse_ref(b or a)
except ValueError:
return None
if r1 <= 0 or c1 <= 0 or r2 <= 0 or c2 <= 0:
return None
return min(r1, r2), min(c1, c2), max(r1, r2), max(c1, c2)
def _expand(spans: list[tuple[int, int]], lo: int, hi: int) -> set[int]:
out: set[int] = set()
for a, b in spans:
a, b = max(a, lo), min(b, hi)
if a <= b:
out.update(range(a, b + 1))
return out
def grid_from(books: Workbooks, sheet: SheetScan, *, max_rows: int | None = None) -> Grid:
"""read_grid 的内部版本:用调用方开好的工作簿(配方执行器一次执行里读多张表时复用)。"""
b = sheet.bounds
if b is None:
return Grid(sheet=sheet.name, bounds=None)
if max_rows is None and sheet.nonempty > GRID_MAX_CELLS:
raise GridTooLarge(
f"工作表「{sheet.name}」有 {sheet.nonempty:,} 个非空单元格,超过网格读取的上限({GRID_MAX_CELLS:,} 个)")
last = b.max_row if max_rows is None else min(b.max_row, b.min_row + max(max_rows, 1) - 1)
truncated = last < b.max_row
cells: dict[tuple[int, int], GridCell] = {}
for r, row in sheet_rows(books, sheet, min_row=b.min_row, max_row=last, min_col=b.min_col, max_col=b.max_col):
if max_rows is not None and len(cells) + len(row) > GRID_MAX_CELLS:
# 前 max_rows 行就已经超过上限(极宽的表):读到上一行为止
truncated, last = True, r - 1
break
for c, cell in row:
cells[(r, c)] = cell
merges = []
for ref in sheet.merged:
span = parse_range(ref)
if span is not None and span[0] <= last and span[2] >= b.min_row:
merges.append(span)
return Grid(
sheet=sheet.name, bounds=(b.min_row, b.min_col, b.max_row, b.max_col), cells=cells, merges=merges,
hidden_rows=_expand(sheet.hidden_rows, b.min_row, last),
hidden_cols=_expand(sheet.hidden_cols, b.min_col, b.max_col),
truncated=truncated,
)
def read_grid(raw: bytes, scan: WorkbookScan, sheet: str, *, max_rows: int | None = None) -> Grid:
"""物化一张可见工作表。
隐藏工作表抛 UnsupportedTable。非空格超过 GRID_MAX_CELLS 且没给 max_rows 抛 GridTooLarge;给了 max_rows
就只读已用区域的前 max_rows 行(起草、预览用),读不完时 truncated=True。Grid.bounds 始终是整张表的真实边界
(界面要知道表有多大),读到哪一行看 truncated 和 cells。隐藏行列、合并区只保留落在读取范围内的。
"""
found = visible_sheet(scan, sheet)
with Workbooks(raw) as books:
return grid_from(books, found, max_rows=max_rows)
# --------------------------------------------------------------------------
# 预览
# --------------------------------------------------------------------------
def display(cell: GridCell) -> tuple[str, str]:
"""一格在预览里的 (写法, 种类)。种类:text / number / date / bool / error / formula / formula_uncached。"""
v = cell.value
if cell.formula is not None:
if v is None:
return "", "formula_uncached"
return _clip(tabular.as_text(v)), "formula"
if isinstance(v, bool):
return ("TRUE" if v else "FALSE"), "bool"
if isinstance(v, (int, float)):
return tabular.as_text(v), "number"
if isinstance(v, str):
text = v.strip()
return _clip(text), ("error" if text in tabular.EXCEL_ERRORS else "text")
text = tabular.as_text(v)
return _clip(text), ("date" if isinstance(v, (_dt.datetime, _dt.date, _dt.time)) else "text")
def _clip(text: str) -> str:
return text if len(text) <= PREVIEW_TEXT_MAX else text[:PREVIEW_TEXT_MAX] + "…"
def _range_a1(span: tuple[int, int, int, int]) -> str:
r1, c1, r2, c2 = span
return cell_ref(r1, c1) if (r1, c1) == (r2, c2) else f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}"
def preview(grid: Grid, *, max_rows: int = PREVIEW_MAX_ROWS, max_cols: int = PREVIEW_MAX_COLS) -> dict[str, Any]:
"""给界面的网格预览(P2-SPEC 7.4 的 GridPreview):左上角起最多 max_rows 行、max_cols 列。
cells 是 [行, 列, 写法, 种类];formulas 是 {坐标: 公式原文}(悬停时显示)。这是给用户看的,数字照给;
发给模型的压缩表示另有一套(recipe_ai.compress),不从这里取。
"""
if grid.bounds is None:
return {"sheet": grid.sheet, "bounds": None, "total_rows": 0, "total_cols": 0, "truncated": grid.truncated,
"rows": [], "cols": [], "cells": [], "formulas": {}, "merges": [], "hidden_rows": [],
"hidden_cols": []}
r1, c1, r2, c2 = grid.bounds
total_rows, total_cols = r2 - r1 + 1, c2 - c1 + 1
last_r = min(r2, r1 + max(max_rows, 1) - 1)
last_c = min(c2, c1 + max(max_cols, 1) - 1)
cells: list[list[Any]] = []
formulas: dict[str, str] = {}
for r in grid.row_numbers():
if r > last_r:
break
for c, cell in grid.row(r):
if c > last_c:
break
text, kind = display(cell)
cells.append([r, c, text, kind])
if cell.formula is not None:
formulas[cell_ref(r, c)] = cell.formula
merges = [_range_a1(m) for m in grid.merges
if m[0] <= last_r and m[2] >= r1 and m[1] <= last_c and m[3] >= c1]
return {
"sheet": grid.sheet,
"bounds": f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}",
"total_rows": total_rows,
"total_cols": total_cols,
"truncated": bool(grid.truncated or last_r < r2 or last_c < c2),
"rows": list(range(r1, last_r + 1)),
"cols": list(range(c1, last_c + 1)),
"cells": cells,
"formulas": formulas,
"merges": merges,
"hidden_rows": sorted(r for r in grid.hidden_rows if r1 <= r <= last_r),
"hidden_cols": sorted(c for c in grid.hidden_cols if c1 <= c <= last_c),
}