mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-17 04:31:59 +00:00
Excel 导入改造期 1:止血与版本底座——写错的列名不再静默得 0,导入不再悄悄读错,每次上传生成不可变的版本,运行固定发起时的数据版本
连接层 - 所有 SQLite 数据源关闭双引号字符串兼容(DQS),写错的双引号列名直接报错;设置失败就拒绝连接,启动时自检 - 连接缓存按数据源加版本快照分键、按源整体失效、最多 32 个;版本快照以 immutable 只读打开,建连接前核对文件哈希,并在借出连接时核对文件指纹,被改过就拒绝查询 - 结构探查遇到取不了列的视图时逐个对象重取,只跳过坏的那个并注明原因 解析(新 xlsx_scan + tabular) - 先流式扫描真实边界和版式(不信 dimension),再按边界读值,不整表进内存;按 Excel 自身上限设硬上限,并发解析最多两个 - 隐藏和深度隐藏的工作表不导入;稀疏区域、无缓存值的公式、同名工作表、DTD 拒收 - 去掉空行和两侧空列;列名表名统一规范化(新 names 模块)并保留原表头,撞名不再互相覆盖 - 前导零、超长数字按文本;千分位按数字;日期不带 T;带单位的数字文本写进回执 - 数字列混入非数字、交叉表和多块结构必须由用户决定(422),可选存空值或按原样导入(未规整);STRICT 建表,整份在一个事务里写完 版本底座(新 table_versions、raw_store) - 每次上传写新库文件、落盘、0444,事务里写构建、导入记录、快照后才切指针;失败时旧版完整可查 - 原件按哈希存档,不提供下载,可以清除;上传源不许改连接字段、不再重新探查;手工 SQLite 源不许指向上传目录和应用自己的数据文件(按文件身份判断) - 启动时清理半成品与孤儿文件、给老上传补初始版本;被运行引用的版本不回收,发起运行与回收互斥 - 上传接口的解析移到线程池 运行固定数据版本 - 发起时把图里(含子工作流)用到的上传源的当前快照记进运行记录和 run.started,续跑、审批恢复沿用;数据源工具按固定的快照生成描述、表结构并查询 - 固定的版本没了、源被删除后同名重建,都明确报错,不改用当前版本或同名的新源;查询快照记 data_version 证据层 - 新报告记 entity_syntax: 2,[[t:明细]]、[[c:明细.金额]] 和 SQL 里不加引号的中文表名都认得,名字按 SQLite 的口径查找;老报告按原规则复核,结果不变 - 写作规则改为中文表名、字段名一律用 [[t:]] / [[c:]];未规整的表在工具描述、写作目录和裁判摘录里标出 前端 - 上传弹窗接上导入前的决定(混合列、交叉表)和导入回执(跳过的隐藏表、类型转换、警告、原表头对照、未规整标记),上传前告知原件会保存 - 数据源卡片按 origin 认上传的表格,显示当前版本;上传的表格不再给「探查结构」
This commit is contained in:
+365
-30
@@ -6,16 +6,20 @@ MySQL 要 charset、asyncpg 不认 sslmode 这些事。哪天要加一种库,
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, AsyncIterator, Awaitable, Callable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import event, text
|
||||
|
||||
from app.core.crypto import decrypt
|
||||
from app.data import guard
|
||||
@@ -134,6 +138,122 @@ def build_url(source: Any, *, reveal: bool = False) -> str:
|
||||
return f"{driver}://{auth}{host}/{source.database or ''}{('?' + query) if query else ''}"
|
||||
|
||||
|
||||
class SqliteHardeningError(sqlite3.OperationalError):
|
||||
"""SQLite 连接没能关掉「双引号当字符串」的兼容行为。这条连接不交出去。
|
||||
|
||||
继承 sqlite3.OperationalError:SQLAlchemy 按驱动异常包装,上层(测试连接、工具报错)
|
||||
照常拿到一个连接失败,而不是一个不认识的异常类型。
|
||||
"""
|
||||
|
||||
|
||||
class _DqsOffConnection(sqlite3.Connection):
|
||||
"""打开时就关掉 DQS 的 sqlite3 连接(经 sqlite3.connect 的 factory 参数接入)。
|
||||
|
||||
DQS(double-quoted string)是 SQLite 的历史兼容:双引号里的名字找不到对应的列时,
|
||||
当成字符串字面量。于是列名写错不报错:`SUM("金颔")` 得 0,`GROUP BY "金颔"`
|
||||
把整张表归成一组,结论错了却没人知道。关掉以后同一条 SQL 报 no such column。
|
||||
|
||||
为什么用 factory 而不是 SQLAlchemy 的 connect 事件:aiosqlite 的连接在它自己的
|
||||
线程里建,connect 事件拿到的是异步适配层,要碰底层 sqlite3 连接只能走 aiosqlite
|
||||
的私有属性(_execute / _conn),升级就可能失效。factory 是 sqlite3.connect 的公开
|
||||
参数,aiosqlite.connect 的 **kwargs 原样转给它,SQLAlchemy 的 connect_args 又原样
|
||||
转给 aiosqlite.connect——三层都是公开接口。设置在建连接的那个线程里同步完成,
|
||||
没设上就在这里抛错,连接根本不会进连接池。而且凡是经 engine_args 建的 engine
|
||||
(连接缓存、表单里的一次性测试连接)都自动带上,不必逐处挂事件。
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
try:
|
||||
_disable_dqs(self)
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
|
||||
#: 要关掉的两项:DML(查询、增删改)和 DDL(建表、建索引)里的双引号字符串
|
||||
_DQS_OPTIONS = (
|
||||
("DML", getattr(sqlite3, "SQLITE_DBCONFIG_DQS_DML", None)),
|
||||
("DDL", getattr(sqlite3, "SQLITE_DBCONFIG_DQS_DDL", None)),
|
||||
)
|
||||
|
||||
|
||||
def _disable_dqs(conn: sqlite3.Connection) -> None:
|
||||
"""关掉 DQS,并读回来确认确实关了。任何一步不成立都抛 SqliteHardeningError。
|
||||
|
||||
Python 3.12 起才有 setconfig / getconfig;没有的话同样拒绝——宁可连不上,
|
||||
也不交出一条会把写错的列名静默当成字符串的连接。
|
||||
"""
|
||||
for label, op in _DQS_OPTIONS:
|
||||
try:
|
||||
if op is None:
|
||||
raise AttributeError(f"sqlite3 模块缺少 SQLITE_DBCONFIG_DQS_{label}")
|
||||
conn.setconfig(op, False)
|
||||
still_on = conn.getconfig(op)
|
||||
except Exception as e: # noqa: BLE001 - 原因原样带出,统一换成拒绝连接
|
||||
raise SqliteHardeningError(_dqs_refusal(f"{type(e).__name__}: {e}")) from e
|
||||
if still_on:
|
||||
raise SqliteHardeningError(_dqs_refusal(f"设置后读回仍为开启({label})"))
|
||||
|
||||
|
||||
def _dqs_refusal(cause: str) -> str:
|
||||
return ("未能关闭 SQLite 的双引号字符串兼容,已拒绝连接"
|
||||
f"(否则写错的列名会被当成文字,查询不报错却给出错误结果):{cause}")
|
||||
|
||||
|
||||
#: 自检先建的探针表
|
||||
_SELF_CHECK_SETUP = "CREATE TABLE _dqs_probe (a INTEGER)"
|
||||
#: 自检的探针:每条都写了一个不存在的双引号列名,关掉 DQS 后都必须报 no such column。
|
||||
#: 查询一条验 DML 那项,建索引一条验 DDL 那项——只关一项时另一条会照常执行。
|
||||
#: 用 ASCII 名字:DQS 对中英文名字一视同仁,SQL 原文也不必进文案检查
|
||||
_SELF_CHECK_PROBES = (
|
||||
'SELECT "no_such_column" FROM _dqs_probe',
|
||||
'CREATE INDEX _dqs_probe_i ON _dqs_probe ("no_such_column")',
|
||||
)
|
||||
|
||||
|
||||
def sqlite_dqs_self_check() -> None:
|
||||
"""启动自检:在内存库上确认 DQS 确实关得掉,写错的双引号列名确实报错。
|
||||
|
||||
连接层的每条 SQLite 连接都靠 _DqsOffConnection 关 DQS;这里用同一个类开一条内存
|
||||
连接,查询和建索引各验一次。不成立就抛 RuntimeError,让服务起不来——Python 或
|
||||
SQLite 换了版本、行为变了,宁可启动失败,也不悄悄退回「列名写错照样出数」。
|
||||
|
||||
只认「no such column」这一种报错:探针因为别的原因报错(表没建上、语法变了),
|
||||
说明它根本没走到双引号名字那一步,什么也没验到,同样算失败。
|
||||
"""
|
||||
problem: str | None = None
|
||||
try:
|
||||
conn = sqlite3.connect(":memory:", factory=_DqsOffConnection)
|
||||
except Exception as e: # noqa: BLE001
|
||||
problem = str(e)
|
||||
else:
|
||||
try:
|
||||
conn.execute(_SELF_CHECK_SETUP)
|
||||
for sql in _SELF_CHECK_PROBES:
|
||||
try:
|
||||
conn.execute(sql)
|
||||
except sqlite3.OperationalError as e:
|
||||
if "no such column" not in str(e):
|
||||
problem = f"{sql} 报错,但不是预期的「no such column」:{e}"
|
||||
break
|
||||
else:
|
||||
problem = f"{sql} 没有报错:双引号里的名字仍被当成字符串"
|
||||
break
|
||||
except Exception as e: # noqa: BLE001 - 探针表都建不起来,同样是没验到
|
||||
problem = f"{type(e).__name__}: {e}"
|
||||
finally:
|
||||
conn.close()
|
||||
if problem:
|
||||
logger.critical("SQLite 自检失败,服务不启动:%s", problem)
|
||||
raise RuntimeError(f"SQLite 自检失败:无法确认写错的双引号列名会报错,服务不启动。{problem}")
|
||||
|
||||
|
||||
def _sqlite_extra() -> dict[str, Any]:
|
||||
"""每个 SQLite engine 都带的连接参数:关 DQS。"""
|
||||
return {"connect_args": {"factory": _DqsOffConnection}}
|
||||
|
||||
|
||||
def engine_args(source: Any) -> tuple[str, dict[str, Any]]:
|
||||
"""真正连库用的连接串和额外参数。只读源在这里拿到**连接层**的只读。
|
||||
|
||||
@@ -146,16 +266,17 @@ def engine_args(source: Any) -> tuple[str, dict[str, Any]]:
|
||||
- PostgreSQL:会话的默认事务只读。能关掉它的 SET / set_config() 守卫都拦。
|
||||
- MySQL / MariaDB:会话事务只读。能关掉它的只有 SET,守卫拦。
|
||||
- Oracle:没有会话级只读开关,只剩守卫这一道——只读数据源请配只读账号。
|
||||
|
||||
SQLite 不论只读与否都关 DQS(见 _DqsOffConnection)。source 带 immutable
|
||||
(上传表格的版本快照,见 table_versions.SourceView)时,以 mode=ro&immutable=1
|
||||
打开,不看 readonly 字段:快照文件发布后就不再变,immutable 让 SQLite 不加锁、
|
||||
不找日志文件,同时也写不进去。
|
||||
"""
|
||||
url = build_url(source, reveal=True)
|
||||
if not source.readonly:
|
||||
return url, {}
|
||||
kind = (source.kind or "").lower()
|
||||
if kind == "sqlite":
|
||||
path = source.database or ""
|
||||
if path and path != ":memory:":
|
||||
from urllib.parse import quote
|
||||
url = f"{_DRIVERS['sqlite']}:///file:{quote(path)}?mode=ro&uri=true"
|
||||
return _sqlite_url(source, url), _sqlite_extra()
|
||||
if not source.readonly:
|
||||
return url, {}
|
||||
if kind in ("postgres", "postgresql"):
|
||||
return url, {"connect_args": {"server_settings": {"default_transaction_read_only": "on"}}}
|
||||
@@ -164,6 +285,19 @@ def engine_args(source: Any) -> tuple[str, dict[str, Any]]:
|
||||
return url, {}
|
||||
|
||||
|
||||
def _sqlite_url(source: Any, url: str) -> str:
|
||||
from urllib.parse import quote
|
||||
|
||||
path = source.database or ""
|
||||
if getattr(source, "immutable", False):
|
||||
if not path or path == ":memory:":
|
||||
raise ValueError("版本快照必须指向一个数据文件")
|
||||
return f"{_DRIVERS['sqlite']}:///file:{quote(path)}?mode=ro&immutable=1&uri=true"
|
||||
if source.readonly and path and path != ":memory:":
|
||||
return f"{_DRIVERS['sqlite']}:///file:{quote(path)}?mode=ro&uri=true"
|
||||
return url
|
||||
|
||||
|
||||
def _timeout_value(value: Any) -> float | None:
|
||||
"""options 里的查询时限:表单存的是字符串,也认数字。填得不对返回 None。"""
|
||||
if value is None or isinstance(value, bool):
|
||||
@@ -218,46 +352,247 @@ def _seconds(value: float) -> str:
|
||||
return str(int(value)) if float(value).is_integer() else f"{value:g}"
|
||||
|
||||
|
||||
#: 连接缓存最多留多少个 engine。版本快照各占一个键,长期运行下键只增不减;
|
||||
#: 超出时关掉最久没用的那个,下次用到再建
|
||||
MAX_ENGINES = 32
|
||||
|
||||
|
||||
class SnapshotTampered(ValueError):
|
||||
"""版本快照的数据文件和登记的哈希对不上(或者文件没了)。拒绝查询,也不缓存。"""
|
||||
|
||||
|
||||
_TAMPERED = "数据文件与登记的版本不一致,可能被修改过,已拒绝查询"
|
||||
_SNAPSHOT_UNREADABLE = "登记版本的数据文件无法读取,无法核对是否被修改过,已拒绝查询"
|
||||
_NO_HASH = "数据版本缺少文件哈希的登记,无法核对数据文件是否被修改过,已拒绝查询"
|
||||
_REPLACED = "数据文件在核对之后被替换或改动过,已拒绝这次查询;再次查询时会重新核对"
|
||||
|
||||
#: 快照文件的指纹:(st_dev, st_ino, st_size, st_mtime_ns)。整份替换(rename 覆盖)换 inode,
|
||||
#: 原地改写换大小或修改时间:两种都认得出,而 stat 只要几微秒,每次取引擎、每次借连接都查得起
|
||||
Fingerprint = tuple[int, int, int, int]
|
||||
|
||||
|
||||
def cache_key(source: Any) -> str:
|
||||
"""连接缓存的键:数据源 id 加版本快照 id(没有快照的源为空)。
|
||||
|
||||
上传表格每次发布都是一个新文件,同一个数据源的新旧版本得各用各的连接池——
|
||||
否则固定在旧版本上的运行会查到新文件,或者反过来。
|
||||
"""
|
||||
return f"{source.id}:{getattr(source, 'snapshot_id', None) or ''}"
|
||||
|
||||
|
||||
def _source_of(key: str) -> str:
|
||||
# 快照 id 是十六进制,不含冒号;数据源 id 里就算有冒号,从右边切也切得对
|
||||
return key.rsplit(":", 1)[0]
|
||||
|
||||
|
||||
def _file_sha256(path: str) -> str:
|
||||
with open(path, "rb") as f:
|
||||
return hashlib.file_digest(f, "sha256").hexdigest()
|
||||
|
||||
|
||||
def _fingerprint(path: str) -> Fingerprint | None:
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns
|
||||
|
||||
|
||||
class _Moving(OSError):
|
||||
"""算哈希的过程中文件变了:算出来的哈希说明不了现在的文件。"""
|
||||
|
||||
|
||||
def _hash_and_print(path: str) -> tuple[str, Fingerprint]:
|
||||
"""算文件哈希,连同它的指纹。前后各 stat 一次:对不上说明读的过程中文件被替换或改写了。
|
||||
|
||||
指纹和哈希必须描述同一个文件:之后每次取引擎、借连接都拿当时的 stat 和这份指纹比,
|
||||
比得上才说明 SQLite 打开的还是核对过的那个文件。
|
||||
"""
|
||||
before = _fingerprint(path)
|
||||
digest = _file_sha256(path)
|
||||
after = _fingerprint(path)
|
||||
if before is None or before != after:
|
||||
raise _Moving(f"文件在核对过程中发生了变化:{path}")
|
||||
return digest, after
|
||||
|
||||
|
||||
async def _verify_snapshot(source: Any, expected: str) -> Fingerprint:
|
||||
"""建 engine 之前核对快照文件的哈希,返回核对时的文件指纹。大文件读一遍要时间,放到线程里,不卡事件循环。"""
|
||||
try:
|
||||
actual, fingerprint = await asyncio.to_thread(_hash_and_print, source.database or "")
|
||||
except _Moving as e:
|
||||
logger.warning("快照文件在核对过程中发生了变化:source=%s snapshot=%s path=%s",
|
||||
source.id, getattr(source, "snapshot_id", None), source.database)
|
||||
raise SnapshotTampered(_TAMPERED) from e
|
||||
except OSError as e:
|
||||
raise SnapshotTampered(_SNAPSHOT_UNREADABLE) from e
|
||||
if actual.lower() != str(expected).strip().lower():
|
||||
logger.warning("快照文件哈希不符:source=%s snapshot=%s path=%s",
|
||||
source.id, getattr(source, "snapshot_id", None), source.database)
|
||||
raise SnapshotTampered(_TAMPERED)
|
||||
return fingerprint
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Pin:
|
||||
"""一个快照引擎核对过的文件:路径和核对时的指纹。stale 表示已经发现对不上,不再可信。"""
|
||||
|
||||
path: str
|
||||
fingerprint: Fingerprint
|
||||
stale: bool = False
|
||||
|
||||
def holds(self) -> bool:
|
||||
return not self.stale and _fingerprint(self.path) == self.fingerprint
|
||||
|
||||
|
||||
def _guard_pool(engine: AsyncEngine, pin: _Pin, key: str) -> None:
|
||||
"""每次从连接池借连接时核对文件指纹,对不上就拒绝这条连接。
|
||||
|
||||
取引擎时核对过一次,但引擎缓存着、连接池会在之后新开连接(并发查询、pool_recycle、
|
||||
探活失败重连),每条新连接都按路径重新打开文件。文件要是在这期间被整份替换(恢复备份、
|
||||
同步工具),新连接读到的就是没核对过的文件。所以借连接(checkout)时再比一次:新开的
|
||||
连接打开的是哪个文件,这时 stat 看到的就是哪个。对不上就把这个键标成不可信并拒绝;
|
||||
抛出的异常让 SQLAlchemy 关掉这条连接,下次取引擎时重新核对哈希(内容没变的恢复会放行)。
|
||||
"""
|
||||
def check(*_args: Any) -> None:
|
||||
if not pin.holds():
|
||||
if not pin.stale:
|
||||
logger.warning("快照文件在核对之后发生了变化,已拒绝连接:key=%s path=%s", key, pin.path)
|
||||
pin.stale = True
|
||||
raise SnapshotTampered(_REPLACED)
|
||||
|
||||
event.listen(engine.sync_engine, "checkout", check)
|
||||
|
||||
|
||||
class EngineCache:
|
||||
"""按数据源 id 缓存 engine。连接池的建立不便宜,不该每次查询都重来一遍。
|
||||
"""按数据源(和版本快照)缓存 engine。连接池的建立不便宜,不该每次查询都重来一遍。
|
||||
|
||||
配置变了要显式 invalidate——改了密码却还在用旧连接,排查起来很费神;
|
||||
改了只读开关却还在用旧连接,只读就形同虚设。
|
||||
改了只读开关却还在用旧连接,只读就形同虚设。invalidate 清掉这个源名下所有版本的连接。
|
||||
|
||||
最多留 MAX_ENGINES 个,按最近使用排序,超出时关掉最久没用的。
|
||||
source 带 expected_sha256 时,这个键新建 engine 前核对一遍文件哈希,并记下核对时的文件指纹
|
||||
(inode、大小、修改时间)。之后缓存命中、连接池借连接时都拿当时的 stat 比这份指纹:快照文件
|
||||
0444 挡不住整份替换(rename 只要目录可写),对不上就摘掉这个引擎、下次重新核对哈希——内容
|
||||
没变(备份恢复回同样的文件)就放行,变了就拒绝。同一个键并发首次使用时只核对一次,大家等同一个结果。
|
||||
source 标着 immutable(版本快照)却没给期望哈希的,一律拒绝:默认放行的话,哪条新路径漏传
|
||||
哈希,快照就不经核对地打开了。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._engines: dict[str, AsyncEngine] = {}
|
||||
def __init__(self, max_engines: int | None = None) -> None:
|
||||
self._engines: OrderedDict[str, AsyncEngine] = OrderedDict()
|
||||
#: 快照引擎核对过的文件,键和 _engines 相同。手工源没有
|
||||
self._pins: dict[str, _Pin] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
#: None 表示按模块常量 MAX_ENGINES(现取,测试里可以改)
|
||||
self._max = max_engines
|
||||
#: 正在进行的快照核对,按(缓存键, 文件, 登记的哈希)合并。核对结束(不论成败)即移除,
|
||||
#: 失败结果不留:下次照常重新核对
|
||||
self._verifying: dict[tuple[str, str, str], asyncio.Future[Fingerprint]] = {}
|
||||
|
||||
def _hit(self, key: str, doomed: list[AsyncEngine]) -> AsyncEngine | None:
|
||||
"""缓存里还可信的 engine。快照文件和核对时的指纹对不上的,摘出来放进 doomed 交给调用方关掉。"""
|
||||
engine = self._engines.get(key)
|
||||
if engine is None:
|
||||
return None
|
||||
pin = self._pins.get(key)
|
||||
if pin is not None and not pin.holds():
|
||||
if not pin.stale:
|
||||
logger.warning("快照文件在核对之后发生了变化,重新核对:key=%s path=%s", key, pin.path)
|
||||
del self._engines[key]
|
||||
self._pins.pop(key, None)
|
||||
doomed.append(engine)
|
||||
return None
|
||||
self._engines.move_to_end(key)
|
||||
return engine
|
||||
|
||||
async def get(self, source: Any) -> AsyncEngine:
|
||||
key = str(source.id)
|
||||
cached = self._engines.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
async with self._lock:
|
||||
if key in self._engines:
|
||||
return self._engines[key]
|
||||
url, extra = engine_args(source)
|
||||
engine = create_async_engine(
|
||||
url,
|
||||
pool_size=3,
|
||||
max_overflow=2,
|
||||
pool_pre_ping=True, # 长时间空闲后连接会被数据库掐掉,先探活再用
|
||||
pool_recycle=1800,
|
||||
**extra,
|
||||
)
|
||||
self._engines[key] = engine
|
||||
key = cache_key(source)
|
||||
doomed: list[AsyncEngine] = []
|
||||
try:
|
||||
cached = self._hit(key, doomed)
|
||||
if cached is not None:
|
||||
return cached
|
||||
expected = str(getattr(source, "expected_sha256", None) or "").strip()
|
||||
if getattr(source, "immutable", False) and not expected:
|
||||
logger.warning("版本快照没有登记文件哈希,已拒绝:key=%s path=%s", key, source.database)
|
||||
raise SnapshotTampered(_NO_HASH)
|
||||
fingerprint: Fingerprint | None = None
|
||||
if expected:
|
||||
# 在锁外核对:哈希一个大文件要几秒,不能让别的数据源都排在它后面
|
||||
fingerprint = await self._verify_once(key, source, expected)
|
||||
async with self._lock:
|
||||
cached = self._hit(key, doomed)
|
||||
if cached is not None:
|
||||
return cached
|
||||
url, extra = engine_args(source)
|
||||
engine = create_async_engine(
|
||||
url,
|
||||
pool_size=3,
|
||||
max_overflow=2,
|
||||
pool_pre_ping=True, # 长时间空闲后连接会被数据库掐掉,先探活再用
|
||||
pool_recycle=1800,
|
||||
**extra,
|
||||
)
|
||||
if fingerprint is not None:
|
||||
pin = _Pin(str(source.database or ""), fingerprint)
|
||||
_guard_pool(engine, pin, key)
|
||||
self._pins[key] = pin
|
||||
self._engines[key] = engine
|
||||
limit = max(1, self._max if self._max is not None else MAX_ENGINES)
|
||||
while len(self._engines) > limit:
|
||||
old_key, old = self._engines.popitem(last=False)
|
||||
self._pins.pop(old_key, None)
|
||||
doomed.append(old)
|
||||
return engine
|
||||
finally:
|
||||
for old in doomed:
|
||||
# 正被借出的连接不受影响:dispose 只关池子里空闲的,借出的还回来时随旧池子回收
|
||||
await old.dispose()
|
||||
|
||||
async def _verify_once(self, key: str, source: Any, expected: str) -> Fingerprint:
|
||||
"""核对快照哈希;同一个键已经有一次在核对,就等那一次的结果。
|
||||
|
||||
一次运行开头往往同时发起几次工具调用,查的是同一个快照。不合并的话每个请求
|
||||
各读一遍整个文件(快照可能几百 MB),还挤占默认线程池。
|
||||
|
||||
合并的范围带上文件路径和登记的哈希:键相同而这两样不同的(不该出现,但不赌)
|
||||
各核各的,绝不拿别人的结论放行。核对本身包成任务、等的时候加 shield:某个
|
||||
请求被取消,不连累别人在等的那一次。核对结束就移除——成功之后到 engine
|
||||
装进缓存之间若恰好来了新请求,它会再核对一次;多算一次,不会少算。
|
||||
"""
|
||||
token = (key, str(source.database or ""), str(expected).strip().lower())
|
||||
loop = asyncio.get_running_loop()
|
||||
pending = self._verifying.get(token)
|
||||
if pending is None or pending.get_loop() is not loop:
|
||||
pending = loop.create_task(_verify_snapshot(source, expected))
|
||||
self._verifying[token] = pending
|
||||
pending.add_done_callback(lambda done: self._settle_verify(token, done))
|
||||
return await asyncio.shield(pending)
|
||||
|
||||
def _settle_verify(self, token: tuple[str, str, str], done: asyncio.Future[Fingerprint]) -> None:
|
||||
# 只移除自己:失效后可能已经有新的一次登记在同一个位置上
|
||||
if self._verifying.get(token) is done:
|
||||
del self._verifying[token]
|
||||
|
||||
async def invalidate(self, source_id: str) -> None:
|
||||
engine = self._engines.pop(str(source_id), None)
|
||||
if engine is not None:
|
||||
sid = str(source_id)
|
||||
# 正在进行的核对也撇开:失效之后来的请求重新核对,不搭失效之前那一次的便车
|
||||
for token in [t for t in self._verifying if _source_of(t[0]) == sid]:
|
||||
del self._verifying[token]
|
||||
keys = [key for key in list(self._engines) if _source_of(key) == sid]
|
||||
doomed = [self._engines.pop(key) for key in keys]
|
||||
for key in keys:
|
||||
self._pins.pop(key, None)
|
||||
for engine in doomed:
|
||||
await engine.dispose()
|
||||
|
||||
async def close(self) -> None:
|
||||
self._verifying.clear()
|
||||
for engine in list(self._engines.values()):
|
||||
await engine.dispose()
|
||||
self._engines.clear()
|
||||
self._pins.clear()
|
||||
|
||||
|
||||
engines = EngineCache()
|
||||
|
||||
@@ -27,6 +27,8 @@ from app.data.engine import engines
|
||||
_MAX_TABLES = 200
|
||||
# summary 里每张表列几个代表字段。够 Copilot 判断"这表是不是我要的"即可。
|
||||
_PREVIEW_COLUMNS = 6
|
||||
# 批量反射失败、逐个对象重试时,开头连着这么多个都取不到就不再试了
|
||||
_GIVE_UP_AFTER = 5
|
||||
|
||||
|
||||
# 各家的系统 schema,发现可用 schema 时要排掉——否则用户面对的是一屏
|
||||
@@ -141,25 +143,49 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any
|
||||
# shop 那次 "Lost connection to MySQL server during query"
|
||||
# (run 554a0f92)多半是偶发的服务端超时——所以真正的对策不在这里,
|
||||
# 在下面:失败要如实记成失败,让工具和界面都说真话。
|
||||
def _multi(fn: Any, **kw: Any) -> dict[Any, Any]:
|
||||
#
|
||||
# 批量是一锤子买卖:其中一个对象反射出错,整批都拿不到。SQLite 连接关了双引号
|
||||
# 字符串兼容以后,老库里一个写着 `"yes" AS flag` 的视图就让 PRAGMA 报错;以前这里
|
||||
# 吞掉异常返回空,好好的表跟着一起消失,探查却显示「成功、0 张表」。所以批量失败
|
||||
# 时逐个对象再取一遍,只跳过真正取不到的那个,名字和原因记进 skipped。
|
||||
failures: dict[str, str] = {}
|
||||
|
||||
def _multi(fn: Any, single: Any, names: list[str], *,
|
||||
errors: dict[str, str] | None = None) -> dict[Any, Any]:
|
||||
try:
|
||||
return dict(fn(schema=target_schema, filter_names=picked,
|
||||
kind=ObjectKind.ANY, **kw))
|
||||
return dict(fn(schema=target_schema, filter_names=names, kind=ObjectKind.ANY))
|
||||
except NotImplementedError:
|
||||
return {} # 表注释这类不是所有方言都支持
|
||||
except Exception: # noqa: BLE001 - 取不到不该让整次探查失败
|
||||
return {}
|
||||
except Exception: # noqa: BLE001 - 批量取不到,逐个对象再试
|
||||
pass
|
||||
out: dict[Any, Any] = {}
|
||||
for name in names:
|
||||
try:
|
||||
out[(target_schema, name)] = single(name, schema=target_schema)
|
||||
except NotImplementedError:
|
||||
break
|
||||
except Exception as e: # noqa: BLE001 - 只跳过这一个对象
|
||||
if errors is not None:
|
||||
errors[name] = first_line(e) or describe_exception(e)
|
||||
if not out and len(errors or ()) >= _GIVE_UP_AFTER:
|
||||
break # 开头几个全取不到,多半是连接本身出了问题,不再挨个耗时间
|
||||
return out
|
||||
|
||||
columns_by = _multi(inspector.get_multi_columns)
|
||||
pk_by = _multi(inspector.get_multi_pk_constraint)
|
||||
comment_by = _multi(inspector.get_multi_table_comment)
|
||||
columns_by = _multi(inspector.get_multi_columns, inspector.get_columns, picked,
|
||||
errors=failures)
|
||||
readable = [n for n in picked if columns_by.get((target_schema, n))]
|
||||
pk_by = _multi(inspector.get_multi_pk_constraint, inspector.get_pk_constraint, readable)
|
||||
comment_by = _multi(inspector.get_multi_table_comment, inspector.get_table_comment, readable)
|
||||
|
||||
tables: dict[str, Any] = {}
|
||||
skipped: dict[str, str] = {}
|
||||
for name in picked:
|
||||
key = (target_schema, name)
|
||||
columns = columns_by.get(key)
|
||||
if not columns:
|
||||
continue # 这张表反射不出来,跳过,别毁掉整次探查
|
||||
# 这个对象反射不出来,跳过,别毁掉整次探查;但要留下名字和原因
|
||||
skipped[name] = failures.get(name) or "读不到列"
|
||||
continue
|
||||
pk = (pk_by.get(key) or {}).get("constrained_columns") or []
|
||||
comment = (comment_by.get(key) or {}).get("text")
|
||||
|
||||
@@ -181,7 +207,14 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any
|
||||
"comment": comment,
|
||||
"is_view": name in views,
|
||||
}
|
||||
return {"tables": tables, "truncated": truncated, "total": len(names) + len(views)}
|
||||
payload: dict[str, Any] = {"tables": tables, "truncated": truncated,
|
||||
"total": len(names) + len(views)}
|
||||
if failures and not tables:
|
||||
# 一个对象也取不到、而且确实出了错:如实记成失败,不能显示成「已同步、0 张表」
|
||||
payload.update(failed=True, error=next(iter(failures.values())))
|
||||
elif skipped:
|
||||
payload["skipped"] = skipped
|
||||
return payload
|
||||
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""表名、列名、维度值的规范化和比较。
|
||||
|
||||
导入(data/tabular.py、data/table_versions.py)和证据层(engine/evidence.py)共用这一份。
|
||||
两边各写一套的话,「导入时认为没撞名、核对时认为是同一个名字」这种口径分歧迟早会出现。
|
||||
|
||||
三个用途,三个函数,不能混用:
|
||||
- canon:比较和存储维度值、主键用的规范写法(全角、各类连字符、格式字符、空白都抹平);
|
||||
- name_key:按名字找表、找列。只对 ASCII 不区分大小写、不做 NFKC,和 SQLite 的标识符比较完全一致
|
||||
(导入的名字都满足 name_problem、等于自身的 NFKC 形式,对它们做不做 NFKC 没有区别);
|
||||
- collide_key:判断两个名字算不算撞名。用 casefold,比 SQLite 严:「Дата」和「дата」
|
||||
在 SQLite 里是两张表,但并存只会让人和模型都分不清,所以也不许。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
#: NFKC 之后仍然不是 ASCII 连字符的各种横线:连字符、不换行连字符、数字线、en dash、
|
||||
#: em dash、水平线、减号。全角「-」NFKC 会归一,不在这里
|
||||
_DASHES = frozenset("‐‑‒–—―−")
|
||||
#: 看起来是空白、却不算 Zs 也不算 Cf 的填充字符(韩文填充符一类),混进名字里肉眼看不出来。
|
||||
#: 证据层核对 [[t:]] / [[c:]] 的写法时也用它(engine/evidence.py)
|
||||
FILLERS = frozenset("ᅟᅠㅤᅠ")
|
||||
_ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
|
||||
#: 名字上限 48 字:工具描述、表结构里要列出来,太长的名字模型和人都读不全
|
||||
NAME_MAX = 48
|
||||
_NAME = re.compile(rf"^[^\W\d]\w{{0,{NAME_MAX - 1}}}$")
|
||||
#: 作名字会让不加引号的 SQL 读错的关键字。不求全,收常见的、表头里可能出现的
|
||||
RESERVED = frozenset({
|
||||
"abort", "add", "all", "alter", "and", "as", "asc", "between", "by", "case", "check",
|
||||
"collate", "column", "commit", "constraint", "create", "cross", "default", "delete",
|
||||
"desc", "distinct", "drop", "else", "end", "escape", "except", "exists", "foreign",
|
||||
"from", "full", "group", "having", "in", "index", "inner", "insert", "intersect",
|
||||
"into", "is", "join", "key", "left", "like", "limit", "natural", "not", "null",
|
||||
"offset", "on", "or", "order", "outer", "primary", "references", "right", "rowid",
|
||||
"select", "set", "table", "then", "to", "transaction", "trigger", "union", "unique",
|
||||
"update", "using", "values", "view", "when", "where", "with",
|
||||
})
|
||||
|
||||
|
||||
def canon(text: str) -> str:
|
||||
"""规范写法:NFKC,各类横线统一成「-」,去掉格式字符(Cf),空白折叠成一个空格、去首尾。
|
||||
|
||||
九月的表把「8-9」写成「8–9」(en dash)或「8-9」(全角),按原文存的话,
|
||||
`WHERE 时段 = '8-9'` 会静默漏掉那个月——所以维度值和主键都按这个存,原文另记。
|
||||
"""
|
||||
s = unicodedata.normalize("NFKC", str(text))
|
||||
s = "".join("-" if ch in _DASHES else ch for ch in s if unicodedata.category(ch) != "Cf")
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
|
||||
def name_problem(name: str) -> str | None:
|
||||
"""表名或列名能不能用。能用返回 None,不能用返回一句给人看的原因。
|
||||
|
||||
规则:等于自身的 NFKC 形式(挡住全角 ASCII「ABC」、带圈数字「①」);不含控制字符、
|
||||
格式字符、空格类和组合附加符号,也不含填充字符;字母或汉字开头,只含字母、数字、汉字和
|
||||
下划线,最长 NAME_MAX 字;不以 sqlite_ 开头(SQLite 保留);不是 SQL 关键字。
|
||||
"""
|
||||
if not name:
|
||||
return "名称不能为空"
|
||||
if name != unicodedata.normalize("NFKC", name):
|
||||
return f"名称「{name}」含全角或兼容字符,请改用常规写法"
|
||||
if any(unicodedata.category(ch) in ("Cc", "Cf", "Zs", "Mn") or ch in FILLERS for ch in name):
|
||||
return f"名称「{name}」含不可见字符或空格"
|
||||
if not _NAME.match(name):
|
||||
return (f"名称「{name}」需以字母或汉字开头,只能包含字母、数字、汉字和下划线,"
|
||||
f"最长 {NAME_MAX} 字")
|
||||
if name.lower().startswith("sqlite_"):
|
||||
return f"名称「{name}」以 sqlite_ 开头,这是 SQLite 的保留前缀"
|
||||
if name.lower() in RESERVED:
|
||||
return f"名称「{name}」是 SQL 关键字"
|
||||
return None
|
||||
|
||||
|
||||
def name_key(name: str) -> str:
|
||||
"""按名字查找时用的键:只把 ASCII 字母转小写,其余逐字比,不做 NFKC。
|
||||
|
||||
SQLite 比较标识符时只对 ASCII 不区分大小写(`ABC` 和 `abc` 是同一张表,`Дата` 和
|
||||
`дата` 不是),也不做任何归一(全角的 `ABC` 和 `abc` 是两张表,库里可以并存)。证据层用这个键
|
||||
找表找列、给 SQL 里的表名去重,和真正执行 SQL 的数据库口径一致。
|
||||
|
||||
不能先做 NFKC:手工接入的库里全角「ORDERS」和「orders」并存时,[[t:orders]] 会被静默绑到
|
||||
全角那张,SQL 台账也会把真正 JOIN 的两张表当成一张。导入产生的名字都满足 name_problem(等于自身的
|
||||
NFKC 形式),对它们这两种写法得出的键一样;证据层的 [[t:]] / [[c:]] 引用也要求 NFKC 稳定。
|
||||
"""
|
||||
return str(name).translate(_ASCII_LOWER)
|
||||
|
||||
|
||||
def collide_key(name: str) -> str:
|
||||
"""撞名检查用的键:NFKC 加 casefold,比 SQLite 严。"""
|
||||
return unicodedata.normalize("NFKC", str(name)).casefold()
|
||||
|
||||
|
||||
def to_sql_name(header: object, *, fallback: str) -> str:
|
||||
"""原表头 → 一个满足 name_problem 的名字。原表头另存作显示名,这里只管能用。
|
||||
|
||||
canon 之后把不是字母、数字、汉字、下划线的字符(括号、斜杠、空格、换行、方括号、竖线)
|
||||
换成「_」并合并,去掉首尾的「_」;数字开头加「c_」,关键字后面加「_」,超长截断。
|
||||
什么都不剩时用 fallback(一般是 col_<位置>)。「金额(元)」→「金额_元」。
|
||||
"""
|
||||
s = canon("" if header is None else str(header))
|
||||
s = re.sub(r"[^\w]+", "_", s)
|
||||
s = "".join(ch for ch in s if unicodedata.category(ch) not in ("Mn", "Cf") and ch not in FILLERS)
|
||||
s = re.sub(r"_+", "_", s).strip("_")
|
||||
if not s:
|
||||
return fallback
|
||||
if s[0].isdigit():
|
||||
s = f"c_{s}"
|
||||
if s.lower().startswith("sqlite_"):
|
||||
s = f"t_{s}"
|
||||
s = s[:NAME_MAX].rstrip("_") or fallback
|
||||
if s.lower() in RESERVED:
|
||||
s = f"{s}_"
|
||||
return s if name_problem(s) is None else fallback
|
||||
@@ -0,0 +1,133 @@
|
||||
"""上传表格的原件存档:按内容哈希存,只写一次,没有任何下载接口。
|
||||
|
||||
**为什么要留原件。** 以前上传接口把文件读进内存、交给解析器,最后只剩一个 .db:
|
||||
解析规则一改只能让用户重传,报告里也说不出「用的是哪份文件」。原件按 sha256 存下来,
|
||||
构建 id 里记着它的哈希,版本才说得清来历(硬约束 H12)。
|
||||
|
||||
**为什么不给下载。** 原件里常有不该外传的东西:隐藏工作表里的口令、没导入的批注
|
||||
(硬约束 H10)。导入时这些都被挡在库外,开一个下载口子等于又放了出去。要删掉它,
|
||||
走「清除原件」(purge_raw),导入记录里留下时间、署名和理由。
|
||||
|
||||
**写法。** 先写同目录的临时文件、fsync,再 os.link 到最终路径——link 不会覆盖已有文件,
|
||||
两个请求同时存同一份内容时,后到的拿到 FileExistsError,校验哈希后直接复用。最终文件
|
||||
改成 0444:存档之后不该再被任何代码路径改写。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SHA = re.compile(r"^[0-9a-f]{64}$")
|
||||
#: 临时文件名里的标记。启动清理按它找上次崩溃留下的半成品(table_versions.startup)
|
||||
TMP_MARK = ".tmp-"
|
||||
|
||||
|
||||
def raw_root() -> Path:
|
||||
"""原件存档的根目录。按调用时的 settings 现算:测试会换数据目录。"""
|
||||
return settings.uploads_dir / "raw"
|
||||
|
||||
|
||||
def raw_path(sha: str) -> Path:
|
||||
"""哈希 → 存档路径(uploads/raw/<前两位>/<sha>)。只认 64 位小写十六进制,挡住路径穿越。"""
|
||||
if not _SHA.match(sha or ""):
|
||||
raise ValueError("原件哈希的格式不对")
|
||||
return raw_root() / sha[:2] / sha
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def fsync_dir(path: Path) -> None:
|
||||
"""目录项也要落盘:只 fsync 文件,断电后可能文件内容在、目录里却没有这个名字。"""
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def put_raw(raw: bytes) -> tuple[str, Path]:
|
||||
"""存一份原件,返回 (sha256, 路径)。同样内容已经存过就校验后复用,不重写。
|
||||
|
||||
已有文件的哈希对不上(磁盘损坏、被人改过)时,用这次的字节原子替换掉它:路径本身就是
|
||||
内容的哈希,这次手里的字节才是这个名字应有的内容。
|
||||
"""
|
||||
sha = hashlib.sha256(raw).hexdigest()
|
||||
path = raw_path(sha)
|
||||
if path.is_file() and sha256_file(path) == sha:
|
||||
return sha, path
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.parent / f".{sha}{TMP_MARK}{secrets.token_hex(6)}"
|
||||
try:
|
||||
with open(tmp, "wb") as fh:
|
||||
fh.write(raw)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
try:
|
||||
os.link(tmp, path)
|
||||
except FileExistsError:
|
||||
if sha256_file(path) == sha:
|
||||
return sha, path # 并发的另一个请求刚存好同一份
|
||||
logger.warning("原件存档 %s 的内容与文件名不符,已用本次上传的内容替换", sha)
|
||||
os.replace(tmp, path)
|
||||
os.chmod(path, 0o444)
|
||||
fsync_dir(path.parent)
|
||||
return sha, path
|
||||
finally:
|
||||
tmp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def raw_exists(sha: str) -> bool:
|
||||
try:
|
||||
return raw_path(sha).is_file()
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def purge_raw(sha: str, *, prune: bool = False) -> bool:
|
||||
"""删掉一份原件。返回是否真的删了文件(本来就不在返回 False)。
|
||||
|
||||
只删文件;导入记录上的 raw_state 和清除记录由调用方写——同一份内容可能被几次导入共用,
|
||||
删不删由调用方看过引用之后决定。
|
||||
|
||||
prune:删完后所在的 raw/<前两位>/ 空了就一并删掉。只有持着版本存储锁的调用方能这么做
|
||||
(table_versions 的清除原件、回收):put_raw 也在锁里,先建目录再写临时文件,锁外删目录
|
||||
会让并发的存档在两步之间扑空。
|
||||
"""
|
||||
try:
|
||||
path = raw_path(sha)
|
||||
except ValueError:
|
||||
return False
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
try:
|
||||
fsync_dir(path.parent)
|
||||
except OSError: # pragma: no cover - 删已经成功,目录落盘失败不改变结论
|
||||
pass
|
||||
if prune:
|
||||
remove_empty_dir(path.parent)
|
||||
return True
|
||||
|
||||
|
||||
def remove_empty_dir(path: Path) -> bool:
|
||||
"""目录空了就删掉,返回删没删。不空、不存在、没权限都不算错:留着一个空目录不影响任何事。"""
|
||||
try:
|
||||
path.rmdir()
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
File diff suppressed because it is too large
Load Diff
+1191
-201
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,995 @@
|
||||
"""一遍流式扫描 .xlsx 的版式元数据:openpyxl read_only 拿不到的,这里补上。
|
||||
|
||||
**为什么要自己扫。** 导入走 openpyxl 的 read_only(流式、整数不失真、内存低),但 read_only
|
||||
没有合并区、隐藏行列、自动筛选、表格对象,`<dimension>` 也可能是陈旧的或者干脆没有;
|
||||
普通模式能拿到这些,代价是整本工作簿进内存(20 万行约 950 MB)。所以元数据另走一遍:
|
||||
标准库 zipfile 加 ElementTree.iterparse,按行流式读,读完一行丢一行。
|
||||
|
||||
**为什么不信 `<dimension>`。** 它是写文件的程序自己填的:openpyxl 的 write_only 根本不写,
|
||||
有的程序删了数据也不更新。真实边界只能从非空单元格算——这一遍顺手就算出来了,
|
||||
导入的第二遍按这个边界读(tabular.load_into)。
|
||||
|
||||
**安全。** 上传的文件不可信:
|
||||
- 解压后的总大小、单个部件的大小、部件个数都设上限(防 zip 炸弹)。只限字节不够:每个字节在
|
||||
内存里能放大几十倍,所以工作表按 Excel 自己的上限数行、列、挂在树上的元素和单元格文字长度,
|
||||
共享字符串表数条目,openpyxl 整份建树的小部件(样式、文档属性、图表工作表……)数元素和格式条目。
|
||||
- 压缩包里**每一个**条目(不看扩展名:工作表改名成 .bin,关系文件照样指着它)都先用 expat 解析到
|
||||
根元素为止,序言里有 DTD 就拒收。OOXML 从不需要 DTD,有 DTD 只可能是实体膨胀之类的攻击。拦在
|
||||
解析器上而不是比字节,是因为字节比较认不出 UTF-16 写的「<!DOCTYPE」。
|
||||
- 这里和 openpyxl 找部件的办法不同(这里按关系文件,openpyxl 按内容类型清单),两边找到的
|
||||
工作簿主部件、共享字符串表必须是同一个;工作表名重复(不区分大小写)拒收。否则扫描看的是
|
||||
一份,导入读的是另一份,「隐藏的工作表已跳过」这类回执就不可信了。
|
||||
|
||||
这里只读不判:哪些情况拒收、怎么提示,是 tabular 的事。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
import zipfile
|
||||
from xml.parsers import expat
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass, field
|
||||
from typing import IO
|
||||
|
||||
#: 解压后的总大小上限。上传本身限 50 MB,正常的 xlsx 压缩比在 5 到 20 倍之间;
|
||||
#: 超过这个数的基本是 zip 炸弹。可调
|
||||
MAX_UNCOMPRESSED = 2 * 1024 ** 3
|
||||
#: 单个 XML 部件(一张工作表、共享字符串表)解压后的上限。可调
|
||||
MAX_XML_BYTES = 1024 ** 3
|
||||
#: 工作表、共享字符串表以外的 XML 部件(样式、主题、文档属性、工作簿、关系文件、表格定义……)
|
||||
#: 解压后的上限。这些部件 openpyxl 一次整份建树(样式每个字节在内存里放大几十倍),正常文件
|
||||
#: 一般不过 1 MB,样式堆积得很厉害的也在 10 MB 上下
|
||||
MAX_PART_BYTES = 16 * 1024 ** 2
|
||||
#: 上述小部件里元素个数的上限:字节上限挡不住「几百万个空元素」这种写法
|
||||
MAX_PART_ELEMENTS = 400_000
|
||||
#: 样式表里格式条目(xf)的上限:Excel 自己最多 65,490 种单元格格式,两组(cellStyleXfs、cellXfs)各算一份
|
||||
MAX_XF = 2 * 65_490
|
||||
#: 压缩包里条目个数的上限。一千张工作表的工作簿也只有几千个部件
|
||||
MAX_ENTRIES = 10_000
|
||||
#: Excel 的行数、列数上限
|
||||
EXCEL_MAX_ROWS = 1_048_576
|
||||
EXCEL_MAX_COLS = 16_384
|
||||
#: Excel 单元格文字、公式长度的上限(字符)
|
||||
EXCEL_MAX_TEXT = 32_767
|
||||
EXCEL_MAX_FORMULA = 8_192
|
||||
#: 扫描一张工作表时,树上同时挂着的元素个数上限。处理完的行清空后行本身的空壳还挂在
|
||||
#: sheetData 下(只要 end 事件就拿不到父元素、摘不下来),行以外的元素(合并区、筛选、扩展、
|
||||
#: 写文件的程序塞进来的未知元素)也一直挂着:每个几十到上百字节,几百万个就是几百 MB。
|
||||
#: 正常的表最多 1,048,576 个行壳加上几万个其他元素,这里再留一百万的余量
|
||||
MAX_ALIVE = EXCEL_MAX_ROWS + 1_000_000
|
||||
#: 共享字符串表的条目上限。openpyxl 会把整张表读进一个列表,每条约一百字节
|
||||
MAX_SHARED_STRINGS = 4_000_000
|
||||
#: 共享字符串表里一个条目(富文本的若干段)的元素个数上限
|
||||
MAX_SI_ELEMENTS = 100_000
|
||||
#: 坐标清单(无缓存值的公式、错误值……)最多记多少个。总数另计,不受这个限制
|
||||
MAX_CELLS = 50
|
||||
#: 合并区最多记多少个。合并区上万的文件也见过(整列逐格合并),全记下来只是占内存
|
||||
MAX_MERGED = 10_000
|
||||
#: 离群格示例最多几个
|
||||
MAX_FAR = 10
|
||||
#: 判「离群」的跳跃幅度:一个非空格比此前见过的最大行号多出这么多行、或比最大列号多出这么多列。
|
||||
#: 正常表格的行是连着的,隔几千行才冒出来一个格子,多半是误输入或残留格式
|
||||
FAR_ROW_JUMP = 1000
|
||||
FAR_COL_JUMP = 50
|
||||
|
||||
_CHUNK = 64 * 1024
|
||||
|
||||
_MAIN_NS = (
|
||||
"http://schemas.openxmlformats.org/spreadsheetml/2006/main",
|
||||
"http://purl.oclc.org/ooxml/spreadsheetml/main", # Strict OOXML
|
||||
)
|
||||
#: 工作表里要看的元素:完整标签 → 本地名。两种命名空间都认,一次字典查找就分派
|
||||
_SHEET_TAGS = {
|
||||
f"{{{ns}}}{local}": local
|
||||
for ns in _MAIN_NS
|
||||
for local in ("row", "c", "v", "f", "is", "mergeCell", "autoFilter", "col", "dimension")
|
||||
}
|
||||
_DIGITS = "0123456789"
|
||||
|
||||
|
||||
class UnsupportedTable(ValueError):
|
||||
"""认不出或读不了。message 直接给用户看。
|
||||
|
||||
定义在这里而不是 tabular:扫描是导入的第一步,同样要能拒收,而 tabular 依赖这个模块。
|
||||
对外仍然从 app.data.tabular 导入(那边原样转出)。
|
||||
"""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 结果
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Bounds:
|
||||
"""真实非空单元格的外接矩形,1 基行列号。"""
|
||||
|
||||
min_row: int
|
||||
min_col: int
|
||||
max_row: int
|
||||
max_col: int
|
||||
|
||||
@property
|
||||
def area(self) -> int:
|
||||
return (self.max_row - self.min_row + 1) * (self.max_col - self.min_col + 1)
|
||||
|
||||
def a1(self) -> str:
|
||||
return f"{cell_ref(self.min_row, self.min_col)}:{cell_ref(self.max_row, self.max_col)}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CellList:
|
||||
"""一类单元格的坐标:前 MAX_CELLS 个的坐标,加总数和最大行号。
|
||||
|
||||
最大行号让调用方不必拿到全部坐标也能精确判断「某行以下有没有」:
|
||||
导入区域从表头行一直到边界底部,有没有落在区域里只看 max_row。
|
||||
"""
|
||||
|
||||
cells: list[str] = field(default_factory=list)
|
||||
total: int = 0
|
||||
max_row: int = 0
|
||||
#: 和 cells 一一对应的原文(目前只有公式类用:公式文本,截断到 80 字)
|
||||
texts: list[str] = field(default_factory=list)
|
||||
|
||||
def add(self, row: int, col: int, text: str | None = None) -> None:
|
||||
self.total += 1
|
||||
if row > self.max_row:
|
||||
self.max_row = row
|
||||
if len(self.cells) < MAX_CELLS:
|
||||
self.cells.append(cell_ref(row, col))
|
||||
if text is not None:
|
||||
self.texts.append(text[:80])
|
||||
|
||||
|
||||
@dataclass
|
||||
class SheetScan:
|
||||
name: str
|
||||
#: visible / hidden / veryHidden
|
||||
state: str
|
||||
#: 工作表 XML 在压缩包里的路径
|
||||
path: str
|
||||
#: 真实非空格的边界;整张表没有非空格时是 None
|
||||
bounds: Bounds | None = None
|
||||
#: 非空格数。有公式的格不算空(即使没有缓存值);只有空白字符的文本算空
|
||||
nonempty: int = 0
|
||||
merged: list[str] = field(default_factory=list)
|
||||
merged_total: int = 0
|
||||
#: 隐藏行、隐藏列,合并成闭区间 (起, 止)。筛选隐藏和手动隐藏在 XML 里分不出来
|
||||
hidden_rows: list[tuple[int, int]] = field(default_factory=list)
|
||||
hidden_cols: list[tuple[int, int]] = field(default_factory=list)
|
||||
autofilter: str | None = None
|
||||
formulas: int = 0
|
||||
#: 有公式、却没有缓存值的格(程序生成、没用 Excel 保存过的文件)。data_only 读出来是空
|
||||
formulas_uncached: CellList = field(default_factory=CellList)
|
||||
#: 错误值格(#DIV/0! 之类)
|
||||
errors: CellList = field(default_factory=CellList)
|
||||
#: 汇总了本格上方连续几行的公式(如 C26 的 SUM(C22:C25)):表内合计行的信号
|
||||
formulas_above: CellList = field(default_factory=CellList)
|
||||
#: 离群格示例:稀疏判定拒收时告诉用户是哪几个远处的格子把区域撑大了
|
||||
far_cells: list[str] = field(default_factory=list)
|
||||
#: 文件自己声明的 <dimension>,只作记录,不用来定边界
|
||||
dimension: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TableScan:
|
||||
"""Excel 的表格对象(插入 → 表格)。ref 连表头带汇总行一起算。"""
|
||||
|
||||
name: str
|
||||
ref: str
|
||||
sheet: str
|
||||
totals_row_count: int = 0
|
||||
header_row_count: int = 1
|
||||
#: 各列的列名,从左到右
|
||||
columns: list[str] = field(default_factory=list)
|
||||
|
||||
def span(self) -> tuple[int, int, int, int] | None:
|
||||
"""ref 的 (首行, 首列, 末行, 末列);ref 写坏了是 None。"""
|
||||
a, _, b = self.ref.replace("$", "").partition(":")
|
||||
try:
|
||||
r1, c1 = parse_ref(a)
|
||||
r2, c2 = parse_ref(b or a)
|
||||
except ValueError:
|
||||
return None
|
||||
return min(r1, r2), min(c1, c2), max(r1, r2), max(c1, c2)
|
||||
|
||||
def totals_rows(self) -> tuple[int, int] | None:
|
||||
"""汇总行的行号区间(闭区间);没有汇总行是 None。"""
|
||||
span = self.span()
|
||||
if span is None or self.totals_row_count <= 0:
|
||||
return None
|
||||
return max(span[0], span[2] - self.totals_row_count + 1), span[2]
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkbookScan:
|
||||
sheets: list[SheetScan] = field(default_factory=list)
|
||||
#: calcPr 的 fullCalcOnLoad:打开时全部重算。XlsxWriter 默认这么写,同时把公式缓存值写成 0
|
||||
full_calc_on_load: bool = False
|
||||
tables: list[TableScan] = field(default_factory=list)
|
||||
#: 定义名称 → 原文。工作表级的名称记作「工作表名!名称」,免得同名互相覆盖
|
||||
defined_names: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def sheet(self, name: str) -> SheetScan | None:
|
||||
return next((s for s in self.sheets if s.name == name), None)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 坐标
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_COL_CACHE: dict[str, int] = {}
|
||||
|
||||
|
||||
def col_letter(col: int) -> str:
|
||||
out = ""
|
||||
while col > 0:
|
||||
col, rem = divmod(col - 1, 26)
|
||||
out = chr(65 + rem) + out
|
||||
return out
|
||||
|
||||
|
||||
def cell_ref(row: int, col: int) -> str:
|
||||
return f"{col_letter(col)}{row}"
|
||||
|
||||
|
||||
def _col_number(letters: str) -> int:
|
||||
hit = _COL_CACHE.get(letters)
|
||||
if hit is None:
|
||||
hit = 0
|
||||
for ch in letters.upper():
|
||||
hit = hit * 26 + ord(ch) - 64
|
||||
if len(_COL_CACHE) < 20_000:
|
||||
_COL_CACHE[letters] = hit
|
||||
return hit
|
||||
|
||||
|
||||
def parse_ref(ref: str) -> tuple[int, int]:
|
||||
"""「AB12」→ (12, 28)。不认 $,调用方给的是单元格的 r 属性,不带 $。"""
|
||||
letters = ref.rstrip(_DIGITS)
|
||||
return int(ref[len(letters):]), _col_number(letters)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 读压缩包
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _Budget:
|
||||
"""整个压缩包解压出来的总字节数。声明的大小先查一遍,读的时候再实数一遍。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.used = 0
|
||||
|
||||
def take(self, n: int) -> None:
|
||||
self.used += n
|
||||
if self.used > MAX_UNCOMPRESSED:
|
||||
raise UnsupportedTable(_too_big())
|
||||
|
||||
|
||||
def _too_big() -> str:
|
||||
return (f"文件解压后超过 {MAX_UNCOMPRESSED // 1024 ** 2} MB,无法导入。"
|
||||
"请删除不需要的工作表,或另存为 .csv 后重新上传")
|
||||
|
||||
|
||||
class _Guarded:
|
||||
"""包住压缩包里的一个 XML 部件:数着字节读,超过大小上限就停。
|
||||
|
||||
iterparse 只要求 source 有 read(n)。DTD 不在这里查:见 _check_prolog。
|
||||
"""
|
||||
|
||||
def __init__(self, fp: IO[bytes], part: str, budget: _Budget, limit: int | None = None) -> None:
|
||||
self._fp = fp
|
||||
self._part = part
|
||||
self._budget = budget
|
||||
self._seen = 0
|
||||
#: 这个部件最多读多少字节。None 表示按 MAX_XML_BYTES(现取,测试里可以改)
|
||||
self._limit = limit
|
||||
|
||||
def read(self, n: int = -1) -> bytes:
|
||||
data = self._fp.read(n if n and n > 0 else _CHUNK)
|
||||
if not data:
|
||||
return data
|
||||
self._seen += len(data)
|
||||
self._budget.take(len(data))
|
||||
limit = self._limit if self._limit is not None else MAX_XML_BYTES
|
||||
if self._seen > limit:
|
||||
raise UnsupportedTable(_part_too_big(self._part, limit))
|
||||
return data
|
||||
|
||||
def close(self) -> None:
|
||||
self._fp.close()
|
||||
|
||||
|
||||
def _part_too_big(part: str, limit: int) -> str:
|
||||
return (f"文件中的「{part}」解压后超过 {limit // 1024 ** 2} MB,无法导入。"
|
||||
"请拆分工作表,或另存为 .csv 后重新上传")
|
||||
|
||||
|
||||
def _abnormal(what: str) -> str:
|
||||
"""文件结构超出 Excel 自己的上限、或者前后矛盾:Excel 不会生成这样的文件。"""
|
||||
return f"{what},不是 Excel 正常保存的文件,已拒绝导入。请在 Excel 中另存为 .xlsx 或 .csv 后重新上传"
|
||||
|
||||
|
||||
def _local(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def _resolve(base_dir: str, target: str) -> str:
|
||||
"""关系文件里的 Target → 压缩包内路径。openpyxl 写绝对路径(/xl/...),Excel 写相对路径。"""
|
||||
if target.startswith("/"):
|
||||
return target.lstrip("/")
|
||||
out: list[str] = []
|
||||
for part in f"{base_dir}/{target}".split("/"):
|
||||
if part == "..":
|
||||
if out:
|
||||
out.pop()
|
||||
elif part and part != ".":
|
||||
out.append(part)
|
||||
return "/".join(out)
|
||||
|
||||
|
||||
class _Archive:
|
||||
def __init__(self, raw: bytes) -> None:
|
||||
try:
|
||||
self.zip = zipfile.ZipFile(io.BytesIO(raw))
|
||||
except zipfile.BadZipFile as e:
|
||||
raise UnsupportedTable(_not_xlsx()) from e
|
||||
infos = self.zip.infolist()
|
||||
if len(infos) > MAX_ENTRIES:
|
||||
raise UnsupportedTable(_abnormal(f"压缩包中的部件超过 {MAX_ENTRIES:,} 个"))
|
||||
self.names = set(self.zip.namelist())
|
||||
self.budget = _Budget()
|
||||
declared = sum(info.file_size for info in infos)
|
||||
if declared > MAX_UNCOMPRESSED:
|
||||
raise UnsupportedTable(_too_big())
|
||||
#: 已经数过元素个数的小部件(_check_small 只做一次)
|
||||
self.checked: set[str] = set()
|
||||
|
||||
def open(self, path: str, limit: int | None = None) -> _Guarded:
|
||||
"""打开一个部件,读出的字节受 limit 限制(None 表示按 MAX_XML_BYTES)。声明的大小先查一遍。"""
|
||||
info = self.zip.getinfo(path)
|
||||
cap = limit if limit is not None else MAX_XML_BYTES
|
||||
if info.file_size > cap:
|
||||
raise UnsupportedTable(_part_too_big(path, cap))
|
||||
return _Guarded(self.zip.open(info), path, self.budget, cap)
|
||||
|
||||
def tree(self, path: str) -> ET.Element:
|
||||
"""小部件(workbook.xml、关系文件、表格定义)整个解析。先按小部件的上限数一遍元素,再建树。"""
|
||||
_check_small(self, path)
|
||||
parser = ET.XMLParser()
|
||||
with closing(self.open(path, MAX_PART_BYTES)) as source:
|
||||
while chunk := source.read(_CHUNK):
|
||||
parser.feed(chunk)
|
||||
return parser.close()
|
||||
|
||||
def rels(self, part: str) -> dict[str, tuple[str, str]]:
|
||||
"""部件的关系表:Id → (类型, 解析后的路径)。没有关系文件时是空的。"""
|
||||
folder, _, name = part.rpartition("/")
|
||||
rels_path = f"{folder}/_rels/{name}.rels" if folder else f"_rels/{name}.rels"
|
||||
if rels_path not in self.names:
|
||||
return {}
|
||||
out: dict[str, tuple[str, str]] = {}
|
||||
for rel in self.tree(rels_path):
|
||||
target = rel.get("Target") or ""
|
||||
if rel.get("TargetMode") == "External":
|
||||
continue
|
||||
out[rel.get("Id") or ""] = (rel.get("Type") or "", _resolve(folder, target))
|
||||
return out
|
||||
|
||||
|
||||
class _RootReached(Exception):
|
||||
"""序言读完了(见到了根元素):_check_prolog 用它提前结束解析。"""
|
||||
|
||||
|
||||
def _no_dtd(*_args: object) -> None:
|
||||
raise UnsupportedTable(_DTD_MESSAGE)
|
||||
|
||||
|
||||
def _root_reached(*_args: object) -> None:
|
||||
raise _RootReached
|
||||
|
||||
|
||||
_DTD_MESSAGE = "文件中含有 Excel 文件不应包含的内容(DTD 声明),已拒绝导入"
|
||||
_PROLOG_CHUNK = 4096
|
||||
|
||||
|
||||
def _check_prolog(archive: _Archive, path: str, *, strict: bool = True) -> bool:
|
||||
"""一个部件的序言里有没有 DTD:有就拒收。返回它是不是 XML(读到了根元素)。
|
||||
|
||||
DOCTYPE 只能出现在根元素之前,所以只解析到根元素的开始标签为止,通常只读几百字节。
|
||||
用 expat 解析而不是比字节:编码由 expat 自己认(UTF-8、带或不带 BOM 的 UTF-16……),
|
||||
和 ElementTree、openpyxl 用的是同一套规则,「换个编码就绕过」行不通。
|
||||
|
||||
**压缩包里每个条目都查,不看扩展名。** 工作表、共享字符串表的路径来自关系文件和内容类型,
|
||||
改名成 sheet1.bin 照样会被这里的扫描和 openpyxl 当 XML 解析,只查 .xml/.rels 就漏了。
|
||||
strict(名字是 .xml/.rels)时序言解析不了也拒收:OOXML 规定部件是 UTF-8 或 UTF-16 的合法
|
||||
XML,expat 连序言都读不过去的,别的解析器读出来是什么样说不准。其余名字(图片、
|
||||
printerSettings、vbaProject.bin)解析不了是正常的,跳过:expat 读不过序言的,ElementTree
|
||||
和 openpyxl 也不可能把它当 XML 读出内容。
|
||||
|
||||
唯一的例外:.svg 图片可以带不含内部子集的 DOCTYPE(老版本 Illustrator 导出的 SVG 都带一行
|
||||
`<!DOCTYPE svg PUBLIC …>`)。没有内部子集就没有实体定义,expat 也不会去取外部 DTD,
|
||||
改名成 .svg 的工作表因此展开不了任何实体;内部子集、实体声明照样拒收。
|
||||
"""
|
||||
svg = path.lower().endswith(".svg")
|
||||
|
||||
def doctype(_name: str, _sysid: str | None, _pubid: str | None, has_internal_subset: int) -> None:
|
||||
if has_internal_subset or not svg:
|
||||
raise UnsupportedTable(_DTD_MESSAGE)
|
||||
|
||||
parser = expat.ParserCreate()
|
||||
parser.StartDoctypeDeclHandler = doctype
|
||||
parser.EntityDeclHandler = _no_dtd
|
||||
parser.StartElementHandler = _root_reached
|
||||
# 读序言不受单个部件的大小上限约束(大图片也只读开头几 KB),读到的字节照样计入总量
|
||||
with closing(archive.open(path, max(archive.zip.getinfo(path).file_size, 1))) as source:
|
||||
try:
|
||||
while chunk := source.read(_PROLOG_CHUNK):
|
||||
parser.Parse(chunk, False)
|
||||
except _RootReached:
|
||||
return True
|
||||
except (expat.ExpatError, LookupError) as e:
|
||||
# LookupError:声明了 Python 不认识的编码,pyexpat 原样抛出
|
||||
if strict:
|
||||
raise UnsupportedTable(_broken()) from e
|
||||
return False
|
||||
# 读到结尾也没见到根元素(空部件之类):没有 DOCTYPE 就不管,读不读得了是真正解析时的事
|
||||
return False
|
||||
|
||||
|
||||
def _check_small(archive: _Archive, path: str) -> None:
|
||||
"""工作表、共享字符串表以外的 XML 部件:字节数和元素个数都设上限,样式里的格式条目单独设上限。
|
||||
|
||||
这些部件 openpyxl 用 lxml 一次整份建树(样式还要给每个格式条目建对象),内存是字节数的
|
||||
几十倍;只限字节,几 MB 的「几十万个空元素」就能吃掉几个 GB。这里流式数一遍:
|
||||
处理完的元素清空、根下的直接子元素摘掉,数的过程本身不占多少内存。
|
||||
"""
|
||||
if path in archive.checked:
|
||||
return
|
||||
total = xf = depth = 0
|
||||
root: ET.Element | None = None
|
||||
with closing(archive.open(path, MAX_PART_BYTES)) as source:
|
||||
for event, el in ET.iterparse(source, events=("start", "end")):
|
||||
if event == "start":
|
||||
depth += 1
|
||||
if root is None:
|
||||
root = el
|
||||
continue
|
||||
depth -= 1
|
||||
total += 1
|
||||
if total > MAX_PART_ELEMENTS:
|
||||
raise UnsupportedTable(_abnormal(f"文件中的「{path}」包含的元素超过 {MAX_PART_ELEMENTS:,} 个"))
|
||||
if _local(el.tag) == "xf":
|
||||
xf += 1
|
||||
if xf > MAX_XF:
|
||||
raise UnsupportedTable(_abnormal(f"样式表中的单元格格式超过 {MAX_XF:,} 种"))
|
||||
el.clear()
|
||||
if depth == 1 and root is not None:
|
||||
root.remove(el)
|
||||
archive.checked.add(path)
|
||||
|
||||
|
||||
#: 工作簿主部件的几种内容类型,按 openpyxl 找主部件的顺序(reader/excel.py _find_workbook_part)
|
||||
_WORKBOOK_TYPES = (
|
||||
"application/vnd.ms-excel.template.macroEnabled.main+xml",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.template.main+xml",
|
||||
"application/vnd.ms-excel.sheet.macroEnabled.main+xml",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml",
|
||||
)
|
||||
_SHARED_STRINGS_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml"
|
||||
_CONTENT_TYPES = "[Content_Types].xml"
|
||||
|
||||
|
||||
def _content_types(archive: _Archive) -> tuple[dict[str, list[str]], set[str]]:
|
||||
"""内容类型清单:(内容类型 → 按出现顺序的部件路径, 默认登记过的内容类型)。"""
|
||||
if _CONTENT_TYPES not in archive.names:
|
||||
raise UnsupportedTable(_not_xlsx())
|
||||
overrides: dict[str, list[str]] = {}
|
||||
defaults: set[str] = set()
|
||||
for el in archive.tree(_CONTENT_TYPES):
|
||||
local = _local(el.tag)
|
||||
if local == "Override":
|
||||
overrides.setdefault(el.get("ContentType") or "", []).append((el.get("PartName") or "").lstrip("/"))
|
||||
elif local == "Default":
|
||||
defaults.add(el.get("ContentType") or "")
|
||||
return overrides, defaults
|
||||
|
||||
|
||||
def _book_path(archive: _Archive, overrides: dict[str, list[str]], defaults: set[str]) -> str:
|
||||
"""工作簿主部件的路径。内容类型清单和包关系文件各指一个:openpyxl 按前者找,这里按后者找,
|
||||
两边不是同一个就拒收——否则扫描看的是一本工作簿,导入读的是另一本。"""
|
||||
claimed = [path for kind in _WORKBOOK_TYPES for path in overrides.get(kind, [])]
|
||||
if len(claimed) > 1:
|
||||
raise UnsupportedTable(_abnormal("文件中登记了不止一个工作簿主部件"))
|
||||
if claimed:
|
||||
by_types = claimed[0]
|
||||
elif defaults & set(_WORKBOOK_TYPES):
|
||||
by_types = "xl/workbook.xml" # openpyxl 的退路:默认内容类型就是工作簿
|
||||
else:
|
||||
raise UnsupportedTable(_not_xlsx())
|
||||
by_rels = next((path for kind, path in archive.rels("").values() if kind.endswith("/officeDocument")), None)
|
||||
if by_rels is not None and by_rels != by_types:
|
||||
raise UnsupportedTable(_abnormal("文件中两处登记的工作簿主部件不一致"))
|
||||
return by_types
|
||||
|
||||
|
||||
def _shared_strings_path(rels: dict[str, tuple[str, str]], overrides: dict[str, list[str]],
|
||||
archive: _Archive) -> str | None:
|
||||
"""共享字符串表的路径。openpyxl 按内容类型清单找,这里按工作簿的关系文件找:两边不一致就拒收,
|
||||
否则扫描判断「格子空不空」用的是一张表,导入读出来的是另一张。"""
|
||||
by_rels = next((path for kind, path in rels.values()
|
||||
if kind.endswith("/sharedStrings") and path in archive.names), None)
|
||||
claimed = overrides.get(_SHARED_STRINGS_TYPE, [])
|
||||
if len(claimed) > 1 or (claimed and claimed[0] != by_rels):
|
||||
raise UnsupportedTable(_abnormal("文件中两处登记的共享字符串表不一致"))
|
||||
return by_rels
|
||||
|
||||
|
||||
def _not_xlsx() -> str:
|
||||
return ("无法打开该 Excel 文件:文件不是有效的 .xlsx(可能已损坏,或只是修改了扩展名)。"
|
||||
"请在 Excel 中另存为 .xlsx 或 .csv 后重新上传")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 扫描
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _broken() -> str:
|
||||
return "无法打开该 Excel 文件:文件内容不完整或格式不正确。请在 Excel 中另存为 .xlsx 或 .csv 后重新上传"
|
||||
|
||||
|
||||
def scan(raw: bytes) -> WorkbookScan:
|
||||
"""xlsx 字节 → 版式元数据。读不了就抛 UnsupportedTable(message 给用户看)。"""
|
||||
archive = _Archive(raw)
|
||||
try:
|
||||
return _scan(archive)
|
||||
except UnsupportedTable:
|
||||
raise
|
||||
except (ET.ParseError, KeyError, ValueError, zipfile.BadZipFile, EOFError) as e:
|
||||
# KeyError:压缩包里缺部件;ValueError:坐标、数字写坏了;ParseError:XML 不完整
|
||||
raise UnsupportedTable(_broken()) from e
|
||||
finally:
|
||||
archive.zip.close()
|
||||
|
||||
|
||||
def _scan(archive: _Archive) -> WorkbookScan:
|
||||
# 先把每个条目的序言查一遍,再开始真正的解析:openpyxl 之后还要读样式、主题这些
|
||||
# 这里不读的部件,DTD 的拒收要覆盖到它们(不看扩展名,见 _check_prolog)
|
||||
xml_parts = [info for info in archive.zip.infolist() if not info.is_dir() and _check_prolog(
|
||||
archive, info.filename, strict=info.filename.lower().endswith((".xml", ".rels")))]
|
||||
overrides, defaults = _content_types(archive)
|
||||
book_path = _book_path(archive, overrides, defaults)
|
||||
book = archive.tree(book_path)
|
||||
rels = archive.rels(book_path)
|
||||
result = WorkbookScan()
|
||||
|
||||
sst_path = _shared_strings_path(rels, overrides, archive)
|
||||
# 工作表和共享字符串表可以很大,按 MAX_XML_BYTES 和逐行的上限管。这里自己整份解析的小部件
|
||||
# (内容类型清单、工作簿、关系文件、表格定义)经 _Archive.tree 按小部件管;openpyxl 整份解析、
|
||||
# 这里不读的,在这里先查
|
||||
for path in _openpyxl_parts(archive, rels, {info.filename for info in xml_parts}):
|
||||
_check_small(archive, path)
|
||||
|
||||
sst_flags = _scan_shared_strings(archive, sst_path) if sst_path else None
|
||||
|
||||
sheet_names: list[str] = []
|
||||
seen_names: dict[str, str] = {}
|
||||
for el in book.iter():
|
||||
local = _local(el.tag)
|
||||
if local == "calcPr":
|
||||
result.full_calc_on_load = el.get("fullCalcOnLoad") in ("1", "true")
|
||||
elif local == "sheet":
|
||||
rid = next((v for k, v in el.attrib.items() if k.endswith("}id")), None)
|
||||
kind, path = rels.get(rid or "", ("", ""))
|
||||
name = el.get("name") or ""
|
||||
# Excel 的工作表名不区分大小写、不许重复。重名时 openpyxl 按名字取表会拿到第一个
|
||||
# (可能是隐藏的那个),导入的内容就和回执说的对不上了
|
||||
folded = name.casefold()
|
||||
if folded in seen_names:
|
||||
raise UnsupportedTable(_abnormal(
|
||||
f"工作簿中有两个工作表同名(「{seen_names[folded]}」与「{name}」,工作表名不区分大小写)"))
|
||||
seen_names[folded] = name
|
||||
sheet_names.append(name)
|
||||
# 图表工作表、对话框工作表没有单元格,openpyxl 的 worksheets 里也不含它们
|
||||
if not kind.endswith("/worksheet") or path not in archive.names:
|
||||
continue
|
||||
sheet = SheetScan(name=name, state=el.get("state") or "visible", path=path)
|
||||
# 表格对象先读:扫单元格时要用它的位置认出「销售表[金额]」这种结构化引用汇总了哪几行
|
||||
tables = _scan_tables(archive, path, name)
|
||||
result.tables.extend(tables)
|
||||
_scan_sheet(archive, sheet, sst_flags, {t.name.casefold(): t for t in tables if t.name})
|
||||
result.sheets.append(sheet)
|
||||
|
||||
for el in book.iter():
|
||||
if _local(el.tag) != "definedName":
|
||||
continue
|
||||
name = el.get("name") or ""
|
||||
local_id = el.get("localSheetId")
|
||||
if local_id is not None and local_id.isdigit() and int(local_id) < len(sheet_names):
|
||||
name = f"{sheet_names[int(local_id)]}!{name}"
|
||||
result.defined_names[name] = (el.text or "").strip()
|
||||
return result
|
||||
|
||||
|
||||
#: openpyxl 按固定路径整份解析的部件:文档属性、自定义属性、样式(reader/excel.py、styles/stylesheet.py)
|
||||
_OPENPYXL_FIXED = ("docProps/core.xml", "docProps/custom.xml", "xl/styles.xml")
|
||||
|
||||
|
||||
def _openpyxl_parts(archive: _Archive, rels: dict[str, tuple[str, str]], xml_names: set[str]) -> list[str]:
|
||||
"""openpyxl 只读打开时整份建树、而这里的扫描不读的 XML 部件。
|
||||
|
||||
只管真会被解析的:计算链、批注、透视表缓存这些部件可以很大(和公式数、数据行数成正比),
|
||||
但没人解析它们,按小部件设限只会误拒正常文件。外部链接 openpyxl 默认也会读,导入时已关掉
|
||||
(tabular._open_book 的 keep_links=False)。图表工作表连同它引用的绘图、图表一并算进来。
|
||||
"""
|
||||
out = [path for path in _OPENPYXL_FIXED if path in xml_names]
|
||||
todo = [path for kind, path in rels.values() if kind.endswith("/chartsheet") and path in xml_names]
|
||||
while todo:
|
||||
path = todo.pop()
|
||||
if path in out:
|
||||
continue
|
||||
out.append(path)
|
||||
folder, _, name = path.rpartition("/")
|
||||
rels_path = f"{folder}/_rels/{name}.rels" if folder else f"_rels/{name}.rels"
|
||||
if rels_path in xml_names:
|
||||
out.append(rels_path)
|
||||
todo += [dep for _kind, dep in archive.rels(path).values() if dep in xml_names and dep not in out]
|
||||
return out
|
||||
|
||||
|
||||
def _scan_tables(archive: _Archive, sheet_path: str, sheet_name: str) -> list[TableScan]:
|
||||
out = []
|
||||
for tkind, tpath in archive.rels(sheet_path).values():
|
||||
if not tkind.endswith("/table") or tpath not in archive.names:
|
||||
continue
|
||||
t = archive.tree(tpath)
|
||||
columns = [c.get("name") or "" for c in t.iter() if _local(c.tag) == "tableColumn"]
|
||||
out.append(TableScan(
|
||||
name=t.get("displayName") or t.get("name") or "",
|
||||
ref=t.get("ref") or "",
|
||||
sheet=sheet_name,
|
||||
totals_row_count=int(t.get("totalsRowCount") or 0),
|
||||
header_row_count=int(t.get("headerRowCount") or 1),
|
||||
columns=columns,
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
def _text_info(container: ET.Element) -> tuple[bool, int]:
|
||||
"""<si> 或 <is> 里的文字:(有没有非空白文字, 总字符数)。直接的 <t>,或者富文本 <r><t>;注音 <rPh> 不算。"""
|
||||
nonblank, length = False, 0
|
||||
for child in container:
|
||||
local = _local(child.tag)
|
||||
if local == "t":
|
||||
text = child.text
|
||||
if text:
|
||||
length += len(text)
|
||||
nonblank = nonblank or not text.isspace()
|
||||
elif local == "r":
|
||||
for t in child:
|
||||
text = t.text
|
||||
if text and _local(t.tag) == "t":
|
||||
length += len(text)
|
||||
nonblank = nonblank or not text.isspace()
|
||||
return nonblank, length
|
||||
|
||||
|
||||
def _text_nonblank(container: ET.Element) -> bool:
|
||||
"""<si> 或 <is> 里有没有非空白文字。"""
|
||||
return _text_info(container)[0]
|
||||
|
||||
|
||||
def _scan_shared_strings(archive: _Archive, path: str) -> bytearray:
|
||||
"""共享字符串表 → 每一项「非空」与否(1/0)。不还原全文:只需要知道引用它的格算不算空。
|
||||
|
||||
条目数、单个条目的文字长度和元素个数都设上限:openpyxl 会把整张表读进一个列表,
|
||||
超出 Excel 自己上限的条目只可能是构造出来撑爆内存的。根下的元素处理完就摘掉
|
||||
(不只是 <si>:写文件的程序塞进来的未知元素也一样),根下不留空壳。
|
||||
"""
|
||||
flags = bytearray()
|
||||
root: ET.Element | None = None
|
||||
depth = inner = 0
|
||||
with closing(archive.open(path)) as source:
|
||||
for event, el in ET.iterparse(source, events=("start", "end")):
|
||||
if event == "start":
|
||||
depth += 1
|
||||
if root is None:
|
||||
root = el
|
||||
continue
|
||||
depth -= 1
|
||||
if depth > 1:
|
||||
inner += 1 # 条目内部的元素:条目结束时随它一起清掉
|
||||
if inner > MAX_SI_ELEMENTS:
|
||||
raise UnsupportedTable(_abnormal("共享字符串表中有一个条目包含的元素过多"))
|
||||
continue
|
||||
if depth == 1:
|
||||
inner = 0
|
||||
if _local(el.tag) == "si":
|
||||
if len(flags) >= MAX_SHARED_STRINGS:
|
||||
raise UnsupportedTable(_abnormal(f"共享字符串表的条目超过 {MAX_SHARED_STRINGS:,} 条"))
|
||||
nonblank, length = _text_info(el)
|
||||
if length > EXCEL_MAX_TEXT:
|
||||
raise UnsupportedTable(_abnormal(
|
||||
f"共享字符串表中有一项文字超过 Excel 单元格的上限({EXCEL_MAX_TEXT:,} 个字符)"))
|
||||
flags.append(1 if nonblank else 0)
|
||||
el.clear()
|
||||
if root is not None:
|
||||
root.remove(el) # 处理完就摘掉,根下不留空壳
|
||||
return flags
|
||||
|
||||
|
||||
#: 公式里的区域引用:C22:C25、$C$22:$C$25。前面紧挨着「!」的是别的工作表,不算
|
||||
_RANGE_REF = re.compile(
|
||||
r"(?<![A-Za-z0-9_.!$'])\$?([A-Za-z]{1,3})\$?(\d{1,7}):\$?([A-Za-z]{1,3})\$?(\d{1,7})(?![A-Za-z0-9_(])",
|
||||
re.ASCII,
|
||||
)
|
||||
#: 任何像区域的写法(包括别的工作表的):找单格引用之前先整个抹掉,免得把「B2:B9」的 B9 当成单格
|
||||
_ANY_RANGE = re.compile(r"\$?[A-Za-z]{1,3}\$?\d{1,7}:\$?[A-Za-z]{1,3}\$?\d{1,7}", re.ASCII)
|
||||
#: 单格引用:B2、$B$2。边界和 _RANGE_REF 一样;后面跟「(」的是函数名(LOG10(),跟「!」的是工作表名
|
||||
_CELL_REF = re.compile(r"(?<![\w.!$'\]])\$?([A-Za-z]{1,3})\$?([0-9]{1,7})(?![\w(!:.\[])")
|
||||
#: 字符串常量和带引号的工作表名:先抹掉,里面的「B2」不是引用。一条正则从左往右扫,谁先开头算谁
|
||||
_QUOTED = re.compile(r'"(?:[^"]|"")*"|\'(?:[^\']|\'\')*\'')
|
||||
#: 方括号(结构化引用的列名、外部工作簿的编号),最多两层
|
||||
_BRACKETS = re.compile(r"\[(?:[^\[\]]|\[[^\[\]]*\])*\]")
|
||||
#: 结构化引用的表名部分:「销售表[」。表名可以是中文
|
||||
_TABLE_REF = re.compile(r"(?<![\w.!$'\]])([^\W\d][\w.]*)\[")
|
||||
_SPECIALS = frozenset({"#all", "#data", "#headers", "#totals", "#this row"})
|
||||
|
||||
|
||||
def _aggregates_above(formula: str, row: int, col: int, tables: dict[str, TableScan] | None = None) -> bool:
|
||||
"""公式是不是汇总了本格正上方的几行。表内合计行(C26 = SUM(C22:C25))是交叉表、多块报表的典型信号。
|
||||
|
||||
认三种写法:
|
||||
- 区域:整个在本行之上、至少两行、列覆盖本格所在列(SUM(C22:C25)、SUBTOTAL(9,D2:D9));
|
||||
- 逐格相加:单格引用至少两个,而且全都在本格正上方的同一列(B2+B3+B4、SUM(C2,C3,C4));
|
||||
- 结构化引用:「销售表[金额]」指的明细行全在本行之上、列覆盖本格(tables 给出本工作表的表格对象,
|
||||
键是 casefold 后的表名)。只引用表头、汇总行或本行([#Totals]、[@金额])的不算。
|
||||
|
||||
逐行算的公式(D3 = B3*C3)、累计(D3 = SUM(C$2:C3),区域含本行)、滚动余额(C3 = C2+B3,
|
||||
引用了别的列)、占比(D5 = C5/SUM(C$5:C$20))都不满足。
|
||||
"""
|
||||
if ":" not in formula and "[" not in formula:
|
||||
letters = col_letter(col)
|
||||
if letters not in formula and letters.lower() not in formula:
|
||||
return False # 快速路径:绝大多数逐行公式根本不引用本列
|
||||
if tables and "[" in formula and _structured_above(formula, row, col, tables):
|
||||
return True
|
||||
text = _BRACKETS.sub("[]", _QUOTED.sub(lambda m: '""' if m.group(0)[0] == '"' else "'_'", formula))
|
||||
for m in _RANGE_REF.finditer(text):
|
||||
c1, r1, c2, r2 = _col_number(m.group(1)), int(m.group(2)), _col_number(m.group(3)), int(m.group(4))
|
||||
if r1 > r2:
|
||||
r1, r2 = r2, r1
|
||||
if c1 > c2:
|
||||
c1, c2 = c2, c1
|
||||
if r2 < row and r2 > r1 and c1 <= col <= c2:
|
||||
return True
|
||||
refs = {(int(m.group(2)), _col_number(m.group(1))) for m in _CELL_REF.finditer(_ANY_RANGE.sub(" ", text))}
|
||||
return len(refs) >= 2 and all(c == col and r < row for r, c in refs)
|
||||
|
||||
|
||||
def _bracket(formula: str, start: int) -> str | None:
|
||||
"""formula[start] 是「[」:返回配对的括号里的内容。「'」转义下一个字符。括号不成对时 None。"""
|
||||
depth = 0
|
||||
i = start
|
||||
while i < len(formula):
|
||||
ch = formula[i]
|
||||
if ch == "'":
|
||||
i += 2
|
||||
continue
|
||||
if ch == "[":
|
||||
depth += 1
|
||||
elif ch == "]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return formula[start + 1:i]
|
||||
i += 1
|
||||
return None
|
||||
|
||||
|
||||
def _structured_above(formula: str, row: int, col: int, tables: dict[str, TableScan]) -> bool:
|
||||
for m in _TABLE_REF.finditer(formula):
|
||||
table = tables.get(m.group(1).casefold())
|
||||
span = table.span() if table is not None else None
|
||||
content = _bracket(formula, m.end() - 1) if span is not None else None
|
||||
if table is None or span is None or content is None:
|
||||
continue
|
||||
content = content.strip()
|
||||
items = re.findall(r"\[((?:[^\]']|'.)*)\]", content) if content.startswith("[") else [content]
|
||||
items = [i.strip() for i in items]
|
||||
specials = {i.lower() for i in items if i.lower() in _SPECIALS}
|
||||
if "#this row" in specials or any(i.startswith("@") for i in items):
|
||||
continue # 本行:逐行计算
|
||||
if specials and not specials & {"#all", "#data"}:
|
||||
continue # 只引用表头或汇总行
|
||||
r1, c1, r2, c2 = span
|
||||
data_lo = r1 + max(table.header_row_count, 0)
|
||||
data_hi = r2 - max(table.totals_row_count, 0)
|
||||
if data_lo > data_hi:
|
||||
continue
|
||||
bottom = r2 if "#all" in specials or "#totals" in specials else data_hi
|
||||
if bottom >= row:
|
||||
continue
|
||||
lo, hi = c1, c2
|
||||
names = [re.sub(r"'(.)", r"\1", i).casefold() for i in items if i and i.lower() not in _SPECIALS]
|
||||
known = [n.casefold() for n in table.columns]
|
||||
if names and all(n in known for n in names):
|
||||
pos = [known.index(n) for n in names]
|
||||
lo, hi = c1 + min(pos), c1 + max(pos)
|
||||
if lo <= col <= hi:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _descendants(el: ET.Element) -> int:
|
||||
return sum(1 for _ in el.iter()) - 1
|
||||
|
||||
|
||||
def _add_span(spans: list[tuple[int, int]], lo: int, hi: int) -> None:
|
||||
if spans and spans[-1][1] + 1 >= lo:
|
||||
spans[-1] = (spans[-1][0], max(spans[-1][1], hi))
|
||||
else:
|
||||
spans.append((lo, hi))
|
||||
|
||||
|
||||
def _scan_sheet(archive: _Archive, sheet: SheetScan, sst_flags: bytearray | None,
|
||||
tables: dict[str, TableScan] | None = None) -> None:
|
||||
"""一张工作表流式扫一遍。按 OOXML 的元素顺序,合并区、筛选在 sheetData 之后,所以必须扫到底。
|
||||
|
||||
只要 end 事件,单元格在所在行结束时一起处理(这时行号、各格的子元素都齐了),处理完把行清空。
|
||||
要 start 事件的话事件数翻倍,20 万行的表要多花一秒;代价是清空后的行还挂在 sheetData 下,
|
||||
每行几十字节,行以外的元素也一直挂着。所以按 Excel 自己的上限设硬上限:行号和行数不超过
|
||||
1,048,576、列号不超过 16,384,树上同时挂着的元素不超过 MAX_ALIVE(行清空时,它下面的
|
||||
元素随之释放,从计数里减掉);单元格文字、公式也按 Excel 的长度上限查。超了就拒收:
|
||||
正常的文件到不了这些数,到了的只可能是构造出来撑爆内存的(几十 KB 的压缩包能解出几百万个空行)。
|
||||
"""
|
||||
tags = _SHEET_TAGS
|
||||
sst_len = len(sst_flags) if sst_flags is not None else 0
|
||||
min_row = min_col = 1 << 30
|
||||
max_row = max_col = 0
|
||||
nonempty = 0
|
||||
far: list[str] = []
|
||||
max_col_cell = max_row_cell = ""
|
||||
cur_row = 0
|
||||
hidden_rows = sheet.hidden_rows
|
||||
formulas_uncached, formulas_above, errors = sheet.formulas_uncached, sheet.formulas_above, sheet.errors
|
||||
|
||||
alive = rows = 0
|
||||
where = f"工作表「{sheet.name}」"
|
||||
# 上限先取到局部变量:每个元素都要比一次,模块属性每次现查要多花时间(测试里照样能改模块常量)
|
||||
max_alive, max_rows, max_cols = MAX_ALIVE, EXCEL_MAX_ROWS, EXCEL_MAX_COLS
|
||||
max_text, max_formula = EXCEL_MAX_TEXT, EXCEL_MAX_FORMULA
|
||||
|
||||
with closing(archive.open(sheet.path)) as source:
|
||||
for _event, el in ET.iterparse(source, events=("end",)):
|
||||
alive += 1
|
||||
if alive > max_alive:
|
||||
raise UnsupportedTable(_abnormal(f"{where}包含的元素过多"))
|
||||
name = tags.get(el.tag)
|
||||
if name is None or name in ("c", "v", "f", "is"):
|
||||
continue
|
||||
if name != "row":
|
||||
if name == "mergeCell":
|
||||
ref = el.get("ref")
|
||||
if ref:
|
||||
sheet.merged_total += 1
|
||||
if len(sheet.merged) < MAX_MERGED:
|
||||
sheet.merged.append(ref)
|
||||
el.clear()
|
||||
elif name == "col":
|
||||
if el.get("hidden") in ("1", "true"):
|
||||
lo, hi = int(el.get("min") or 0), int(el.get("max") or 0)
|
||||
if max(lo, hi) > max_cols:
|
||||
raise UnsupportedTable(_abnormal(f"{where}超过 Excel 的列数上限({max_cols:,} 列)"))
|
||||
if lo and hi:
|
||||
_add_span(sheet.hidden_cols, lo, hi)
|
||||
elif name == "autoFilter":
|
||||
sheet.autofilter = el.get("ref")
|
||||
elif name == "dimension":
|
||||
sheet.dimension = el.get("ref")
|
||||
continue
|
||||
|
||||
r = el.get("r")
|
||||
row = int(r) if r else cur_row + 1
|
||||
cur_row = row
|
||||
rows += 1
|
||||
if row > max_rows or rows > max_rows:
|
||||
raise UnsupportedTable(_abnormal(f"{where}超过 Excel 的行数上限({max_rows:,} 行)"))
|
||||
if el.get("hidden") in ("1", "true"):
|
||||
_add_span(hidden_rows, row, row)
|
||||
col = 0
|
||||
#: 这一行清空时随之释放的元素个数(行本身的空壳留着,不算)
|
||||
freed = 0
|
||||
for cell in el:
|
||||
freed += 1
|
||||
if tags.get(cell.tag) != "c":
|
||||
if len(cell):
|
||||
freed += _descendants(cell)
|
||||
continue
|
||||
ref = cell.get("r")
|
||||
if ref:
|
||||
letters = ref.rstrip(_DIGITS)
|
||||
col = _col_number(letters)
|
||||
else:
|
||||
col += 1
|
||||
if col > max_cols:
|
||||
raise UnsupportedTable(_abnormal(f"{where}超过 Excel 的列数上限({max_cols:,} 列)"))
|
||||
ctype = cell.get("t") or "n"
|
||||
v_el = f_el = is_el = None
|
||||
inline_filled, inline_len = False, 0
|
||||
for child in cell:
|
||||
freed += 1
|
||||
cname = tags.get(child.tag)
|
||||
if cname == "v":
|
||||
v_el = child
|
||||
elif cname == "f":
|
||||
f_el = child
|
||||
else:
|
||||
# <is> 下的文字、未知元素下的子孙也随行释放。<v>、<f> 正常没有子元素:
|
||||
# 构造出来的子元素不减,只会让计数偏大、更早拒收
|
||||
if len(child):
|
||||
freed += _descendants(child)
|
||||
if cname == "is":
|
||||
is_el = child
|
||||
inline_filled, inline_len = _text_info(child)
|
||||
vtext = v_el.text if v_el is not None else None
|
||||
if inline_len > max_text or (vtext is not None and len(vtext) > max_text):
|
||||
raise UnsupportedTable(_abnormal(
|
||||
f"{where}的单元格 {cell_ref(row, col)} 中的文字超过 Excel 的上限({max_text:,} 个字符)"))
|
||||
if f_el is not None and f_el.text and len(f_el.text) > max_formula:
|
||||
raise UnsupportedTable(_abnormal(
|
||||
f"{where}的单元格 {cell_ref(row, col)} 中的公式超过 Excel 的上限({max_formula:,} 个字符)"))
|
||||
if ctype == "s":
|
||||
if vtext:
|
||||
idx = int(vtext)
|
||||
filled = sst_flags[idx] == 1 if 0 <= idx < sst_len and sst_flags is not None else True
|
||||
else:
|
||||
filled = False
|
||||
elif ctype == "inlineStr":
|
||||
filled = inline_filled
|
||||
elif ctype == "str":
|
||||
filled = bool(vtext and vtext.strip())
|
||||
else:
|
||||
filled = bool(vtext)
|
||||
if ctype == "e" and vtext:
|
||||
errors.add(row, col)
|
||||
if f_el is not None:
|
||||
sheet.formulas += 1
|
||||
# 公式返回空字符串时 Excel 写 t="str" 加空的 <v>:那是有缓存的空值,不算「无缓存」
|
||||
if v_el is None or (not vtext and ctype != "str"):
|
||||
formulas_uncached.add(row, col)
|
||||
ftext = f_el.text
|
||||
if ftext and _aggregates_above(ftext, row, col, tables):
|
||||
formulas_above.add(row, col, ftext)
|
||||
filled = True # 有公式的格不算空,哪怕缓存值是空的
|
||||
if not filled:
|
||||
continue
|
||||
nonempty += 1
|
||||
if row < min_row:
|
||||
min_row = row
|
||||
if col < min_col:
|
||||
min_col = col
|
||||
# 同一个格子可能既往右又往下跳得很远(XFD1048576):只记一次
|
||||
jumped = False
|
||||
if col > max_col:
|
||||
jumped = bool(max_col) and col - max_col > FAR_COL_JUMP
|
||||
max_col = col
|
||||
max_col_cell = cell_ref(row, col)
|
||||
if row > max_row:
|
||||
jumped = jumped or (bool(max_row) and row - max_row > FAR_ROW_JUMP)
|
||||
max_row = row
|
||||
max_row_cell = cell_ref(row, col)
|
||||
if jumped and len(far) < MAX_FAR:
|
||||
far.append(cell_ref(row, col))
|
||||
el.clear()
|
||||
alive -= freed
|
||||
|
||||
sheet.nonempty = nonempty
|
||||
if nonempty:
|
||||
sheet.bounds = Bounds(min_row, min_col, max_row, max_col)
|
||||
# 没有明显的跳跃(比如区域是慢慢变稀的),就拿撑出右边界、下边界的格子当示例
|
||||
sheet.far_cells = far or list(dict.fromkeys(c for c in (max_col_cell, max_row_cell) if c))
|
||||
Reference in New Issue
Block a user