mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 14:12:00 +00:00
全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理
- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费 - 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定 - 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
@@ -28,6 +28,7 @@ import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timedelta
|
||||
@@ -51,7 +52,9 @@ MAX_COLUMNS = 12
|
||||
#: 现查到的表的范围在进程里记多久(秒):同一张表一会儿内查好几次(一张图里几个取数节点、助手自查几轮)不再全表 MIN /
|
||||
#: MAX。滚动表一天才换一次数据,几分钟的旧不影响判断;查不成的、退回剖析的不记
|
||||
LIVE_TTL_S = 300.0
|
||||
_LIVE: dict[tuple[Any, ...], tuple[float, Any]] = {}
|
||||
SNAPSHOT_LIVE_TTL_S = 86400.0
|
||||
_LIVE_MAX = 512
|
||||
_LIVE: OrderedDict[tuple[Any, ...], tuple[float, Any]] = OrderedDict()
|
||||
#: 现查没成功(大表全表 MIN / MAX 超时、库一时连不上)也记 LIVE_TTL_S:这段时间里同一张表直接退回剖析、取值观察,
|
||||
#: 不然取数 agent 连查几次、每次结果里有整列 0,每次都要白等 PROBE_TIMEOUT_S 秒才退回
|
||||
_PROBE_FAILED = object()
|
||||
@@ -68,7 +71,7 @@ NO_DATA_SHORT = "查询区间里这张表没有数据"
|
||||
_COMPARE = {"GT": ">", "GTE": ">=", "LT": "<", "LTE": "<=", "EQ": "="}
|
||||
_FLIP = {">": "<", ">=": "<=", "<": ">", "<=": ">=", "=": "="}
|
||||
_ISO = re.compile(r"^(\d{4})-(\d{1,2})-(\d{1,2})(?:[ T](\d{1,2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?)?"
|
||||
r"(?:Z|[+-]\d{2}:?\d{2})?$")
|
||||
r"(?:Z|[+-]\d{2}(?::?\d{2})?)?$")
|
||||
_COMPACT = re.compile(r"^(\d{4})(\d{2})(\d{2})$")
|
||||
|
||||
|
||||
@@ -399,14 +402,19 @@ async def _range(source: Any, flt: _Filter, read: str | None, cache: Cache, fact
|
||||
key = (str(getattr(source, "name", "")), name_key(flt.table_name), name_key(flt.column_name))
|
||||
if key in cache.ranges:
|
||||
return cache.ranges[key]
|
||||
live = (getattr(source, "id", None), getattr(source, "snapshot_id", None), *key)
|
||||
snapshot_id = getattr(source, "snapshot_id", None)
|
||||
live = (getattr(source, "id", None), snapshot_id, *key)
|
||||
held = _LIVE.get(live)
|
||||
failed_before = False
|
||||
if held is not None and held[0] > time.monotonic():
|
||||
if held[1] is not _PROBE_FAILED:
|
||||
cache.ranges[key] = held[1]
|
||||
return held[1]
|
||||
failed_before = True
|
||||
if held is not None:
|
||||
if held[0] > time.monotonic():
|
||||
_LIVE.move_to_end(live)
|
||||
if held[1] is not _PROBE_FAILED:
|
||||
cache.ranges[key] = held[1]
|
||||
return held[1]
|
||||
failed_before = True
|
||||
else:
|
||||
_LIVE.pop(live, None)
|
||||
table = flt.table.copy()
|
||||
table.set("alias", None)
|
||||
ident = flt.column.this.copy() if isinstance(flt.column.this, exp.Identifier) \
|
||||
@@ -429,16 +437,18 @@ async def _range(source: Any, flt: _Filter, read: str | None, cache: Cache, fact
|
||||
out = await _fallback(flt, facts)
|
||||
cache.ranges[key] = out
|
||||
if found is None and not failed_before:
|
||||
_remember(live, _PROBE_FAILED)
|
||||
_remember(live, _PROBE_FAILED, ttl=LIVE_TTL_S)
|
||||
elif out is not None and out.basis == "live":
|
||||
_remember(live, out)
|
||||
ttl = SNAPSHOT_LIVE_TTL_S if snapshot_id else LIVE_TTL_S
|
||||
_remember(live, out, ttl=ttl)
|
||||
return out
|
||||
|
||||
|
||||
def _remember(live: tuple[Any, ...], value: Any) -> None:
|
||||
if len(_LIVE) > 512:
|
||||
_LIVE.clear()
|
||||
_LIVE[live] = (time.monotonic() + LIVE_TTL_S, value)
|
||||
def _remember(live: tuple[Any, ...], value: Any, *, ttl: float = LIVE_TTL_S) -> None:
|
||||
_LIVE.pop(live, None)
|
||||
while len(_LIVE) >= _LIVE_MAX:
|
||||
_LIVE.popitem(last=False)
|
||||
_LIVE[live] = (time.monotonic() + ttl, value)
|
||||
|
||||
|
||||
async def _fallback(flt: _Filter, facts: Facts) -> _Range | None:
|
||||
|
||||
@@ -110,29 +110,44 @@ class CsvTable:
|
||||
delimiter: str
|
||||
|
||||
|
||||
def _resolve_csv_meta(raw: bytes, delimiter: str | None) -> tuple[str, str, str]:
|
||||
from app.data import tabular
|
||||
|
||||
enc, errors = tabular._detect_csv_encoding(raw)
|
||||
if delimiter is None:
|
||||
with tabular._open_csv_text(raw, encoding=enc, errors=errors, newline="") as probe_stream:
|
||||
head = probe_stream.read(8192)
|
||||
try:
|
||||
delimiter = csv.Sniffer().sniff(head, delimiters=_DELIMITERS).delimiter
|
||||
except csv.Error:
|
||||
delimiter = "," # 嗅不出来就按最常见的来
|
||||
return enc, errors, delimiter
|
||||
|
||||
|
||||
def _iter_csv_records(raw: bytes, *, enc: str, errors: str, delimiter: str) -> Iterator[list[str]]:
|
||||
from app.data import tabular
|
||||
|
||||
row_count = 0
|
||||
try:
|
||||
with tabular._open_csv_text(raw, encoding=enc, errors=errors, newline="") as stream:
|
||||
for record in csv.reader(stream, delimiter=delimiter):
|
||||
if row_count >= MAX_ROWS:
|
||||
raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行,超出可以导入的上限")
|
||||
if len(record) > MAX_COLS:
|
||||
raise UnsupportedTable(f"CSV 文件第 {row_count + 1} 行有 {len(record):,} 列,超过 {MAX_COLS:,} 列的上限")
|
||||
row_count += 1
|
||||
yield record
|
||||
except csv.Error as e:
|
||||
raise UnsupportedTable(f"无法按 CSV 读取这个文件:第 {row_count + 1} 行的写法不正确({e})。"
|
||||
"请在 Excel 中重新另存为 CSV 后上传") from e
|
||||
|
||||
|
||||
def parse(raw: bytes, delimiter: str | None = None) -> CsvTable:
|
||||
"""原件 → 记录。编码照 tabular 读 CSV 的做法;delimiter 不给时嗅探(同 tabular)。字段太长、引号不配对读不了时抛
|
||||
UnsupportedTable。"""
|
||||
from app.data import tabular
|
||||
|
||||
text = tabular._decode_csv(raw)
|
||||
if delimiter is None:
|
||||
try:
|
||||
delimiter = csv.Sniffer().sniff(text[:8192], delimiters=_DELIMITERS).delimiter
|
||||
except csv.Error:
|
||||
delimiter = "," # 嗅不出来就按最常见的来
|
||||
records: list[list[str]] = []
|
||||
try:
|
||||
for record in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter):
|
||||
if len(records) >= MAX_ROWS:
|
||||
raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行,超出可以导入的上限")
|
||||
if len(record) > MAX_COLS:
|
||||
raise UnsupportedTable(f"CSV 文件第 {len(records) + 1} 行有 {len(record):,} 列,超过 {MAX_COLS:,} 列的上限")
|
||||
records.append(record)
|
||||
except csv.Error as e:
|
||||
raise UnsupportedTable(f"无法按 CSV 读取这个文件:第 {len(records) + 1} 行的写法不正确({e})。"
|
||||
"请在 Excel 中重新另存为 CSV 后上传") from e
|
||||
return CsvTable(records=records, delimiter=delimiter)
|
||||
enc, errors, resolved_delim = _resolve_csv_meta(raw, delimiter)
|
||||
records = list(_iter_csv_records(raw, enc=enc, errors=errors, delimiter=resolved_delim))
|
||||
return CsvTable(records=records, delimiter=resolved_delim)
|
||||
|
||||
|
||||
def _cells(table: CsvTable) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]:
|
||||
@@ -155,15 +170,18 @@ def sheet_delimiter(sheet: Any) -> str | None:
|
||||
|
||||
def scan(raw: bytes, *, delimiter: str | None = None) -> WorkbookScan:
|
||||
"""CSV 原件 → 扫描结果:一张可见工作表(名字 CSV_SHEET、路径 CSV_PATH 加分隔符),边界、非空格数、逐行非空格数的
|
||||
游程。没有合并区、隐藏行列、公式、错误值、离群格(CSV 里每一格都是写出来的,没有残留格式)。"""
|
||||
table = parse(raw, delimiter)
|
||||
sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + table.delimiter)
|
||||
游程。流式统计,不再先把整张 CSV 物化成全量字符串与二维列表。"""
|
||||
enc, errors, resolved_delim = _resolve_csv_meta(raw, delimiter)
|
||||
sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + resolved_delim)
|
||||
lo_r = lo_c = hi_r = hi_c = 0
|
||||
runs: list[tuple[int, int, int]] = []
|
||||
for r, cells in _cells(table):
|
||||
n = len(cells)
|
||||
for r, record in enumerate(_iter_csv_records(raw, enc=enc, errors=errors, delimiter=resolved_delim), 1):
|
||||
nonempty_cols = [c for c, field in enumerate(record, 1) if cell_value(field) is not None]
|
||||
if not nonempty_cols:
|
||||
continue
|
||||
n = len(nonempty_cols)
|
||||
sheet.nonempty += n
|
||||
first, last = cells[0][0], cells[-1][0]
|
||||
first, last = nonempty_cols[0], nonempty_cols[-1]
|
||||
if not lo_r:
|
||||
lo_r, lo_c, hi_c = r, first, last
|
||||
lo_c, hi_c, hi_r = min(lo_c, first), max(hi_c, last), r
|
||||
|
||||
+108
-46
@@ -14,7 +14,7 @@ import re
|
||||
import sqlite3
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from contextlib import asynccontextmanager
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, AsyncIterator, Awaitable, Callable
|
||||
|
||||
@@ -297,7 +297,7 @@ def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = F
|
||||
mode = "ro" if readonly else "rwc"
|
||||
extra = "&immutable=1" if immutable else ""
|
||||
return sqlite3.connect(f"file:{quote(os.fspath(path))}?mode={mode}{extra}", uri=True,
|
||||
factory=_DqsOffConnection)
|
||||
timeout=5.0, factory=_DqsOffConnection)
|
||||
|
||||
|
||||
def open_memory_sqlite() -> sqlite3.Connection:
|
||||
@@ -549,6 +549,8 @@ class EngineCache:
|
||||
#: 正在进行的快照核对,按(缓存键, 文件, 登记的哈希)合并。核对结束(不论成败)即移除,
|
||||
#: 失败结果不留:下次照常重新核对
|
||||
self._verifying: dict[tuple[str, str, str], asyncio.Future[Fingerprint]] = {}
|
||||
#: 每个数据源的失效世代:get() 在锁外核对快照期间若赶上 invalidate(),进锁后看到世代变了就重来,绝不把旧连接装回缓存
|
||||
self._epochs: dict[str, int] = {}
|
||||
|
||||
def _hit(self, key: str, doomed: list[AsyncEngine]) -> AsyncEngine | None:
|
||||
"""缓存里还可信的 engine。快照文件和核对时的指纹对不上的,摘出来放进 doomed 交给调用方关掉。"""
|
||||
@@ -568,6 +570,7 @@ class EngineCache:
|
||||
|
||||
async def get(self, source: Any) -> AsyncEngine:
|
||||
key = cache_key(source)
|
||||
sid = _source_of(key)
|
||||
doomed: list[AsyncEngine] = []
|
||||
try:
|
||||
cached = self._hit(key, doomed)
|
||||
@@ -577,34 +580,44 @@ class EngineCache:
|
||||
if getattr(source, "immutable", False) and not expected:
|
||||
logger.warning("版本快照没有登记文件哈希,已拒绝:key=%s path=%s", key, source.database)
|
||||
raise SnapshotTampered(_NO_HASH)
|
||||
fingerprint: Fingerprint | None = None
|
||||
if expected:
|
||||
# 在锁外核对:哈希一个大文件要几秒,不能让别的数据源都排在它后面
|
||||
fingerprint = await self._verify_once(key, source, expected)
|
||||
async with self._lock:
|
||||
cached = self._hit(key, doomed)
|
||||
if cached is not None:
|
||||
return cached
|
||||
url, extra = engine_args(source)
|
||||
engine = create_async_engine(
|
||||
url,
|
||||
pool_size=3,
|
||||
max_overflow=2,
|
||||
pool_pre_ping=True, # 长时间空闲后连接会被数据库掐掉,先探活再用
|
||||
pool_recycle=1800,
|
||||
**extra,
|
||||
)
|
||||
if fingerprint is not None:
|
||||
pin = _Pin(str(source.database or ""), fingerprint)
|
||||
_guard_pool(engine, pin, key)
|
||||
self._pins[key] = pin
|
||||
self._engines[key] = engine
|
||||
limit = max(1, self._max if self._max is not None else MAX_ENGINES)
|
||||
while len(self._engines) > limit:
|
||||
old_key, old = self._engines.popitem(last=False)
|
||||
self._pins.pop(old_key, None)
|
||||
doomed.append(old)
|
||||
return engine
|
||||
while True:
|
||||
epoch = self._epochs.get(sid, 0)
|
||||
fingerprint: Fingerprint | None = None
|
||||
if expected:
|
||||
# 在锁外核对:哈希一个大文件要几秒,不能让别的数据源都排在它后面
|
||||
fingerprint = await self._verify_once(key, source, expected)
|
||||
async with self._lock:
|
||||
cached = self._hit(key, doomed)
|
||||
if cached is not None:
|
||||
return cached
|
||||
if self._epochs.get(sid, 0) != epoch:
|
||||
continue
|
||||
url, extra = engine_args(source)
|
||||
is_memory_sqlite = (source.kind or "").lower() == "sqlite" and (
|
||||
(source.database or "") == ":memory:" or url.endswith("/:memory:")
|
||||
)
|
||||
pool_kwargs: dict[str, Any] = {} if is_memory_sqlite else {
|
||||
"pool_size": 3,
|
||||
"max_overflow": 2,
|
||||
"pool_pre_ping": True, # 长时间空闲后连接会被数据库掐掉,先探活再用
|
||||
"pool_recycle": 1800,
|
||||
}
|
||||
engine = create_async_engine(
|
||||
url,
|
||||
**pool_kwargs,
|
||||
**extra,
|
||||
)
|
||||
if fingerprint is not None:
|
||||
pin = _Pin(str(source.database or ""), fingerprint)
|
||||
_guard_pool(engine, pin, key)
|
||||
self._pins[key] = pin
|
||||
self._engines[key] = engine
|
||||
limit = max(1, self._max if self._max is not None else MAX_ENGINES)
|
||||
while len(self._engines) > limit:
|
||||
old_key, old = self._engines.popitem(last=False)
|
||||
self._pins.pop(old_key, None)
|
||||
doomed.append(old)
|
||||
return engine
|
||||
finally:
|
||||
for old in doomed:
|
||||
# 正被借出的连接不受影响:dispose 只关池子里空闲的,借出的还回来时随旧池子回收
|
||||
@@ -653,22 +666,27 @@ class EngineCache:
|
||||
from app.cluster import caches
|
||||
|
||||
await caches.broadcast("datasource_engine", sid)
|
||||
# 正在进行的核对也撇开:失效之后来的请求重新核对,不搭失效之前那一次的便车
|
||||
for token in [t for t in self._verifying if _source_of(t[0]) == sid]:
|
||||
del self._verifying[token]
|
||||
keys = [key for key in list(self._engines) if _source_of(key) == sid]
|
||||
doomed = [self._engines.pop(key) for key in keys]
|
||||
for key in keys:
|
||||
self._pins.pop(key, None)
|
||||
async with self._lock:
|
||||
self._epochs[sid] = self._epochs.get(sid, 0) + 1
|
||||
# 正在进行的核对也撇开:失效之后来的请求重新核对,不搭失效之前那一次的便车
|
||||
for token in [t for t in self._verifying if _source_of(t[0]) == sid]:
|
||||
del self._verifying[token]
|
||||
keys = [key for key in list(self._engines) if _source_of(key) == sid]
|
||||
doomed = [self._engines.pop(key) for key in keys]
|
||||
for key in keys:
|
||||
self._pins.pop(key, None)
|
||||
for engine in doomed:
|
||||
await engine.dispose()
|
||||
|
||||
async def close(self) -> None:
|
||||
self._verifying.clear()
|
||||
for engine in list(self._engines.values()):
|
||||
async with self._lock:
|
||||
self._verifying.clear()
|
||||
doomed = list(self._engines.values())
|
||||
self._engines.clear()
|
||||
self._pins.clear()
|
||||
self._epochs.clear()
|
||||
for engine in doomed:
|
||||
await engine.dispose()
|
||||
self._engines.clear()
|
||||
self._pins.clear()
|
||||
|
||||
|
||||
engines = EngineCache()
|
||||
@@ -743,10 +761,20 @@ class _Deadline:
|
||||
def __init__(self, seconds: float) -> None:
|
||||
self.seconds = seconds
|
||||
self.ends = time.monotonic() + seconds
|
||||
#: SQLite 的进度回调真的掐过
|
||||
#: SQLite 的进度回调真的按时限掐过
|
||||
self.fired = False
|
||||
#: 调用方主动取消(CancelledError 等),进度回调同样立即中止工作线程里的语句
|
||||
self.cancelled = False
|
||||
#: 语句已经结束。回调这时要是还挂在连接上(摘除没成功),也不能再掐别人的语句
|
||||
self.over = False
|
||||
#: 由 _server_deadline 登记的底层驱动打断回调(如 sqlite3.Connection.interrupt / oracledb.cancel)
|
||||
self._aborter: Callable[[], None] | None = None
|
||||
|
||||
def abort(self) -> None:
|
||||
self.cancelled = True
|
||||
if self._aborter is not None:
|
||||
with suppress(Exception):
|
||||
self._aborter()
|
||||
|
||||
def remaining_ms(self) -> int:
|
||||
return max(1, int((self.ends - time.monotonic()) * 1000))
|
||||
@@ -759,7 +787,11 @@ class _Deadline:
|
||||
|
||||
def sqlite_progress(self) -> int:
|
||||
# 在 SQLite 执行语句的那个线程里被调用;返回非零,SQLite 就中止当前语句
|
||||
if not self.over and time.monotonic() >= self.ends:
|
||||
if self.over:
|
||||
return 0
|
||||
if self.cancelled:
|
||||
return 1
|
||||
if time.monotonic() >= self.ends:
|
||||
self.fired = True
|
||||
return 1
|
||||
return 0
|
||||
@@ -843,9 +875,16 @@ async def _server_deadline(conn: AsyncConnection, kind: str, deadline: _Deadline
|
||||
logger.warning("Oracle call_timeout 没设上(%s),只剩后端兜底", e)
|
||||
yield
|
||||
return
|
||||
cancel_fn = getattr(driver, "cancel", None) or getattr(driver, "breakexecution", None)
|
||||
if callable(cancel_fn):
|
||||
deadline._aborter = cancel_fn
|
||||
try:
|
||||
yield
|
||||
except asyncio.CancelledError:
|
||||
deadline.abort()
|
||||
raise
|
||||
finally:
|
||||
deadline._aborter = None
|
||||
try:
|
||||
driver.call_timeout = before
|
||||
except Exception: # noqa: BLE001 - 超时后连接可能已经被驱动关掉
|
||||
@@ -854,15 +893,23 @@ async def _server_deadline(conn: AsyncConnection, kind: str, deadline: _Deadline
|
||||
|
||||
if kind == "sqlite":
|
||||
driver = await _driver(conn)
|
||||
raw_conn = getattr(driver, "_conn", None)
|
||||
if raw_conn is not None:
|
||||
deadline._aborter = raw_conn.interrupt
|
||||
await driver.set_progress_handler(deadline.sqlite_progress, _SQLITE_STEPS)
|
||||
try:
|
||||
yield
|
||||
except BaseException:
|
||||
deadline.abort()
|
||||
raise
|
||||
finally:
|
||||
deadline.over = True
|
||||
deadline._aborter = None
|
||||
try:
|
||||
await driver.set_progress_handler(None, 0)
|
||||
except Exception: # noqa: BLE001 - over 已经让它不再掐人
|
||||
except Exception: # noqa: BLE001 - over 保底不再掐后面的语句
|
||||
pass
|
||||
finally:
|
||||
deadline.over = True
|
||||
return
|
||||
|
||||
yield
|
||||
@@ -887,7 +934,22 @@ async def _bounded(
|
||||
if deadline.expired():
|
||||
raise SqlRejected(over)
|
||||
async with _server_deadline(conn, kind, deadline):
|
||||
return await work(conn)
|
||||
# 用 shield 保护内层 greenlet:收到 CancelledError 时先经 deadline.abort() 通知底层驱动
|
||||
# (如 sqlite3_interrupt)正常中止查询,避免 CancelledError 直接打进 SQLAlchemy greenlet
|
||||
# 触发半关闭连接上的 rollback ValueError("Connection closed") 与连接池泄漏。
|
||||
work_task = asyncio.create_task(work(conn))
|
||||
try:
|
||||
return await asyncio.shield(work_task)
|
||||
except asyncio.CancelledError:
|
||||
deadline.abort()
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.shield(work_task), timeout=1.0)
|
||||
except BaseException:
|
||||
if not work_task.done():
|
||||
work_task.cancel()
|
||||
with suppress(BaseException):
|
||||
await work_task
|
||||
raise
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_exec(), timeout=limits.timeout_seconds + BACKSTOP_S)
|
||||
|
||||
@@ -8496,47 +8496,182 @@ def _slot_of(expr: str, index: int, context: str) -> TemplateSlot:
|
||||
return TemplateSlot(index=index, field=found.group(1), filters=filters, context=context)
|
||||
|
||||
|
||||
def _template_slots(template: str) -> tuple[list[tuple[int, int, TemplateSlot]], str]:
|
||||
"""模板里每一处 {{ … }}:[(起, 止, 槽位)],以及按哨兵渲染的 SQL。逐字看引号和注释,定下每一处在哪:单引号里是字,
|
||||
双引号、反引号里是名字,-- 和 /* */ 里是注释,别处是裸写。{{ … }} 里面的字不参与判断。"""
|
||||
_DOLLAR_QUOTE_RE = re.compile(r"\$(?:[^\W\d]\w*)?\$")
|
||||
_ORACLE_Q_CLOSE = {"[": "]", "(": ")", "{": "}", "<": ">"}
|
||||
|
||||
|
||||
def _is_word_char(ch: str) -> bool:
|
||||
return ch.isalnum() or ch in "_$"
|
||||
|
||||
|
||||
def _is_e_string_at(sql: str, i: int) -> bool:
|
||||
return i >= 1 and sql[i - 1] in "eE" and not (i >= 2 and _is_word_char(sql[i - 2]))
|
||||
|
||||
|
||||
def _oracle_q_close_at(sql: str, i: int) -> str | None:
|
||||
if i + 1 >= len(sql) or sql[i + 1].isspace() or not (i >= 1 and sql[i - 1] in "qQ"):
|
||||
return None
|
||||
j = i - 2
|
||||
if j >= 0 and sql[j] in "nN":
|
||||
j -= 1
|
||||
if j >= 0 and _is_word_char(sql[j]):
|
||||
return None
|
||||
opener = sql[i + 1]
|
||||
return _ORACLE_Q_CLOSE.get(opener, opener)
|
||||
|
||||
|
||||
def _is_bracket_ident_at(sql: str, i: int, dialect_key: str) -> bool:
|
||||
if dialect_key == "sqlite":
|
||||
return True
|
||||
if dialect_key in ("postgres", "postgresql", "mysql", "mariadb", "oracle"):
|
||||
return False
|
||||
j = i - 1
|
||||
while j >= 0 and sql[j].isspace():
|
||||
j -= 1
|
||||
return j < 0 or not (_is_word_char(sql[j]) or sql[j] in ")]")
|
||||
|
||||
|
||||
def _template_slots(
|
||||
template: str, *, dialect: str | None = None,
|
||||
) -> tuple[list[tuple[int, int, TemplateSlot]], str]:
|
||||
"""模板里每一处 {{ … }}:[(起, 止, 槽位)],以及按哨兵渲染的 SQL。
|
||||
|
||||
按方言词法逐字扫描引号与注释,定下每一处在哪:
|
||||
- 单引号(含 PostgreSQL E'…'、MySQL 反斜杠转义、PostgreSQL $$…$$ / $tag$…$tag$、Oracle q'[…]')里是字(quoted);
|
||||
- 双引号、反引号、SQLite 方括号 `[…]` 里是名字(name;MySQL 默认模式下双引号是字符串);
|
||||
- `--`、`/* … */`(PostgreSQL 支持嵌套)与 MySQL `#` 里是注释(comment);
|
||||
- 其余代码位置是裸写(bare)。`{{ … }}` 里面的字不参与外层引号/注释状态切换。
|
||||
"""
|
||||
dialect_key = (dialect or "").strip().lower()
|
||||
is_mysql = dialect_key in ("mysql", "mariadb")
|
||||
is_pg = dialect_key in ("postgres", "postgresql")
|
||||
allow_dollar = dialect_key in ("", "postgres", "postgresql")
|
||||
allow_q_quote = dialect_key in ("", "oracle")
|
||||
|
||||
spans: list[tuple[int, int, TemplateSlot]] = []
|
||||
out: list[str] = []
|
||||
state = "code"
|
||||
quote = ""
|
||||
backslash_escape = False
|
||||
dollar_tag = ""
|
||||
q_close = ""
|
||||
block_depth = 0
|
||||
|
||||
i, n = 0, len(template)
|
||||
while i < n:
|
||||
if template.startswith("{{", i):
|
||||
m = _SLOT_RE.match(template, i)
|
||||
if m is not None:
|
||||
context = {"code": "bare", "string": "quoted", "name": "name", "line": "comment",
|
||||
"block": "comment"}[state]
|
||||
context = {
|
||||
"code": "bare",
|
||||
"string": "quoted",
|
||||
"dollar": "quoted",
|
||||
"q_string": "quoted",
|
||||
"name": "name",
|
||||
"bracket_name": "name",
|
||||
"line": "comment",
|
||||
"block": "comment",
|
||||
}[state]
|
||||
slot = _slot_of(m.group(1).strip(), len(spans), context)
|
||||
spans.append((m.start(), m.end(), slot))
|
||||
out.append(_STRING_SENTINEL.format(slot.index) if context == "quoted"
|
||||
else str(_NUMBER_SENTINEL + slot.index) if context == "bare" else "")
|
||||
out.append(
|
||||
_STRING_SENTINEL.format(slot.index)
|
||||
if context == "quoted"
|
||||
else str(_NUMBER_SENTINEL + slot.index)
|
||||
if context == "bare"
|
||||
else ""
|
||||
)
|
||||
i = m.end()
|
||||
continue
|
||||
ch = template[i]
|
||||
if state == "code":
|
||||
if ch == "'":
|
||||
q_closer = _oracle_q_close_at(template, i) if allow_q_quote else None
|
||||
if q_closer is not None:
|
||||
state = "q_string"
|
||||
q_close = q_closer
|
||||
out.append(template[i:i + 2])
|
||||
i += 2
|
||||
continue
|
||||
state = "string"
|
||||
elif ch in "\"`":
|
||||
quote = "'"
|
||||
backslash_escape = is_mysql or _is_e_string_at(template, i)
|
||||
elif ch == '"' and is_mysql:
|
||||
state = "string"
|
||||
quote = '"'
|
||||
backslash_escape = True
|
||||
elif ch in '"`':
|
||||
state = "name"
|
||||
quote = ch
|
||||
elif ch == "[" and _is_bracket_ident_at(template, i, dialect_key):
|
||||
state = "bracket_name"
|
||||
elif allow_dollar and ch == "$" and (dm := _DOLLAR_QUOTE_RE.match(template, i)) is not None:
|
||||
state = "dollar"
|
||||
dollar_tag = dm.group(0)
|
||||
out.append(dollar_tag)
|
||||
i = dm.end()
|
||||
continue
|
||||
elif template.startswith("--", i):
|
||||
state = "line"
|
||||
out.append("--")
|
||||
i += 2
|
||||
continue
|
||||
elif is_mysql and ch == "#":
|
||||
state = "line"
|
||||
elif template.startswith("/*", i):
|
||||
state = "block"
|
||||
elif state == "string" and ch == "'":
|
||||
state = "code" # 成对的 '' 是转义:关上又马上打开,结果一样
|
||||
elif state == "name" and ch == quote:
|
||||
state = "code"
|
||||
block_depth = 1
|
||||
out.append("/*")
|
||||
i += 2
|
||||
continue
|
||||
elif state == "string":
|
||||
if backslash_escape and ch == "\\" and i + 1 < n:
|
||||
out.append(template[i:i + 2])
|
||||
i += 2
|
||||
continue
|
||||
if ch == quote:
|
||||
if i + 1 < n and template[i + 1] == quote:
|
||||
out.append(quote * 2)
|
||||
i += 2
|
||||
continue
|
||||
state = "code"
|
||||
elif state == "dollar":
|
||||
if template.startswith(dollar_tag, i):
|
||||
out.append(dollar_tag)
|
||||
i += len(dollar_tag)
|
||||
state = "code"
|
||||
continue
|
||||
elif state == "q_string":
|
||||
if ch == q_close and i + 1 < n and template[i + 1] == "'":
|
||||
out.append(template[i:i + 2])
|
||||
i += 2
|
||||
state = "code"
|
||||
continue
|
||||
elif state == "name":
|
||||
if ch == quote:
|
||||
if i + 1 < n and template[i + 1] == quote:
|
||||
out.append(quote * 2)
|
||||
i += 2
|
||||
continue
|
||||
state = "code"
|
||||
elif state == "bracket_name":
|
||||
if ch == "]":
|
||||
state = "code"
|
||||
elif state == "line" and ch == "\n":
|
||||
state = "code"
|
||||
elif state == "block" and template.startswith("*/", i):
|
||||
out.append("*/")
|
||||
i += 2
|
||||
state = "code"
|
||||
continue
|
||||
elif state == "block":
|
||||
if is_pg and template.startswith("/*", i):
|
||||
block_depth += 1
|
||||
out.append("/*")
|
||||
i += 2
|
||||
continue
|
||||
if template.startswith("*/", i):
|
||||
out.append("*/")
|
||||
i += 2
|
||||
block_depth -= 1
|
||||
if block_depth <= 0:
|
||||
state = "code"
|
||||
continue
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return spans, "".join(out)
|
||||
@@ -8734,7 +8869,7 @@ def _bindings(template_sql: str, sql: str, read: str | None, info: _Schema
|
||||
) -> Iterator[tuple[str, dict[str, Any], tuple[str, ...], tuple[TemplateSlot, ...]]]:
|
||||
"""按 BINDING_LEVELS 的先后给出配得上的绑定:(哪一级, {字段: 值}, 没取到值的字段, 槽位),同样的绑定只给一次。
|
||||
模板没有槽位时只给一个空绑定(level 为 parsed:用不着配)。不可比抛 _NotComparable。"""
|
||||
spans, sentinel_sql = _template_slots(template_sql)
|
||||
spans, sentinel_sql = _template_slots(template_sql, dialect=read)
|
||||
slots = tuple(slot for _, _, slot in spans)
|
||||
fields = tuple(dict.fromkeys(s.field for s in slots))
|
||||
if not slots:
|
||||
@@ -8839,7 +8974,7 @@ def bind_template(template_sql: str, slots: Any, sql: str, dialect: str | None =
|
||||
try:
|
||||
for level, values, unbound, found in _bindings(template_sql or "", sql or "", read, info):
|
||||
return TemplateBinding(True, values, unbound, level, None, found)
|
||||
spans, _ = _template_slots(template_sql or "")
|
||||
spans, _ = _template_slots(template_sql or "", dialect=read)
|
||||
except _NotComparable as e:
|
||||
return TemplateBinding(False, reason=str(e))
|
||||
except Exception: # noqa: BLE001 - 绑定本身的缺陷不能连累调用方:当作没配上
|
||||
@@ -8924,7 +9059,7 @@ def match_template(template_sql: str, slots: Any, sql: str, dialect: str | None
|
||||
level, values, unbound, diff, rendered = tried[0]
|
||||
return TemplateMatch(False, binding=values, unbound=unbound, level=level, diff=diff, rendered=rendered)
|
||||
try:
|
||||
spans, _ = _template_slots(template_sql)
|
||||
spans, _ = _template_slots(template_sql, dialect=read)
|
||||
except _NotComparable as e:
|
||||
return TemplateMatch(False, comparable=False, reason=str(e))
|
||||
fields = tuple(dict.fromkeys(slot.field for _, _, slot in spans))
|
||||
@@ -8954,7 +9089,7 @@ def _now_slots(template_sql: str, sql: str, read: str | None) -> list[tuple[str,
|
||||
from sqlglot import exp
|
||||
|
||||
try:
|
||||
spans, sentinel = _template_slots(template_sql)
|
||||
spans, sentinel = _template_slots(template_sql, dialect=read)
|
||||
except _NotComparable:
|
||||
return []
|
||||
by_index = {slot.index: slot for _, _, slot in spans}
|
||||
@@ -9344,7 +9479,7 @@ def project_template(template_sql: str, keep: Sequence[int], dialect: str | None
|
||||
template = (template_sql or "").strip()
|
||||
keep = tuple(keep)
|
||||
try:
|
||||
slots, sentinel = _template_slots(template)
|
||||
slots, sentinel = _template_slots(template, dialect=read)
|
||||
root, spans, expected = _plan_projection(sentinel, keep, read, None)
|
||||
items = list(root.expressions)
|
||||
names = tuple(_output_name(items[k], read) for k in keep)
|
||||
@@ -9358,7 +9493,7 @@ def project_template(template_sql: str, keep: Sequence[int], dialect: str | None
|
||||
raise _NotProjectable(_PROJ_TEXT)
|
||||
out = _cut(template, mapped, keep) # type: ignore[arg-type]
|
||||
# 换回模板之后再按哨兵渲染一遍:结构(字面量只看位置,槽位的编号会因为去掉了几处而变)要和投影后的一样
|
||||
_, again = _template_slots(out)
|
||||
_, again = _template_slots(out, dialect=read)
|
||||
root_again, _ = _parse(again, read)
|
||||
lits_a: list[Any] = []
|
||||
lits_b: list[Any] = []
|
||||
@@ -9420,7 +9555,7 @@ def output_names(sql: str, dialect: str | None = None, *, template: bool = False
|
||||
text = (sql or "").strip()
|
||||
try:
|
||||
if template:
|
||||
_, text = _template_slots(text)
|
||||
_, text = _template_slots(text, dialect=read)
|
||||
root, _ = _parse(_mysql_dashes(text) if read == "mysql" else text, read)
|
||||
except _NotComparable:
|
||||
return None
|
||||
|
||||
+48
-16
@@ -28,6 +28,7 @@ _ALWAYS_DENIED 里有 create,那是对的——模型不该能建表。而导
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
import csv
|
||||
import datetime as _dt
|
||||
import io
|
||||
@@ -865,33 +866,64 @@ def _close(it: Any) -> None:
|
||||
_ENCODINGS = ("utf-8-sig", "utf-8", "gb18030", "latin-1")
|
||||
|
||||
|
||||
def _decode_csv(raw: bytes) -> str:
|
||||
# 带 BOM 的 UTF-16(Excel 的「Unicode 文本」):按 latin-1 读会混进 NUL,列名全乱
|
||||
if raw[:2] in (b"\xff\xfe", b"\xfe\xff"):
|
||||
try:
|
||||
return raw.decode("utf-16")
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
_CSV_DECODE_CHUNK = 1024 * 1024
|
||||
|
||||
|
||||
def _can_decode_csv(raw: bytes, enc: str) -> bool:
|
||||
dec = codecs.getincrementaldecoder(enc)()
|
||||
mv = memoryview(raw)
|
||||
try:
|
||||
for offset in range(0, len(mv), _CSV_DECODE_CHUNK):
|
||||
end = offset + _CSV_DECODE_CHUNK
|
||||
dec.decode(mv[offset:end], final=end >= len(mv))
|
||||
if not len(mv):
|
||||
dec.decode(b"", final=True)
|
||||
return True
|
||||
except UnicodeDecodeError:
|
||||
return False
|
||||
|
||||
|
||||
def _detect_csv_encoding(raw: bytes) -> tuple[str, str]:
|
||||
"""增量分块探测 CSV 编码,避免为判编码而一次性物化整份全量字符串。"""
|
||||
if raw[:2] in (b"\xff\xfe", b"\xfe\xff") and _can_decode_csv(raw, "utf-16"):
|
||||
return "utf-16", "strict"
|
||||
for enc in _ENCODINGS:
|
||||
try:
|
||||
return raw.decode(enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return raw.decode("latin-1", errors="replace")
|
||||
if _can_decode_csv(raw, enc):
|
||||
return enc, "strict"
|
||||
return "latin-1", "replace"
|
||||
|
||||
|
||||
def _open_csv_text(
|
||||
raw: bytes,
|
||||
*,
|
||||
encoding: str | None = None,
|
||||
errors: str = "strict",
|
||||
newline: str | None = "",
|
||||
) -> io.TextIOWrapper:
|
||||
if encoding is None:
|
||||
encoding, errors = _detect_csv_encoding(raw)
|
||||
return io.TextIOWrapper(io.BytesIO(raw), encoding=encoding, errors=errors, newline=newline)
|
||||
|
||||
|
||||
def _decode_csv(raw: bytes) -> str:
|
||||
enc, errors = _detect_csv_encoding(raw)
|
||||
return raw.decode(enc, errors=errors)
|
||||
|
||||
|
||||
def _csv_sources(raw: bytes, filename: str, header_row: int) -> list[_Source]:
|
||||
text = _decode_csv(raw)
|
||||
enc, errors = _detect_csv_encoding(raw)
|
||||
delimiter = "\t" if _ext(filename) == ".tsv" else None
|
||||
if delimiter is None:
|
||||
with _open_csv_text(raw, encoding=enc, errors=errors, newline="") as probe_stream:
|
||||
head = probe_stream.read(8192)
|
||||
try:
|
||||
delimiter = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|").delimiter
|
||||
delimiter = csv.Sniffer().sniff(head, delimiters=",;\t|").delimiter
|
||||
except csv.Error:
|
||||
delimiter = "," # 嗅不出来就按最常见的来
|
||||
|
||||
def rows() -> Iterator[Sequence[Any]]:
|
||||
reader = csv.reader(io.StringIO(text), delimiter=delimiter)
|
||||
return itertools.islice(reader, header_row - 1, None)
|
||||
with _open_csv_text(raw, encoding=enc, errors=errors, newline="") as stream:
|
||||
yield from itertools.islice(csv.reader(stream, delimiter=delimiter), header_row - 1, None)
|
||||
|
||||
stem = filename.rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||||
# 分号分隔是欧洲区域设置的 CSV:那里逗号是小数点,「1,500」是 1.5 而不是一千五
|
||||
|
||||
Reference in New Issue
Block a user