全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理

- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费
- 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定
- 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
ThearchyHelios
2026-11-13 14:51:36 +08:00
parent 09949fc8ac
commit eddeded2a4
106 changed files with 7178 additions and 3913 deletions
+48 -16
View File
@@ -28,6 +28,7 @@ _ALWAYS_DENIED 里有 create,那是对的——模型不该能建表。而导
from __future__ import annotations
import codecs
import csv
import datetime as _dt
import io
@@ -865,33 +866,64 @@ def _close(it: Any) -> None:
_ENCODINGS = ("utf-8-sig", "utf-8", "gb18030", "latin-1")
def _decode_csv(raw: bytes) -> str:
# 带 BOM 的 UTF-16(Excel 的「Unicode 文本」):按 latin-1 读会混进 NUL,列名全乱
if raw[:2] in (b"\xff\xfe", b"\xfe\xff"):
try:
return raw.decode("utf-16")
except UnicodeDecodeError:
pass
_CSV_DECODE_CHUNK = 1024 * 1024
def _can_decode_csv(raw: bytes, enc: str) -> bool:
dec = codecs.getincrementaldecoder(enc)()
mv = memoryview(raw)
try:
for offset in range(0, len(mv), _CSV_DECODE_CHUNK):
end = offset + _CSV_DECODE_CHUNK
dec.decode(mv[offset:end], final=end >= len(mv))
if not len(mv):
dec.decode(b"", final=True)
return True
except UnicodeDecodeError:
return False
def _detect_csv_encoding(raw: bytes) -> tuple[str, str]:
"""增量分块探测 CSV 编码,避免为判编码而一次性物化整份全量字符串。"""
if raw[:2] in (b"\xff\xfe", b"\xfe\xff") and _can_decode_csv(raw, "utf-16"):
return "utf-16", "strict"
for enc in _ENCODINGS:
try:
return raw.decode(enc)
except UnicodeDecodeError:
continue
return raw.decode("latin-1", errors="replace")
if _can_decode_csv(raw, enc):
return enc, "strict"
return "latin-1", "replace"
def _open_csv_text(
raw: bytes,
*,
encoding: str | None = None,
errors: str = "strict",
newline: str | None = "",
) -> io.TextIOWrapper:
if encoding is None:
encoding, errors = _detect_csv_encoding(raw)
return io.TextIOWrapper(io.BytesIO(raw), encoding=encoding, errors=errors, newline=newline)
def _decode_csv(raw: bytes) -> str:
enc, errors = _detect_csv_encoding(raw)
return raw.decode(enc, errors=errors)
def _csv_sources(raw: bytes, filename: str, header_row: int) -> list[_Source]:
text = _decode_csv(raw)
enc, errors = _detect_csv_encoding(raw)
delimiter = "\t" if _ext(filename) == ".tsv" else None
if delimiter is None:
with _open_csv_text(raw, encoding=enc, errors=errors, newline="") as probe_stream:
head = probe_stream.read(8192)
try:
delimiter = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|").delimiter
delimiter = csv.Sniffer().sniff(head, delimiters=",;\t|").delimiter
except csv.Error:
delimiter = "," # 嗅不出来就按最常见的来
def rows() -> Iterator[Sequence[Any]]:
reader = csv.reader(io.StringIO(text), delimiter=delimiter)
return itertools.islice(reader, header_row - 1, None)
with _open_csv_text(raw, encoding=enc, errors=errors, newline="") as stream:
yield from itertools.islice(csv.reader(stream, delimiter=delimiter), header_row - 1, None)
stem = filename.rsplit("/", 1)[-1].rsplit(".", 1)[0]
# 分号分隔是欧洲区域设置的 CSV:那里逗号是小数点,「1,500」是 1.5 而不是一千五