mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-19 01:51:59 +00:00
全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理
- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费 - 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定 - 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
@@ -110,29 +110,44 @@ class CsvTable:
|
||||
delimiter: str
|
||||
|
||||
|
||||
def _resolve_csv_meta(raw: bytes, delimiter: str | None) -> tuple[str, str, str]:
|
||||
from app.data import tabular
|
||||
|
||||
enc, errors = tabular._detect_csv_encoding(raw)
|
||||
if delimiter is None:
|
||||
with tabular._open_csv_text(raw, encoding=enc, errors=errors, newline="") as probe_stream:
|
||||
head = probe_stream.read(8192)
|
||||
try:
|
||||
delimiter = csv.Sniffer().sniff(head, delimiters=_DELIMITERS).delimiter
|
||||
except csv.Error:
|
||||
delimiter = "," # 嗅不出来就按最常见的来
|
||||
return enc, errors, delimiter
|
||||
|
||||
|
||||
def _iter_csv_records(raw: bytes, *, enc: str, errors: str, delimiter: str) -> Iterator[list[str]]:
|
||||
from app.data import tabular
|
||||
|
||||
row_count = 0
|
||||
try:
|
||||
with tabular._open_csv_text(raw, encoding=enc, errors=errors, newline="") as stream:
|
||||
for record in csv.reader(stream, delimiter=delimiter):
|
||||
if row_count >= MAX_ROWS:
|
||||
raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行,超出可以导入的上限")
|
||||
if len(record) > MAX_COLS:
|
||||
raise UnsupportedTable(f"CSV 文件第 {row_count + 1} 行有 {len(record):,} 列,超过 {MAX_COLS:,} 列的上限")
|
||||
row_count += 1
|
||||
yield record
|
||||
except csv.Error as e:
|
||||
raise UnsupportedTable(f"无法按 CSV 读取这个文件:第 {row_count + 1} 行的写法不正确({e})。"
|
||||
"请在 Excel 中重新另存为 CSV 后上传") from e
|
||||
|
||||
|
||||
def parse(raw: bytes, delimiter: str | None = None) -> CsvTable:
|
||||
"""原件 → 记录。编码照 tabular 读 CSV 的做法;delimiter 不给时嗅探(同 tabular)。字段太长、引号不配对读不了时抛
|
||||
UnsupportedTable。"""
|
||||
from app.data import tabular
|
||||
|
||||
text = tabular._decode_csv(raw)
|
||||
if delimiter is None:
|
||||
try:
|
||||
delimiter = csv.Sniffer().sniff(text[:8192], delimiters=_DELIMITERS).delimiter
|
||||
except csv.Error:
|
||||
delimiter = "," # 嗅不出来就按最常见的来
|
||||
records: list[list[str]] = []
|
||||
try:
|
||||
for record in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter):
|
||||
if len(records) >= MAX_ROWS:
|
||||
raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行,超出可以导入的上限")
|
||||
if len(record) > MAX_COLS:
|
||||
raise UnsupportedTable(f"CSV 文件第 {len(records) + 1} 行有 {len(record):,} 列,超过 {MAX_COLS:,} 列的上限")
|
||||
records.append(record)
|
||||
except csv.Error as e:
|
||||
raise UnsupportedTable(f"无法按 CSV 读取这个文件:第 {len(records) + 1} 行的写法不正确({e})。"
|
||||
"请在 Excel 中重新另存为 CSV 后上传") from e
|
||||
return CsvTable(records=records, delimiter=delimiter)
|
||||
enc, errors, resolved_delim = _resolve_csv_meta(raw, delimiter)
|
||||
records = list(_iter_csv_records(raw, enc=enc, errors=errors, delimiter=resolved_delim))
|
||||
return CsvTable(records=records, delimiter=resolved_delim)
|
||||
|
||||
|
||||
def _cells(table: CsvTable) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]:
|
||||
@@ -155,15 +170,18 @@ def sheet_delimiter(sheet: Any) -> str | None:
|
||||
|
||||
def scan(raw: bytes, *, delimiter: str | None = None) -> WorkbookScan:
|
||||
"""CSV 原件 → 扫描结果:一张可见工作表(名字 CSV_SHEET、路径 CSV_PATH 加分隔符),边界、非空格数、逐行非空格数的
|
||||
游程。没有合并区、隐藏行列、公式、错误值、离群格(CSV 里每一格都是写出来的,没有残留格式)。"""
|
||||
table = parse(raw, delimiter)
|
||||
sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + table.delimiter)
|
||||
游程。流式统计,不再先把整张 CSV 物化成全量字符串与二维列表。"""
|
||||
enc, errors, resolved_delim = _resolve_csv_meta(raw, delimiter)
|
||||
sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + resolved_delim)
|
||||
lo_r = lo_c = hi_r = hi_c = 0
|
||||
runs: list[tuple[int, int, int]] = []
|
||||
for r, cells in _cells(table):
|
||||
n = len(cells)
|
||||
for r, record in enumerate(_iter_csv_records(raw, enc=enc, errors=errors, delimiter=resolved_delim), 1):
|
||||
nonempty_cols = [c for c, field in enumerate(record, 1) if cell_value(field) is not None]
|
||||
if not nonempty_cols:
|
||||
continue
|
||||
n = len(nonempty_cols)
|
||||
sheet.nonempty += n
|
||||
first, last = cells[0][0], cells[-1][0]
|
||||
first, last = nonempty_cols[0], nonempty_cols[-1]
|
||||
if not lo_r:
|
||||
lo_r, lo_c, hi_c = r, first, last
|
||||
lo_c, hi_c, hi_r = min(lo_c, first), max(hi_c, last), r
|
||||
|
||||
Reference in New Issue
Block a user