mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 04:12:04 +00:00
全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理
- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费 - 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定 - 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
@@ -22,6 +22,8 @@ from app.memory.embeddings import _tokens
|
||||
|
||||
_K1 = 1.5
|
||||
_B = 0.75
|
||||
_IN_BATCH = 500
|
||||
_REBUILD_BATCH = 256
|
||||
|
||||
|
||||
def term_freqs(text: str) -> dict[str, int]:
|
||||
@@ -47,11 +49,69 @@ async def drop_chunk_terms(session: AsyncSession, chunk_ids: list[str]) -> None:
|
||||
"""删片段时一起删倒排。
|
||||
|
||||
外键写了 ondelete=CASCADE,但 SQLite 的级联要连接级 PRAGMA 打开才生效
|
||||
(base.py 里是打开的)——索引一致性不该依赖一个开关,显式删。
|
||||
(base.py 里是打开的)——索引一致性不该依赖一个开关,显式按批删,避免超过
|
||||
SQLite / PostgreSQL 单条语句绑定参数上限(32,766)。
|
||||
"""
|
||||
if chunk_ids:
|
||||
await session.execute(
|
||||
delete(ChunkTerm).where(ChunkTerm.chunk_id.in_(chunk_ids)))
|
||||
for i in range(0, len(chunk_ids), _IN_BATCH):
|
||||
part = chunk_ids[i:i + _IN_BATCH]
|
||||
if part:
|
||||
await session.execute(
|
||||
delete(ChunkTerm).where(ChunkTerm.chunk_id.in_(part)))
|
||||
|
||||
|
||||
async def candidates_with_scores(
|
||||
session: AsyncSession, *, collection: str | None, query: str, limit: int,
|
||||
) -> tuple[list[tuple[str, float]], dict[str, float]]:
|
||||
"""按 BM25 取候选,返回 (前 limit 条 [(chunk_id, 分数)], 全部命中片段的精确 BM25 分数字典)。
|
||||
|
||||
直接通过 ChunkTerm JOIN Chunk 一次性取回 token_len,彻底消除大集合高频词命中数万条时
|
||||
`Chunk.id.in_(...)` 超出数据库 32,767 参数上限崩溃的问题。
|
||||
"""
|
||||
# 按查询词的出现次数加权,不能去重。原来的 BM25.score 遍历的是带重复的
|
||||
# 词表:「导出订单最多多少行」里「多」出现两次,它的贡献就加两次。去重会让
|
||||
# 同一个查询在倒排和全表扫两条路上给出不同的分——实测顶部那条差了 2.32
|
||||
qcounts = Counter(t for t in _tokens(query) if len(t) <= 64)
|
||||
terms = list(qcounts)
|
||||
if not terms:
|
||||
return [], {}
|
||||
|
||||
rows: list[Any] = []
|
||||
for i in range(0, len(terms), _IN_BATCH):
|
||||
part = terms[i:i + _IN_BATCH]
|
||||
stmt = (
|
||||
select(ChunkTerm.chunk_id, ChunkTerm.term, ChunkTerm.tf, Chunk.token_len)
|
||||
.join(Chunk, Chunk.id == ChunkTerm.chunk_id)
|
||||
.where(ChunkTerm.term.in_(part))
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(ChunkTerm.collection == collection)
|
||||
rows.extend((await session.execute(stmt)).all())
|
||||
if not rows:
|
||||
return [], {}
|
||||
|
||||
# df 直接从命中里数:含有该词的片段数,正是 BM25 要的那个 df
|
||||
df: dict[str, int] = defaultdict(int)
|
||||
for _chunk_id, term, _tf, _tlen in rows:
|
||||
df[term] += 1
|
||||
|
||||
stat = select(func.count(Chunk.id), func.sum(Chunk.token_len))
|
||||
if collection:
|
||||
stat = stat.where(Chunk.collection == collection)
|
||||
total, total_len = (await session.execute(stat)).one()
|
||||
total = int(total or 0)
|
||||
if not total:
|
||||
return [], {}
|
||||
avg_len = float(total_len or 0) / total or 1.0
|
||||
|
||||
scores: dict[str, float] = defaultdict(float)
|
||||
for chunk_id, term, tf, token_len in rows:
|
||||
idf = math.log(1 + (total - df[term] + 0.5) / (df[term] + 0.5))
|
||||
length = int(token_len or 0) or 1
|
||||
denom = tf + _K1 * (1 - _B + _B * length / avg_len)
|
||||
scores[chunk_id] += qcounts[term] * idf * (tf * (_K1 + 1)) / denom
|
||||
|
||||
ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
return ranked[:limit], dict(scores)
|
||||
|
||||
|
||||
async def candidates(
|
||||
@@ -61,51 +121,16 @@ async def candidates(
|
||||
|
||||
没有任何查询词命中时返回空——调用方据此决定是不是要退回全表。
|
||||
"""
|
||||
# 按查询词的出现次数加权,不能去重。原来的 BM25.score 遍历的是带重复的
|
||||
# 词表:「导出订单最多多少行」里「多」出现两次,它的贡献就加两次。去重会让
|
||||
# 同一个查询在倒排和全表扫两条路上给出不同的分——实测顶部那条差了 2.32
|
||||
qcounts = Counter(t for t in _tokens(query) if len(t) <= 64)
|
||||
terms = list(qcounts)
|
||||
if not terms:
|
||||
return []
|
||||
top, _ = await candidates_with_scores(session, collection=collection, query=query, limit=limit)
|
||||
return top
|
||||
|
||||
stmt = select(ChunkTerm.chunk_id, ChunkTerm.term, ChunkTerm.tf).where(
|
||||
ChunkTerm.term.in_(terms))
|
||||
|
||||
async def has_terms(session: AsyncSession, collection: str | None = None) -> bool:
|
||||
"""集合里是否已建有倒排词项(区分「真没命中查询词」与「存量升级尚未建倒排」)。"""
|
||||
stmt = select(ChunkTerm.id).limit(1)
|
||||
if collection:
|
||||
stmt = stmt.where(ChunkTerm.collection == collection)
|
||||
rows = list(await session.execute(stmt))
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
# df 直接从命中里数:含有该词的片段数,正是 BM25 要的那个 df
|
||||
df: dict[str, int] = defaultdict(int)
|
||||
for _chunk_id, term, _tf in rows:
|
||||
df[term] += 1
|
||||
|
||||
stat = select(func.count(Chunk.id), func.sum(Chunk.token_len))
|
||||
if collection:
|
||||
stat = stat.where(Chunk.collection == collection)
|
||||
total, total_len = (await session.execute(stat)).one()
|
||||
total = int(total or 0)
|
||||
if not total:
|
||||
return []
|
||||
avg_len = float(total_len or 0) / total or 1.0
|
||||
|
||||
# 要 .all():Result 自带 keys(),dict() 会当它是映射去下标取值,
|
||||
# 抛 'ChunkedIteratorResult' object is not subscriptable
|
||||
lengths = dict((await session.execute(
|
||||
select(Chunk.id, Chunk.token_len).where(
|
||||
Chunk.id.in_({r[0] for r in rows})))).all())
|
||||
|
||||
scores: dict[str, float] = defaultdict(float)
|
||||
for chunk_id, term, tf in rows:
|
||||
idf = math.log(1 + (total - df[term] + 0.5) / (df[term] + 0.5))
|
||||
length = lengths.get(chunk_id) or 1
|
||||
denom = tf + _K1 * (1 - _B + _B * length / avg_len)
|
||||
scores[chunk_id] += qcounts[term] * idf * (tf * (_K1 + 1)) / denom
|
||||
|
||||
ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
|
||||
return ranked[:limit]
|
||||
return (await session.execute(stmt)).first() is not None
|
||||
|
||||
|
||||
async def missing_count(session: AsyncSession, collection: str | None = None) -> int:
|
||||
@@ -117,15 +142,20 @@ async def missing_count(session: AsyncSession, collection: str | None = None) ->
|
||||
|
||||
|
||||
async def rebuild(session: AsyncSession, collection: str | None = None) -> int:
|
||||
"""重建倒排。存量片段升级后要跑一次,改了分词器也要。"""
|
||||
stmt = select(Chunk)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return 0
|
||||
await drop_chunk_terms(session, [r.id for r in rows])
|
||||
for row in rows:
|
||||
await index_chunk(session, row)
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
"""按游标分批重建倒排,避免一次性把全库片段加载进内存。"""
|
||||
total = 0
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = select(Chunk).where(Chunk.id > last_id).order_by(Chunk.id.asc()).limit(_REBUILD_BATCH)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
batch = list((await session.execute(stmt)).scalars())
|
||||
if not batch:
|
||||
break
|
||||
last_id = batch[-1].id
|
||||
await drop_chunk_terms(session, [r.id for r in batch])
|
||||
for row in batch:
|
||||
await index_chunk(session, row)
|
||||
await session.commit()
|
||||
total += len(batch)
|
||||
return total
|
||||
|
||||
Reference in New Issue
Block a user