mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-19 17:32:01 +00:00
全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理
- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费 - 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定 - 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
@@ -22,6 +22,8 @@ from app.memory.embeddings import _tokens
|
||||
|
||||
_K1 = 1.5
|
||||
_B = 0.75
|
||||
_IN_BATCH = 500
|
||||
_REBUILD_BATCH = 256
|
||||
|
||||
|
||||
def term_freqs(text: str) -> dict[str, int]:
|
||||
@@ -47,11 +49,69 @@ async def drop_chunk_terms(session: AsyncSession, chunk_ids: list[str]) -> None:
|
||||
"""删片段时一起删倒排。
|
||||
|
||||
外键写了 ondelete=CASCADE,但 SQLite 的级联要连接级 PRAGMA 打开才生效
|
||||
(base.py 里是打开的)——索引一致性不该依赖一个开关,显式删。
|
||||
(base.py 里是打开的)——索引一致性不该依赖一个开关,显式按批删,避免超过
|
||||
SQLite / PostgreSQL 单条语句绑定参数上限(32,766)。
|
||||
"""
|
||||
if chunk_ids:
|
||||
await session.execute(
|
||||
delete(ChunkTerm).where(ChunkTerm.chunk_id.in_(chunk_ids)))
|
||||
for i in range(0, len(chunk_ids), _IN_BATCH):
|
||||
part = chunk_ids[i:i + _IN_BATCH]
|
||||
if part:
|
||||
await session.execute(
|
||||
delete(ChunkTerm).where(ChunkTerm.chunk_id.in_(part)))
|
||||
|
||||
|
||||
async def candidates_with_scores(
|
||||
session: AsyncSession, *, collection: str | None, query: str, limit: int,
|
||||
) -> tuple[list[tuple[str, float]], dict[str, float]]:
|
||||
"""按 BM25 取候选,返回 (前 limit 条 [(chunk_id, 分数)], 全部命中片段的精确 BM25 分数字典)。
|
||||
|
||||
直接通过 ChunkTerm JOIN Chunk 一次性取回 token_len,彻底消除大集合高频词命中数万条时
|
||||
`Chunk.id.in_(...)` 超出数据库 32,767 参数上限崩溃的问题。
|
||||
"""
|
||||
# 按查询词的出现次数加权,不能去重。原来的 BM25.score 遍历的是带重复的
|
||||
# 词表:「导出订单最多多少行」里「多」出现两次,它的贡献就加两次。去重会让
|
||||
# 同一个查询在倒排和全表扫两条路上给出不同的分——实测顶部那条差了 2.32
|
||||
qcounts = Counter(t for t in _tokens(query) if len(t) <= 64)
|
||||
terms = list(qcounts)
|
||||
if not terms:
|
||||
return [], {}
|
||||
|
||||
rows: list[Any] = []
|
||||
for i in range(0, len(terms), _IN_BATCH):
|
||||
part = terms[i:i + _IN_BATCH]
|
||||
stmt = (
|
||||
select(ChunkTerm.chunk_id, ChunkTerm.term, ChunkTerm.tf, Chunk.token_len)
|
||||
.join(Chunk, Chunk.id == ChunkTerm.chunk_id)
|
||||
.where(ChunkTerm.term.in_(part))
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(ChunkTerm.collection == collection)
|
||||
rows.extend((await session.execute(stmt)).all())
|
||||
if not rows:
|
||||
return [], {}
|
||||
|
||||
# df 直接从命中里数:含有该词的片段数,正是 BM25 要的那个 df
|
||||
df: dict[str, int] = defaultdict(int)
|
||||
for _chunk_id, term, _tf, _tlen in rows:
|
||||
df[term] += 1
|
||||
|
||||
stat = select(func.count(Chunk.id), func.sum(Chunk.token_len))
|
||||
if collection:
|
||||
stat = stat.where(Chunk.collection == collection)
|
||||
total, total_len = (await session.execute(stat)).one()
|
||||
total = int(total or 0)
|
||||
if not total:
|
||||
return [], {}
|
||||
avg_len = float(total_len or 0) / total or 1.0
|
||||
|
||||
scores: dict[str, float] = defaultdict(float)
|
||||
for chunk_id, term, tf, token_len in rows:
|
||||
idf = math.log(1 + (total - df[term] + 0.5) / (df[term] + 0.5))
|
||||
length = int(token_len or 0) or 1
|
||||
denom = tf + _K1 * (1 - _B + _B * length / avg_len)
|
||||
scores[chunk_id] += qcounts[term] * idf * (tf * (_K1 + 1)) / denom
|
||||
|
||||
ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
return ranked[:limit], dict(scores)
|
||||
|
||||
|
||||
async def candidates(
|
||||
@@ -61,51 +121,16 @@ async def candidates(
|
||||
|
||||
没有任何查询词命中时返回空——调用方据此决定是不是要退回全表。
|
||||
"""
|
||||
# 按查询词的出现次数加权,不能去重。原来的 BM25.score 遍历的是带重复的
|
||||
# 词表:「导出订单最多多少行」里「多」出现两次,它的贡献就加两次。去重会让
|
||||
# 同一个查询在倒排和全表扫两条路上给出不同的分——实测顶部那条差了 2.32
|
||||
qcounts = Counter(t for t in _tokens(query) if len(t) <= 64)
|
||||
terms = list(qcounts)
|
||||
if not terms:
|
||||
return []
|
||||
top, _ = await candidates_with_scores(session, collection=collection, query=query, limit=limit)
|
||||
return top
|
||||
|
||||
stmt = select(ChunkTerm.chunk_id, ChunkTerm.term, ChunkTerm.tf).where(
|
||||
ChunkTerm.term.in_(terms))
|
||||
|
||||
async def has_terms(session: AsyncSession, collection: str | None = None) -> bool:
|
||||
"""集合里是否已建有倒排词项(区分「真没命中查询词」与「存量升级尚未建倒排」)。"""
|
||||
stmt = select(ChunkTerm.id).limit(1)
|
||||
if collection:
|
||||
stmt = stmt.where(ChunkTerm.collection == collection)
|
||||
rows = list(await session.execute(stmt))
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
# df 直接从命中里数:含有该词的片段数,正是 BM25 要的那个 df
|
||||
df: dict[str, int] = defaultdict(int)
|
||||
for _chunk_id, term, _tf in rows:
|
||||
df[term] += 1
|
||||
|
||||
stat = select(func.count(Chunk.id), func.sum(Chunk.token_len))
|
||||
if collection:
|
||||
stat = stat.where(Chunk.collection == collection)
|
||||
total, total_len = (await session.execute(stat)).one()
|
||||
total = int(total or 0)
|
||||
if not total:
|
||||
return []
|
||||
avg_len = float(total_len or 0) / total or 1.0
|
||||
|
||||
# 要 .all():Result 自带 keys(),dict() 会当它是映射去下标取值,
|
||||
# 抛 'ChunkedIteratorResult' object is not subscriptable
|
||||
lengths = dict((await session.execute(
|
||||
select(Chunk.id, Chunk.token_len).where(
|
||||
Chunk.id.in_({r[0] for r in rows})))).all())
|
||||
|
||||
scores: dict[str, float] = defaultdict(float)
|
||||
for chunk_id, term, tf in rows:
|
||||
idf = math.log(1 + (total - df[term] + 0.5) / (df[term] + 0.5))
|
||||
length = lengths.get(chunk_id) or 1
|
||||
denom = tf + _K1 * (1 - _B + _B * length / avg_len)
|
||||
scores[chunk_id] += qcounts[term] * idf * (tf * (_K1 + 1)) / denom
|
||||
|
||||
ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
|
||||
return ranked[:limit]
|
||||
return (await session.execute(stmt)).first() is not None
|
||||
|
||||
|
||||
async def missing_count(session: AsyncSession, collection: str | None = None) -> int:
|
||||
@@ -117,15 +142,20 @@ async def missing_count(session: AsyncSession, collection: str | None = None) ->
|
||||
|
||||
|
||||
async def rebuild(session: AsyncSession, collection: str | None = None) -> int:
|
||||
"""重建倒排。存量片段升级后要跑一次,改了分词器也要。"""
|
||||
stmt = select(Chunk)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return 0
|
||||
await drop_chunk_terms(session, [r.id for r in rows])
|
||||
for row in rows:
|
||||
await index_chunk(session, row)
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
"""按游标分批重建倒排,避免一次性把全库片段加载进内存。"""
|
||||
total = 0
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = select(Chunk).where(Chunk.id > last_id).order_by(Chunk.id.asc()).limit(_REBUILD_BATCH)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
batch = list((await session.execute(stmt)).scalars())
|
||||
if not batch:
|
||||
break
|
||||
last_id = batch[-1].id
|
||||
await drop_chunk_terms(session, [r.id for r in batch])
|
||||
for row in batch:
|
||||
await index_chunk(session, row)
|
||||
await session.commit()
|
||||
total += len(batch)
|
||||
return total
|
||||
|
||||
+237
-78
@@ -1,9 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
import heapq
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
import numpy as np
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Chunk, Document
|
||||
@@ -18,6 +22,29 @@ _OVERLAP = 120
|
||||
#: 每批送多少段去算向量。远端接口单次批量有上限(OpenAI 是 2048),而且
|
||||
#: 一次把上万段全发出去,要么超时要么被限流,中途失败还没有断点
|
||||
_EMBED_BATCH = 64
|
||||
_ID_FETCH_BATCH = 500
|
||||
_VEC_SCAN_BATCH = 512
|
||||
_VEC_CACHE_MAX_COLLECTIONS = 16
|
||||
_VEC_CACHE_MAX_CHUNKS = 200_000
|
||||
|
||||
|
||||
@dataclass
|
||||
class _VectorCacheEntry:
|
||||
stamp: tuple[int, str, str, int]
|
||||
chunk_ids: list[str]
|
||||
matrix: np.ndarray
|
||||
|
||||
|
||||
_VEC_CACHE: OrderedDict[str, _VectorCacheEntry] = OrderedDict()
|
||||
|
||||
|
||||
def invalidate_vector_cache(collection: str | None = None) -> None:
|
||||
"""文档增删或重建索引后失效进程内归一化向量矩阵缓存。"""
|
||||
if collection is None:
|
||||
_VEC_CACHE.clear()
|
||||
return
|
||||
_VEC_CACHE.pop(collection, None)
|
||||
_VEC_CACHE.pop("__all__", None)
|
||||
|
||||
|
||||
def chunk_text(text: str, target: int = _TARGET, overlap: int = _OVERLAP) -> list[str]:
|
||||
@@ -216,10 +243,137 @@ async def process_document(
|
||||
doc.status = "ready"
|
||||
doc.error = ""
|
||||
await session.commit()
|
||||
invalidate_vector_cache(collection)
|
||||
await session.refresh(doc)
|
||||
return doc
|
||||
|
||||
|
||||
async def _fetch_chunks_by_ids(session: AsyncSession, chunk_ids: list[str]) -> dict[str, Chunk]:
|
||||
found: dict[str, Chunk] = {}
|
||||
for i in range(0, len(chunk_ids), _ID_FETCH_BATCH):
|
||||
part = chunk_ids[i:i + _ID_FETCH_BATCH]
|
||||
if not part:
|
||||
continue
|
||||
for row in (await session.execute(select(Chunk).where(Chunk.id.in_(part)))).scalars():
|
||||
found[row.id] = row
|
||||
return found
|
||||
|
||||
|
||||
def _unit_vector(vec: np.ndarray) -> np.ndarray:
|
||||
arr = np.asarray(vec, dtype=np.float32)
|
||||
norm = float(np.linalg.norm(arr))
|
||||
return arr / norm if norm > 0 else np.zeros_like(arr, dtype=np.float32)
|
||||
|
||||
|
||||
async def _vector_top_ids(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
collection: str | None,
|
||||
query_vec: np.ndarray,
|
||||
limit: int,
|
||||
) -> list[str]:
|
||||
"""全集合语义向量路独立 Top-K 召回(不被 BM25 Top-50 前置截断)。
|
||||
|
||||
中小型集合缓存 L2 归一化 `float32` 矩阵,单次检索仅需一次 BLAS 矩阵乘法;
|
||||
超大集合(> 20 万块)自动退化为游标分批扫描 + 小顶堆,保证常驻内存有界。
|
||||
"""
|
||||
if limit <= 0:
|
||||
return []
|
||||
model_id, dim = embedder_id(), embedder_dim()
|
||||
stat_stmt = select(func.count(Chunk.id), func.max(Chunk.id))
|
||||
if collection:
|
||||
stat_stmt = stat_stmt.where(Chunk.collection == collection)
|
||||
total_raw, max_id_raw = (await session.execute(stat_stmt)).one()
|
||||
total = int(total_raw or 0)
|
||||
if total == 0:
|
||||
return []
|
||||
q_unit = _unit_vector(query_vec)
|
||||
if q_unit.shape[0] != dim:
|
||||
return []
|
||||
|
||||
cache_key = collection or "__all__"
|
||||
stamp = (total, str(max_id_raw or ""), model_id, dim)
|
||||
|
||||
if total <= _VEC_CACHE_MAX_CHUNKS:
|
||||
entry = _VEC_CACHE.get(cache_key)
|
||||
if entry is None or entry.stamp != stamp:
|
||||
ids: list[str] = []
|
||||
vecs: list[np.ndarray] = []
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = (
|
||||
select(Chunk.id, Chunk.embedding)
|
||||
.where(Chunk.id > last_id)
|
||||
.order_by(Chunk.id.asc())
|
||||
.limit(_VEC_SCAN_BATCH)
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
batch = (await session.execute(stmt)).all()
|
||||
if not batch:
|
||||
break
|
||||
last_id = str(batch[-1][0])
|
||||
for cid, blob in batch:
|
||||
v = from_blob(blob, dim)
|
||||
ids.append(str(cid))
|
||||
vecs.append(_unit_vector(v) if v is not None else np.zeros(dim, dtype=np.float32))
|
||||
matrix = np.vstack(vecs).astype(np.float32, copy=False) if vecs else np.zeros((0, dim), dtype=np.float32)
|
||||
entry = _VectorCacheEntry(stamp=stamp, chunk_ids=ids, matrix=matrix)
|
||||
_VEC_CACHE[cache_key] = entry
|
||||
_VEC_CACHE.move_to_end(cache_key)
|
||||
while len(_VEC_CACHE) > _VEC_CACHE_MAX_COLLECTIONS:
|
||||
_VEC_CACHE.popitem(last=False)
|
||||
else:
|
||||
_VEC_CACHE.move_to_end(cache_key)
|
||||
|
||||
if entry.matrix.shape[0] == 0:
|
||||
return []
|
||||
scores = entry.matrix @ q_unit
|
||||
n = scores.shape[0]
|
||||
if n <= limit:
|
||||
order = sorted(range(n), key=lambda idx: (-float(scores[idx]), entry.chunk_ids[idx]))
|
||||
return [entry.chunk_ids[idx] for idx in order]
|
||||
top_idx = np.argpartition(-scores, limit - 1)[:limit]
|
||||
ordered = sorted((int(idx) for idx in top_idx), key=lambda idx: (-float(scores[idx]), entry.chunk_ids[idx]))
|
||||
return [entry.chunk_ids[idx] for idx in ordered]
|
||||
|
||||
# 超大集合流式扫描兜底
|
||||
heap: list[tuple[float, str]] = []
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = (
|
||||
select(Chunk.id, Chunk.embedding)
|
||||
.where(Chunk.id > last_id)
|
||||
.order_by(Chunk.id.asc())
|
||||
.limit(_VEC_SCAN_BATCH)
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
batch = (await session.execute(stmt)).all()
|
||||
if not batch:
|
||||
break
|
||||
last_id = str(batch[-1][0])
|
||||
for cid, blob in batch:
|
||||
v = from_blob(blob, dim)
|
||||
sc = float(np.dot(_unit_vector(v), q_unit)) if v is not None else 0.0
|
||||
if len(heap) < limit:
|
||||
heapq.heappush(heap, (sc, str(cid)))
|
||||
elif sc > heap[0][0]:
|
||||
heapq.heapreplace(heap, (sc, str(cid)))
|
||||
heap.sort(key=lambda item: (-item[0], item[1]))
|
||||
return [cid for _, cid in heap]
|
||||
|
||||
|
||||
async def _stale_model_names(session: AsyncSession, collection: str | None) -> list[str]:
|
||||
stmt = select(Chunk.embed_model).distinct().where(
|
||||
or_(Chunk.embed_model != embedder_id(), Chunk.embed_dim != embedder_dim())
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
names = [str(m or "未知模型") for m in (await session.execute(stmt)).scalars()]
|
||||
return sorted(set(names)) or ["未知模型"]
|
||||
|
||||
|
||||
async def search(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
@@ -229,76 +383,79 @@ async def search(
|
||||
alpha: float | None = None,
|
||||
on_degrade: Any = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""混合检索。
|
||||
"""真·双路混合检索(词面 BM25 独立召回 + 全集合向量矩阵独立召回)。
|
||||
|
||||
存量向量和当前 embedder 对不上时**整个查询退回纯关键词**,并通过
|
||||
on_degrade 把原因交出去。不把对不上的那些当 0 分:那会让它们静默沉底,
|
||||
用户看到的是结果莫名其妙变差,而不是"这里有一批索引该重建了"。
|
||||
|
||||
在此之前这里连维度都不查,换个 embedder 直接 ValueError
|
||||
(shapes (1536,) and (512,) not aligned),整个检索 500。
|
||||
"""
|
||||
# 没显式指定就跟着 embedder 走——写死 0.5 是在给一个没有语义能力的信号
|
||||
# 一半权重,实测会把 hit@1 从 88% 拉到 81%
|
||||
if alpha is None:
|
||||
alpha = default_alpha()
|
||||
|
||||
# 先用倒排把候选收窄。以前这里是把整个集合载入内存、再在 Python 里重建
|
||||
# 一遍 BM25 倒排——每次查询都重新分词一遍所有片段。
|
||||
#
|
||||
# 多捞一些(limit 的 8 倍):倒排只按关键词收,最终名次还要混进向量那一路,
|
||||
# 卡得太紧会把"关键词一般但语义很近"的那些提前筛掉。
|
||||
rows: list[Chunk] = []
|
||||
kw_scores: list[float] | None = None
|
||||
picked = await inverted.candidates(
|
||||
session, collection=collection, query=query, limit=max(limit * 8, 50))
|
||||
if picked:
|
||||
found = dict(
|
||||
(c.id, c) for c in (await session.execute(
|
||||
select(Chunk).where(Chunk.id.in_([cid for cid, _ in picked])))).scalars())
|
||||
pairs = [(found[cid], sc) for cid, sc in picked if cid in found]
|
||||
rows = [c for c, _ in pairs]
|
||||
# 倒排算好的分要带下去。在候选子集上重算 BM25,df 和平均长度是另一套数
|
||||
kw_scores = [sc for _, sc in pairs]
|
||||
|
||||
if not rows:
|
||||
# 一个查询词都没命中,或者这批片段还没建倒排(存量数据升级上来就是这样)。
|
||||
# 退回全表扫,慢但不会漏——宁可慢,也不要因为索引没建好就说"没搜到"
|
||||
stmt = select(Chunk)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
kw_scores = None # 全表扫这条路照旧自己算
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
# 按 id 定序再排。hybrid_rank 用的是稳定排序,分数并列时保留输入顺序——
|
||||
# 而倒排那条路和全表扫那条路喂进去的顺序天生不同,同样的查询会因为走了
|
||||
# 哪条路给出不同的名次。并列本来就该有个确定的先后,用 id 就够了
|
||||
if kw_scores is not None:
|
||||
paired = sorted(zip(rows, kw_scores), key=lambda t: t[0].id)
|
||||
rows = [r for r, _ in paired]
|
||||
kw_scores = [s2 for _, s2 in paired]
|
||||
else:
|
||||
rows = sorted(rows, key=lambda r: r.id)
|
||||
|
||||
stale = [r for r in rows if not usable(r.embed_model, r.embed_dim)]
|
||||
if stale:
|
||||
total_stale = await stale_count(session, collection)
|
||||
if total_stale > 0:
|
||||
if on_degrade:
|
||||
names = sorted({r.embed_model or "未知模型" for r in stale})
|
||||
# 报的是整个集合里要重建多少条,不是本次候选里有几条——候选被倒排
|
||||
# 收窄过,用它会把问题说小,而用户要照着做的恰恰是"重建多少"
|
||||
total_stale = await stale_count(session, collection)
|
||||
names = await _stale_model_names(session, collection)
|
||||
on_degrade(
|
||||
f"{collection or '全部集合'}里有 {total_stale} 条向量由"
|
||||
f"「{'、'.join(names)}」建立,与当前的「{embedder_id()}」不一致,"
|
||||
f"本次已退回纯关键词检索。重建索引后恢复。"
|
||||
)
|
||||
alpha = 0.0 # 纯关键词
|
||||
vectors: list[Any] = [None] * len(rows)
|
||||
query_vec = await embed_text(query)
|
||||
alpha = 0.0
|
||||
|
||||
query_vec = await embed_text(query)
|
||||
cand_limit = max(limit * 8, 50)
|
||||
|
||||
# 第一路:倒排索引词面 BM25 Top-K(返回精确全集合 BM25 得分字典)
|
||||
lex_picked, all_kw_scores = await inverted.candidates_with_scores(
|
||||
session, collection=collection, query=query, limit=cand_limit,
|
||||
)
|
||||
lex_ids = [cid for cid, _ in lex_picked]
|
||||
|
||||
# 第二路:当启用语义权重时,独立从全集合归一化向量索引召回语义 Top-K,不再被 BM25 Top-50 卡死
|
||||
vec_ids = (
|
||||
await _vector_top_ids(session, collection=collection, query_vec=query_vec, limit=cand_limit)
|
||||
if alpha > 0.0
|
||||
else []
|
||||
)
|
||||
|
||||
merged_ids = sorted(set(lex_ids) | set(vec_ids))
|
||||
rows: list[Chunk] = []
|
||||
kw_scores: list[float] | None = None
|
||||
|
||||
if merged_ids:
|
||||
found = await _fetch_chunks_by_ids(session, merged_ids)
|
||||
rows = [found[cid] for cid in merged_ids if cid in found]
|
||||
if lex_picked or await inverted.has_terms(session, collection):
|
||||
kw_scores = [all_kw_scores.get(r.id, 0.0) for r in rows]
|
||||
else:
|
||||
kw_scores = None
|
||||
else:
|
||||
if await inverted.has_terms(session, collection):
|
||||
# 倒排已就绪但没有任何查询词命中且 alpha == 0:所有片段词面分均为 0,只需按 id 定序取前 limit 条
|
||||
stmt = select(Chunk).order_by(Chunk.id.asc()).limit(limit)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
kw_scores = [0.0] * len(rows)
|
||||
else:
|
||||
# 存量数据升级尚未建倒排:退回全表扫现算 BM25,保证不漏检
|
||||
stmt = select(Chunk).order_by(Chunk.id.asc())
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
kw_scores = None
|
||||
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
if total_stale > 0 or alpha == 0.0:
|
||||
vectors: list[Any] = (
|
||||
[None] * len(rows)
|
||||
if total_stale > 0
|
||||
else [from_blob(r.embedding, embedder_dim()) for r in rows]
|
||||
)
|
||||
else:
|
||||
query_vec = await embed_text(query)
|
||||
vectors = [from_blob(r.embedding, embedder_dim()) for r in rows]
|
||||
|
||||
ranked = hybrid_contrib(
|
||||
@@ -313,7 +470,6 @@ async def search(
|
||||
for hit in ranked[:limit]:
|
||||
row = rows[hit.index]
|
||||
score = round(hit.score, 4)
|
||||
# 关键词那份用减法得出:各自四舍五入之后再相加,会和总分差出一个末位
|
||||
by_vec = min(round(hit.contrib["vector"], 4), score)
|
||||
out.append(
|
||||
{
|
||||
@@ -324,7 +480,6 @@ async def search(
|
||||
"content": row.content,
|
||||
"score": score,
|
||||
"signals": {k: round(v, 4) for k, v in hit.signals.items()},
|
||||
# 两路各贡献了多少分,两者之和等于 score。调试台据此画贡献条
|
||||
"contrib": {"vector": by_vec, "keyword": round(score - by_vec, 4)},
|
||||
}
|
||||
)
|
||||
@@ -334,34 +489,38 @@ async def search(
|
||||
async def reindex(
|
||||
session: AsyncSession, collection: str | None = None, *, on_progress: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
"""用当前 embedder 重算向量。
|
||||
|
||||
换了 embedding 模型之后唯一的恢复手段。按批做:一次几千条 chunk 全塞给
|
||||
远端 embedding 接口,要么超时要么被限流,而中途失败又没有断点。
|
||||
on_progress(已完成, 总数) 每批提交之后回调一次。
|
||||
"""
|
||||
stmt = select(Chunk)
|
||||
"""用当前 embedder 按游标分批重算向量并重建倒排,避免全表一次性载入内存。"""
|
||||
count_stmt = select(func.count(Chunk.id))
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return {"collection": collection, "reindexed": 0, "embedder": embedder_id()}
|
||||
|
||||
count_stmt = count_stmt.where(Chunk.collection == collection)
|
||||
total = int((await session.execute(count_stmt)).scalar_one() or 0)
|
||||
model_id, dim = embedder_id(), embedder_dim()
|
||||
batch = _EMBED_BATCH
|
||||
for i in range(0, len(rows), batch):
|
||||
part = rows[i:i + batch]
|
||||
if total == 0:
|
||||
invalidate_vector_cache(collection)
|
||||
return {"collection": collection, "reindexed": 0, "embedder": model_id}
|
||||
|
||||
done = 0
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = select(Chunk).where(Chunk.id > last_id).order_by(Chunk.id.asc()).limit(_EMBED_BATCH)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
part = list((await session.execute(stmt)).scalars())
|
||||
if not part:
|
||||
break
|
||||
last_id = part[-1].id
|
||||
vectors = await embed_texts([r.content for r in part])
|
||||
for row, vec in zip(part, vectors):
|
||||
row.embedding = to_blob(vec)
|
||||
row.embed_model = model_id
|
||||
row.embed_dim = dim
|
||||
await session.commit()
|
||||
done += len(part)
|
||||
if on_progress:
|
||||
on_progress(min(i + batch, len(rows)), len(rows))
|
||||
# 倒排一起重建:两者都是"存量数据升级"的恢复手段,分成两个按钮只会漏点一个
|
||||
on_progress(min(done, total), total)
|
||||
await inverted.rebuild(session, collection)
|
||||
return {"collection": collection, "reindexed": len(rows), "embedder": model_id}
|
||||
invalidate_vector_cache(collection)
|
||||
return {"collection": collection, "reindexed": done, "embedder": model_id}
|
||||
|
||||
|
||||
async def stale_count(session: AsyncSession, collection: str | None = None) -> int:
|
||||
@@ -370,8 +529,6 @@ async def stale_count(session: AsyncSession, collection: str | None = None) -> i
|
||||
用 SQL 数而不是把片段全载进来再在 Python 里过一遍——这个数在检索降级时
|
||||
也要取,那条路径上不该再来一次全表扫。
|
||||
"""
|
||||
from sqlalchemy import func, or_
|
||||
|
||||
stmt = select(func.count(Chunk.id)).where(
|
||||
or_(Chunk.embed_model != embedder_id(), Chunk.embed_dim != embedder_dim()))
|
||||
if collection:
|
||||
@@ -383,11 +540,13 @@ async def delete_document(session: AsyncSession, doc_id: str) -> bool:
|
||||
doc = await session.get(Document, doc_id)
|
||||
if not doc:
|
||||
return False
|
||||
collection = doc.collection
|
||||
chunk_ids = list((await session.execute(
|
||||
select(Chunk.id).where(Chunk.document_id == doc_id))).scalars())
|
||||
await inverted.drop_chunk_terms(session, chunk_ids)
|
||||
await session.delete(doc)
|
||||
await session.commit()
|
||||
invalidate_vector_cache(collection)
|
||||
return True
|
||||
|
||||
|
||||
|
||||
+22
-13
@@ -133,6 +133,7 @@ async def recall(
|
||||
)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
picked_rows: list[MemoryItem] = []
|
||||
now = datetime.now(timezone.utc)
|
||||
for idx, score, parts in ranked:
|
||||
row = rows[idx]
|
||||
@@ -140,6 +141,7 @@ async def recall(
|
||||
boosted = score * (0.8 + 0.4 * row.importance) * (1 + min(row.use_count, 10) * 0.01)
|
||||
if boosted < min_score:
|
||||
continue
|
||||
picked_rows.append(row)
|
||||
results.append(
|
||||
{
|
||||
"id": row.id,
|
||||
@@ -158,11 +160,9 @@ async def recall(
|
||||
if not track:
|
||||
return results
|
||||
# 记一下被召回过,作为下次排序的信号
|
||||
for r in results:
|
||||
item = await session.get(MemoryItem, r["id"])
|
||||
if item:
|
||||
item.use_count += 1
|
||||
item.last_used_at = now
|
||||
for item in picked_rows:
|
||||
item.use_count += 1
|
||||
item.last_used_at = now
|
||||
await session.commit()
|
||||
return results
|
||||
|
||||
@@ -241,28 +241,37 @@ async def reindex(
|
||||
路径——记忆这边悄悄退回关键词,而且没有任何恢复手段。
|
||||
on_progress(已完成, 总数) 每批提交之后回调一次。
|
||||
"""
|
||||
stmt = select(MemoryItem)
|
||||
count_stmt = select(func.count(MemoryItem.id))
|
||||
if scope:
|
||||
stmt = stmt.where(MemoryItem.scope == scope)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
count_stmt = count_stmt.where(MemoryItem.scope == scope)
|
||||
total = int((await session.execute(count_stmt)).scalar_one() or 0)
|
||||
if total == 0:
|
||||
return 0
|
||||
|
||||
model_id, dim = embedder_id(), embedder_dim()
|
||||
from app.memory import kb
|
||||
|
||||
batch = kb._EMBED_BATCH # 和 kb 那边统一:一次几千条全发给远端接口必被限流
|
||||
for i in range(0, len(rows), batch):
|
||||
part = rows[i:i + batch]
|
||||
done = 0
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = select(MemoryItem).where(MemoryItem.id > last_id).order_by(MemoryItem.id.asc()).limit(batch)
|
||||
if scope:
|
||||
stmt = stmt.where(MemoryItem.scope == scope)
|
||||
part = list((await session.execute(stmt)).scalars())
|
||||
if not part:
|
||||
break
|
||||
last_id = part[-1].id
|
||||
vectors = await embed_texts([r.content for r in part])
|
||||
for row, vec in zip(part, vectors):
|
||||
row.embedding = to_blob(vec)
|
||||
row.embed_model = model_id
|
||||
row.embed_dim = dim
|
||||
await session.commit()
|
||||
done += len(part)
|
||||
if on_progress:
|
||||
on_progress(min(i + batch, len(rows)), len(rows))
|
||||
return len(rows)
|
||||
on_progress(min(done, total), total)
|
||||
return done
|
||||
|
||||
|
||||
async def list_scopes(session: AsyncSession) -> list[dict[str, Any]]:
|
||||
|
||||
Reference in New Issue
Block a user