Files
AgentLab/backend/app/memory/kb.py
T
ThearchyHelios 913380f738 倒排索引取代全表重算;PDF/Word 终于传得进来
**倒排**:以前每次检索都 select(Chunk) 全量载入,再在 Python 里重建一遍 BM25
——不只是扫表,是对每一段重新分词、重新统计词频,每次查询都来一遍。几百段
无感,上万段就是秒级,而那正是知识库开始有用的规模。

没用 SQLite 的 FTS5。方案里写了实施前先验证,验了:trigram 分词器要求查询
至少三个字符,中文的「商家」「订单」「权限」这些两字词一个都召不回。照搬会
让召回大幅倒退。现有分词器本来就做中文单字 + 二元组,自己存一张 chunk_terms
既保住召回,也不把知识库锁死在 SQLite 上。

快是次要的,一致才是前提——所以先写了"倒排必须和全表扫给出同一个名次"的
测试。它当场抓到两个真问题:

1. BM25 被算了两次:倒排按全集统计(N、avg_len、df)算一次,hybrid_rank
   又在候选子集上重建一次,而子集的 df 和平均长度是另一套数。改成把倒排
   算好的分传下去,不再重算。
2. 原来的 BM25.score 遍历的是**带重复**的查询词表:「导出订单最多多少行」
   里「多」出现两次,贡献就加两次。倒排用 set() 去重,顶部那条差了 2.32。
   改成按查询词频加权。

还有一处是排序确定性:分数并列时稳定排序保留输入顺序,而两条路径喂进去的
顺序天生不同——同一个查询会因为走了哪条路给出不同名次。两边都按 id 定序。

倒排跟着 ingest 建、跟着删文档清、跟着 reindex 重建;没建倒排的片段(存量
数据升级上来就是这样)退回全表扫——宁可慢,也不要因为索引没建好就说"没搜到"。

**解析**:upload 以前只收 UTF-8,PDF 传上去直接 415,而 mime 字段记了却从不
参与判断。现在认 PDF / DOCX / HTML。解析库走可选依赖 [docs](几十兆,只用
纯文本的人不该为此付出安装时间),HTML 用已在依赖里的 bs4。

几处是实际验出来的,不是想出来的:DOCX 要连表格一起取(合同和口径说明的关键
信息常常整个在表里);PDF 每行拖着排版填充空格,不清掉会混进切块和词频统计
(实测 416 字里只有 68 字是正文);扫描件提不出文字要明说,否则悄悄存一份
空文档,用户以为传成功了却永远搜不到。

PDF/DOCX 两条路径在临时 venv 里用真文件验过(含中文、表格、扫描件、坏文件),
没有动开发环境。
2026-09-19 09:44:09 +08:00

293 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import re
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Chunk, Document
from app.memory import inverted
from app.memory.embeddings import (
default_alpha, embed_text, embed_texts, embedder_dim, embedder_id, from_blob,
hybrid_rank, to_blob, usable,
)
_TARGET = 800
_OVERLAP = 120
def chunk_text(text: str, target: int = _TARGET, overlap: int = _OVERLAP) -> list[str]:
"""按语义边界切块。
先按空行分段,段落塞不下再按句子切,实在超长才硬切。
这样切出来的块不会把一句话拦腰截断,检索命中后读起来是完整的。
"""
text = text.replace("\r\n", "\n").strip()
if not text:
return []
paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
blocks: list[str] = []
for para in paragraphs:
if len(para) <= target:
blocks.append(para)
continue
sentences = re.split(r"(?<=[。!?.!?\n])\s*", para)
current = ""
for sentence in sentences:
if not sentence:
continue
if len(current) + len(sentence) <= target:
current += sentence
else:
if current:
blocks.append(current)
while len(sentence) > target:
blocks.append(sentence[:target])
sentence = sentence[target:]
current = sentence
if current:
blocks.append(current)
# 把相邻小块合并到接近目标长度,并留一点重叠保住上下文
merged: list[str] = []
buf = ""
for block in blocks:
if len(buf) + len(block) + 1 <= target:
buf = f"{buf}\n{block}" if buf else block
else:
if buf:
merged.append(buf)
tail = buf[-overlap:] if overlap else ""
buf = f"{tail}\n{block}" if tail else block
else:
merged.append(block)
buf = ""
if buf:
merged.append(buf)
return [m.strip() for m in merged if m.strip()]
async def ingest_document(
session: AsyncSession,
*,
collection: str,
title: str,
content: str,
source: str = "",
mime: str = "text/plain",
meta: dict[str, Any] | None = None,
) -> Document:
doc = Document(
collection=collection,
title=title or source or "未命名文档",
source=source,
mime=mime,
content=content,
meta=meta or {},
)
session.add(doc)
await session.flush()
pieces = chunk_text(content)
made: list[Chunk] = []
if pieces:
vectors = await embed_texts(pieces)
for i, (piece, vec) in enumerate(zip(pieces, vectors)):
chunk = Chunk(
document_id=doc.id,
collection=collection,
ordinal=i,
content=piece,
embedding=to_blob(vec),
embed_model=embedder_id(),
embed_dim=embedder_dim(),
meta={"title": doc.title},
)
session.add(chunk)
made.append(chunk)
# 先 flush 拿到 chunk.id,倒排行要引用它
await session.flush()
for chunk in made:
await inverted.index_chunk(session, chunk)
doc.chunk_count = len(pieces)
await session.commit()
await session.refresh(doc)
return doc
async def search(
session: AsyncSession,
*,
collection: str | None,
query: str,
limit: int = 5,
alpha: float | None = None,
on_degrade: Any = None,
) -> list[dict[str, Any]]:
"""混合检索。
存量向量和当前 embedder 对不上时**整个查询退回纯关键词**,并通过
on_degrade 把原因交出去。不把对不上的那些当 0 分:那会让它们静默沉底,
用户看到的是结果莫名其妙变差,而不是"这里有一批索引该重建了"。
在此之前这里连维度都不查,换个 embedder 直接 ValueError
(shapes (1536,) and (512,) not aligned),整个检索 500。
"""
# 没显式指定就跟着 embedder 走——写死 0.5 是在给一个没有语义能力的信号
# 一半权重,实测会把 hit@1 从 88% 拉到 81%
if alpha is None:
alpha = default_alpha()
# 先用倒排把候选收窄。以前这里是把整个集合载入内存、再在 Python 里重建
# 一遍 BM25 倒排——每次查询都重新分词一遍所有片段。
#
# 多捞一些(limit 的 8 倍):倒排只按关键词收,最终名次还要混进向量那一路,
# 卡得太紧会把"关键词一般但语义很近"的那些提前筛掉。
rows: list[Chunk] = []
kw_scores: list[float] | None = None
picked = await inverted.candidates(
session, collection=collection, query=query, limit=max(limit * 8, 50))
if picked:
found = dict(
(c.id, c) for c in (await session.execute(
select(Chunk).where(Chunk.id.in_([cid for cid, _ in picked])))).scalars())
pairs = [(found[cid], sc) for cid, sc in picked if cid in found]
rows = [c for c, _ in pairs]
# 倒排算好的分要带下去。在候选子集上重算 BM25,df 和平均长度是另一套数
kw_scores = [sc for _, sc in pairs]
if not rows:
# 一个查询词都没命中,或者这批片段还没建倒排(存量数据升级上来就是这样)。
# 退回全表扫,慢但不会漏——宁可慢,也不要因为索引没建好就说"没搜到"
stmt = select(Chunk)
if collection:
stmt = stmt.where(Chunk.collection == collection)
rows = list((await session.execute(stmt)).scalars())
kw_scores = None # 全表扫这条路照旧自己算
if not rows:
return []
# 按 id 定序再排。hybrid_rank 用的是稳定排序,分数并列时保留输入顺序——
# 而倒排那条路和全表扫那条路喂进去的顺序天生不同,同样的查询会因为走了
# 哪条路给出不同的名次。并列本来就该有个确定的先后,用 id 就够了
if kw_scores is not None:
paired = sorted(zip(rows, kw_scores), key=lambda t: t[0].id)
rows = [r for r, _ in paired]
kw_scores = [s2 for _, s2 in paired]
else:
rows = sorted(rows, key=lambda r: r.id)
stale = [r for r in rows if not usable(r.embed_model, r.embed_dim)]
if stale:
if on_degrade:
names = sorted({r.embed_model or "未知模型" for r in stale})
# 报的是整个集合里要重建多少条,不是本次候选里有几条——候选被倒排
# 收窄过,用它会把问题说小,而用户要照着做的恰恰是"重建多少"
total_stale = await stale_count(session, collection)
on_degrade(
f"{collection or '全部集合'}里有 {total_stale} 条向量由"
f"「{'、'.join(names)}」建立,和当前的「{embedder_id()}」对不上,"
f"本次已退回纯关键词检索。重建索引后恢复。"
)
alpha = 0.0 # 纯关键词
vectors: list[Any] = [None] * len(rows)
query_vec = await embed_text(query)
else:
query_vec = await embed_text(query)
vectors = [from_blob(r.embedding, embedder_dim()) for r in rows]
ranked = hybrid_rank(
query,
[r.content for r in rows],
vectors,
query_vec,
alpha=alpha,
keyword_scores=kw_scores,
)
out: list[dict[str, Any]] = []
for idx, score, parts in ranked[:limit]:
row = rows[idx]
out.append(
{
"chunk_id": row.id,
"document_id": row.document_id,
"title": (row.meta or {}).get("title", ""),
"ordinal": row.ordinal,
"content": row.content,
"score": round(score, 4),
"signals": {k: round(v, 4) for k, v in parts.items()},
}
)
return out
async def reindex(session: AsyncSession, collection: str | None = None) -> dict[str, Any]:
"""用当前 embedder 重算向量。
换了 embedding 模型之后唯一的恢复手段。按批做:一次几千条 chunk 全塞给
远端 embedding 接口,要么超时要么被限流,而中途失败又没有断点。
"""
stmt = select(Chunk)
if collection:
stmt = stmt.where(Chunk.collection == collection)
rows = list((await session.execute(stmt)).scalars())
if not rows:
return {"collection": collection, "reindexed": 0, "embedder": embedder_id()}
model_id, dim = embedder_id(), embedder_dim()
batch = 64
for i in range(0, len(rows), batch):
part = rows[i:i + batch]
vectors = await embed_texts([r.content for r in part])
for row, vec in zip(part, vectors):
row.embedding = to_blob(vec)
row.embed_model = model_id
row.embed_dim = dim
await session.commit()
# 倒排一起重建:两者都是"存量数据升级"的恢复手段,分成两个按钮只会漏点一个
await inverted.rebuild(session, collection)
return {"collection": collection, "reindexed": len(rows), "embedder": model_id}
async def stale_count(session: AsyncSession, collection: str | None = None) -> int:
"""有多少条向量和当前 embedder 对不上。知识库页据此提示要不要重建。
用 SQL 数而不是把片段全载进来再在 Python 里过一遍——这个数在检索降级时
也要取,那条路径上不该再来一次全表扫。
"""
from sqlalchemy import func, or_
stmt = select(func.count(Chunk.id)).where(
or_(Chunk.embed_model != embedder_id(), Chunk.embed_dim != embedder_dim()))
if collection:
stmt = stmt.where(Chunk.collection == collection)
return int((await session.execute(stmt)).scalar_one() or 0)
async def delete_document(session: AsyncSession, doc_id: str) -> bool:
doc = await session.get(Document, doc_id)
if not doc:
return False
chunk_ids = list((await session.execute(
select(Chunk.id).where(Chunk.document_id == doc_id))).scalars())
await inverted.drop_chunk_terms(session, chunk_ids)
await session.delete(doc)
await session.commit()
return True
async def list_collections(session: AsyncSession) -> list[dict[str, Any]]:
rows = await session.execute(
select(
Document.collection,
func.count(Document.id),
func.sum(Document.chunk_count),
).group_by(Document.collection)
)
return [
{"collection": name, "documents": docs, "chunks": chunks or 0}
for name, docs, chunks in rows
]