Files
AgentLab/backend/app/memory/kb.py
T
ThearchyHelios e8ef6ff9a8 检索质量有数了:评测集 + 重排,alpha 不再写死
先做评测再做重排——否则重排的收益只能靠"搜两下感觉准了"。

**评测集**(test_retrieval_quality.py):24 篇语料 + 16 个问题,问题分成字面
重叠和纯语义两类。第一版是 8 篇、主指标 hit@3,结果各档 alpha 全是 100%,
等于什么都没量——8 选 3 瞎猜都有 37%。扩语料、加干扰、主指标换成 hit@1
之后才开始能分辨。

量出来的第一个结论就推翻了我原来的猜测:

    alpha   hit@1   hit@5   MRR     字面    语义
    0.0     88%     100%    0.911   100%    75%
    0.2     81%     100%    0.880   88%     75%
    0.5     81%     100%    0.880   88%     75%
    1.0     81%     100%    0.880   88%     75%

本地哈希向量那一路不是"弱一点",是**纯噪音**:给它任何权重,hit@1 都从 88%
掉到 81%。而 alpha 的默认值一直写死 0.5,等于主动把检索做差。改成跟着
embedder 能力走(default_alpha:有语义泛化才给 0.5,否则 0),节点上的
alpha 也不再写死默认值。

**重排**(memory/rerank.py):初筛多捞一些(20 条),让模型按"对回答这个问题
有多大帮助"重排,再取 top_n。只捞 5 条的话重排最多把这 5 条换个顺序,第 6
条是对的也救不回来。

用已配置的对话模型做,不引入本地 cross-encoder——后者要下模型权重,和"零配置
能跑起来"的基线冲突。代价是每次检索多一次调用,所以默认关闭。

失败一律退回初筛顺序并说一声:解析不出分数、个数对不上、模型抛异常、没配
provider,四条路径都有测试。重排是增强,不该因为模型抽风就让整个检索挂掉。
2026-09-19 09:33:09 +08:00

240 lines
7.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import re
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Chunk, Document
from app.memory.embeddings import (
default_alpha, embed_text, embed_texts, embedder_dim, embedder_id, from_blob,
hybrid_rank, to_blob, usable,
)
_TARGET = 800
_OVERLAP = 120
def chunk_text(text: str, target: int = _TARGET, overlap: int = _OVERLAP) -> list[str]:
"""按语义边界切块。
先按空行分段,段落塞不下再按句子切,实在超长才硬切。
这样切出来的块不会把一句话拦腰截断,检索命中后读起来是完整的。
"""
text = text.replace("\r\n", "\n").strip()
if not text:
return []
paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
blocks: list[str] = []
for para in paragraphs:
if len(para) <= target:
blocks.append(para)
continue
sentences = re.split(r"(?<=[。!?.!?\n])\s*", para)
current = ""
for sentence in sentences:
if not sentence:
continue
if len(current) + len(sentence) <= target:
current += sentence
else:
if current:
blocks.append(current)
while len(sentence) > target:
blocks.append(sentence[:target])
sentence = sentence[target:]
current = sentence
if current:
blocks.append(current)
# 把相邻小块合并到接近目标长度,并留一点重叠保住上下文
merged: list[str] = []
buf = ""
for block in blocks:
if len(buf) + len(block) + 1 <= target:
buf = f"{buf}\n{block}" if buf else block
else:
if buf:
merged.append(buf)
tail = buf[-overlap:] if overlap else ""
buf = f"{tail}\n{block}" if tail else block
else:
merged.append(block)
buf = ""
if buf:
merged.append(buf)
return [m.strip() for m in merged if m.strip()]
async def ingest_document(
session: AsyncSession,
*,
collection: str,
title: str,
content: str,
source: str = "",
mime: str = "text/plain",
meta: dict[str, Any] | None = None,
) -> Document:
doc = Document(
collection=collection,
title=title or source or "未命名文档",
source=source,
mime=mime,
content=content,
meta=meta or {},
)
session.add(doc)
await session.flush()
pieces = chunk_text(content)
if pieces:
vectors = await embed_texts(pieces)
for i, (piece, vec) in enumerate(zip(pieces, vectors)):
session.add(
Chunk(
document_id=doc.id,
collection=collection,
ordinal=i,
content=piece,
embedding=to_blob(vec),
embed_model=embedder_id(),
embed_dim=embedder_dim(),
meta={"title": doc.title},
)
)
doc.chunk_count = len(pieces)
await session.commit()
await session.refresh(doc)
return doc
async def search(
session: AsyncSession,
*,
collection: str | None,
query: str,
limit: int = 5,
alpha: float | None = None,
on_degrade: Any = None,
) -> list[dict[str, Any]]:
"""混合检索。
存量向量和当前 embedder 对不上时**整个查询退回纯关键词**,并通过
on_degrade 把原因交出去。不把对不上的那些当 0 分:那会让它们静默沉底,
用户看到的是结果莫名其妙变差,而不是"这里有一批索引该重建了"。
在此之前这里连维度都不查,换个 embedder 直接 ValueError
(shapes (1536,) and (512,) not aligned),整个检索 500。
"""
# 没显式指定就跟着 embedder 走——写死 0.5 是在给一个没有语义能力的信号
# 一半权重,实测会把 hit@1 从 88% 拉到 81%
if alpha is None:
alpha = default_alpha()
stmt = select(Chunk)
if collection:
stmt = stmt.where(Chunk.collection == collection)
rows = list((await session.execute(stmt)).scalars())
if not rows:
return []
stale = [r for r in rows if not usable(r.embed_model, r.embed_dim)]
if stale:
if on_degrade:
names = sorted({r.embed_model or "未知模型" for r in stale})
on_degrade(
f"{collection or '全部集合'}里有 {len(stale)}/{len(rows)} 条向量由"
f"「{'、'.join(names)}」建立,和当前的「{embedder_id()}」对不上,"
f"本次已退回纯关键词检索。重建索引后恢复。"
)
alpha = 0.0 # 纯关键词
vectors: list[Any] = [None] * len(rows)
query_vec = await embed_text(query)
else:
query_vec = await embed_text(query)
vectors = [from_blob(r.embedding, embedder_dim()) for r in rows]
ranked = hybrid_rank(
query,
[r.content for r in rows],
vectors,
query_vec,
alpha=alpha,
)
out: list[dict[str, Any]] = []
for idx, score, parts in ranked[:limit]:
row = rows[idx]
out.append(
{
"chunk_id": row.id,
"document_id": row.document_id,
"title": (row.meta or {}).get("title", ""),
"ordinal": row.ordinal,
"content": row.content,
"score": round(score, 4),
"signals": {k: round(v, 4) for k, v in parts.items()},
}
)
return out
async def reindex(session: AsyncSession, collection: str | None = None) -> dict[str, Any]:
"""用当前 embedder 重算向量。
换了 embedding 模型之后唯一的恢复手段。按批做:一次几千条 chunk 全塞给
远端 embedding 接口,要么超时要么被限流,而中途失败又没有断点。
"""
stmt = select(Chunk)
if collection:
stmt = stmt.where(Chunk.collection == collection)
rows = list((await session.execute(stmt)).scalars())
if not rows:
return {"collection": collection, "reindexed": 0, "embedder": embedder_id()}
model_id, dim = embedder_id(), embedder_dim()
batch = 64
for i in range(0, len(rows), batch):
part = rows[i:i + batch]
vectors = await embed_texts([r.content for r in part])
for row, vec in zip(part, vectors):
row.embedding = to_blob(vec)
row.embed_model = model_id
row.embed_dim = dim
await session.commit()
return {"collection": collection, "reindexed": len(rows), "embedder": model_id}
async def stale_count(session: AsyncSession, collection: str | None = None) -> int:
"""有多少条向量和当前 embedder 对不上。知识库页据此提示要不要重建。"""
stmt = select(Chunk)
if collection:
stmt = stmt.where(Chunk.collection == collection)
rows = list((await session.execute(stmt)).scalars())
return sum(1 for r in rows if not usable(r.embed_model, r.embed_dim))
async def delete_document(session: AsyncSession, doc_id: str) -> bool:
doc = await session.get(Document, doc_id)
if not doc:
return False
await session.delete(doc)
await session.commit()
return True
async def list_collections(session: AsyncSession) -> list[dict[str, Any]]:
rows = await session.execute(
select(
Document.collection,
func.count(Document.id),
func.sum(Document.chunk_count),
).group_by(Document.collection)
)
return [
{"collection": name, "documents": docs, "chunks": chunks or 0}
for name, docs, chunks in rows
]