mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-19 05:32:01 +00:00
知识库:换 embedding 模型不再 500,检索能回指了
三件事,都是"这三个模块一条测试都没有"的后果。 **换 embedder 会让检索整体 500。** 库里是 512 维的本地向量、查询是 1536 维的 远端向量时,cosine 直接 `shapes (1536,) and (512,) not aligned`。from_blob 有 dim 参数,但 kb.py 和 memory/store.py 调用时都没传;而且存量向量属于哪个模型 代码里没有任何地方记着,所以也无从降级。 Chunk / MemoryItem 各加 embed_model + embed_dim(走现成的 _COLUMN_MIGRATIONS, 存量回填空值,正好表示"不知道是谁建的")。检索遇到对不上的向量整体退回纯 关键词并说破,不是把它们当 0 分静默沉底——那样用户看到的是结果莫名其妙变差, 而不是"这里有一批索引该重建了"。配套 kb.reindex + POST /api/kb/reindex。 长期记忆那份一起修了。判重那条更危险:维度碰巧相同但模型不同时 cosine 算出来 是随机数,会把不相关的旧记忆认成"重复",然后把新内容丢掉。 **默认 embedder 没有语义能力,而开关藏在一个环境变量里。** LocalEmbedder 是 特征哈希 + TF,和 BM25 吃同一批词;alpha 默认 0.5 等于给了它一半权重,同义 改写一条都召不回。开启真模型的唯一开关是 AGENTLAB_USE_OPENAI_EMBEDDINGS—— 没界面、没人会发现。改成设置项(照 COPILOT_SETTING_KEY 的路子),知识库页顶部 能选能看能重建。 配置失败必须响:第一版写成静默退回本地,但选了 OpenAI 却拿到哈希向量的人, 在搜不出同义表达时完全无从怀疑到这里。现在抛 EmbedderUnavailable,API 400 原样交回"检查 OPENAI_API_KEY 是否配了",且不会把配置改掉。 **检索结果不进工件库。** db_query、工具节点、agent 工具调用都落 put_json 快照, run_retrieve 什么都不落——于是结论只要来自知识库,"这个数是哪来的"就断在这儿, 而数字回指正是这个项目的签名能力。现在落 kind="retrieval_snapshot",并发一条 retrieve.end 带 artifact id。解码层补了正式映射(原来会掉进"未知事件"兜底, 画成一坨原始 JSON)。 新增 test_kb.py 11 条,补上这三个模块此前的零覆盖,包括 embeddings.py 里 注释记着、但一直没有测试守的那个归一化边界(单条候选被 min_score 误杀)。 端到端在真实服务上验过:撞上存量向量返回 200 + 降级说明(以前 500)、 重建后 stale 归零、配不起来的远端模型 400 且配置不变。
This commit is contained in:
@@ -175,8 +175,76 @@ async def search_kb(
|
||||
alpha: float = Query(default=0.5, ge=0, le=1, description="1=纯向量, 0=纯关键词"),
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> dict[str, Any]:
|
||||
hits = await kb.search(session, collection=collection, query=q, limit=limit, alpha=alpha)
|
||||
return {"query": q, "collection": collection, "results": hits}
|
||||
notes: list[str] = []
|
||||
hits = await kb.search(session, collection=collection, query=q, limit=limit,
|
||||
alpha=alpha, on_degrade=notes.append)
|
||||
# 退回关键词这件事要跟着结果一起交出去,而不是只写进服务端日志——
|
||||
# 调用方(设置页的"试一下"、外部脚本)看到的是结果变差,得知道为什么
|
||||
return {"query": q, "collection": collection, "results": hits, "degraded": notes}
|
||||
|
||||
|
||||
@kb_router.get("/embedding")
|
||||
async def embedding_status(
|
||||
collection: str | None = None, session: AsyncSession = Depends(get_session)
|
||||
) -> dict[str, Any]:
|
||||
"""当前用的是哪个 embedder,以及有多少条向量已经对不上了。"""
|
||||
from app.memory.embeddings import EMBEDDING_SETTING_KEY, embedder_dim, embedder_id
|
||||
from app.db.models import Setting
|
||||
|
||||
row = await session.get(Setting, EMBEDDING_SETTING_KEY)
|
||||
saved = (row.value if row else {}) or {}
|
||||
return {
|
||||
"embedder": embedder_id(),
|
||||
"dim": embedder_dim(),
|
||||
"configured": bool(saved.get("kind")),
|
||||
"kind": saved.get("kind", "local"),
|
||||
"model": saved.get("model", ""),
|
||||
"stale_chunks": await kb.stale_count(session, collection),
|
||||
}
|
||||
|
||||
|
||||
class EmbeddingIn(BaseModel):
|
||||
kind: str = Field(default="local", pattern="^(local|openai)$")
|
||||
model: str = ""
|
||||
|
||||
|
||||
@kb_router.put("/embedding")
|
||||
async def set_embedding(
|
||||
payload: EmbeddingIn, session: AsyncSession = Depends(get_session)
|
||||
) -> dict[str, Any]:
|
||||
"""换 embedding 模型。
|
||||
|
||||
建不起来就 400 原样交回原因——选了远端却静默退回本地哈希向量,
|
||||
用户会以为自己在用语义检索,而搜不出同义表达时完全无从怀疑到这里。
|
||||
"""
|
||||
from app.memory import embeddings as emb
|
||||
from app.db.models import Setting
|
||||
|
||||
try:
|
||||
emb.configure(payload.kind, payload.model)
|
||||
except emb.EmbedderUnavailable as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
row = await session.get(Setting, emb.EMBEDDING_SETTING_KEY)
|
||||
value = {"kind": payload.kind, "model": payload.model}
|
||||
if row:
|
||||
row.value = value
|
||||
else:
|
||||
session.add(Setting(key=emb.EMBEDDING_SETTING_KEY, value=value))
|
||||
await session.commit()
|
||||
return {
|
||||
"embedder": emb.embedder_id(),
|
||||
"dim": emb.embedder_dim(),
|
||||
"stale_chunks": await kb.stale_count(session),
|
||||
}
|
||||
|
||||
|
||||
@kb_router.post("/reindex")
|
||||
async def reindex(
|
||||
collection: str | None = None, session: AsyncSession = Depends(get_session)
|
||||
) -> dict[str, Any]:
|
||||
"""用当前 embedder 重算向量。换了模型之后唯一的恢复手段。"""
|
||||
return await kb.reindex(session, collection)
|
||||
|
||||
|
||||
@kb_router.delete("/documents/{doc_id}", status_code=204)
|
||||
|
||||
@@ -39,6 +39,9 @@ class EventType(StrEnum):
|
||||
TOOL_START = "tool.start"
|
||||
TOOL_END = "tool.end"
|
||||
TOOL_ERROR = "tool.error"
|
||||
# 知识检索。以前只发一条 info 日志,于是"这句结论依据的是哪份文档的哪一段"
|
||||
# 在轨迹里查不到——而 SQL 取数那条路早就有工件快照可以下钻
|
||||
RETRIEVE_END = "retrieve.end"
|
||||
SANDBOX_START = "sandbox.start"
|
||||
SANDBOX_END = "sandbox.end"
|
||||
|
||||
|
||||
@@ -75,6 +75,12 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [
|
||||
("workflows", "published_by", "VARCHAR(100)"),
|
||||
("workflow_versions", "graph_hash", "VARCHAR(64)"),
|
||||
("approvals", "resolved_by", "VARCHAR(100)"),
|
||||
# 存量向量没有这两列,回填成空串 / 0 —— 正好表示"不知道是谁建的",
|
||||
# 检索侧会把它们当成对不上当前 embedder 处理
|
||||
("chunks", "embed_model", "VARCHAR(100) DEFAULT ''"),
|
||||
("chunks", "embed_dim", "INTEGER DEFAULT 0"),
|
||||
("memories", "embed_model", "VARCHAR(100) DEFAULT ''"),
|
||||
("memories", "embed_dim", "INTEGER DEFAULT 0"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -256,6 +256,11 @@ class MemoryItem(Base, TimestampMixin):
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
meta: Mapped[dict[str, Any]] = mapped_column(default=dict)
|
||||
embedding: Mapped[bytes | None] = mapped_column(LargeBinary, default=None)
|
||||
# 这条向量是谁产出的。没有它就无法判断存量向量还能不能和当前查询比对——
|
||||
# 换 embedder 之后维度对不上,cosine 直接 ValueError(512 vs 1536),
|
||||
# 而在此之前代码里没有任何地方能看出"这条是旧模型建的"
|
||||
embed_model: Mapped[str] = mapped_column(String(100), default="")
|
||||
embed_dim: Mapped[int] = mapped_column(Integer, default=0)
|
||||
importance: Mapped[float] = mapped_column(Float, default=0.5)
|
||||
last_used_at: Mapped[datetime | None] = mapped_column(default=None)
|
||||
use_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
@@ -290,6 +295,11 @@ class Chunk(Base):
|
||||
ordinal: Mapped[int] = mapped_column(Integer, default=0)
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
embedding: Mapped[bytes | None] = mapped_column(LargeBinary, default=None)
|
||||
# 这条向量是谁产出的。没有它就无法判断存量向量还能不能和当前查询比对——
|
||||
# 换 embedder 之后维度对不上,cosine 直接 ValueError(512 vs 1536),
|
||||
# 而在此之前代码里没有任何地方能看出"这条是旧模型建的"
|
||||
embed_model: Mapped[str] = mapped_column(String(100), default="")
|
||||
embed_dim: Mapped[int] = mapped_column(Integer, default=0)
|
||||
meta: Mapped[dict[str, Any]] = mapped_column(default=dict)
|
||||
|
||||
document: Mapped[Document] = relationship(back_populates="chunks")
|
||||
|
||||
@@ -72,10 +72,15 @@ async def run_retrieve(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
limit = int(ctx.cfg("limit", 5) or 5)
|
||||
alpha = float(ctx.cfg("alpha", 0.5) or 0.5) # 1=纯向量,0=纯关键词
|
||||
|
||||
degraded: list[str] = []
|
||||
async with SessionLocal() as session:
|
||||
hits = await kb.search(
|
||||
session, collection=collection, query=query, limit=limit, alpha=alpha
|
||||
session, collection=collection, query=query, limit=limit, alpha=alpha,
|
||||
on_degrade=degraded.append,
|
||||
)
|
||||
for note in degraded:
|
||||
# 检索悄悄少了一半能力,不说的话用户只会觉得"最近搜得不准"
|
||||
ctx.emit(EventType.LOG, level="warn", message=note)
|
||||
|
||||
min_score = float(ctx.cfg("min_score", 0.0) or 0.0)
|
||||
hits = [h for h in hits if h["score"] >= min_score]
|
||||
@@ -85,15 +90,44 @@ async def run_retrieve(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
f"[{i + 1}] {h['title']}(片段 {h['ordinal']})\n{h['content']}"
|
||||
for i, h in enumerate(hits)
|
||||
)
|
||||
# 命中落工件库。SQL 取数、工具调用、agent 工具调用都有快照,检索一直没有,
|
||||
# 于是结论只要来自知识库,"这个数是哪来的"这条链就断在这儿
|
||||
from app.core.artifact_store import put_json
|
||||
from app.memory.embeddings import embedder_id
|
||||
|
||||
snapshot: str | None = None
|
||||
try:
|
||||
snapshot = await put_json(
|
||||
{
|
||||
"query": query, "collection": collection,
|
||||
"alpha": alpha, "min_score": min_score,
|
||||
"embedder": embedder_id(), "degraded": degraded,
|
||||
"hits": hits,
|
||||
},
|
||||
kind="retrieval_snapshot",
|
||||
run_id=ctx.run.run_id, node_id=ctx.node.id,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - 存不下不影响这次检索本身
|
||||
snapshot = None
|
||||
|
||||
ctx.emit(
|
||||
EventType.RETRIEVE_END,
|
||||
collection=collection,
|
||||
query=query[:200],
|
||||
count=len(hits),
|
||||
top_score=hits[0]["score"] if hits else 0,
|
||||
degraded=bool(degraded),
|
||||
artifact=snapshot,
|
||||
)
|
||||
|
||||
result = {
|
||||
"hits": hits,
|
||||
"count": len(hits),
|
||||
"text": context_text,
|
||||
"query": query,
|
||||
"collection": collection,
|
||||
"artifact": snapshot,
|
||||
}
|
||||
ctx.emit(EventType.LOG, level="info",
|
||||
message=f"检索 {collection}:命中 {len(hits)} 条,最高分 {hits[0]['score'] if hits else 0}")
|
||||
|
||||
updates: dict[str, Any] = {"nodes": {ctx.node.id: result}}
|
||||
var_name = ctx.cfg("assign_to", "")
|
||||
|
||||
@@ -30,6 +30,19 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
settings.ensure_dirs()
|
||||
await init_db()
|
||||
await seed_defaults()
|
||||
|
||||
# 把存的 embedding 配置读进来。不读的话重启后会悄悄退回本地哈希向量,
|
||||
# 而库里的向量是上次那个模型建的——检索会整体退回关键词,
|
||||
# 用户只看到"重启之后搜得不准了"
|
||||
from app.db.base import SessionLocal
|
||||
from app.memory.embeddings import load_setting
|
||||
|
||||
async with SessionLocal() as _session:
|
||||
try:
|
||||
await load_setting(_session)
|
||||
except Exception as e: # noqa: BLE001 - key 没了不该拦住整个服务起不来
|
||||
logger.warning("embedding 配置没能生效,暂用本地向量:%s", e)
|
||||
|
||||
await run_manager.setup()
|
||||
from app.sandbox.manager import sandbox_manager as _sbm
|
||||
|
||||
|
||||
@@ -61,7 +61,9 @@ class OpenAIEmbedder:
|
||||
base_url: str | None = None) -> None:
|
||||
from langchain_openai import OpenAIEmbeddings
|
||||
|
||||
self.dim = 1536
|
||||
self.model = model
|
||||
# text-embedding-3-large 是 3072,别写死
|
||||
self.dim = 3072 if "3-large" in model else 1536
|
||||
self._impl = OpenAIEmbeddings(
|
||||
model=model,
|
||||
api_key=api_key or os.environ.get("OPENAI_API_KEY"),
|
||||
@@ -79,17 +81,92 @@ class OpenAIEmbedder:
|
||||
|
||||
_local = LocalEmbedder()
|
||||
|
||||
#: 设置表里存 embedding 配置的键。和 Copilot 模型同一套路子
|
||||
EMBEDDING_SETTING_KEY = "embedding"
|
||||
|
||||
def get_embedder() -> LocalEmbedder | OpenAIEmbedder:
|
||||
"""默认走本地;显式打开 AGENTLAB_USE_OPENAI_EMBEDDINGS 且有 key 时才用远端。"""
|
||||
#: 进程内缓存。get_embedder 在检索热路径上被反复调用,不该每次都去读库
|
||||
_cached: tuple[str, Any] | None = None
|
||||
|
||||
|
||||
class EmbedderUnavailable(RuntimeError):
|
||||
"""配了远端 embedder 但建不起来。message 直接给用户看。"""
|
||||
|
||||
|
||||
def _make(kind: str, model: str, *, strict: bool) -> "LocalEmbedder | OpenAIEmbedder":
|
||||
if kind != "openai":
|
||||
return _local
|
||||
try:
|
||||
return OpenAIEmbedder(model=model or "text-embedding-3-small")
|
||||
except Exception as e: # noqa: BLE001
|
||||
if strict:
|
||||
# 人明确选了远端就不能偷偷退回本地:他会以为自己在用语义检索,
|
||||
# 而实际拿到的是哈希词袋——搜不出同义表达时完全无从怀疑到这里
|
||||
raise EmbedderUnavailable(
|
||||
f"用不了 embedding 模型 {model or 'text-embedding-3-small'}:"
|
||||
f"{type(e).__name__}: {e}。检查 OPENAI_API_KEY 是否配了。"
|
||||
) from e
|
||||
return _local
|
||||
|
||||
|
||||
def configure(kind: str, model: str = "") -> None:
|
||||
"""切换当前 embedder。设置页保存后调用,也用于测试。建不起来会抛。"""
|
||||
global _cached
|
||||
_cached = (f"{kind}:{model}", _make(kind, model, strict=True))
|
||||
|
||||
|
||||
def get_embedder() -> "LocalEmbedder | OpenAIEmbedder":
|
||||
"""当前 embedder。
|
||||
|
||||
优先级:进程内已配置 > 环境变量兜底 > 本地。设置页写库后会调 configure,
|
||||
所以这里不再每次读库——检索热路径上一次查询要走几百次。
|
||||
|
||||
以前这里只认 AGENTLAB_USE_OPENAI_EMBEDDINGS 一个环境变量:没有界面、
|
||||
没人会发现,于是所有人都在用那个没有语义能力的哈希向量,而 alpha 默认
|
||||
0.5 还给了它一半权重。
|
||||
"""
|
||||
if _cached is not None:
|
||||
return _cached[1]
|
||||
if os.environ.get("AGENTLAB_USE_OPENAI_EMBEDDINGS") and os.environ.get("OPENAI_API_KEY"):
|
||||
try:
|
||||
return OpenAIEmbedder()
|
||||
except Exception: # noqa: BLE001
|
||||
return _local
|
||||
return _make("openai", "", strict=False)
|
||||
return _local
|
||||
|
||||
|
||||
def embedder_id() -> str:
|
||||
"""当前 embedder 的身份。存进 chunk 那一列,检索时拿它判断存量向量还能不能用。"""
|
||||
emb = get_embedder()
|
||||
name = getattr(emb, "name", "unknown")
|
||||
model = getattr(emb, "model", "")
|
||||
return f"{name}:{model}" if model else name
|
||||
|
||||
|
||||
def embedder_dim() -> int:
|
||||
return int(getattr(get_embedder(), "dim", _DIM))
|
||||
|
||||
|
||||
def usable(embed_model: str | None, embed_dim: int | None) -> bool:
|
||||
"""这条存量向量还能不能和当前查询比对。
|
||||
|
||||
维度对不上 cosine 会直接抛 ValueError(shapes (1536,) and (512,) not
|
||||
aligned),所以这不是"质量差一点",是会 500。模型名也要比:同维度不同
|
||||
模型的向量空间没有可比性,算出来的相似度是随机数。
|
||||
|
||||
存量数据这两列是空的(迁移回填的默认值),一律当成不可用——保守,
|
||||
但总比拿旧模型的向量冒充当前空间里的坐标强。
|
||||
"""
|
||||
return bool(embed_model) and embed_model == embedder_id() and embed_dim == embedder_dim()
|
||||
|
||||
|
||||
async def load_setting(session: Any) -> None:
|
||||
"""从库里读 embedding 配置并生效。应用启动时调一次。"""
|
||||
from app.db.models import Setting
|
||||
|
||||
row = await session.get(Setting, EMBEDDING_SETTING_KEY)
|
||||
saved = (row.value if row else {}) or {}
|
||||
kind = saved.get("kind")
|
||||
if kind:
|
||||
configure(kind, saved.get("model", ""))
|
||||
|
||||
|
||||
async def embed_text(text: str) -> np.ndarray:
|
||||
emb = get_embedder()
|
||||
if isinstance(emb, OpenAIEmbedder):
|
||||
|
||||
@@ -7,7 +7,9 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Chunk, Document
|
||||
from app.memory.embeddings import embed_text, embed_texts, from_blob, hybrid_rank, to_blob
|
||||
from app.memory.embeddings import (
|
||||
embed_text, embed_texts, embedder_dim, embedder_id, from_blob, hybrid_rank, to_blob, usable,
|
||||
)
|
||||
|
||||
_TARGET = 800
|
||||
_OVERLAP = 120
|
||||
@@ -97,6 +99,8 @@ async def ingest_document(
|
||||
ordinal=i,
|
||||
content=piece,
|
||||
embedding=to_blob(vec),
|
||||
embed_model=embedder_id(),
|
||||
embed_dim=embedder_dim(),
|
||||
meta={"title": doc.title},
|
||||
)
|
||||
)
|
||||
@@ -113,7 +117,17 @@ async def search(
|
||||
query: str,
|
||||
limit: int = 5,
|
||||
alpha: float = 0.5,
|
||||
on_degrade: Any = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""混合检索。
|
||||
|
||||
存量向量和当前 embedder 对不上时**整个查询退回纯关键词**,并通过
|
||||
on_degrade 把原因交出去。不把对不上的那些当 0 分:那会让它们静默沉底,
|
||||
用户看到的是结果莫名其妙变差,而不是"这里有一批索引该重建了"。
|
||||
|
||||
在此之前这里连维度都不查,换个 embedder 直接 ValueError
|
||||
(shapes (1536,) and (512,) not aligned),整个检索 500。
|
||||
"""
|
||||
stmt = select(Chunk)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
@@ -121,11 +135,26 @@ async def search(
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
query_vec = await embed_text(query)
|
||||
stale = [r for r in rows if not usable(r.embed_model, r.embed_dim)]
|
||||
if stale:
|
||||
if on_degrade:
|
||||
names = sorted({r.embed_model or "未知模型" for r in stale})
|
||||
on_degrade(
|
||||
f"{collection or '全部集合'}里有 {len(stale)}/{len(rows)} 条向量由"
|
||||
f"「{'、'.join(names)}」建立,和当前的「{embedder_id()}」对不上,"
|
||||
f"本次已退回纯关键词检索。重建索引后恢复。"
|
||||
)
|
||||
alpha = 0.0 # 纯关键词
|
||||
vectors: list[Any] = [None] * len(rows)
|
||||
query_vec = await embed_text(query)
|
||||
else:
|
||||
query_vec = await embed_text(query)
|
||||
vectors = [from_blob(r.embedding, embedder_dim()) for r in rows]
|
||||
|
||||
ranked = hybrid_rank(
|
||||
query,
|
||||
[r.content for r in rows],
|
||||
[from_blob(r.embedding) for r in rows],
|
||||
vectors,
|
||||
query_vec,
|
||||
alpha=alpha,
|
||||
)
|
||||
@@ -146,6 +175,41 @@ async def search(
|
||||
return out
|
||||
|
||||
|
||||
async def reindex(session: AsyncSession, collection: str | None = None) -> dict[str, Any]:
|
||||
"""用当前 embedder 重算向量。
|
||||
|
||||
换了 embedding 模型之后唯一的恢复手段。按批做:一次几千条 chunk 全塞给
|
||||
远端 embedding 接口,要么超时要么被限流,而中途失败又没有断点。
|
||||
"""
|
||||
stmt = select(Chunk)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return {"collection": collection, "reindexed": 0, "embedder": embedder_id()}
|
||||
|
||||
model_id, dim = embedder_id(), embedder_dim()
|
||||
batch = 64
|
||||
for i in range(0, len(rows), batch):
|
||||
part = rows[i:i + batch]
|
||||
vectors = await embed_texts([r.content for r in part])
|
||||
for row, vec in zip(part, vectors):
|
||||
row.embedding = to_blob(vec)
|
||||
row.embed_model = model_id
|
||||
row.embed_dim = dim
|
||||
await session.commit()
|
||||
return {"collection": collection, "reindexed": len(rows), "embedder": model_id}
|
||||
|
||||
|
||||
async def stale_count(session: AsyncSession, collection: str | None = None) -> int:
|
||||
"""有多少条向量和当前 embedder 对不上。知识库页据此提示要不要重建。"""
|
||||
stmt = select(Chunk)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
return sum(1 for r in rows if not usable(r.embed_model, r.embed_dim))
|
||||
|
||||
|
||||
async def delete_document(session: AsyncSession, doc_id: str) -> bool:
|
||||
doc = await session.get(Document, doc_id)
|
||||
if not doc:
|
||||
|
||||
@@ -7,7 +7,9 @@ from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import MemoryItem
|
||||
from app.memory.embeddings import cosine, embed_text, from_blob, hybrid_rank, to_blob
|
||||
from app.memory.embeddings import (
|
||||
cosine, embed_text, embedder_dim, embedder_id, from_blob, hybrid_rank, to_blob, usable,
|
||||
)
|
||||
|
||||
# 判重阈值:余弦相似度高于它才算同一件事。比排名分数严格得多,
|
||||
# 因为这里比的是真实语义距离,不是"谁排第一"。
|
||||
@@ -44,7 +46,12 @@ async def remember(
|
||||
candidates = list((await session.execute(stmt)).scalars())
|
||||
best, best_score = None, 0.0
|
||||
for row in candidates:
|
||||
other = from_blob(row.embedding)
|
||||
# 对不上当前 embedder 的直接跳过:维度不同 cosine 会抛,维度碰巧
|
||||
# 相同但模型不同则更糟——两个空间之间的相似度是个随机数,
|
||||
# 拿它判重会把不相关的旧记忆认成"重复"然后丢掉新内容
|
||||
if not usable(row.embed_model, row.embed_dim):
|
||||
continue
|
||||
other = from_blob(row.embedding, embedder_dim())
|
||||
if other is None:
|
||||
continue
|
||||
score = cosine(vec, other)
|
||||
@@ -64,6 +71,8 @@ async def remember(
|
||||
meta=meta or {},
|
||||
importance=importance,
|
||||
embedding=to_blob(vec),
|
||||
embed_model=embedder_id(),
|
||||
embed_dim=embedder_dim(),
|
||||
)
|
||||
session.add(item)
|
||||
await session.commit()
|
||||
@@ -89,11 +98,19 @@ async def recall(
|
||||
return []
|
||||
|
||||
query_vec = await embed_text(query)
|
||||
# 和 kb.search 同一个道理:有对不上的就整体退回关键词,而不是让它们静默沉底
|
||||
if any(not usable(r.embed_model, r.embed_dim) for r in rows):
|
||||
vectors: list[Any] = [None] * len(rows)
|
||||
alpha = 0.0
|
||||
else:
|
||||
vectors = [from_blob(r.embedding, embedder_dim()) for r in rows]
|
||||
alpha = 0.5
|
||||
ranked = hybrid_rank(
|
||||
query,
|
||||
[r.content for r in rows],
|
||||
[from_blob(r.embedding) for r in rows],
|
||||
vectors,
|
||||
query_vec,
|
||||
alpha=alpha,
|
||||
)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
"""知识库:切块、混合检索、换 embedder 之后还能不能用。
|
||||
|
||||
这三个模块(kb.py / embeddings.py / memory/store.py,共 526 行检索逻辑)
|
||||
在此之前**一条测试都没有**。于是有两件事一直没人发现:
|
||||
|
||||
- 换 embedding 模型会让检索整体 500:库里是 512 维的本地向量、查询是 1536 维
|
||||
的远端向量,cosine 直接 `shapes (1536,) and (512,) not aligned`
|
||||
- 存量向量属于哪个模型,代码里没有任何地方记着,所以也无从降级
|
||||
|
||||
现在这两件事都有守的了。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Chunk
|
||||
from app.memory import embeddings as emb
|
||||
from app.memory import kb
|
||||
from app.memory.embeddings import hybrid_rank, to_blob
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _local_embedder():
|
||||
"""每个用例都从本地 embedder 起步,别让上一个用例的 configure 漏过来。"""
|
||||
emb.configure("local")
|
||||
yield
|
||||
emb.configure("local")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 切块
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_short_paragraphs_stay_whole() -> None:
|
||||
blocks = kb.chunk_text("第一段。\n\n第二段。\n\n第三段。")
|
||||
assert blocks and all("第" in b for b in blocks)
|
||||
|
||||
|
||||
def test_a_long_paragraph_is_cut_on_sentence_boundaries() -> None:
|
||||
"""命中之后读起来要是完整的句子,不能拦腰截断。"""
|
||||
text = "。".join(f"这是第{i}句话,讲了一点内容" for i in range(120)) + "。"
|
||||
blocks = kb.chunk_text(text, target=200, overlap=0)
|
||||
assert len(blocks) > 1
|
||||
# 除了最后一块,其余都该收在句号上
|
||||
assert all(b.rstrip().endswith("。") for b in blocks[:-1])
|
||||
|
||||
|
||||
def test_a_sentence_longer_than_the_target_is_hard_cut() -> None:
|
||||
"""没有标点可切时也不能无限膨胀,硬切是最后的兜底。"""
|
||||
blocks = kb.chunk_text("啊" * 2000, target=300, overlap=0)
|
||||
assert len(blocks) > 1 and max(len(b) for b in blocks) <= 300
|
||||
|
||||
|
||||
def test_empty_input_yields_nothing() -> None:
|
||||
assert kb.chunk_text("") == [] and kb.chunk_text(" \n\n ") == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# hybrid_rank 的归一化边界
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_single_candidate_is_not_scored_zero() -> None:
|
||||
"""min-max 归一化在只有一条候选时 hi==lo。
|
||||
|
||||
返回 0 的话它的合并分恒为 0,会被 min_score 过滤掉——于是新建集合写入
|
||||
第一条之后永远搜不到,而"先写一条、下一轮再检索"正是 agent 最常见的用法。
|
||||
embeddings.py 里为此写了一段注释,但一直没有测试守着。
|
||||
"""
|
||||
vec = np.ones(4, dtype=np.float32)
|
||||
ranked = hybrid_rank("查询", ["只有这一条"], [vec], vec)
|
||||
assert len(ranked) == 1 and ranked[0][1] > 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 换 embedder:以前这里直接 500
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _seed(collection: str, texts: list[str]) -> None:
|
||||
async with SessionLocal() as session:
|
||||
for t in texts:
|
||||
await kb.ingest_document(
|
||||
session, collection=collection, title=t[:10], content=t)
|
||||
|
||||
|
||||
def test_stale_vectors_are_recognised() -> None:
|
||||
"""存量数据那两列是空的,一律当成对不上——保守但不会拿旧坐标冒充新空间。"""
|
||||
assert emb.usable("", 0) is False
|
||||
assert emb.usable("openai:text-embedding-3-small", 1536) is False
|
||||
assert emb.usable(emb.embedder_id(), emb.embedder_dim()) is True
|
||||
|
||||
|
||||
async def test_ingest_records_who_built_the_vector() -> None:
|
||||
await _seed("kbtest_mark", ["平台管理员一共有六个人。"])
|
||||
async with SessionLocal() as session:
|
||||
from sqlalchemy import select
|
||||
row = (await session.execute(
|
||||
select(Chunk).where(Chunk.collection == "kbtest_mark"))).scalars().first()
|
||||
assert row.embed_model == emb.embedder_id()
|
||||
assert row.embed_dim == emb.embedder_dim()
|
||||
|
||||
|
||||
async def test_search_degrades_instead_of_crashing_after_a_model_switch() -> None:
|
||||
"""整件事的核心。
|
||||
|
||||
以前:cosine(1536 维查询, 512 维存量) → ValueError → 检索 500。
|
||||
现在:整体退回纯关键词,并把原因说出来。
|
||||
"""
|
||||
await _seed("kbtest_switch", ["平台管理员一共有六个人。", "商家账户共五个。"])
|
||||
|
||||
# 把库里的向量伪装成别的模型建的(维度也不同)
|
||||
async with SessionLocal() as session:
|
||||
from sqlalchemy import select
|
||||
rows = list((await session.execute(
|
||||
select(Chunk).where(Chunk.collection == "kbtest_switch"))).scalars())
|
||||
for r in rows:
|
||||
r.embedding = to_blob(np.ones(1536, dtype=np.float32))
|
||||
r.embed_model = "openai:text-embedding-3-small"
|
||||
r.embed_dim = 1536
|
||||
await session.commit()
|
||||
|
||||
notes: list[str] = []
|
||||
async with SessionLocal() as session:
|
||||
hits = await kb.search(session, collection="kbtest_switch", query="管理员",
|
||||
limit=5, on_degrade=notes.append)
|
||||
|
||||
assert hits, "退回关键词也要能搜到东西,而不是空手而归"
|
||||
assert notes, "少了一半能力却不吭声,用户只会觉得最近搜得不准"
|
||||
assert "重建索引" in notes[0] and "2/2" in notes[0]
|
||||
|
||||
|
||||
async def test_reindex_restores_hybrid_search() -> None:
|
||||
await _seed("kbtest_reindex", ["平台管理员一共有六个人。"])
|
||||
async with SessionLocal() as session:
|
||||
from sqlalchemy import select
|
||||
row = (await session.execute(
|
||||
select(Chunk).where(Chunk.collection == "kbtest_reindex"))).scalars().first()
|
||||
row.embed_model, row.embed_dim = "别的模型", 99
|
||||
await session.commit()
|
||||
|
||||
async with SessionLocal() as session:
|
||||
assert await kb.stale_count(session, "kbtest_reindex") == 1
|
||||
out = await kb.reindex(session, "kbtest_reindex")
|
||||
assert out["reindexed"] == 1
|
||||
assert await kb.stale_count(session, "kbtest_reindex") == 0
|
||||
|
||||
notes: list[str] = []
|
||||
async with SessionLocal() as session:
|
||||
hits = await kb.search(session, collection="kbtest_reindex", query="管理员",
|
||||
on_degrade=notes.append)
|
||||
assert hits and not notes, "重建之后不该再降级"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 配置 embedder
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_choosing_a_remote_embedder_that_cannot_be_built_says_so(monkeypatch) -> None:
|
||||
"""选了远端却静默退回本地哈希向量,用户会以为自己在用语义检索。"""
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
with pytest.raises(emb.EmbedderUnavailable) as e:
|
||||
emb.configure("openai", "text-embedding-3-small")
|
||||
assert "OPENAI_API_KEY" in str(e.value)
|
||||
|
||||
|
||||
def test_the_local_embedder_is_always_available() -> None:
|
||||
emb.configure("local")
|
||||
assert emb.embedder_id() == "local-hashing" and emb.embedder_dim() == 512
|
||||
Reference in New Issue
Block a user