Files
AgentLab/backend/tests/test_kb.py
T
ThearchyHelios 913380f738 倒排索引取代全表重算;PDF/Word 终于传得进来
**倒排**:以前每次检索都 select(Chunk) 全量载入,再在 Python 里重建一遍 BM25
——不只是扫表,是对每一段重新分词、重新统计词频,每次查询都来一遍。几百段
无感,上万段就是秒级,而那正是知识库开始有用的规模。

没用 SQLite 的 FTS5。方案里写了实施前先验证,验了:trigram 分词器要求查询
至少三个字符,中文的「商家」「订单」「权限」这些两字词一个都召不回。照搬会
让召回大幅倒退。现有分词器本来就做中文单字 + 二元组,自己存一张 chunk_terms
既保住召回,也不把知识库锁死在 SQLite 上。

快是次要的,一致才是前提——所以先写了"倒排必须和全表扫给出同一个名次"的
测试。它当场抓到两个真问题:

1. BM25 被算了两次:倒排按全集统计(N、avg_len、df)算一次,hybrid_rank
   又在候选子集上重建一次,而子集的 df 和平均长度是另一套数。改成把倒排
   算好的分传下去,不再重算。
2. 原来的 BM25.score 遍历的是**带重复**的查询词表:「导出订单最多多少行」
   里「多」出现两次,贡献就加两次。倒排用 set() 去重,顶部那条差了 2.32。
   改成按查询词频加权。

还有一处是排序确定性:分数并列时稳定排序保留输入顺序,而两条路径喂进去的
顺序天生不同——同一个查询会因为走了哪条路给出不同名次。两边都按 id 定序。

倒排跟着 ingest 建、跟着删文档清、跟着 reindex 重建;没建倒排的片段(存量
数据升级上来就是这样)退回全表扫——宁可慢,也不要因为索引没建好就说"没搜到"。

**解析**:upload 以前只收 UTF-8,PDF 传上去直接 415,而 mime 字段记了却从不
参与判断。现在认 PDF / DOCX / HTML。解析库走可选依赖 [docs](几十兆,只用
纯文本的人不该为此付出安装时间),HTML 用已在依赖里的 bs4。

几处是实际验出来的,不是想出来的:DOCX 要连表格一起取(合同和口径说明的关键
信息常常整个在表里);PDF 每行拖着排版填充空格,不清掉会混进切块和词频统计
(实测 416 字里只有 68 字是正文);扫描件提不出文字要明说,否则悄悄存一份
空文档,用户以为传成功了却永远搜不到。

PDF/DOCX 两条路径在临时 venv 里用真文件验过(含中文、表格、扫描件、坏文件),
没有动开发环境。
2026-09-19 09:44:09 +08:00

248 lines
9.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""知识库:切块、混合检索、换 embedder 之后还能不能用。
这三个模块(kb.py / embeddings.py / memory/store.py,共 526 行检索逻辑)
在此之前**一条测试都没有**。于是有两件事一直没人发现:
- 换 embedding 模型会让检索整体 500:库里是 512 维的本地向量、查询是 1536 维
的远端向量,cosine 直接 `shapes (1536,) and (512,) not aligned`
- 存量向量属于哪个模型,代码里没有任何地方记着,所以也无从降级
现在这两件事都有守的了。
"""
from __future__ import annotations
import numpy as np
import pytest
from app.db.base import SessionLocal
from app.db.models import Chunk
from app.memory import embeddings as emb
from app.memory import kb
from app.memory.embeddings import hybrid_rank, to_blob
@pytest.fixture(autouse=True)
def _local_embedder():
"""每个用例都从本地 embedder 起步,别让上一个用例的 configure 漏过来。"""
emb.configure("local")
yield
emb.configure("local")
# --------------------------------------------------------------------------
# 切块
# --------------------------------------------------------------------------
def test_short_paragraphs_stay_whole() -> None:
blocks = kb.chunk_text("第一段。\n\n第二段。\n\n第三段。")
assert blocks and all("第" in b for b in blocks)
def test_a_long_paragraph_is_cut_on_sentence_boundaries() -> None:
"""命中之后读起来要是完整的句子,不能拦腰截断。"""
text = "。".join(f"这是第{i}句话,讲了一点内容" for i in range(120)) + "。"
blocks = kb.chunk_text(text, target=200, overlap=0)
assert len(blocks) > 1
# 除了最后一块,其余都该收在句号上
assert all(b.rstrip().endswith("。") for b in blocks[:-1])
def test_a_sentence_longer_than_the_target_is_hard_cut() -> None:
"""没有标点可切时也不能无限膨胀,硬切是最后的兜底。"""
blocks = kb.chunk_text("啊" * 2000, target=300, overlap=0)
assert len(blocks) > 1 and max(len(b) for b in blocks) <= 300
def test_empty_input_yields_nothing() -> None:
assert kb.chunk_text("") == [] and kb.chunk_text(" \n\n ") == []
# --------------------------------------------------------------------------
# hybrid_rank 的归一化边界
# --------------------------------------------------------------------------
def test_a_single_candidate_is_not_scored_zero() -> None:
"""min-max 归一化在只有一条候选时 hi==lo。
返回 0 的话它的合并分恒为 0,会被 min_score 过滤掉——于是新建集合写入
第一条之后永远搜不到,而"先写一条、下一轮再检索"正是 agent 最常见的用法。
embeddings.py 里为此写了一段注释,但一直没有测试守着。
"""
vec = np.ones(4, dtype=np.float32)
ranked = hybrid_rank("查询", ["只有这一条"], [vec], vec)
assert len(ranked) == 1 and ranked[0][1] > 0
# --------------------------------------------------------------------------
# 换 embedder:以前这里直接 500
# --------------------------------------------------------------------------
async def _seed(collection: str, texts: list[str]) -> None:
async with SessionLocal() as session:
for t in texts:
await kb.ingest_document(
session, collection=collection, title=t[:10], content=t)
def test_stale_vectors_are_recognised() -> None:
"""存量数据那两列是空的,一律当成对不上——保守但不会拿旧坐标冒充新空间。"""
assert emb.usable("", 0) is False
assert emb.usable("openai:text-embedding-3-small", 1536) is False
assert emb.usable(emb.embedder_id(), emb.embedder_dim()) is True
async def test_ingest_records_who_built_the_vector() -> None:
await _seed("kbtest_mark", ["平台管理员一共有六个人。"])
async with SessionLocal() as session:
from sqlalchemy import select
row = (await session.execute(
select(Chunk).where(Chunk.collection == "kbtest_mark"))).scalars().first()
assert row.embed_model == emb.embedder_id()
assert row.embed_dim == emb.embedder_dim()
async def test_search_degrades_instead_of_crashing_after_a_model_switch() -> None:
"""整件事的核心。
以前:cosine(1536 维查询, 512 维存量) → ValueError → 检索 500。
现在:整体退回纯关键词,并把原因说出来。
"""
await _seed("kbtest_switch", ["平台管理员一共有六个人。", "商家账户共五个。"])
# 把库里的向量伪装成别的模型建的(维度也不同)
async with SessionLocal() as session:
from sqlalchemy import select
rows = list((await session.execute(
select(Chunk).where(Chunk.collection == "kbtest_switch"))).scalars())
for r in rows:
r.embedding = to_blob(np.ones(1536, dtype=np.float32))
r.embed_model = "openai:text-embedding-3-small"
r.embed_dim = 1536
await session.commit()
notes: list[str] = []
async with SessionLocal() as session:
hits = await kb.search(session, collection="kbtest_switch", query="管理员",
limit=5, on_degrade=notes.append)
assert hits, "退回关键词也要能搜到东西,而不是空手而归"
assert notes, "少了一半能力却不吭声,用户只会觉得最近搜得不准"
assert "重建索引" in notes[0] and "2 条" in notes[0], notes[0]
async def test_reindex_restores_hybrid_search() -> None:
await _seed("kbtest_reindex", ["平台管理员一共有六个人。"])
async with SessionLocal() as session:
from sqlalchemy import select
row = (await session.execute(
select(Chunk).where(Chunk.collection == "kbtest_reindex"))).scalars().first()
row.embed_model, row.embed_dim = "别的模型", 99
await session.commit()
async with SessionLocal() as session:
assert await kb.stale_count(session, "kbtest_reindex") == 1
out = await kb.reindex(session, "kbtest_reindex")
assert out["reindexed"] == 1
assert await kb.stale_count(session, "kbtest_reindex") == 0
notes: list[str] = []
async with SessionLocal() as session:
hits = await kb.search(session, collection="kbtest_reindex", query="管理员",
on_degrade=notes.append)
assert hits and not notes, "重建之后不该再降级"
# --------------------------------------------------------------------------
# 配置 embedder
# --------------------------------------------------------------------------
def test_choosing_a_remote_embedder_that_cannot_be_built_says_so(monkeypatch) -> None:
"""选了远端却静默退回本地哈希向量,用户会以为自己在用语义检索。"""
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
with pytest.raises(emb.EmbedderUnavailable) as e:
emb.configure("openai", "text-embedding-3-small")
assert "OPENAI_API_KEY" in str(e.value)
def test_the_local_embedder_is_always_available() -> None:
emb.configure("local")
assert emb.embedder_id() == "local-hashing" and emb.embedder_dim() == 512
# --------------------------------------------------------------------------
# 文档解析
# --------------------------------------------------------------------------
#
# 以前 upload 只接受 UTF-8,PDF 传上去直接 415;而 mime 字段记了却从不参与
# 任何判断。知识库里最常见的恰恰是 PDF 和 Word。
def test_plain_text_still_goes_straight_through() -> None:
from app.memory.parsing import extract
assert extract("就是一段文本".encode(), "note.txt") == "就是一段文本"
assert extract("# 标题".encode(), "readme.md") == "# 标题"
def test_html_is_stripped_to_text() -> None:
from app.memory.parsing import extract
html = """<html><head><style>p{color:red}</style></head>
<body><nav>导航</nav><h1>权限说明</h1><p>只有管理员能改角色。</p>
<script>alert(1)</script><footer>版权</footer></body></html>""".encode()
out = extract(html, "doc.html")
assert "权限说明" in out and "只有管理员能改角色" in out
# 脚本样式导航页脚都不是正文,留着只会污染检索
assert "alert" not in out and "color:red" not in out
assert "导航" not in out and "版权" not in out
def test_an_unreadable_binary_says_what_to_do() -> None:
"""报错要能照着做,不是一句 UnicodeDecodeError。"""
from app.memory.parsing import UnsupportedDocument, extract
with pytest.raises(UnsupportedDocument) as e:
extract(b"\x00\x01\x02\xff\xfe", "mystery.bin")
assert "UTF-8" in str(e.value) and "PDF" in str(e.value)
def test_a_missing_parser_tells_you_which_package(monkeypatch) -> None:
"""没装可选依赖时给的必须是"装哪个包",而不是 ImportError。"""
import builtins
from app.memory.parsing import UnsupportedDocument, extract
real_import = builtins.__import__
def _no_pypdf(name, *a, **kw):
if name == "pypdf":
raise ImportError("no pypdf")
return real_import(name, *a, **kw)
monkeypatch.setattr(builtins, "__import__", _no_pypdf)
with pytest.raises(UnsupportedDocument) as e:
extract(b"%PDF-1.4 fake", "doc.pdf")
assert "agentlab-backend[docs]" in str(e.value)
def test_a_scanned_pdf_is_called_out_rather_than_ingested_empty() -> None:
"""扫描件提不出文字。悄悄存一份空文档,用户会以为传成功了、却永远搜不到。"""
pytest.importorskip("pypdf")
import pypdf
from app.memory.parsing import UnsupportedDocument, extract
writer = pypdf.PdfWriter()
writer.add_blank_page(width=200, height=200)
buf = __import__("io").BytesIO()
writer.write(buf)
with pytest.raises(UnsupportedDocument) as e:
extract(buf.getvalue(), "scan.pdf")
assert "扫描件" in str(e.value)