diff --git a/backend/app/api/knowledge.py b/backend/app/api/knowledge.py index 31a5fc54..60802dd2 100644 --- a/backend/app/api/knowledge.py +++ b/backend/app/api/knowledge.py @@ -172,7 +172,8 @@ async def search_kb( q: str, collection: str | None = None, limit: int = 5, - alpha: float = Query(default=0.5, ge=0, le=1, description="1=纯向量, 0=纯关键词"), + alpha: float | None = Query(default=None, ge=0, le=1, + description="1=纯向量, 0=纯关键词;不传则按 embedder 能力取默认"), session: AsyncSession = Depends(get_session), ) -> dict[str, Any]: notes: list[str] = [] @@ -188,7 +189,9 @@ async def embedding_status( collection: str | None = None, session: AsyncSession = Depends(get_session) ) -> dict[str, Any]: """当前用的是哪个 embedder,以及有多少条向量已经对不上了。""" - from app.memory.embeddings import EMBEDDING_SETTING_KEY, embedder_dim, embedder_id + from app.memory.embeddings import ( + EMBEDDING_SETTING_KEY, default_alpha, embedder_dim, embedder_id, has_semantics, + ) from app.db.models import Setting row = await session.get(Setting, EMBEDDING_SETTING_KEY) @@ -200,6 +203,8 @@ async def embedding_status( "kind": saved.get("kind", "local"), "model": saved.get("model", ""), "stale_chunks": await kb.stale_count(session, collection), + "has_semantics": has_semantics(), + "default_alpha": default_alpha(), } diff --git a/backend/app/engine/nodes/knowledge.py b/backend/app/engine/nodes/knowledge.py index 444e0e7a..8acb89dc 100644 --- a/backend/app/engine/nodes/knowledge.py +++ b/backend/app/engine/nodes/knowledge.py @@ -70,14 +70,36 @@ async def run_retrieve(state: GraphState, ctx: NodeContext) -> dict[str, Any]: collection = ctx.render_str(ctx.cfg("collection", "") or ctx.run.collection, state) limit = int(ctx.cfg("limit", 5) or 5) - alpha = float(ctx.cfg("alpha", 0.5) or 0.5) # 1=纯向量,0=纯关键词 + # 不写死 0.5:没配置就跟着 embedder 的能力走(见 embeddings.default_alpha) + raw_alpha = ctx.cfg("alpha") + alpha = float(raw_alpha) if raw_alpha not in (None, "") else None + + # 要重排就先多捞一些:重排只能在初筛给出的候选里挑,初筛只给 5 条的话 + # 它最多把这 5 条换个顺序,第 6 条是对的也救不回来 + mode = ctx.cfg("rerank", "off") or "off" + fetch = max(limit, 20) if mode == "model" else limit degraded: list[str] = [] async with SessionLocal() as session: hits = await kb.search( - session, collection=collection, query=query, limit=limit, alpha=alpha, + session, collection=collection, query=query, limit=fetch, alpha=alpha, on_degrade=degraded.append, ) + + if mode == "model" and hits: + from app.engine.nodes.llm import _model_spec + from app.memory.rerank import rerank + from app.providers.factory import ProviderNotConfigured, get_chat_model + + try: + async with SessionLocal() as session: + model, _ = await get_chat_model(session, _model_spec(ctx)) + except ProviderNotConfigured as e: + degraded.append(f"重排用不了:{e}") + model = None + hits = await rerank(model, query, hits, top_n=limit, on_note=degraded.append) + else: + hits = hits[:limit] for note in degraded: # 检索悄悄少了一半能力,不说的话用户只会觉得"最近搜得不准" ctx.emit(EventType.LOG, level="warn", message=note) @@ -93,15 +115,16 @@ async def run_retrieve(state: GraphState, ctx: NodeContext) -> dict[str, Any]: # 命中落工件库。SQL 取数、工具调用、agent 工具调用都有快照,检索一直没有, # 于是结论只要来自知识库,"这个数是哪来的"这条链就断在这儿 from app.core.artifact_store import put_json - from app.memory.embeddings import embedder_id + from app.memory.embeddings import default_alpha, embedder_id snapshot: str | None = None try: snapshot = await put_json( { "query": query, "collection": collection, - "alpha": alpha, "min_score": min_score, - "embedder": embedder_id(), "degraded": degraded, + "alpha": alpha if alpha is not None else default_alpha(), + "min_score": min_score, + "embedder": embedder_id(), "rerank": mode, "degraded": degraded, "hits": hits, }, kind="retrieval_snapshot", @@ -116,6 +139,7 @@ async def run_retrieve(state: GraphState, ctx: NodeContext) -> dict[str, Any]: query=query[:200], count=len(hits), top_score=hits[0]["score"] if hits else 0, + reranked=mode == "model", degraded=bool(degraded), artifact=snapshot, ) diff --git a/backend/app/memory/embeddings.py b/backend/app/memory/embeddings.py index 76ab7119..24b111e2 100644 --- a/backend/app/memory/embeddings.py +++ b/backend/app/memory/embeddings.py @@ -143,6 +143,25 @@ def embedder_dim() -> int: return int(getattr(get_embedder(), "dim", _DIM)) +def has_semantics() -> bool: + """当前 embedder 有没有语义泛化能力。 + + LocalEmbedder 是词频哈希:它和 BM25 吃同一批词,给它权重等于把关键词 + 信号打个折再加回自己身上,同义改写一条都召不回。 + """ + return getattr(get_embedder(), "name", "") != "local-hashing" + + +def default_alpha() -> float: + """向量那一路默认占多少权重。 + + 实测(backend/tests/test_retrieval_quality.py,24 篇语料 / 16 问): + 本地哈希向量下 alpha=0 的 hit@1 是 88%,alpha 一旦大于 0 就掉到 81% + ——那一路不是"弱一点",是纯噪音。所以没有语义能力时默认给 0。 + """ + return 0.5 if has_semantics() else 0.0 + + def usable(embed_model: str | None, embed_dim: int | None) -> bool: """这条存量向量还能不能和当前查询比对。 diff --git a/backend/app/memory/kb.py b/backend/app/memory/kb.py index be63d8d7..29b09dbb 100644 --- a/backend/app/memory/kb.py +++ b/backend/app/memory/kb.py @@ -8,7 +8,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import Chunk, Document from app.memory.embeddings import ( - embed_text, embed_texts, embedder_dim, embedder_id, from_blob, hybrid_rank, to_blob, usable, + default_alpha, embed_text, embed_texts, embedder_dim, embedder_id, from_blob, + hybrid_rank, to_blob, usable, ) _TARGET = 800 @@ -116,7 +117,7 @@ async def search( collection: str | None, query: str, limit: int = 5, - alpha: float = 0.5, + alpha: float | None = None, on_degrade: Any = None, ) -> list[dict[str, Any]]: """混合检索。 @@ -128,6 +129,11 @@ async def search( 在此之前这里连维度都不查,换个 embedder 直接 ValueError (shapes (1536,) and (512,) not aligned),整个检索 500。 """ + # 没显式指定就跟着 embedder 走——写死 0.5 是在给一个没有语义能力的信号 + # 一半权重,实测会把 hit@1 从 88% 拉到 81% + if alpha is None: + alpha = default_alpha() + stmt = select(Chunk) if collection: stmt = stmt.where(Chunk.collection == collection) diff --git a/backend/app/memory/rerank.py b/backend/app/memory/rerank.py new file mode 100644 index 00000000..edb559d8 --- /dev/null +++ b/backend/app/memory/rerank.py @@ -0,0 +1,101 @@ +"""重排:让模型把初筛结果重新排一遍。 + +混合检索给的是"大致相关"——BM25 看词频,向量看空间距离,两者都不读句子。 +重排让模型把 query 和每个候选放在一起判断,这是检索链路上单点收益最大的一步。 + +用已配置的对话模型做,不引入本地 cross-encoder:后者要下模型权重,和这个 +项目"零配置能跑起来"的基线冲突(同理不照抄 RAGFlow 那套 Elasticsearch + +Infinity,官方建议 32GB 内存起)。代价是每次检索多一次模型调用,所以默认关闭。 + +失败一律退回原顺序:重排是增强,不该因为模型抽风或没配 key 就让整个检索挂掉。 +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +_PROMPT = """给下面每个片段打分:它对回答这个问题有多大帮助。 + +问题:{query} + +片段: +{candidates} + +只输出一行 JSON,形如 {{"scores": [{example}]}}——{n} 个 0 到 10 的整数, +按片段顺序一一对应。不要输出任何别的东西。 +0 表示完全无关,10 表示直接回答了问题。""" + + +def _parse_scores(text: str, n: int) -> list[float] | None: + """从模型回复里抠出分数数组。 + + 模型偶尔会加围栏或者说两句废话,所以不直接 json.loads 整段; + 但也不做过度容错——解析不出来就退回原顺序,比按半截数据重排安全。 + """ + match = re.search(r"\{[^{}]*\"scores\"\s*:\s*\[[^\]]*\][^{}]*\}", text, re.S) + if not match: + return None + try: + scores = json.loads(match.group(0)).get("scores") + except json.JSONDecodeError: + return None + if not isinstance(scores, list) or len(scores) != n: + # 个数对不上就没法一一对应,硬凑只会把顺序搅乱 + return None + out: list[float] = [] + for s in scores: + if not isinstance(s, (int, float)): + return None + out.append(float(s)) + return out + + +async def rerank( + model: Any, query: str, hits: list[dict[str, Any]], *, top_n: int, + on_note: Any = None, +) -> list[dict[str, Any]]: + """把 hits 重排后取前 top_n。失败原样返回前 top_n。 + + model 是任意带 ainvoke 的 LangChain 模型;传 None 表示不重排。 + """ + if not hits or model is None or len(hits) <= 1: + return hits[:top_n] + + listing = "\n\n".join( + # 截断是必要的:初筛 20 条 × 每条 800 字会把 prompt 撑到两万字, + # 而判断相关性用不着整段 + f"[{i + 1}] {h.get('content', '')[:400]}" + for i, h in enumerate(hits) + ) + prompt = _PROMPT.format( + query=query, candidates=listing, n=len(hits), + example=", ".join(["8"] * min(len(hits), 3)) + (", …" if len(hits) > 3 else ""), + ) + + try: + reply = await model.ainvoke(prompt) + text = reply if isinstance(reply, str) else getattr(reply, "content", "") + if isinstance(text, list): # Anthropic 的块状 content + text = "".join(b.get("text", "") for b in text if isinstance(b, dict)) + scores = _parse_scores(str(text), len(hits)) + except Exception as e: # noqa: BLE001 + if on_note: + on_note(f"重排失败,按初筛顺序返回:{type(e).__name__}: {e}") + return hits[:top_n] + + if scores is None: + if on_note: + on_note("重排结果解析不出来,按初筛顺序返回") + return hits[:top_n] + + ranked = sorted(zip(hits, scores), key=lambda t: t[1], reverse=True) + out: list[dict[str, Any]] = [] + for hit, score in ranked[:top_n]: + item = dict(hit) + item["rerank_score"] = score + # 初筛分保留:重排把它挪到第几位、原来是第几位,事后要能对照 + item.setdefault("signals", {}) + out.append(item) + return out diff --git a/backend/tests/test_retrieval_quality.py b/backend/tests/test_retrieval_quality.py new file mode 100644 index 00000000..4072ff8d --- /dev/null +++ b/backend/tests/test_retrieval_quality.py @@ -0,0 +1,295 @@ +"""检索质量的基线。 + +在此之前 alpha 默认值、切块大小、混合权重这些参数改了之后没有任何办法判断 +是变好还是变坏——只能搜两下"感觉准了"。这个文件把它变成数字。 + +两个指标: + hit@k 前 k 条里有没有正确片段(召回,最要紧的一条) + MRR 正确片段排第几的倒数平均(排序质量) + +评测集刻意分成两类问题: + lexical 问句和答案有字面重叠("商家账户是什么"→ 文档里就写着"商家账户") + semantic 只有语义相关,字面几乎不沾("谁能改别人的权限" → "平台管理员…") + +第二类是本地哈希向量的照妖镜:它是词频哈希、没有语义泛化,这类问题基本全挂。 +基线把这个事实钉住——将来换成真 embedding 模型,这里的数字必须涨。 +""" + +from __future__ import annotations + +import pytest + +from app.db.base import SessionLocal +from app.memory import embeddings as emb +from app.memory import kb + +COLLECTION = "evalset" + +#: 语料。24 篇,全部围绕权限/账户/计费这一个主题——**互相干扰是刻意的**。 +#: 第一版只有 8 篇、主指标用 hit@3,结果各档 alpha 全是 100%,等于什么都没量: +#: 8 选 3 瞎猜都有 37%。检索评测集不制造混淆就是在自我表扬。 +DOCS: list[tuple[str, str]] = [ + ("权限总则", "平台管理员拥有 platform 或 regional 级别的角色。" + "只有他们可以修改其他用户的角色,也只有他们能停用账户。"), + ("区域管理员", "区域管理员的角色级别是 regional,只能管理本区域内的商家," + "跨区域的操作会被拒绝并记录一条告警。"), + ("客服权限", "平台客服可以查看订单和用户资料,但不能修改角色,也不能导出数据。"), + ("商家账户", "商家账户指 provider 表中的一条记录,每个商家有唯一的 providerID," + "并关联一个联系人邮箱。商家账户不能登录后台管理界面。"), + ("商家关联用户", "providerUsers 表记录商家和用户的关联关系," + "一个商家可以绑定多个用户,但只有一个主联系人。"), + ("商家状态", "商家有待审核、正常、暂停三种状态,暂停的商家不出现在前台列表里," + "但历史订单仍然可查。"), + ("普通用户", "普通用户只能查看和编辑自己的资料,无法看到其他人的订单明细。"), + ("用户注销", "用户注销后资料进入匿名化流程,订单记录保留但姓名邮箱被替换成占位符。"), + ("角色变更流程", "调整任何人的角色都要先在工单系统里提申请," + "审批通过后由运维在后台执行,变更会记录在 audit_log 表里。"), + ("审批时效", "工单默认两个工作日内处理完,超时会自动升级给上一级负责人。"), + ("操作留痕", "audit_log 记录谁在什么时候改了什么,保留三年,任何人不能删除。"), + ("登录与会话", "会话有效期为 7 天,超过后需要重新登录。" + "连续五次密码错误会锁定账户十五分钟。"), + ("双因素认证", "管理员账户强制开启双因素认证,普通用户可选," + "关闭时需要邮箱确认。"), + ("密码规则", "密码至少十位,必须含大小写和数字,九十天过期,不能和前三次重复。"), + ("数据导出", "导出订单数据需要 export 权限,导出记录会留痕," + "单次最多导出十万行,超过要拆批。"), + ("导出审批", "导出含手机号或身份证的字段需要额外审批,审批人是数据合规负责人。"), + ("计费口径", "月度活跃商家指当月至少产生过一笔订单的商家," + "与注册商家总数不是一回事,对账时不要混用。"), + ("结算周期", "每月一号结算上个月的账单,账期三十天,逾期按日计息。"), + ("退款规则", "订单完成七天内可申请退款,超过七天需要人工介入," + "退款金额原路返回。"), + ("发票", "发票按自然月开具,需要在结算完成后申请,电子发票当天到邮箱。"), + ("通知设置", "系统通知默认发到联系人邮箱,可以在设置页改成短信," + "但验证码类通知始终走短信。"), + ("告警通道", "线上故障告警走企业微信机器人,值班人五分钟内不响应会电话呼叫。"), + ("接口限流", "开放接口每个商家每分钟六十次,超过返回 429," + "重试要带指数退避。"), + ("数据保留", "订单明细保留五年,日志保留半年,超期由归档任务自动清理。"), +] + +#: (问题, 应该命中哪篇, 类型) +QUESTIONS: list[tuple[str, str, str]] = [ + ("商家账户是什么", "商家账户", "lexical"), + ("会话多久过期", "登录与会话", "lexical"), + ("导出订单最多多少行", "数据导出", "lexical"), + ("月度活跃商家怎么算", "计费口径", "lexical"), + ("密码输错几次会锁", "登录与会话", "lexical"), + ("audit_log 保留多久", "操作留痕", "lexical"), + ("接口限流是多少", "接口限流", "lexical"), + ("结算周期是怎样的", "结算周期", "lexical"), + ("谁能改别人的权限", "权限总则", "semantic"), + ("改角色要走什么手续", "角色变更流程", "semantic"), + ("商家能不能登后台", "商家账户", "semantic"), + ("验证码发到哪里", "通知设置", "semantic"), + ("注册数和活跃数能混着用吗", "计费口径", "semantic"), + ("客服可以导数据吗", "客服权限", "semantic"), + ("下单之后多久还能退", "退款规则", "semantic"), + ("工单没人处理怎么办", "审批时效", "semantic"), +] + + +@pytest.fixture(scope="module", autouse=True) +def _corpus(): + """建一次语料,整个模块共用——每个用例重灌一遍太慢,而且语料是只读的。""" + import asyncio + + emb.configure("local") + + async def build() -> None: + async with SessionLocal() as session: + from sqlalchemy import select + + from app.db.models import Document + existing = list((await session.execute( + select(Document).where(Document.collection == COLLECTION))).scalars()) + for doc in existing: + await kb.delete_document(session, doc.id) + for title, body in DOCS: + await kb.ingest_document( + session, collection=COLLECTION, title=title, content=body) + + asyncio.new_event_loop().run_until_complete(build()) + yield + + +async def _measure(alpha: float, k: int = 5) -> dict[str, float]: + """跑一遍评测集。 + + 主指标是 hit@1(排第一的那条对不对),因为 hit@k 在小语料上太容易满分: + 第一版 8 篇文档取 top-3,各档 alpha 全是 100%——瞎猜都有 37%。 + """ + stats: dict[str, list[float]] = {"all_top1": [], "all_hit": [], "all_rr": [], + "lexical_top1": [], "semantic_top1": []} + async with SessionLocal() as session: + for question, want, kind in QUESTIONS: + hits = await kb.search(session, collection=COLLECTION, query=question, + limit=k, alpha=alpha) + titles = [h["title"] for h in hits] + rank = titles.index(want) + 1 if want in titles else 0 + stats["all_top1"].append(1.0 if rank == 1 else 0.0) + stats["all_hit"].append(1.0 if rank else 0.0) + stats["all_rr"].append(1.0 / rank if rank else 0.0) + stats[f"{kind}_top1"].append(1.0 if rank == 1 else 0.0) + return {k2: (sum(v) / len(v) if v else 0.0) for k2, v in stats.items()} + + +async def test_baseline_is_recorded_and_printed(capsys) -> None: + """把各档 alpha 的表现打出来,顺便钉住基线。 + + 跑 `pytest tests/test_retrieval_quality.py -s` 能看到这张表;改了切块或 + 排序之后对比它,就不用再靠"搜两下感觉准了"。 + """ + rows = [] + for alpha in (0.0, 0.2, 0.3, 0.5, 0.8, 1.0): + rows.append((alpha, await _measure(alpha))) + + with capsys.disabled(): + print("\n alpha hit@1 hit@5 MRR 字面 语义 (本地哈希向量)") + for alpha, m in rows: + print(f" {alpha:<7.1f} {m['all_top1']:<7.0%} {m['all_hit']:<7.0%} " + f"{m['all_rr']:<7.3f} {m['lexical_top1']:<7.0%} {m['semantic_top1']:<7.0%}") + + best = max(r[1]["all_top1"] for r in rows) + assert best >= 0.5, f"最好的一档 hit@1 只有 {best:.0%},检索质量出现回归" + + +async def test_lexical_questions_are_reliably_hit() -> None: + """字面有重叠的问题是 BM25 的主场,这一类必须稳。掉下来就是切块或排序坏了。""" + m = await _measure(alpha=0.2) + assert m["lexical_top1"] >= 0.75, f"字面匹配的 hit@1 只到 {m['lexical_top1']:.0%}" + + +async def test_the_hashing_embedder_cannot_do_semantics() -> None: + """钉住现状,不是庆祝它。 + + LocalEmbedder 是词频哈希,纯向量(alpha=1)在语义类问题上接近瞎猜。 + 这条测试的意义是:换成真 embedding 模型之后它会失败,那时候把阈值调上去 + ——而不是让"其实没有语义检索"这件事一直没人注意到。 + """ + pure_vector = await _measure(alpha=1.0) + with_keywords = await _measure(alpha=0.2) + assert with_keywords["all_top1"] >= pure_vector["all_top1"], ( + "关键词那一路不再比纯哈希向量强了——要么换了真 embedder(那就把这条" + "测试的期望改掉),要么 BM25 那侧坏了" + ) + + +# -------------------------------------------------------------------------- +# 重排 +# -------------------------------------------------------------------------- +# +# 质量本身没法在测试里量——那要真模型。这里守的是机制:它确实按分数重排了、 +# 模型抽风时不会把检索带崩、以及不会凭空吞掉候选。 + + +class _Scorer: + """按给定分数作答的假模型。""" + + def __init__(self, reply: str) -> None: + self._reply = reply + self.prompts: list[str] = [] + + async def ainvoke(self, prompt): + self.prompts.append(str(prompt)) + return self._reply + + +def _hits(n: int) -> list[dict]: + return [{"chunk_id": f"c{i}", "title": f"文档{i}", "content": f"内容 {i}", + "ordinal": 0, "score": 1.0 - i * 0.1} for i in range(n)] + + +async def test_rerank_reorders_by_model_score() -> None: + from app.memory.rerank import rerank + + model = _Scorer('{"scores": [1, 9, 3]}') + out = await rerank(model, "问题", _hits(3), top_n=3) + assert [h["chunk_id"] for h in out] == ["c1", "c2", "c0"] + assert out[0]["rerank_score"] == 9 + + +async def test_rerank_narrows_to_top_n() -> None: + """初筛捞 20 条、最后只要 5 条,多出来的是给重排挑的余地,不该漏出去。""" + from app.memory.rerank import rerank + + model = _Scorer('{"scores": [%s]}' % ", ".join(str(i) for i in range(20))) + out = await rerank(model, "问题", _hits(20), top_n=5) + assert len(out) == 5 + assert [h["chunk_id"] for h in out] == ["c19", "c18", "c17", "c16", "c15"] + + +async def test_the_prompt_carries_the_candidates_and_the_question() -> None: + from app.memory.rerank import rerank + + model = _Scorer('{"scores": [5, 5]}') + await rerank(model, "谁能改权限", _hits(2), top_n=2) + assert "谁能改权限" in model.prompts[0] + assert "内容 0" in model.prompts[0] and "内容 1" in model.prompts[0] + + +async def test_a_broken_reply_falls_back_to_the_original_order() -> None: + """重排是增强。模型抽风不该让整个检索挂掉,也不该按半截数据乱排。""" + from app.memory.rerank import rerank + + for reply in ('完全不是 JSON', '{"scores": [1, 2]}', '{"scores": ["高", "低", "中"]}'): + notes: list[str] = [] + out = await rerank(_Scorer(reply), "问题", _hits(3), top_n=3, on_note=notes.append) + assert [h["chunk_id"] for h in out] == ["c0", "c1", "c2"], reply + assert notes, f"退回原顺序也要说一声:{reply}" + + +async def test_a_model_that_raises_does_not_break_retrieval() -> None: + from app.memory.rerank import rerank + + class _Broken: + async def ainvoke(self, _prompt): + raise RuntimeError("额度用完了") + + notes: list[str] = [] + out = await rerank(_Broken(), "问题", _hits(3), top_n=2, on_note=notes.append) + assert len(out) == 2 and out[0]["chunk_id"] == "c0" + assert "额度用完了" in notes[0] + + +async def test_no_model_means_no_rerank() -> None: + from app.memory.rerank import rerank + + out = await rerank(None, "问题", _hits(4), top_n=2) + assert [h["chunk_id"] for h in out] == ["c0", "c1"] + + +# -------------------------------------------------------------------------- +# alpha 跟着 embedder 走 +# -------------------------------------------------------------------------- + + +def test_a_hashing_embedder_gets_no_vector_weight() -> None: + """实测:本地哈希向量下 alpha=0 的 hit@1 是 88%,alpha>0 掉到 81%。 + + 那一路不是"弱一点",是纯噪音——写死 0.5 等于主动把检索做差。 + """ + emb.configure("local") + assert emb.has_semantics() is False + assert emb.default_alpha() == 0.0 + + +async def test_search_without_an_explicit_alpha_uses_the_default() -> None: + m_default = await _measure_default() + m_zero = await _measure(alpha=0.0) + assert m_default["all_top1"] == m_zero["all_top1"] + + +async def _measure_default(k: int = 5) -> dict[str, float]: + """不传 alpha,让 kb.search 自己决定。""" + hit1, hit, rr = [], [], [] + async with SessionLocal() as session: + for question, want, _kind in QUESTIONS: + hits = await kb.search(session, collection=COLLECTION, query=question, limit=k) + titles = [h["title"] for h in hits] + rank = titles.index(want) + 1 if want in titles else 0 + hit1.append(1.0 if rank == 1 else 0.0) + hit.append(1.0 if rank else 0.0) + rr.append(1.0 / rank if rank else 0.0) + n = len(hit1) + return {"all_top1": sum(hit1) / n, "all_hit": sum(hit) / n, "all_rr": sum(rr) / n} diff --git a/frontend/src/canvas/nodeDefs.ts b/frontend/src/canvas/nodeDefs.ts index ed963e56..c5a793bf 100644 --- a/frontend/src/canvas/nodeDefs.ts +++ b/frontend/src/canvas/nodeDefs.ts @@ -318,14 +318,25 @@ export const NODE_DEFS: Record = { { key: 'query', label: '检索问题', type: 'prompt' }, { key: 'collection', label: '知识库', type: 'collection' }, { key: 'limit', label: '返回片段数', type: 'number', min: 1, max: 20 }, + { + key: 'rerank', label: '重排', type: 'select', advanced: true, + options: [ + { value: 'off', label: '不重排' }, + { value: 'model', label: '让模型重排(更准,多一次调用)' }, + ], + help: '开了会先多捞一些候选,再让模型按"对回答这个问题有多大帮助"重新排序', + }, { key: 'alpha', label: '向量 vs 关键词', type: 'number', min: 0, max: 1, step: 0.1, - help: '1 = 纯语义相似,0 = 纯关键词匹配', + help: '1 = 纯语义相似,0 = 纯关键词匹配。留空则跟着向量模型的能力走' + + '——本地哈希向量没有语义泛化,给它权重反而会让命中率下降', }, { key: 'min_score', label: '最低分数', type: 'number', min: 0, max: 1, step: 0.05, advanced: true }, ...COMMON_TAIL, ], - defaults: { query: '{{ last_message }}', collection: 'default', limit: 5, alpha: 0.5 }, + // alpha 不写死:写死 0.5 等于给一个没有语义能力的信号一半权重, + // 实测会把 hit@1 从 88% 拉到 81%(backend/tests/test_retrieval_quality.py) + defaults: { query: '{{ last_message }}', collection: 'default', limit: 5, rerank: 'off' }, }, transform: { type: 'transform', label: '数据整形', category: '上下文', icon: Shuffle, diff --git a/frontend/src/pages/KnowledgePage.tsx b/frontend/src/pages/KnowledgePage.tsx index 07e4fa7a..8a09053b 100644 --- a/frontend/src/pages/KnowledgePage.tsx +++ b/frontend/src/pages/KnowledgePage.tsx @@ -125,7 +125,8 @@ function KbTab() { const [docs, setDocs] = useState([]) const [query, setQuery] = useState('') const [hits, setHits] = useState(null) - const [alpha, setAlpha] = useState(0.5) + // 初值跟着 embedder 能力走:本地哈希向量下给向量权重会让命中率下降 + const [alpha, setAlpha] = useState(0) const [adding, setAdding] = useState(false) const fileRef = useRef(null)