from __future__ import annotations import logging from typing import Any from fastapi import ( APIRouter, BackgroundTasks, Depends, File, HTTPException, Query, UploadFile, ) from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.core.errors import explain, raw from app.core.config import settings from app.db.base import get_session from app.db.models import Document, MemoryItem, Skill from app.memory import inverted, kb, store logger = logging.getLogger(__name__) # -------------------------------------------------------------------------- # 长期记忆 # -------------------------------------------------------------------------- memory_router = APIRouter(prefix="/api/memory", tags=["memory"]) class MemoryIn(BaseModel): content: str scope: str = "default" kind: str = "fact" importance: float = Field(default=0.5, ge=0, le=1) meta: dict[str, Any] = Field(default_factory=dict) class MemoryPatch(BaseModel): content: str | None = None kind: str | None = None importance: float | None = Field(default=None, ge=0, le=1) class MemoryOut(BaseModel): id: str scope: str kind: str content: str importance: float use_count: int meta: dict[str, Any] #: 这条是哪来的:kind=run 时带运行、节点和工作流名;manual 是手动添加的; #: playground 是在工具库里直接调 remember 写进来的 source: dict[str, Any] = Field(default_factory=dict) created_at: Any = None # 不给 updated_at:每次召回记计数也是一次写,它跟着召回走,当「修改于」 # 显示就是错的。界面要的「记下于」「上次召回」各有字段 last_used_at: Any = None model_config = {"from_attributes": True} def _content(text: str) -> str: """只有空白的内容和空串是一回事——存下去就是一条什么都召回不到的空记忆。""" if not text.strip(): raise HTTPException(422, "记忆内容为空,请填写要记住的内容后再保存") return text async def _with_source(session: AsyncSession, rows: list[MemoryItem]) -> list[MemoryOut]: """给记忆补上来源。 长期记忆里存的是口径规则,审查时得知道「这条是哪次运行、哪个节点写进来的」 才能判断能不能信。meta 里只有 id,节点名和工作流名要回运行记录里查—— 运行被删了就照实标 run_exists=false,不编一个出来。 """ from app.db.models import Run run_ids = {str((r.meta or {}).get("run_id") or "") for r in rows} - {"", "playground"} runs: dict[str, Any] = {} if run_ids: for row in await session.execute( select(Run.id, Run.workflow_name, Run.graph).where(Run.id.in_(run_ids)) ): runs[row.id] = row out: list[MemoryOut] = [] for item in rows: meta = item.meta or {} run_id = str(meta.get("run_id") or "") node_id = meta.get("node_id") or None if not run_id: source: dict[str, Any] = {"kind": "manual"} elif run_id == "playground": source = {"kind": "playground"} else: run = runs.get(run_id) nodes = ((run.graph if run else None) or {}).get("nodes") or [] label = next( ((n.get("data") or {}).get("label") for n in nodes if n.get("id") == node_id), None ) source = { "kind": "run", "run_id": run_id, "node_id": node_id, "node_label": label or None, "workflow_name": (run.workflow_name or None) if run else None, "run_exists": run is not None, } view = MemoryOut.model_validate(item) view.source = source out.append(view) return out @memory_router.get("/scopes") async def memory_scopes(session: AsyncSession = Depends(get_session)) -> list[dict[str, Any]]: return await store.list_scopes(session) @memory_router.get("", response_model=list[MemoryOut]) async def list_memory( scope: str | None = None, limit: int = Query(default=200, le=500), session: AsyncSession = Depends(get_session), ) -> list[MemoryOut]: return await _with_source(session, await store.list_memories(session, scope=scope, limit=limit)) @memory_router.post("", response_model=MemoryOut, status_code=201) async def add_memory( payload: MemoryIn, session: AsyncSession = Depends(get_session) ) -> MemoryOut: item = await store.remember( session, scope=payload.scope, content=_content(payload.content), kind=payload.kind, importance=payload.importance, meta=payload.meta, ) return (await _with_source(session, [item]))[0] @memory_router.patch("/{memory_id}", response_model=MemoryOut) async def update_memory( memory_id: str, payload: MemoryPatch, session: AsyncSession = Depends(get_session) ) -> MemoryOut: """写错了原地改,不必删掉重写——重写会丢掉来源和召回记录。""" item = await store.revise( session, memory_id, content=None if payload.content is None else _content(payload.content), kind=payload.kind, importance=payload.importance, ) if item is None: raise HTTPException(404, "记忆不存在,可能已被删除") return (await _with_source(session, [item]))[0] @memory_router.get("/search") async def search_memory( q: str, scope: str = "default", limit: int = 10, peek: bool = Query(default=False, description="只看不记:调试台用,不计入召回次数"), session: AsyncSession = Depends(get_session), ) -> dict[str, Any]: hits = await store.recall(session, scope=scope, query=q, limit=limit, track=not peek) return {"query": q, "scope": scope, "results": hits, "peek": peek} @memory_router.delete("/{memory_id}", status_code=204) async def delete_memory( memory_id: str, session: AsyncSession = Depends(get_session) ) -> None: if not await store.forget(session, memory_id): raise HTTPException(404, "记忆不存在,可能已被删除") @memory_router.delete("/scope/{scope}") async def clear_scope(scope: str, session: AsyncSession = Depends(get_session)) -> dict[str, int]: return {"removed": await store.clear_scope(session, scope)} # -------------------------------------------------------------------------- # 知识库 # -------------------------------------------------------------------------- kb_router = APIRouter(prefix="/api/kb", tags=["knowledge"]) class DocumentOut(BaseModel): id: str collection: str title: str source: str mime: str chunk_count: int #: ready | processing | failed。前端据此显示"处理中 3200/7914 段" status: str = "ready" error: str = "" meta: dict[str, Any] = Field(default_factory=dict) created_at: Any = None model_config = {"from_attributes": True} @kb_router.get("/collections") async def collections(session: AsyncSession = Depends(get_session)) -> list[dict[str, Any]]: return await kb.list_collections(session) @kb_router.get("/formats") async def formats() -> dict[str, Any]: """上传框收哪些文件。前端的 accept 从这里取,和后端的解析判断是同一份清单。""" from app.memory.parsing import supported_formats out: dict[str, Any] = supported_formats() out["accept"] = ",".join(out["extensions"]) return out @kb_router.get("/documents", response_model=list[DocumentOut]) async def list_documents( collection: str | None = None, session: AsyncSession = Depends(get_session), ) -> list[Document]: stmt = select(Document).order_by(Document.created_at.desc()).limit(300) if collection: stmt = stmt.where(Document.collection == collection) return list((await session.execute(stmt)).scalars()) class IngestIn(BaseModel): collection: str = "default" title: str = "" content: str = Field(min_length=1) source: str = "" @kb_router.post("/documents", response_model=DocumentOut, status_code=201) async def ingest(payload: IngestIn, session: AsyncSession = Depends(get_session)) -> Document: return await kb.ingest_document( session, collection=payload.collection, title=payload.title, content=payload.content, source=payload.source, ) @kb_router.post("/upload", response_model=DocumentOut, status_code=201) async def upload( background: BackgroundTasks, file: UploadFile = File(...), collection: str = "default", session: AsyncSession = Depends(get_session), ) -> Document: # 边读边数,超了立刻停——原来是 raw = await file.read() 读完再判断, # 传个 1GB 的文件,内存在报 413 之前就已经吃掉了。那个检查拦的是"入库", # 不是"占内存" limit = settings.max_upload_mb * 1024 * 1024 chunks: list[bytes] = [] total = 0 while piece := await file.read(1024 * 1024): total += len(piece) if total > limit: raise HTTPException(413, f"文件超过 {settings.max_upload_mb} MB") chunks.append(piece) raw = b"".join(chunks) from app.memory.parsing import UnsupportedDocument, extract try: content = extract(raw, file.filename or "", file.content_type or "") except UnsupportedDocument as e: # 原样交回:里面写的是"装哪个包"或"这是扫描件",都能照着做 raise HTTPException(415, str(e)) from None doc = await kb.create_document( session, collection=collection, title=file.filename or "上传文档", content=content, source=file.filename or "", mime=file.content_type or "text/plain", status="processing", ) # 立刻返回,切块和算向量在后台跑。实测一个 10MB 文档用本地哈希要两分半, # 换成远端 embedding 是几十分钟——压在请求里必然超时,而后端其实还在跑, # 界面显示失败、数据其实成功,比直接拒绝还糟 background.add_task(_process_in_background, doc.id) return doc async def _process_in_background(doc_id: str) -> None: """后台切块。进度写回文档行,失败也写回——不然它会永远停在"处理中"。""" from app.db.base import SessionLocal async with SessionLocal() as session: doc = await session.get(Document, doc_id) if not doc: return try: # 进度由 process_document 自己按批提交——它就在主事务里, # 不会和另一个 session 抢 SQLite 的写锁 await kb.process_document(session, doc) except Exception as e: # noqa: BLE001 logger.warning("文档 %s 处理失败:%s", doc_id, raw(e)) await session.rollback() doc = await session.get(Document, doc_id) if doc: reason, hint = explain(e) doc.status = "failed" doc.error = (f"文档分块或生成向量时出错:{reason}" + (f"。{hint}" if hint else ""))[:500] await session.commit() @kb_router.get("/search") async def search_kb( q: str, collection: str | None = None, limit: int = 5, alpha: float | None = Query(default=None, ge=0, le=1, description="1=纯向量, 0=纯关键词;不传则按 embedder 能力取默认"), session: AsyncSession = Depends(get_session), ) -> dict[str, Any]: from app.memory.embeddings import default_alpha notes: list[str] = [] # 没传就是运行时的默认值。把实际用的那个交回去:调试台的用处是预演 agent # 会拿到什么,默认值对不上,人会照着错的命中去调知识库 used = default_alpha() if alpha is None else alpha hits = await kb.search(session, collection=collection, query=q, limit=limit, alpha=used, on_degrade=notes.append) # 退回关键词这件事要跟着结果一起交出去,而不是只写进服务端日志—— # 调用方(设置页的"试一下"、外部脚本)看到的是结果变差,得知道为什么 return {"query": q, "collection": collection, "results": hits, "degraded": notes, "alpha": used} @kb_router.get("/embedding") async def embedding_status( collection: str | None = None, session: AsyncSession = Depends(get_session) ) -> dict[str, Any]: """当前用的是哪个 embedder,以及有多少条向量已经对不上了。""" from app.memory.embeddings import ( EMBEDDING_SETTING_KEY, default_alpha, embedder_dim, embedder_id, has_semantics, unavailable_reason, ) from app.db.models import Setting row = await session.get(Setting, EMBEDDING_SETTING_KEY) saved = (row.value if row else {}) or {} # 配的是语义模型,实际在用本地哈希:保存的配置没生效(多半是启动时本机的模型 # 服务还没起)。这时"对不上"的存量向量正是那个模型建好的,界面不能再劝人重建—— # 那等于拿哈希把它们覆盖掉,连上之后还得再重建一遍 fallback = saved.get("kind") == "openai" and not has_semantics() return { "embedder": embedder_id(), "dim": embedder_dim(), "configured": bool(saved.get("kind")), "kind": saved.get("kind", "local"), "model": saved.get("model", ""), "base_url": saved.get("base_url", ""), "stale_chunks": await kb.stale_count(session, collection), # 记忆和知识库共用一个 embedder,换模型时一起失效——报一个漏一个 # 的话,用户点完重建还是想不起事,而且不知道为什么 "stale_memories": await store.stale_count(session), "has_semantics": has_semantics(), "fallback": fallback, "fallback_reason": (unavailable_reason() or "") if fallback else "", "default_alpha": default_alpha(), # 没建倒排的片段会走全表扫——结果对,但慢 "unindexed_chunks": await inverted.missing_count(session, collection), } class EmbeddingIn(BaseModel): kind: str = Field(default="local", pattern="^(local|openai)$") model: str = "" #: 任何讲 OpenAI /v1/embeddings 的地址:本机 LM Studio / Ollama / vLLM / TEI, #: 或者你自己的网关。留空才走 api.openai.com base_url: str = "" @kb_router.put("/embedding") async def set_embedding( payload: EmbeddingIn, session: AsyncSession = Depends(get_session) ) -> dict[str, Any]: """换 embedding 模型。 建不起来就 400 原样交回原因——选了远端却静默退回本地哈希向量, 用户会以为自己在用语义检索,而搜不出同义表达时完全无从怀疑到这里。 """ from app.memory import embeddings as emb from app.db.models import Setting try: emb.configure(payload.kind, payload.model, payload.base_url) except emb.EmbedderUnavailable as e: raise HTTPException(400, str(e)) from e row = await session.get(Setting, emb.EMBEDDING_SETTING_KEY) value = {"kind": payload.kind, "model": payload.model, "base_url": payload.base_url} if row: row.value = value else: session.add(Setting(key=emb.EMBEDDING_SETTING_KEY, value=value)) await session.commit() # 多进程:别的进程照新的设置换向量模型(不换的话它们按旧模型检索、写旧模型的向量) from app.cluster import caches await caches.broadcast("embeddings") return { "embedder": emb.embedder_id(), "dim": emb.embedder_dim(), "stale_chunks": await kb.stale_count(session), "stale_memories": await store.stale_count(session), } @kb_router.get("/embedding/probe") async def probe_embedding(base_url: str) -> dict[str, Any]: """问问这个地址上有哪些模型。 自己填模型名太容易写错(LM Studio 里叫 text-embedding-qwen3-embedding-4b,不是 Qwen3-Embedding-4B), 而填错的表现是切换时报一句 404,人还以为是服务没起。 """ import httpx url = base_url.rstrip("/") + "/models" try: async with httpx.AsyncClient(timeout=8) as client: data = (await client.get(url)).json() except Exception as e: # noqa: BLE001 reason, hint = explain(e) raise HTTPException(400, f"无法获取 {url} 上的模型列表:{reason}" + (f"。{hint}" if hint else "")) from e models = [m.get("id", "") for m in (data.get("data") or []) if m.get("id")] return {"base_url": base_url, "models": models} class _ReindexJob: """一次重建索引。进度按批更新:kb / store 每提交一批回调一次,不插值。""" def __init__(self, collection: str | None, chunks: int, memories: int) -> None: from app.db.base import new_id, utcnow self.id = new_id() self.collection = collection self.state = "running" # running | done | failed # chunks → index(倒排重建,没有分批进度)→ memories → done self.phase = "chunks" self.chunks = {"total": chunks, "done": 0} self.memories = {"total": memories, "done": 0} self.started_at = utcnow() self.finished_at: Any = None self.error: str | None = None self.hint: str | None = None self.result: dict[str, Any] | None = None def chunk_step(self, done: int, total: int) -> None: self.chunks = {"total": total, "done": done} if done >= total: self.phase = "index" def memory_step(self, done: int, total: int) -> None: self.memories = {"total": total, "done": done} def view(self) -> dict[str, Any]: return { "id": self.id, "state": self.state, "phase": self.phase, "collection": self.collection, "total": self.chunks["total"] + self.memories["total"], "done": self.chunks["done"] + self.memories["done"], "chunks": dict(self.chunks), "memories": dict(self.memories), "started_at": self.started_at.isoformat(), "finished_at": self.finished_at.isoformat() if self.finished_at else None, "error": self.error, "hint": self.hint, "result": self.result, } #: 最近一次重建(本进程里的)。重建是幂等的,服务重启丢了进度,再点一次就是。多进程时进度另记在设置表里(REINDEX_KEY, #: 带上做它的进程),别的进程上的「有没有在跑」「到哪了」读那一份;同一时间只跑一个靠跨进程的锁(kb-reindex) _REINDEX: _ReindexJob | None = None REINDEX_KEY = "cluster:kb_reindex" #: 后台任务要有人拿着引用,否则跑到一半可能被回收 _REINDEX_TASK: Any = None def reindex_job() -> _ReindexJob | None: return _REINDEX async def _save_state(job: _ReindexJob) -> None: """把进度记进设置表(多进程时别的进程读它;SQLite 只有一个进程,不记)。写不进去只记日志。""" from app.cluster import counters, identity if not counters.shared(): return from app.db.base import SessionLocal from app.db.models import Setting try: async with SessionLocal() as session: row = await session.get(Setting, REINDEX_KEY) value = {**job.view(), "owner": identity.owner()} if row is None: session.add(Setting(key=REINDEX_KEY, value=value)) else: row.value = value await session.commit() except Exception: # noqa: BLE001 logger.exception("重建索引的进度没记进库") async def _stored_state() -> dict[str, Any] | None: """别的进程记的那一份:还写着在跑、做它的进程却不在了的,照实说没做完。SQLite 只有一个进程,没有这一份。""" from app.cluster import counters, state if not counters.shared(): return None from app.db.base import SessionLocal from app.db.models import Setting async with SessionLocal() as session: row = await session.get(Setting, REINDEX_KEY) value = dict(row.value) if row is not None and isinstance(row.value, dict) else None if value is None: return None owner = value.pop("owner", None) if value.get("state") == "running" and not (await state.live_for([owner])).alive(owner, local=False): value.update(state="failed", error="重建索引未完成:做这件事的服务进程停止了", hint="再点一次重建就是") return value async def _run_reindex(job: _ReindexJob, lock: Any = None) -> None: import asyncio from app.db.base import SessionLocal, utcnow async def report() -> None: while True: await _save_state(job) await asyncio.sleep(1.0) reporter = asyncio.ensure_future(report()) try: async with SessionLocal() as session: out = await kb.reindex(session, job.collection, on_progress=job.chunk_step) job.phase = "memories" # 记忆不按 collection 分,重建就是全量 out["memories_reindexed"] = await store.reindex(session, on_progress=job.memory_step) job.result, job.state, job.phase = out, "done", "done" except Exception as e: # noqa: BLE001 - 失败要落到任务状态上,不能只进日志 logger.warning("重建索引失败:%s", raw(e)) reason, hint = explain(e) job.state, job.error, job.hint = "failed", f"重建索引未完成:{reason}", hint or None finally: job.finished_at = utcnow() reporter.cancel() await _save_state(job) if lock is not None: await lock.give_back() @kb_router.post("/reindex") async def reindex( collection: str | None = None, background: bool = Query(default=False, description="后台跑,立刻返回任务;进度用 GET /kb/reindex 查"), session: AsyncSession = Depends(get_session), ): """用当前 embedder 重算向量。换了模型之后唯一的恢复手段。 知识库和长期记忆一起重建:它们共用一个 embedder,换模型时一起失效。 分成两个按钮的结果是点了一个、以为好了,另一个还在悄悄退回关键词—— 真踩过,记忆那边连个提示都没有。 几千段配远端 embedding 要好几分钟,压在请求里界面只能干转圈。background=true 时立刻返回 202 和任务,按批更新进度;不带它仍然等做完再返回,老调用方不受影响。 同一时间只跑一个:两次重建交错写同一批向量,谁最后写赢都说不清。 """ global _REINDEX, _REINDEX_TASK import asyncio from fastapi.responses import JSONResponse from sqlalchemy import func from app.db.models import Chunk count = select(func.count(Chunk.id)) if collection: count = count.where(Chunk.collection == collection) chunks = int((await session.execute(count)).scalar_one() or 0) memories = int((await session.execute(select(func.count(MemoryItem.id)))).scalar_one() or 0) # 先数完再查「有没有在跑」,查和占位之间不能有 await:以前先查后数,两个同时 # 到的请求都在计数那里让出去,回来都以为没在跑,于是两个都起了 if _REINDEX is not None and _REINDEX.state == "running": running = _REINDEX.view() raise HTTPException( 409, f"正在重建索引({_REINDEX.collection or '全部集合'}," f"已完成 {running['done']}/{running['total']}),请等待完成后再操作", ) # 多进程:别的进程上正在重建的,跨进程的锁占不到(app/cluster/mutex),同样 409,说法照它记在库里的进度 from app.cluster import mutex lock = mutex.lock("kb-reindex") if not await lock.try_take(): other = await _stored_state() or {} raise HTTPException( 409, f"正在重建索引({other.get('collection') or '全部集合'}," f"已完成 {other.get('done', 0)}/{other.get('total', 0)}),请等待完成后再操作", ) job = _REINDEX = _ReindexJob(collection, chunks, memories) if background: _REINDEX_TASK = asyncio.create_task(_run_reindex(job, lock)) return JSONResponse(job.view(), status_code=202) await _run_reindex(job, lock) if job.state == "failed": raise HTTPException(502, job.error + (f"。{job.hint}" if job.hint else "")) return job.result @kb_router.get("/reindex") async def reindex_status() -> dict[str, Any]: """最近一次重建的进度:{state, phase, total, done, chunks, memories, result, error…}。 还没重建过返回 {state: "idle"}。多进程时本进程没在做的,读做它的进程记在库里的那一份。 """ if _REINDEX is not None and _REINDEX.state == "running": return _REINDEX.view() stored = await _stored_state() if stored is not None and (_REINDEX is None or str(stored.get("started_at") or "") > _REINDEX.view()["started_at"]): return stored return _REINDEX.view() if _REINDEX is not None else {"state": "idle"} #: 单段返回多少字。切块本来就在千字量级,详情页是给人扫一眼"切得对不对"的, #: 不是给人通读原文的——整篇几千段全量吐出去,一份大文档能有几兆 _CHUNK_PREVIEW = 1200 @kb_router.get("/documents/{doc_id}") async def get_document( doc_id: str, session: AsyncSession = Depends(get_session) ) -> dict[str, Any]: """一份文档被切成了什么样。 切块这一层做了不少事——按行边界留重叠、跨块把表头续上(memory/kb.py 的 chunk_text),而在此之前结果在界面上一处都看不见。检索不准的时候第一个 该看的就是它:表头有没有续上、重叠对不对、哪一段被截断了。 """ from app.db.models import Chunk doc = await session.get(Document, doc_id) if not doc: raise HTTPException(404, "文档不存在,可能已被删除") rows = list((await session.execute( select(Chunk).where(Chunk.document_id == doc_id).order_by(Chunk.ordinal) )).scalars()) return { "document": DocumentOut.model_validate(doc).model_dump(mode="json"), "chunks": [ { "id": c.id, "ordinal": c.ordinal, "content": c.content[:_CHUNK_PREVIEW], "truncated": len(c.content) > _CHUNK_PREVIEW, "chars": len(c.content), "token_len": c.token_len, # 哪条向量是谁建的。一份文档里混着两个模型建的向量是真会发生的 # (换模型之后只重建了一部分),而那会让检索悄悄退回关键词 "embed_model": c.embed_model or "", "has_vector": c.embedding is not None, } for c in rows ], } @kb_router.delete("/documents/{doc_id}", status_code=204) async def delete_document(doc_id: str, session: AsyncSession = Depends(get_session)) -> None: if not await kb.delete_document(session, doc_id): raise HTTPException(404, "文档不存在,可能已被删除") # -------------------------------------------------------------------------- # Skill(方法论) # -------------------------------------------------------------------------- skills_router = APIRouter(prefix="/api/skills", tags=["knowledge"]) class SkillIn(BaseModel): name: str = Field(min_length=1, max_length=100) description: str = "" instructions: str = "" examples: list[dict[str, Any]] = Field(default_factory=list) suggested_tools: list[str] = Field(default_factory=list) tags: list[str] = Field(default_factory=list) enabled: bool = True class SkillOut(SkillIn): id: str created_at: Any = None model_config = {"from_attributes": True} @skills_router.get("", response_model=list[SkillOut]) async def list_skills(session: AsyncSession = Depends(get_session)) -> list[Skill]: return list((await session.execute(select(Skill).order_by(Skill.created_at.desc()))).scalars()) @skills_router.post("", response_model=SkillOut, status_code=201) async def create_skill(payload: SkillIn, session: AsyncSession = Depends(get_session)) -> Skill: if (await session.execute(select(Skill).where(Skill.name == payload.name))).scalar_one_or_none(): raise HTTPException(409, "同名 Skill 已存在") row = Skill(**payload.model_dump()) session.add(row) await session.commit() await session.refresh(row) return row @skills_router.patch("/{skill_id}", response_model=SkillOut) async def update_skill( skill_id: str, payload: SkillIn, session: AsyncSession = Depends(get_session) ) -> Skill: row = await session.get(Skill, skill_id) if not row: raise HTTPException(404, "Skill 不存在,可能已被删除") for key, value in payload.model_dump().items(): setattr(row, key, value) await session.commit() await session.refresh(row) return row @skills_router.delete("/{skill_id}", status_code=204) async def delete_skill(skill_id: str, session: AsyncSession = Depends(get_session)) -> None: row = await session.get(Skill, skill_id) if not row: raise HTTPException(404, "Skill 不存在,可能已被删除") await session.delete(row) await session.commit()