mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-20 23:42:03 +00:00
6a46e4b55c
压测量出来的三个问题,10MB 文本(切 7914 段):
上传耗时 150s(本地哈希) 换成 Qwen3-Embedding-4B 要 40 分钟以上
而这 150 秒是压在一个 HTTP 请求里同步跑完的
浏览器和反向代理的默认超时都在 60 秒上下,也就是 5MB 以上的文件用户必然
看到超时——而后端其实还在跑,跑完文档也确实入了库。界面显示失败、数据其实
成功,比直接拒绝还糟。
**分批算向量。** 原来 ingest 是 embed_texts(pieces) 一次全发。本地哈希只是慢,
换成远端模型就是必挂:近八千段一次请求塞不下(OpenAI 单次上限 2048)。
reindex 那边早就分批了,ingest 这条一直漏着——而我上一轮刚把"换真模型"做成
设置页里点两下的事。
**上传转后台。** create_document / process_document 拆开,上传立刻返回
status=processing,切块和算向量在后台跑,进度写回文档行,前端轮询显示
"正在切块并算向量… 4288/7914 段"。实测上传返回 0.07 秒。
进度这件事返工过一次:先用另一个 session 单独写,结果整个过程界面上都是空的、
只在结束时闪一下——SQLite 只允许一个写者,主事务开着的时候那次写被堵住了。
改成主事务自己按批提交,顺手把进度写进去。
失败要写回 failed,否则永远停在"处理中"转圈。服务重启时也把残留的
processing 收尾成 failed——和 runner 对 running 的 run 是同一个道理。
**边读边数大小。** 原来是 raw = await file.read() 读完再判断,传个 1GB 的
文件内存在报 413 之前就吃掉了——那个检查拦的是"入库",不是"占内存"。改成
按块读、超了立刻停。上限从写死的 10MB 变成配置项,默认 50MB。
端到端验过:10MB 上传返回 0.07s,后台进度 768→1472→…→7914 实时可见,
最终 ready / 7914 段。
105 lines
3.7 KiB
Python
105 lines
3.7 KiB
Python
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from typing import Any, AsyncIterator
|
|
|
|
from sqlalchemy import JSON, DateTime, MetaData, event
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
|
|
|
from app.core.config import settings
|
|
|
|
NAMING_CONVENTION = {
|
|
"ix": "ix_%(column_0_label)s",
|
|
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
|
"fk": "fk_%(table_name)s_%(column_0_name)s",
|
|
"pk": "pk_%(table_name)s",
|
|
}
|
|
|
|
|
|
def utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def new_id() -> str:
|
|
return uuid.uuid4().hex
|
|
|
|
|
|
class Base(DeclarativeBase):
|
|
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
|
|
|
# 图定义、节点配置、事件负载都是任意形状的 JSON,统一映射到 SQLite 的 JSON 列
|
|
type_annotation_map = {dict[str, Any]: JSON, list[Any]: JSON}
|
|
|
|
|
|
class TimestampMixin:
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), default=utcnow, onupdate=utcnow
|
|
)
|
|
|
|
|
|
engine = create_async_engine(settings.db_url, echo=False, future=True)
|
|
|
|
|
|
@event.listens_for(engine.sync_engine, "connect")
|
|
def _set_sqlite_pragma(dbapi_conn, _record): # pragma: no cover - 驱动回调
|
|
"""WAL + 外键约束。WAL 让读写不互相阻塞,事件写入和前端轮询能并行。"""
|
|
cur = dbapi_conn.cursor()
|
|
cur.execute("PRAGMA journal_mode=WAL")
|
|
cur.execute("PRAGMA foreign_keys=ON")
|
|
cur.execute("PRAGMA busy_timeout=5000")
|
|
cur.close()
|
|
|
|
|
|
SessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
|
|
|
|
|
|
async def get_session() -> AsyncIterator[AsyncSession]:
|
|
"""FastAPI 依赖注入用的会话。"""
|
|
async with SessionLocal() as session:
|
|
yield session
|
|
|
|
|
|
# 轻量迁移:create_all 只建新表,不给已有表加列。SQLite 的 ADD COLUMN
|
|
# 便宜且带默认值回填,够用;等真需要改列类型再考虑 alembic。
|
|
_COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [
|
|
("runs", "run_class", "TEXT DEFAULT 'exploratory'"),
|
|
("runs", "version", "INTEGER"),
|
|
("runs", "version_hash", "VARCHAR(64)"),
|
|
("runs", "manifest_hash", "VARCHAR(64)"),
|
|
("runs", "started_by", "VARCHAR(100)"),
|
|
("workflows", "status", "TEXT DEFAULT 'draft'"),
|
|
("workflows", "published_version", "INTEGER"),
|
|
("workflows", "published_by", "VARCHAR(100)"),
|
|
("workflow_versions", "graph_hash", "VARCHAR(64)"),
|
|
("approvals", "resolved_by", "VARCHAR(100)"),
|
|
# 存量向量没有这两列,回填成空串 / 0 —— 正好表示"不知道是谁建的",
|
|
# 检索侧会把它们当成对不上当前 embedder 处理
|
|
("chunks", "embed_model", "VARCHAR(100) DEFAULT ''"),
|
|
("chunks", "embed_dim", "INTEGER DEFAULT 0"),
|
|
("memories", "embed_model", "VARCHAR(100) DEFAULT ''"),
|
|
("memories", "embed_dim", "INTEGER DEFAULT 0"),
|
|
("chunks", "token_len", "INTEGER DEFAULT 0"),
|
|
("documents", "status", "TEXT DEFAULT 'ready'"),
|
|
("documents", "error", "TEXT DEFAULT ''"),
|
|
]
|
|
|
|
|
|
def _migrate(conn) -> None: # pragma: no cover - 同步回调
|
|
from sqlalchemy import text
|
|
|
|
for table, column, ddl in _COLUMN_MIGRATIONS:
|
|
cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
|
|
if cols and column not in cols:
|
|
conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}"))
|
|
|
|
|
|
async def init_db() -> None:
|
|
from app.db import models # noqa: F401 确保模型已注册到 metadata
|
|
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
await conn.run_sync(_migrate)
|