diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 7d2d2454..df49d9f8 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -8,11 +8,12 @@ from __future__ import annotations from datetime import datetime, timezone from typing import Any -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from app.core.config import settings from app.core.crypto import encrypt, mask from app.data import introspect as introspect_mod from app.data.engine import SUPPORTED_KINDS, engines, test_connection @@ -206,3 +207,111 @@ async def get_schema(source_id: str, table: str | None = None, "summary": introspect_mod.summary(row), "synced_at": row.schema_synced_at.isoformat() if row.schema_synced_at else None, } + + +# -------------------------------------------------------------------------- +# 上传表格 +# -------------------------------------------------------------------------- + + +@router.post("/upload", response_model=dict, status_code=201) +async def upload_table( + file: UploadFile = File(...), + name: str = Form(...), + description: str = Form(""), + header_row: int = Form(1), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + """把 Excel / CSV 变成一个可以用 SQL 查的数据源。 + + 走 SQLite：落成一个 .db 文件，再建一条 kind='sqlite' 的数据源指过去。 + 下游一行都不用改——SQL 守卫、结构探查、db_query__ 工具、查询快照进 + 工件库、Copilot 的数据源清单，全都白拿。 + + 这条路存在的理由不是"多支持一种格式"：表格传进知识库只能被切块检索， + 数字就成了模型从片段里读出来的，而这个项目的地基是"所有算术下沉到 + SQL 或口径卡"。表格必须变成表。 + + **同名就地替换**，不新建。数据源名字会成为工具名（db_query__sales）， + 而工具名写进了保存过的工作流——重传时另起一个 sales_2 等于悄悄让那些图失效。 + """ + import re as _re + + if not _re.match(_NAME_PATTERN, name or ""): + raise HTTPException( + 400, "名字只能用小写字母开头的字母数字下划线——它会成为工具名的一部分" + ) + + # 边读边数，超了立刻停。和知识库上传同一套：那个检查拦的是"入库"， + # 不是"占内存"，读完再判等于先把内存吃掉 + limit = settings.max_upload_mb * 1024 * 1024 + pieces: list[bytes] = [] + total = 0 + while piece := await file.read(1024 * 1024): + total += len(piece) + if total > limit: + raise HTTPException(413, f"文件超过 {settings.max_upload_mb}MB") + pieces.append(piece) + raw = b"".join(pieces) + + tables_dir = settings.uploads_dir / "tables" + tables_dir.mkdir(parents=True, exist_ok=True) + db_path = tables_dir / f"{name}.db" + + from app.data.tabular import UnsupportedTable, load_into + + existing = (await session.execute( + select(DataSource).where(DataSource.name == name) + )).scalar_one_or_none() + if existing and existing.kind != "sqlite": + raise HTTPException( + 409, f"已经有一个叫 {name} 的数据源了，而且它不是上传来的表格。换个名字。" + ) + + try: + report = load_into(str(db_path), raw, file.filename or name, header_row=header_row) + except UnsupportedTable as e: + raise HTTPException(400, str(e)) from e + except Exception as e: # noqa: BLE001 + raise HTTPException(400, f"导入失败：{type(e).__name__}: {e}") from e + + row = existing + if row is None: + row = DataSource( + name=name, kind="sqlite", database=str(db_path), + readonly=True, description=description, + ) + session.add(row) + else: + row.database = str(db_path) + if description: + row.description = description + # 换了文件内容，缓存的连接还指着旧的那个 engine + await engines.invalidate(row.id) + + await session.flush() + # 建完立刻探查：不探的话 db_query 工具的 description 里没有表清单， + # 模型得先花一步去问"有哪些表" + try: + row.schema_cache = await introspect_mod.introspect(row) + row.schema_synced_at = datetime.now(timezone.utc) + except Exception: # noqa: BLE001 - 探查失败不该让导入白做，用户可以手动再探 + pass + await session.commit() + await session.refresh(row) + + return { + "source": _to_out(row).model_dump(), + "replaced": existing is not None, + # 推断出的列名和类型原样回显：表头行取错了（比如文件前两行是标题）， + # 这里当场就能看出来——列名会变成一行数据 + "tables": [ + { + "name": t.name, "sheet": t.source_name, "rows": t.rows, + "columns": [ + {"name": c, "type": ty} for c, ty in zip(t.columns, t.types) + ], + } + for t in report.tables + ], + } diff --git a/backend/app/api/knowledge.py b/backend/app/api/knowledge.py index b5dd711c..5914d97a 100644 --- a/backend/app/api/knowledge.py +++ b/backend/app/api/knowledge.py @@ -339,6 +339,51 @@ async def reindex( return out +#: 单段返回多少字。切块本来就在千字量级，详情页是给人扫一眼"切得对不对"的， +#: 不是给人通读原文的——整篇几千段全量吐出去，一份大文档能有几兆 +_CHUNK_PREVIEW = 1200 + + +@kb_router.get("/documents/{doc_id}") +async def get_document( + doc_id: str, session: AsyncSession = Depends(get_session) +) -> dict[str, Any]: + """一份文档被切成了什么样。 + + 切块这一层做了不少事——按行边界留重叠、跨块把表头续上（memory/kb.py 的 + chunk_text），而在此之前结果在界面上一处都看不见。检索不准的时候第一个 + 该看的就是它：表头有没有续上、重叠对不对、哪一段被截断了。 + """ + from app.db.models import Chunk + + doc = await session.get(Document, doc_id) + if not doc: + raise HTTPException(404, "文档不存在") + + rows = list((await session.execute( + select(Chunk).where(Chunk.document_id == doc_id).order_by(Chunk.ordinal) + )).scalars()) + + return { + "document": DocumentOut.model_validate(doc).model_dump(mode="json"), + "chunks": [ + { + "id": c.id, + "ordinal": c.ordinal, + "content": c.content[:_CHUNK_PREVIEW], + "truncated": len(c.content) > _CHUNK_PREVIEW, + "chars": len(c.content), + "token_len": c.token_len, + # 哪条向量是谁建的。一份文档里混着两个模型建的向量是真会发生的 + # （换模型之后只重建了一部分），而那会让检索悄悄退回关键词 + "embed_model": c.embed_model or "", + "has_vector": c.embedding is not None, + } + for c in rows + ], + } + + @kb_router.delete("/documents/{doc_id}", status_code=204) async def delete_document(doc_id: str, session: AsyncSession = Depends(get_session)) -> None: if not await kb.delete_document(session, doc_id): diff --git a/backend/app/data/tabular.py b/backend/app/data/tabular.py new file mode 100644 index 00000000..ae68262f --- /dev/null +++ b/backend/app/data/tabular.py @@ -0,0 +1,343 @@ +"""把 Excel / CSV 变成一个可以用 SQL 查的 SQLite 库。 + +**为什么不走知识库那条路。** 这个项目有一条不可回退的决策：叙述层无算术权限， +所有算术下沉到 SQL 或口径卡，叙述只能引用（见 engine/issuance.py）。表格数据 +传进知识库只能被切块检索，于是数字是模型从片段里"读"出来的——复核层 +（engine/review.py）拦得住凭空多出的数字，拦不住"从一堆片段里读错了一格"。 +所以表格必须变成表。 + +**为什么落成 SQLite 而不是新造一种数据源。** kind='sqlite' 早就支持， +engine.build_url 拿 database 当文件路径。落成 .db 文件再建一条 DataSource， +下游一行都不用改：SQL 守卫、结构探查、db_query__ 工具、查询快照进 +工件库、Copilot 的数据源清单，全都白拿。 + +**建表必须用这里的直连，不能走 data.engine.run_query。** guard 的 +_ALWAYS_DENIED 里有 create，那是对的——模型不该能建表。而导入是我们自己 +发起的可信操作，两件事不该共用一条通道。 +""" + +from __future__ import annotations + +import csv +import io +import re +import sqlite3 +from dataclasses import dataclass, field +from typing import Any + +#: 扫多少行来推断列类型。全表扫一遍对几十万行的文件太慢，而类型这种东西 +#: 前几千行看不出来的，后面多半也看不出来——真看错了，SQLite 的动态类型 +#: 也还能把值原样存下去，不会丢数据 +_TYPE_SAMPLE = 5000 + +_CSV = {".csv", ".tsv"} +_EXCEL = {".xlsx", ".xlsm"} +_LEGACY_EXCEL = {".xls"} + + +class UnsupportedTable(ValueError): + """认不出或读不了。message 直接给用户看。""" + + +@dataclass +class LoadedTable: + name: str + columns: list[str] + types: list[str] + rows: int + #: 原始 sheet 名。清洗成表名之后，用户要能对上是哪一张 + source_name: str = "" + + +@dataclass +class LoadReport: + tables: list[LoadedTable] = field(default_factory=list) + + @property + def total_rows(self) -> int: + return sum(t.rows for t in self.tables) + + +def _ext(filename: str) -> str: + idx = filename.rfind(".") + return filename[idx:].lower() if idx >= 0 else "" + + +# -------------------------------------------------------------------------- +# 表名 / 列名 +# -------------------------------------------------------------------------- + + +def _table_name(raw: str, taken: set[str]) -> str: + """sheet 名 → 表名。中文原样保留，只替换真正会断句的字符。 + + 一开始清洗成了纯 ASCII，结果「明细」「汇总」两张表都变成 sheet、sheet_2—— + 模型和用户都认不出哪张是哪张。而 SQLite 的标识符本来就允许非 ASCII， + `FROM 明细` 不加引号也解析得了，所以那道清洗纯属自找麻烦。 + + 真需要处理的只有空白和标点：`销售 明细` 不引号就断成两个 token。 + 正则的词字符类按 Unicode 判定，汉字算词字符，正好留下。 + """ + slug = re.sub(r"[^\w]+", "_", (raw or "").strip(), flags=re.UNICODE).strip("_") + if not slug: + slug = "sheet" + elif slug[0].isdigit(): + slug = f"t_{slug}" # 标识符不能数字开头 + base, n = slug[:48], 2 + slug = base + while slug in taken: + slug = f"{base}_{n}" + n += 1 + taken.add(slug) + return slug + + +def _clean_headers(raw_row: list[Any]) -> list[str]: + """表头单元格 → 列名。**保持原样**，只治两种病态。 + + 不转写、不 slug 化：Excel 表头几乎都是中文，转写是有损的，而 SQLite 认 + 带引号的标识符，db_schema__ 又会把准确列名交给模型。 + """ + out: list[str] = [] + seen: dict[str, int] = {} + for i, cell in enumerate(raw_row): + name = str(cell).strip() if cell is not None else "" + # 空表头给个位置名。合并单元格、末尾多出来的空列都会走到这里 + if not name: + name = f"col_{i + 1}" + # 双引号会把带引号的标识符打断，是唯一必须动的字符 + name = name.replace('"', "'") + if name in seen: + seen[name] += 1 + name = f"{name}_{seen[name]}" + else: + seen[name] = 1 + out.append(name) + return out + + +# -------------------------------------------------------------------------- +# 类型推断 +# -------------------------------------------------------------------------- + +_INT_RE = re.compile(r"^[+-]?\d{1,18}$") +_FLOAT_RE = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$") + + +def _norm(value: Any) -> Any: + """单元格 → 要存进去的值。空的一律 NULL。 + + 空字符串和 NULL 必须分清楚：存成 "" 的话，一个本该是数字的列就被一个 + 空单元格拖成了 TEXT，然后 SUM() 静默地不再是你以为的那个 SUM()。 + """ + if value is None: + return None + if isinstance(value, str): + text = value.strip() + return text or None + return value + + +def infer_type(values: list[Any]) -> str: + """一列的 SQLite 类型。 + + SQLite 是动态类型，但列**亲和性**决定比较和排序的行为：整数列存成 TEXT， + ORDER BY 就变成字典序，'10' 排在 '9' 前面；而这正是"Excel 里的数字能不能 + 真的被 SQL 算"的分界线。所以这一步不能省。 + + 日期统一成 TEXT——SQLite 没有日期类型，而 ISO 格式的字典序恰好等于时间序， + 存成 ISO 字符串既能排序也能比较。 + """ + import datetime as _dt + + seen = False + all_int = all_float = True + for value in values: + norm = _norm(value) + if norm is None: + continue + seen = True + if isinstance(norm, bool): + # bool 是 int 的子类，但"真/假"不是数量，不该被当成 INTEGER 去求和 + return "TEXT" + if isinstance(norm, (_dt.datetime, _dt.date, _dt.time)): + return "TEXT" + if isinstance(norm, int): + all_float = all_float and True + continue + if isinstance(norm, float): + all_int = False + continue + text = str(norm) + if all_int and not _INT_RE.match(text): + all_int = False + if all_float and not _FLOAT_RE.match(text): + all_float = False + if not all_float: + return "TEXT" + if not seen: + return "TEXT" # 整列都是空的，别猜 + return "INTEGER" if all_int else ("REAL" if all_float else "TEXT") + + +def _coerce(value: Any, sql_type: str) -> Any: + """按推断出的类型把值转过去。转不动就原样存——宁可类型不齐，不可丢数据。""" + import datetime as _dt + + norm = _norm(value) + if norm is None: + return None + if isinstance(norm, (_dt.datetime, _dt.date, _dt.time)): + return norm.isoformat() + if sql_type == "INTEGER": + try: + return int(str(norm)) + except (TypeError, ValueError): + return str(norm) + if sql_type == "REAL": + try: + return float(str(norm)) + except (TypeError, ValueError): + return str(norm) + return norm if isinstance(norm, (int, float)) else str(norm) + + +# -------------------------------------------------------------------------- +# 读文件 +# -------------------------------------------------------------------------- + +#: CSV 的编码。中文环境里 GBK 系的 CSV 极其常见（Excel 另存为 CSV 的默认行为）， +#: 一律按 UTF-8 读会得到一堆乱码列名——而乱码是不报错的，只是搜不到、看不懂。 +#: 不引入 chardet：按真实场景的频次逐个试就够了，最后 latin-1 保证不抛异常。 +_ENCODINGS = ("utf-8-sig", "utf-8", "gb18030", "latin-1") + + +def _decode_csv(raw: bytes) -> str: + for enc in _ENCODINGS: + try: + return raw.decode(enc) + except UnicodeDecodeError: + continue + return raw.decode("latin-1", errors="replace") + + +def _read_csv(raw: bytes, filename: str, *, header_row: int) -> list[tuple[str, list[list[Any]]]]: + text = _decode_csv(raw) + delimiter = "\t" if _ext(filename) == ".tsv" else None + if delimiter is None: + try: + delimiter = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|").delimiter + except csv.Error: + delimiter = "," # 嗅不出来就按最常见的来 + rows = [list(r) for r in csv.reader(io.StringIO(text), delimiter=delimiter)] + stem = filename.rsplit("/", 1)[-1].rsplit(".", 1)[0] + return [(stem, rows[header_row - 1:])] + + +def _read_excel(raw: bytes, *, header_row: int) -> list[tuple[str, list[list[Any]]]]: + try: + from openpyxl import load_workbook + except ImportError as e: + raise UnsupportedTable( + "要读 Excel 得先装解析库：pip install 'agentlab-backend[docs]'" + ) from e + try: + # read_only 走流式，几十万行也不会把整个工作簿读进内存； + # data_only 取公式算出来的值而不是公式本身——用户要的是数 + book = load_workbook(io.BytesIO(raw), read_only=True, data_only=True) + except Exception as e: # noqa: BLE001 + raise UnsupportedTable(f"这个 Excel 读不开：{type(e).__name__}: {e}") from e + + out: list[tuple[str, list[list[Any]]]] = [] + for sheet in book.worksheets: + rows = [list(r) for r in sheet.iter_rows(min_row=header_row, values_only=True)] + # 整张空的 sheet 就跳过，不建空表。Excel 里常挂着几张没删的空白页 + if any(any(_norm(c) is not None for c in row) for row in rows): + out.append((sheet.title, rows)) + book.close() + if not out: + raise UnsupportedTable("这个 Excel 里没有任何有内容的工作表") + return out + + +def read_tables( + raw: bytes, filename: str, *, header_row: int = 1 +) -> list[tuple[str, list[list[Any]]]]: + """文件字节 → [(sheet 名, 行)]。第一行是表头。""" + if header_row < 1: + raise UnsupportedTable("表头行号从 1 开始") + ext = _ext(filename) + if ext in _EXCEL: + return _read_excel(raw, header_row=header_row) + if ext in _CSV: + return _read_csv(raw, filename, header_row=header_row) + if ext in _LEGACY_EXCEL: + raise UnsupportedTable( + f"读不了 {filename}：.xls 是老的二进制格式，需要外部转换器。" + "请在 Excel 里另存为 .xlsx 或 .csv 再传。" + ) + raise UnsupportedTable( + f"读不了 {filename}：只认 Excel（.xlsx）和 CSV / TSV。" + ) + + +# -------------------------------------------------------------------------- +# 建库 +# -------------------------------------------------------------------------- + + +def load_into( + db_path: str, raw: bytes, filename: str, *, header_row: int = 1 +) -> LoadReport: + """把文件里的每张表写进 db_path 指的 SQLite 库。 + + 同名表**整张替换**（DROP + CREATE）。重传的语义是"这份数据更新了"， + 追加会让行数悄悄翻倍，而没有任何地方会提示。 + + 用 sqlite3 直连而不是 data.engine：那条路上有 SQL 守卫，而守卫拒绝 + CREATE——那是对的，模型不该能建表。导入是我们自己发起的可信操作。 + """ + sheets = read_tables(raw, filename, header_row=header_row) + report = LoadReport() + taken: set[str] = set() + + conn = sqlite3.connect(db_path) + try: + for sheet_name, rows in sheets: + if not rows: + continue + headers = _clean_headers(rows[0]) + body = rows[1:] + if not headers: + continue + + # 行长参差是常态：Excel 的末尾空列、CSV 的缺尾逗号。按表头补齐/截断 + width = len(headers) + body = [(row + [None] * width)[:width] for row in body] + + sample = body[:_TYPE_SAMPLE] + types = [infer_type([row[i] for row in sample]) for i in range(width)] + + table = _table_name(sheet_name, taken) + cols = ", ".join(f'"{h}" {t}' for h, t in zip(headers, types)) + conn.execute(f'DROP TABLE IF EXISTS "{table}"') + conn.execute(f'CREATE TABLE "{table}" ({cols})') + if body: + placeholders = ", ".join("?" * width) + conn.executemany( + f'INSERT INTO "{table}" VALUES ({placeholders})', + ([_coerce(row[i], types[i]) for i in range(width)] for row in body), + ) + report.tables.append( + LoadedTable( + name=table, columns=headers, types=types, + rows=len(body), source_name=sheet_name, + ) + ) + conn.commit() + finally: + conn.close() + + if not report.tables: + raise UnsupportedTable("文件里没有读到任何数据行") + return report diff --git a/backend/app/memory/parsing.py b/backend/app/memory/parsing.py index d6d85e1c..195871d9 100644 --- a/backend/app/memory/parsing.py +++ b/backend/app/memory/parsing.py @@ -18,8 +18,19 @@ import re #: 认得出的扩展名 → 处理方式。mime 不可靠（浏览器给 .md 报 application/octet-stream） _PDF = {".pdf"} _DOCX = {".docx"} +_PPTX = {".pptx"} _HTML = {".html", ".htm"} +#: 表格数据。它们有更好的去处——数据源那条路会把它变成可以真查的表， +#: 而传进知识库只能当文本检索：数字是模型"读"出来的不是"算"出来的， +#: 而这个项目的地基恰恰是"所有算术下沉到 SQL 或口径卡"（见 engine/issuance.py） +_TABULAR = {".xlsx", ".xls", ".csv", ".tsv"} + +#: 老的二进制 Office 格式。解析它们要装 libreoffice 之类的外部转换器，代价和 +#: 收益不成比例——但认出来并说清楚"另存为新格式"，比让用户对着 ImportError +#: 或者"不是 UTF-8 文本"发愣强得多 +_LEGACY_OFFICE = {".ppt": "PowerPoint", ".doc": "Word", ".xls": "Excel"} + class UnsupportedDocument(ValueError): """认不出或解析不了。message 直接给用户看。""" @@ -91,6 +102,47 @@ def _from_docx(raw: bytes) -> str: return text +def _from_pptx(raw: bytes) -> str: + try: + from pptx import Presentation + except ImportError as e: + raise UnsupportedDocument( + "要读 PowerPoint 得先装解析库：pip install 'agentlab-backend[docs]'" + ) from e + try: + deck = Presentation(io.BytesIO(raw)) + except Exception as e: # noqa: BLE001 + raise UnsupportedDocument(f"这个 PowerPoint 读不开：{type(e).__name__}: {e}") from e + + slides: list[str] = [] + for page in deck.slides: + parts: list[str] = [] + for shape in page.shapes: + # 表格和文本框要分开取：一张幻灯片的关键信息常常整个在表里， + # 而 has_text_frame 对表格是 False + if getattr(shape, "has_table", False): + for row in shape.table.rows: + cells = [c.text.strip() for c in row.cells if c.text.strip()] + if cells: + parts.append(" | ".join(cells)) + elif getattr(shape, "has_text_frame", False): + text = shape.text_frame.text.strip() + if text: + parts.append(text) + if parts: + slides.append("\n".join(parts)) + + # 幻灯片之间留空行。和 PDF 按页拼是同一个道理：页/片的边界本来就是 + # 天然的段落边界，而切块是按段落切的 + text = "\n\n".join(slides) + if not text.strip(): + raise UnsupportedDocument( + "这个 PowerPoint 里没有可提取的文字——多半整页都是图片。" + "图里的字要先过 OCR，这里不做图像识别。" + ) + return text + + def _from_html(raw: bytes) -> str: from bs4 import BeautifulSoup @@ -109,12 +161,27 @@ def extract(raw: bytes, filename: str, mime: str = "") -> str: return _from_pdf(raw) if ext in _DOCX or "wordprocessingml" in mime: return _from_docx(raw) + if ext in _PPTX or "presentationml" in mime: + return _from_pptx(raw) if ext in _HTML or mime == "text/html": return _from_html(raw) + if ext in _TABULAR: + raise UnsupportedDocument( + f"{filename} 是表格数据，请传到「设置 → 数据源」那里——" + "它会变成一张可以用 SQL 查的表，数字是算出来的、能追溯到哪条查询。" + "传进知识库只能当文本检索，模型只是把数字读出来，容易读错也查不出来源。" + ) + if ext in _LEGACY_OFFICE: + # 认出来再拒，而不是让它掉进下面那个"不是 UTF-8"的兜底—— + # 后者对着一个 .ppt 说"不是文本"，用户根本不知道该怎么办 + raise UnsupportedDocument( + f"读不了 {filename}：{_LEGACY_OFFICE[ext]} 的老二进制格式（{ext}）需要" + f"外部转换器才能解析。请在 Office 里另存为 {ext}x 再传。" + ) try: return raw.decode("utf-8") except UnicodeDecodeError as e: raise UnsupportedDocument( f"读不了 {filename or '这个文件'}：它不是 UTF-8 文本，" - f"也不是认得出的 PDF / Word / HTML。" + f"也不是认得出的 PDF / Word / PowerPoint / HTML。" ) from e diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 144adca7..edf3f9a3 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -35,7 +35,7 @@ microvm = ["microsandbox>=0.6"] dev = ["pytest>=8.0", "pytest-asyncio>=0.24"] # 文档解析。PDF 和 Word 的解析库加起来几十兆，只用纯文本的人不该为此 # 付出安装时间——和 microvm、db 一样做成可选。HTML 用已在依赖里的 bs4。 -docs = ["pypdf>=5.0", "python-docx>=1.1"] +docs = ["pypdf>=5.0", "python-docx>=1.1", "python-pptx>=1.0", "openpyxl>=3.1"] # 数据库接入。三个驱动都是 asyncio 原生且免装数据库客户端—— # oracledb 走 thin 模式，省掉了 Oracle Instant Client 这个历来最劝退的一步。 db = ["aiomysql>=0.2", "oracledb>=2.0", "asyncpg>=0.30"] diff --git a/backend/tests/test_tabular.py b/backend/tests/test_tabular.py new file mode 100644 index 00000000..29a034d5 --- /dev/null +++ b/backend/tests/test_tabular.py @@ -0,0 +1,284 @@ +"""Excel / CSV → 可以用 SQL 查的表。 + +这块存在的理由不是"多支持一种格式"。这个项目有一条不可回退的决策：叙述层 +无算术权限，所有算术下沉到 SQL 或口径卡（engine/issuance.py）。表格传进知识库 +只能被切块检索，数字就成了模型从片段里"读"出来的——复核层拦得住凭空多出的 +数字，拦不住"从一堆片段里读错了一格"。 + +所以这里最要紧的一条不是"能不能导进去"，是**导进去的数字能不能真的被 SQL 算**： +一个整数列如果落成 TEXT，ORDER BY 就变成字典序（'10' < '9'），而没有任何 +地方会报错。 +""" + +from __future__ import annotations + +import io +import sqlite3 + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data.tabular import UnsupportedTable, infer_type, load_into +from app.main import app + + +@pytest.fixture +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as c: + yield c + + +def xlsx(sheets: dict[str, list[list]]) -> bytes: + from openpyxl import Workbook + + book = Workbook() + book.remove(book.active) + for title, rows in sheets.items(): + sheet = book.create_sheet(title) + for row in rows: + sheet.append(row) + buf = io.BytesIO() + book.save(buf) + return buf.getvalue() + + +def query(db, sql): + conn = sqlite3.connect(str(db)) + try: + return conn.execute(sql).fetchall() + finally: + conn.close() + + +def col_types(db, table): + return {r[1]: r[2] for r in query(db, f'PRAGMA table_info("{table}")')} + + +# -------------------------------------------------------------------------- +# 类型推断：这块的全部意义 +# -------------------------------------------------------------------------- + + +def test_infer_type_basics(): + assert infer_type([1, 2, 3]) == "INTEGER" + assert infer_type(["1", "2", "3"]) == "INTEGER" # CSV 读出来全是字符串 + assert infer_type([1, 2.5]) == "REAL" + assert infer_type(["1.5", "2"]) == "REAL" + assert infer_type([1, "两个"]) == "TEXT" + assert infer_type([]) == "TEXT" # 整列空的，别猜 + assert infer_type([None, " ", None]) == "TEXT" + # 空单元格不该把一个数字列拖成 TEXT——那正是 SUM() 静默出错的起点 + assert infer_type([1, None, 3]) == "INTEGER" + assert infer_type([1, "", 3]) == "INTEGER" + # 真/假不是数量，不该被当成 INTEGER 去求和 + assert infer_type([True, False]) == "TEXT" + + +def test_numbers_sort_as_numbers_not_as_text(tmp_path): + """整块设计的落点：ORDER BY 一个数字列要是 9 < 10，不是 '10' < '9'。 + + 列落成 TEXT 的话这条会反过来，而且不报任何错——只是从此以后每一次 + 排序、每一次 MAX()、每一次区间筛选都悄悄错着。 + """ + db = tmp_path / "t.db" + load_into(str(db), xlsx({"数据": [["名称", "销量"], ["甲", 9], ["乙", 10], ["丙", 100]]}), "a.xlsx") + + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + assert col_types(db, table)["销量"] == "INTEGER" + assert [r[0] for r in query(db, f'SELECT "名称" FROM "{table}" ORDER BY "销量"')] == \ + ["甲", "乙", "丙"] + assert query(db, f'SELECT SUM("销量") FROM "{table}"')[0][0] == 119 + + +def test_empty_cells_become_null_not_empty_string(tmp_path): + db = tmp_path / "t.db" + load_into(str(db), xlsx({"s": [["a", "b"], [1, None], [2, " "]]}), "a.xlsx") + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + assert query(db, f'SELECT COUNT("b") FROM "{table}"')[0][0] == 0 # COUNT 不数 NULL + assert query(db, f'SELECT SUM("a") FROM "{table}"')[0][0] == 3 + + +# -------------------------------------------------------------------------- +# 表 / 列 +# -------------------------------------------------------------------------- + + +def test_multiple_sheets_become_multiple_tables(tmp_path): + db = tmp_path / "t.db" + report = load_into(str(db), xlsx({ + "销售明细": [["月份", "金额"], ["1月", 100]], + "Summary": [["k", "v"], ["总计", 100]], + }), "a.xlsx") + assert len(report.tables) == 2 + # 中文 sheet 名原样留着。清洗成 ASCII 的话「明细」「汇总」都会变成 + # sheet、sheet_2——模型和用户都认不出哪张是哪张 + assert {t.name for t in report.tables} == {"销售明细", "Summary"} + assert query(db, 'SELECT "金额" FROM 销售明细')[0][0] == 100 # 不加引号也查得了 + + +def test_chinese_columns_are_kept_verbatim(tmp_path): + """列名保持原样。转写是有损的，而 SQLite 认带引号的标识符。""" + db = tmp_path / "t.db" + load_into(str(db), xlsx({"s": [["本月销量 (万元)", "x"], [12, "a"]]}), "a.xlsx") + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + assert "本月销量 (万元)" in col_types(db, table) + assert query(db, f'SELECT "本月销量 (万元)" FROM "{table}"')[0][0] == 12 + + +def test_broken_headers_get_usable_names(tmp_path): + db = tmp_path / "t.db" + load_into(str(db), xlsx({"s": [["名称", None, "名称"], ["a", "b", "c"]]}), "a.xlsx") + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + cols = list(col_types(db, table)) + assert cols == ["名称", "col_2", "名称_2"], cols + + +def test_header_row_can_be_moved(tmp_path): + """不猜表头行。猜错的表现是列名变成一行数据，而且没有任何征兆。""" + db = tmp_path / "t.db" + sheets = {"s": [["2026 年销售报表", None], ["月份", "金额"], ["1月", 100]]} + + load_into(str(db), xlsx(sheets), "a.xlsx", header_row=2) + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + assert list(col_types(db, table)) == ["月份", "金额"] + + # 取默认第 1 行就会把标题当表头——这正是要让用户当场看见的那种错 + load_into(str(db), xlsx(sheets), "a.xlsx") + assert "2026 年销售报表" in col_types(db, table) + + +def test_ragged_rows_are_padded(tmp_path): + db = tmp_path / "t.db" + load_into(str(db), xlsx({"s": [["a", "b", "c"], [1], [1, 2, 3, 4]]}), "a.xlsx") + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + assert query(db, f'SELECT COUNT(*) FROM "{table}"')[0][0] == 2 + + +# -------------------------------------------------------------------------- +# CSV +# -------------------------------------------------------------------------- + + +def test_csv_with_gbk_encoding(tmp_path): + """中文环境里 GBK 的 CSV 极常见（Excel 另存为 CSV 的默认行为）。 + + 一律按 UTF-8 读会得到乱码列名，而乱码是不报错的——只是从此搜不到、看不懂。 + """ + db = tmp_path / "t.db" + raw = "月份,金额\n1月,100\n".encode("gb18030") + load_into(str(db), raw, "销售.csv") + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + assert "月份" in col_types(db, table) + assert query(db, f'SELECT "金额" FROM "{table}"')[0][0] == 100 + + +def test_tsv_and_semicolon_csv(tmp_path): + db = tmp_path / "t.db" + load_into(str(db), b"a\tb\n1\t2\n", "x.tsv") + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + assert list(col_types(db, table)) == ["a", "b"] + + +def test_unsupported_formats_say_what_to_do(tmp_path): + db = tmp_path / "t.db" + with pytest.raises(UnsupportedTable, match="另存为"): + load_into(str(db), b"x", "old.xls") + with pytest.raises(UnsupportedTable, match="只认"): + load_into(str(db), b"x", "a.pdf") + + +# -------------------------------------------------------------------------- +# 重传 +# -------------------------------------------------------------------------- + + +def test_reupload_replaces_instead_of_appending(tmp_path): + """重传的语义是"这份数据更新了"。追加会让行数悄悄翻倍，没人会发现。""" + db = tmp_path / "t.db" + load_into(str(db), xlsx({"s": [["a"], [1], [2]]}), "a.xlsx") + load_into(str(db), xlsx({"s": [["a"], [9]]}), "a.xlsx") + table = query(db, "SELECT name FROM sqlite_master WHERE type='table'")[0][0] + assert [r[0] for r in query(db, f'SELECT "a" FROM "{table}"')] == [9] + + +# -------------------------------------------------------------------------- +# 接口 +# -------------------------------------------------------------------------- + + +async def _upload(client, name, content, filename="a.xlsx", **form): + return await client.post( + "/api/datasources/upload", + files={"file": (filename, content, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")}, + data={"name": name, **form}, + ) + + +@pytest.mark.asyncio +async def test_upload_creates_a_queryable_source(client): + resp = await _upload( + client, "tab_sales", + xlsx({"明细": [["月份", "金额"], ["1月", 100], ["2月", 250]]}), + ) + assert resp.status_code == 201, resp.text + body = resp.json() + assert body["source"]["kind"] == "sqlite" + assert body["replaced"] is False + # 列名和类型要原样回显：表头行取错了，用户当场就该看见 + cols = {c["name"]: c["type"] for c in body["tables"][0]["columns"]} + assert cols == {"月份": "TEXT", "金额": "INTEGER"} + assert body["tables"][0]["rows"] == 2 + # 建完立刻探查过，否则 db_query 工具的描述里没有表清单 + assert body["source"]["table_count"] >= 1 + + # 真能查——而且走的是和别的数据源完全相同的那条路 + from app.data.engine import run_query + from app.db.base import SessionLocal + from app.db.models import DataSource + from sqlalchemy import select as _select + + async with SessionLocal() as session: + row = (await session.execute( + _select(DataSource).where(DataSource.name == "tab_sales") + )).scalar_one() + result = await run_query(row, 'SELECT SUM("金额") AS s FROM "明细"') + assert result.rows[0][0] == 350 + + +@pytest.mark.asyncio +async def test_reupload_keeps_the_same_source_and_name(client): + """重传不能换 id 或名字——名字是工具名（db_query__x），写进了保存过的图。""" + first = await _upload(client, "tab_keep", xlsx({"s": [["a"], [1]]})) + assert first.status_code == 201 + src_id = first.json()["source"]["id"] + + again = await _upload(client, "tab_keep", xlsx({"s": [["a"], [7], [8]]})) + assert again.status_code == 201, again.text + assert again.json()["replaced"] is True + assert again.json()["source"]["id"] == src_id + assert again.json()["source"]["name"] == "tab_keep" + assert again.json()["tables"][0]["rows"] == 2 + + +@pytest.mark.asyncio +async def test_upload_rejects_a_name_that_cannot_be_a_tool(client): + resp = await _upload(client, "带中文的名字", xlsx({"s": [["a"], [1]]})) + assert resp.status_code == 400 + assert "工具名" in resp.json()["detail"] + + +@pytest.mark.asyncio +async def test_upload_will_not_hijack_a_real_database(client): + """同名的真数据库不能被一个 Excel 顶掉。""" + from app.db.base import SessionLocal + from app.db.models import DataSource + + async with SessionLocal() as session: + session.add(DataSource(name="tab_real", kind="mysql", host="h", database="d")) + await session.commit() + + resp = await _upload(client, "tab_real", xlsx({"s": [["a"], [1]]})) + assert resp.status_code == 409 + assert "换个名字" in resp.json()["detail"] diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 3c50f32f..6ea70195 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -90,8 +90,10 @@ export default function App() { } /> } /> } /> + } /> } /> } /> + } /> } /> } /> diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index d3512674..3fa45591 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -88,6 +88,19 @@ export const api = { update: (id: string, body: any) => patch(`/datasources/${id}`, body), remove: (id: string) => del(`/datasources/${id}`), test: (id: string) => post(`/datasources/${id}/test`, {}), + /** 上传 Excel / CSV，变成一个可以用 SQL 查的数据源。同名就地替换 */ + uploadTable: (file: File, body: { name: string; description?: string; header_row?: number }) => { + const form = new FormData() + form.append('file', file) + form.append('name', body.name) + form.append('description', body.description ?? '') + form.append('header_row', String(body.header_row ?? 1)) + return request<{ + source: any; replaced: boolean + tables: { name: string; sheet: string; rows: number + columns: { name: string; type: string }[] }[] + }>('/datasources/upload', { method: 'POST', body: form }) + }, introspect: (id: string, schema?: string) => post(`/datasources/${id}/introspect${schema ? `?schema=${encodeURIComponent(schema)}` : ''}`, {}), schema: (id: string, table?: string) => @@ -193,6 +206,14 @@ export const api = { }, search: (q: string, collection?: string, alpha = 0.5) => get(`/kb/search?q=${encodeURIComponent(q)}&alpha=${alpha}${collection ? `&collection=${collection}` : ''}`), + /** 一份文档被切成了什么样。检索不准时第一个该看的就是它 */ + document: (id: string) => get<{ + document: KbDocument + chunks: { + id: string; ordinal: number; content: string; truncated: boolean + chars: number; token_len: number; embed_model: string; has_vector: boolean + }[] + }>(`/kb/documents/${id}`), remove: (id: string) => del(`/kb/documents/${id}`), embedding: (collection?: string) => get<{ embedder: string; dim: number; configured: boolean diff --git a/frontend/src/pages/ChatPage.tsx b/frontend/src/pages/ChatPage.tsx index 168b0820..c8013e05 100644 --- a/frontend/src/pages/ChatPage.tsx +++ b/frontend/src/pages/ChatPage.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { useNavigate, useParams } from 'react-router-dom' -import { Database, ListTree, MessageSquare, Play, Send, Square } from 'lucide-react' +import { Database, ListTree, MessageSquare, Play, RotateCw, Send, Square } from 'lucide-react' import { api } from '../api/client' import { ApprovalCard } from '../run/RunPanel' import { AssistantStream, StreamEmpty, type StreamTurn } from '../run/AssistantStream' @@ -26,7 +26,7 @@ import { useToast } from '../components/ui' * dense 一个开关。两边各写一套的话，同一次运行在两个页面会讲出不同的故事。 */ export function ChatPage() { - const { busy, ask, stop, load, loadSteps, runNow, turnsOf } = useChat() + const { busy, ask, stop, load, loadSteps, runNow, continueTurn, turnsOf } = useChat() const { conversationId } = useParams() const navigate = useNavigate() const currentId = useConversations((s) => s.currentId) @@ -110,6 +110,13 @@ export function ChatPage() { ? () => void loadSteps(currentId, t.id) : undefined } + onContinue={ + // 只在"跑到一半挂了"时出现：建图阶段就失败的那些没有 run， + // 没有断点可续，给个按钮只会让人白点一次 + currentId && t.phase === 'error' && t.runId + ? () => continueTurn(currentId, t.id) + : undefined + } onRun={ currentId && turnsOf(currentId).find((x) => x.id === t.id)?.pendingRun ? () => runNow(currentId, t.id) @@ -200,20 +207,31 @@ function toStreamTurn(turn: ChatTurn): StreamTurn { * 从库里恢复的轮次没有事件流——一次会话几十轮，把每轮的事件都预加载回来 * 是几十个请求换一堆没人看的步骤。所以默认只显示问题和答案，想看过程再取。 */ -function Approvals({ turnId, runId, onShowSteps, onRun }: { +function Approvals({ turnId, runId, onShowSteps, onRun, onContinue }: { turnId: string runId?: string onShowSteps?: () => void onRun?: () => void + onContinue?: () => void | Promise }) { const conversationId = useConversations((s) => s.currentId) const approvals = useCatalog((s) => s.approvals) const reattach = useChat((s) => s.reattach) + const busy = useChat((s) => s.busy) const pending = approvals.filter((a) => a.run_id === runId && a.status === 'pending') if (!runId && !onRun) return null return ( <> + {onContinue && ( + // 跑挂了不该只剩一行红字。断点一直在 checkpoint 里躺着，前面查过的表、 + // 跑过的 SQL 都还在——重问一遍等于让它们全部重来 + + )} {onRun && ( + + + } + > +
+
+ + pick(e.target.files?.[0] ?? null)} /> +

+ Excel（.xlsx）或 CSV / TSV。每个工作表变成一张表。 + .xls 和 .ppt 一样是老的二进制格式，请先另存为 .xlsx +

+
+ +
+
+ + setName(e.target.value)} /> +

+ 会成为工具名的一部分（db_query__{name || 'sales'}），只能用小写字母数字下划线 +

+
+
+ + setHeaderRow(Math.max(1, Number(e.target.value) || 1))} /> +
+
+ +
+ + setDescription(e.target.value)} /> +
+ + {/* 推断出的列名和类型要当场回显。表头行取错了（比如文件前两行是标题）， + 列名会变成一行数据——不给看的话，这个错要等到有人发现汇总一直 + 少一行才暴露 */} + {result && ( +
+
+ {result.replaced ? '已替换' : '已导入'} {result.tables.length} 张表 + ——对一眼列名，不对就改上面的表头行号重传 +
+ {result.tables.map((t: any) => ( +
+
+ {t.name} + {t.sheet !== t.name && （{t.sheet}）} + · {t.rows} 行 +
+
+ {t.columns.map((c: any) => ( + + {c.name} + {c.type} + + ))} +
+
+ ))} +
+ )} +
+ + ) +} + function SourceCard({ row, onChanged, onEdit }: { row: any; onChanged: () => Promise; onEdit: () => void }) { diff --git a/frontend/src/pages/KnowledgePage.tsx b/frontend/src/pages/KnowledgePage.tsx index e9e72e3d..09c8cb95 100644 --- a/frontend/src/pages/KnowledgePage.tsx +++ b/frontend/src/pages/KnowledgePage.tsx @@ -1,4 +1,5 @@ import { useEffect, useRef, useState } from 'react' +import { useNavigate, useParams } from 'react-router-dom' import { BookOpen, Brain, Plus, Search, Sparkles, Trash2, Upload } from 'lucide-react' import { api } from '../api/client' import { useCatalog } from '../store/catalog' @@ -196,7 +197,100 @@ function EmbedderBar({ collection, onChanged }: { // ------------------------------------------------------------------------- function KbTab() { + // 地址上带了文档 id 就看那一份的详情。集合是筛选条件不是位置，所以不进 URL + const { id } = useParams() + if (id) return + return +} + + +/** + * 一份文档被切成了什么样。 + * + * 切块这一层做了不少事——按行边界留重叠、跨块把表头续上（后端 memory/kb.py + * 的 chunk_text），而在此之前结果在界面上一处都看不见。检索不准的时候第一个 + * 该看的就是它：表头有没有续上、重叠对不对、哪一段被截断了。 + */ +function DocDetail({ docId }: { docId: string }) { + const navigate = useNavigate() + const [data, setData] = useState(null) + const [error, setError] = useState('') + + useEffect(() => { + let live = true + void api.kb.document(docId) + .then((d) => { if (live) setData(d) }) + .catch((e) => { if (live) setError(e?.message ?? '取不到这份文档') }) + return () => { live = false } + }, [docId]) + + if (error) { + return ( +
+ + } title="这份文档不在了" hint={error} /> +
+ ) + } + if (!data) return
正在取…
+ + const doc = data.document + // 一份文档里混着两个模型建的向量是真会发生的（换模型之后只重建了一部分）， + // 而那会让检索对这部分内容悄悄退回关键词 + const models = [...new Set(data.chunks.map((c: any) => c.embed_model).filter(Boolean))] + const noVector = data.chunks.filter((c: any) => !c.has_vector).length + + return ( +
+ + +
{doc.title}
+
+ {doc.collection} + {doc.source} + {data.chunks.length} 个片段 + {models.map((m: any) => {m})} + {models.length > 1 && ( + + 这份文档里混着两个模型建的向量，对不上当前模型的那些会退回关键词检索 + + )} + {noVector > 0 && ( + {noVector} 段还没有向量 + )} +
+ + {!data.chunks.length && ( + } title="还没有切块" + hint={doc.status === 'processing' ? '正在处理，稍后刷新' : doc.error || '这份文档没有产生任何片段'} /> + )} + +
+ {data.chunks.map((c: any) => ( +
+
+ 片段 {c.ordinal} + {c.chars} 字 + {c.token_len > 0 && {c.token_len} tokens} + {!c.has_vector && 没有向量} + {c.truncated && （这里只显示开头一部分）} +
+
{c.content}
+
+ ))} +
+
+ ) +} + + +function KbList() { const toast = useToast() + const navigate = useNavigate() const refreshCatalog = useCatalog((s) => s.refresh) const [collection, setCollection] = useState('default') const [collections, setCollections] = useState([]) @@ -316,7 +410,9 @@ function KbTab() {
{docs.map((doc) => (
-
+
+