mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 09:41:59 +00:00
7b04162540
四件事,其中 Excel 那条不是"多支持一种格式",是补一个和地基冲突的缺口。
**Excel / CSV → 可以用 SQL 查的表。**
这个项目有条不可回退的决策:叙述层无算术权限,所有算术下沉到 SQL 或口径卡,
叙述只能引用(engine/issuance.py)。表格传进知识库只能被切块检索,数字就成了
模型从片段里"读"出来的——复核层拦得住凭空多出的数字,拦不住"从一堆片段里
读错了一格"。所以表格必须变成表。
不新增数据源类型:kind='sqlite' 早就支持,build_url 拿 database 当文件路径。
落成 .db 文件再建一条 DataSource 指过去,下游一行都不用改——SQL 守卫、结构
探查、db_query__<name> 工具、查询快照进工件库、Copilot 的数据源清单,全都白拿。
建表走 sqlite3 直连而不是 run_query:guard 的 _ALWAYS_DENIED 里有 create,
那是对的(模型不该能建表),而导入是我们自己发起的可信操作。
四个语义是这块的全部分量:
- **类型要推,不能全存 TEXT。** SQLite 是动态类型,但列亲和性决定排序和比较:
整数列落成 TEXT 的话 '10' < '9',SUM() 在混入空串时静默出错,而没有任何
地方会报。逐列扫描推断 INTEGER/REAL/TEXT,空单元格存 NULL 不存 "",
日期存 ISO(字典序恰好等于时间序)。测试里那条 ORDER BY 要是 9<10 而不是
'10'<'9',守的就是整块的意义。
- **表头取第 1 行,但不猜。** 不做"自动找表头"的启发式:猜错的表现是列名变成
一行数据,没有任何征兆,直到某天有人发现汇总一直少一行。改成 header_row
可填,并把推断出的列名和类型当场回显——错了一眼看见。
- **同名重传原地替换,id 和 name 都不变。** 名字会成为工具名(db_query__sales)
而工具名写进了保存过的图,另起一个 sales_2 等于悄悄让那些图失效。
- **中文原样保留。** 一开始把 sheet 名清洗成了 ASCII,结果「明细」「汇总」都变成
sheet、sheet_2,模型和用户都认不出哪张是哪张——而 SQLite 的标识符本来就允许
非 ASCII,那道清洗纯属自找麻烦。现在只替换会断句的空白和标点。
实测:两 sheet 的中文 xlsx → 两张表,金额列 INTEGER,跨表 JOIN + SUM 跑通,
Copilot 的工具清单里带着表名。CSV 按 utf-8-sig / utf-8 / gb18030 依次试——
GBK 的 CSV 是 Excel 另存为的默认行为,按 UTF-8 读会得到乱码列名,而乱码不报错。
知识库那边顺手改了指路:传 .xlsx/.csv 进去不再说"不是 UTF-8 文本",
而是告诉你该传到数据源那边、以及为什么。
**PPT。** 文本框和表格分开取(表格的 has_text_frame 是 False),幻灯片之间
留空行当段落边界。.ppt/.doc/.xls 这些老二进制格式认出来再拒,说清楚"另存为
新格式"——以前它们会掉进"不是 UTF-8 文本"那个兜底,对着一个 .ppt 说这话,
用户根本不知道该怎么办。
**问数据页的「接着跑」。** 挂了那一轮不该只剩一行红字:断点一直在 checkpoint
里躺着。按钮挂在 approvalsFor 那个每轮插槽里(本来就托着「看执行过程」和
「跑一下」)。不带图——问数据页改不了图,这里就是"用原来的配置重试失败的
那一步",配置本身错了要到画布上改。实测:续的是同一次运行(8→13 条事件),
只跑了 lp 往后,轨迹里有 run.resumed。
**文档 / 工具的地址。** /knowledge/kb/:docId 是新页面——文档原来连详情视图都
没有,而切块这层做了不少事(按行边界留重叠、跨块续表头),结果一处都看不见。
现在逐段列出来带字数和 token 数,还标出"这份文档里混着两个模型建的向量"
(换模型只重建了一部分时真会发生,而那会让这部分内容悄悄退回关键词检索)。
新增 GET /kb/documents/{id}。工具那边 tab key 从 tools 改成 library,
否则地址是 /tools/tools/<id>。
验证:后端 265 passed(新增 test_tabular.py 16 条);tsc 干净;三个 check 全过,
check-ui 补了文档/工具两节;四块都在真浏览器/真后端上端到端跑过。
开发库里造的演示数据源、会话和运行已清干净。
318 lines
12 KiB
Python
318 lines
12 KiB
Python
"""数据源的增删改查、连接测试、结构探查。
|
||
|
||
密码的处理规则和 Provider 完全一致(app/api/settings.py):入库加密、
|
||
出站只给掩码、PATCH 时不传表示不动、传空串表示清空。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timezone
|
||
from typing import Any
|
||
|
||
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
||
from pydantic import BaseModel, Field
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.config import settings
|
||
from app.core.crypto import encrypt, mask
|
||
from app.data import introspect as introspect_mod
|
||
from app.data.engine import SUPPORTED_KINDS, engines, test_connection
|
||
from app.db.base import get_session
|
||
from app.db.models import DataSource
|
||
from app.tools.datasource import tool_names
|
||
|
||
router = APIRouter(prefix="/api/datasources", tags=["datasources"])
|
||
|
||
# 名字会成为工具名的一部分(db_query__<name>),而模型的 function name 只认
|
||
# ASCII。所以这里收紧成 slug,中文说明放 description 字段。
|
||
_NAME_PATTERN = r"^[a-z][a-z0-9_]{0,40}$"
|
||
|
||
|
||
class DataSourceIn(BaseModel):
|
||
name: str = Field(pattern=_NAME_PATTERN,
|
||
description="英文标识,会成为工具名的一部分,如 sales")
|
||
kind: str
|
||
host: str | None = None
|
||
port: int | None = None
|
||
database: str | None = None
|
||
username: str | None = None
|
||
password: str | None = None
|
||
options: dict[str, Any] = Field(default_factory=dict)
|
||
readonly: bool = True
|
||
description: str = ""
|
||
enabled: bool = True
|
||
|
||
|
||
class DataSourcePatch(BaseModel):
|
||
kind: str | None = None
|
||
host: str | None = None
|
||
port: int | None = None
|
||
database: str | None = None
|
||
username: str | None = None
|
||
password: str | None = None # 空串=清空;不传=不动
|
||
options: dict[str, Any] | None = None
|
||
readonly: bool | None = None
|
||
description: str | None = None
|
||
enabled: bool | None = None
|
||
|
||
|
||
class DataSourceOut(BaseModel):
|
||
id: str
|
||
name: str
|
||
kind: str
|
||
host: str | None
|
||
port: int | None
|
||
database: str | None
|
||
username: str | None
|
||
options: dict[str, Any]
|
||
readonly: bool
|
||
description: str
|
||
enabled: bool
|
||
password_masked: str = ""
|
||
has_password: bool = False
|
||
table_count: int = 0
|
||
schema_synced_at: str | None = None
|
||
tools: list[str] = Field(default_factory=list)
|
||
|
||
|
||
def _to_out(row: DataSource) -> DataSourceOut:
|
||
tables = (row.schema_cache or {}).get("tables") or {}
|
||
return DataSourceOut(
|
||
id=row.id, name=row.name, kind=row.kind, host=row.host, port=row.port,
|
||
database=row.database, username=row.username, options=row.options or {},
|
||
readonly=row.readonly, description=row.description, enabled=row.enabled,
|
||
password_masked=mask(row.password),
|
||
has_password=bool(row.password),
|
||
table_count=len(tables),
|
||
schema_synced_at=row.schema_synced_at.isoformat() if row.schema_synced_at else None,
|
||
tools=tool_names(row),
|
||
)
|
||
|
||
|
||
async def _get_or_404(session: AsyncSession, source_id: str) -> DataSource:
|
||
row = await session.get(DataSource, source_id)
|
||
if not row:
|
||
raise HTTPException(404, "数据源不存在")
|
||
return row
|
||
|
||
|
||
@router.get("", response_model=list[DataSourceOut])
|
||
async def list_sources(session: AsyncSession = Depends(get_session)) -> list[DataSourceOut]:
|
||
rows = (await session.execute(select(DataSource).order_by(DataSource.name))).scalars()
|
||
return [_to_out(r) for r in rows]
|
||
|
||
|
||
@router.get("/kinds")
|
||
async def list_kinds() -> dict[str, Any]:
|
||
"""支持哪些数据库,以及各自需要填什么——前端表单据此渲染。"""
|
||
return {
|
||
"kinds": [
|
||
{"value": "mysql", "label": "MySQL / MariaDB", "default_port": 3306,
|
||
"needs": ["host", "database", "username", "password"],
|
||
"hint": "options 可填 charset(默认 utf8mb4)"},
|
||
{"value": "postgres", "label": "PostgreSQL", "default_port": 5432,
|
||
"needs": ["host", "database", "username", "password"], "hint": ""},
|
||
{"value": "oracle", "label": "Oracle", "default_port": 1521,
|
||
"needs": ["host", "username", "password"],
|
||
"hint": "库名填 service_name,或在 options 里给 service_name / sid。"
|
||
"驱动走 thin 模式,不需要装 Instant Client。"
|
||
"只读账号名下通常没有对象——数据在别的 schema 里,"
|
||
"用 options.schema 指定(如 ANALYTICS),否则探查结果是空的"},
|
||
{"value": "sqlite", "label": "SQLite(文件)", "default_port": None,
|
||
"needs": ["database"], "hint": "库名填数据库文件的绝对路径"},
|
||
],
|
||
"supported": list(SUPPORTED_KINDS),
|
||
}
|
||
|
||
|
||
@router.post("", response_model=DataSourceOut, status_code=201)
|
||
async def create_source(
|
||
payload: DataSourceIn, session: AsyncSession = Depends(get_session)
|
||
) -> DataSourceOut:
|
||
if payload.kind not in SUPPORTED_KINDS:
|
||
raise HTTPException(400, f"不支持的类型 {payload.kind},支持:{', '.join(SUPPORTED_KINDS)}")
|
||
exists = (await session.execute(
|
||
select(DataSource).where(DataSource.name == payload.name)
|
||
)).scalar_one_or_none()
|
||
if exists:
|
||
raise HTTPException(409, f"已经有叫 {payload.name} 的数据源了")
|
||
|
||
row = DataSource(
|
||
**payload.model_dump(exclude={"password"}),
|
||
password=encrypt(payload.password),
|
||
)
|
||
session.add(row)
|
||
await session.commit()
|
||
await session.refresh(row)
|
||
return _to_out(row)
|
||
|
||
|
||
@router.patch("/{source_id}", response_model=DataSourceOut)
|
||
async def update_source(
|
||
source_id: str, payload: DataSourcePatch, session: AsyncSession = Depends(get_session)
|
||
) -> DataSourceOut:
|
||
row = await _get_or_404(session, source_id)
|
||
data = payload.model_dump(exclude_unset=True)
|
||
if "password" in data:
|
||
# 空串表示清空,非空表示换新的
|
||
data["password"] = encrypt(data["password"]) if data["password"] else None
|
||
for key, value in data.items():
|
||
setattr(row, key, value)
|
||
await session.commit()
|
||
await session.refresh(row)
|
||
# 连接参数可能变了,旧连接池不能再用——否则改完密码还在用旧连接,很难排查
|
||
await engines.invalidate(source_id)
|
||
return _to_out(row)
|
||
|
||
|
||
@router.delete("/{source_id}", status_code=204)
|
||
async def delete_source(source_id: str, session: AsyncSession = Depends(get_session)) -> None:
|
||
row = await _get_or_404(session, source_id)
|
||
await session.delete(row)
|
||
await session.commit()
|
||
await engines.invalidate(source_id)
|
||
|
||
|
||
@router.post("/{source_id}/test")
|
||
async def test_source(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||
"""测连接。失败时把驱动的原始报错带出来——这类问题九成靠错误信息定位。"""
|
||
row = await _get_or_404(session, source_id)
|
||
return await test_connection(row)
|
||
|
||
|
||
@router.post("/{source_id}/introspect", response_model=DataSourceOut)
|
||
async def introspect_source(
|
||
source_id: str, schema: str | None = None, session: AsyncSession = Depends(get_session)
|
||
) -> DataSourceOut:
|
||
"""探查结构并缓存。显式动作,不做后台轮询——生产库不该被实验工具定时扫。"""
|
||
row = await _get_or_404(session, source_id)
|
||
try:
|
||
row.schema_cache = await introspect_mod.introspect(row, schema=schema)
|
||
except Exception as e: # noqa: BLE001
|
||
raise HTTPException(400, f"探查失败:{type(e).__name__}: {e}") from e
|
||
row.schema_synced_at = datetime.now(timezone.utc)
|
||
await session.commit()
|
||
await session.refresh(row)
|
||
return _to_out(row)
|
||
|
||
|
||
@router.get("/{source_id}/schema")
|
||
async def get_schema(source_id: str, table: str | None = None,
|
||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||
row = await _get_or_404(session, source_id)
|
||
if table:
|
||
return {"table": table, "detail": introspect_mod.describe_table(row, table)}
|
||
return {
|
||
"tables": introspect_mod.table_names(row),
|
||
"summary": introspect_mod.summary(row),
|
||
"synced_at": row.schema_synced_at.isoformat() if row.schema_synced_at else None,
|
||
}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 上传表格
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@router.post("/upload", response_model=dict, status_code=201)
|
||
async def upload_table(
|
||
file: UploadFile = File(...),
|
||
name: str = Form(...),
|
||
description: str = Form(""),
|
||
header_row: int = Form(1),
|
||
session: AsyncSession = Depends(get_session),
|
||
) -> dict[str, Any]:
|
||
"""把 Excel / CSV 变成一个可以用 SQL 查的数据源。
|
||
|
||
走 SQLite:落成一个 .db 文件,再建一条 kind='sqlite' 的数据源指过去。
|
||
下游一行都不用改——SQL 守卫、结构探查、db_query__<name> 工具、查询快照进
|
||
工件库、Copilot 的数据源清单,全都白拿。
|
||
|
||
这条路存在的理由不是"多支持一种格式":表格传进知识库只能被切块检索,
|
||
数字就成了模型从片段里读出来的,而这个项目的地基是"所有算术下沉到
|
||
SQL 或口径卡"。表格必须变成表。
|
||
|
||
**同名就地替换**,不新建。数据源名字会成为工具名(db_query__sales),
|
||
而工具名写进了保存过的工作流——重传时另起一个 sales_2 等于悄悄让那些图失效。
|
||
"""
|
||
import re as _re
|
||
|
||
if not _re.match(_NAME_PATTERN, name or ""):
|
||
raise HTTPException(
|
||
400, "名字只能用小写字母开头的字母数字下划线——它会成为工具名的一部分"
|
||
)
|
||
|
||
# 边读边数,超了立刻停。和知识库上传同一套:那个检查拦的是"入库",
|
||
# 不是"占内存",读完再判等于先把内存吃掉
|
||
limit = settings.max_upload_mb * 1024 * 1024
|
||
pieces: list[bytes] = []
|
||
total = 0
|
||
while piece := await file.read(1024 * 1024):
|
||
total += len(piece)
|
||
if total > limit:
|
||
raise HTTPException(413, f"文件超过 {settings.max_upload_mb}MB")
|
||
pieces.append(piece)
|
||
raw = b"".join(pieces)
|
||
|
||
tables_dir = settings.uploads_dir / "tables"
|
||
tables_dir.mkdir(parents=True, exist_ok=True)
|
||
db_path = tables_dir / f"{name}.db"
|
||
|
||
from app.data.tabular import UnsupportedTable, load_into
|
||
|
||
existing = (await session.execute(
|
||
select(DataSource).where(DataSource.name == name)
|
||
)).scalar_one_or_none()
|
||
if existing and existing.kind != "sqlite":
|
||
raise HTTPException(
|
||
409, f"已经有一个叫 {name} 的数据源了,而且它不是上传来的表格。换个名字。"
|
||
)
|
||
|
||
try:
|
||
report = load_into(str(db_path), raw, file.filename or name, header_row=header_row)
|
||
except UnsupportedTable as e:
|
||
raise HTTPException(400, str(e)) from e
|
||
except Exception as e: # noqa: BLE001
|
||
raise HTTPException(400, f"导入失败:{type(e).__name__}: {e}") from e
|
||
|
||
row = existing
|
||
if row is None:
|
||
row = DataSource(
|
||
name=name, kind="sqlite", database=str(db_path),
|
||
readonly=True, description=description,
|
||
)
|
||
session.add(row)
|
||
else:
|
||
row.database = str(db_path)
|
||
if description:
|
||
row.description = description
|
||
# 换了文件内容,缓存的连接还指着旧的那个 engine
|
||
await engines.invalidate(row.id)
|
||
|
||
await session.flush()
|
||
# 建完立刻探查:不探的话 db_query 工具的 description 里没有表清单,
|
||
# 模型得先花一步去问"有哪些表"
|
||
try:
|
||
row.schema_cache = await introspect_mod.introspect(row)
|
||
row.schema_synced_at = datetime.now(timezone.utc)
|
||
except Exception: # noqa: BLE001 - 探查失败不该让导入白做,用户可以手动再探
|
||
pass
|
||
await session.commit()
|
||
await session.refresh(row)
|
||
|
||
return {
|
||
"source": _to_out(row).model_dump(),
|
||
"replaced": existing is not None,
|
||
# 推断出的列名和类型原样回显:表头行取错了(比如文件前两行是标题),
|
||
# 这里当场就能看出来——列名会变成一行数据
|
||
"tables": [
|
||
{
|
||
"name": t.name, "sheet": t.source_name, "rows": t.rows,
|
||
"columns": [
|
||
{"name": c, "type": ty} for c, ty in zip(t.columns, t.types)
|
||
],
|
||
}
|
||
for t in report.tables
|
||
],
|
||
}
|