Files
AgentLab/backend/app/api/datasources.py
T
ThearchyHelios 39e7a5b014 数据源接入:MySQL / Oracle / PostgreSQL / SQLite
阶段①。产品原先能取数的只有 http_request 和沙箱里自己写代码——面对"分析上季度
华东区销量下滑"这类需求,用户得自己知道数据在哪张表、字段叫什么,再手写 SQL 塞进
代码节点。最难的部分留给了用户。

## 连接层(app/data/)

方言差异全收敛在 engine.py 一处:Oracle 要 service_name/sid、MySQL 要 charset、
各家默认端口不同。三个驱动都是 asyncio 原生且免装客户端——oracledb 走 thin 模式,
不需要 Instant Client。密码复用 app/core/crypto 的 Fernet,和 Provider 同一套;
build_url 默认把密码替换成星号,日志和错误信息里用这个版本。

introspect.py 产出两种粒度:summary()(表名+代表字段,几百 token,给 Copilot 的
system prompt 用)和 describe_table()(单表完整字段,按需取)。分两种是因为一个
中等业务库全量塞进 prompt 轻松几万 token,既贵又会把真正的需求淹掉。

## 守卫(app/data/guard.py)

这是唯一不能出错的地方:SQL 由模型生成,模型会写出什么事前无法穷举,防线不能
建立在"它应该不会这么干"上。三层:

- 只读判定用**白名单**而非黑名单——黑名单永远漏,MERGE/CALL/GRANT/各家私货写不完
- 多语句在到达驱动前就断掉,不赌驱动默认只跑第一条(MySQL 开了
  CLIENT_MULTI_STATEMENTS 就会全跑)
- 行数/字节/超时三重上限,一条 SELECT * 不该把内存和模型上下文一起打爆

刻意不引 sqlparse:完整解析 SQL 方言是无底洞,而这里的判断很窄——"首关键字是
什么"和"有没有第二条语句"。窄问题用窄解法。难点是不能误杀:`SELECT ';'` 只是
一条语句,简单 split(";") 会把它切两半然后拒掉,那比漏杀更让人无从下手。

## 工具(app/tools/datasource.py)

每个数据源动态生成一对 db_query__<源> / db_schema__<源>,而不是一个通用工具带
datasource 参数:工具名自带语义,模型不会把查销售库的 SQL 发给财务库;description
里直接塞表清单,省掉"先问有哪些表"那一轮。接进 registry.build_tools 作为第四来源
(前三个是内置/MCP/自定义)。

查询结果落 put_json(kind="query_snapshot"),把 SQL 原文 + 结果集 + 数据源一起存,
出具体系的数字回指能下钻到"这个数是哪条 SQL 查出来的"。

## 实测

真实 SQLite 业务库(600 行订单,埋了华东 6 月断崖的信号)+ 真实模型:
agent 8 轮自主查询——列表 → 查三张表结构 → 探数据时间范围 → 查地区维度 →
按月汇总 → 按地区+月份下钻,准确定位到华东 6 月从 1187 件跌到 354 件。
只读守卫在 agent 路径上同样生效(UPDATE/DROP/多语句全被拒且数据未变)。

测试 55 项:守卫 30 项(含"字符串里的分号不能误杀")、连接与探查 15 项、
URL 构造 6 项、Copilot 流式 4 项。
2026-09-17 10:39:23 +08:00

207 lines
7.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""数据源的增删改查、连接测试、结构探查。
密码的处理规则和 Provider 完全一致(app/api/settings.py):入库加密、
出站只给掩码、PATCH 时不传表示不动、传空串表示清空。
"""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.crypto import encrypt, mask
from app.data import introspect as introspect_mod
from app.data.engine import SUPPORTED_KINDS, engines, test_connection
from app.db.base import get_session
from app.db.models import DataSource
from app.tools.datasource import tool_names
router = APIRouter(prefix="/api/datasources", tags=["datasources"])
# 名字会成为工具名的一部分(db_query__<name>),而模型的 function name 只认
# ASCII。所以这里收紧成 slug,中文说明放 description 字段。
_NAME_PATTERN = r"^[a-z][a-z0-9_]{0,40}$"
class DataSourceIn(BaseModel):
name: str = Field(pattern=_NAME_PATTERN,
description="英文标识,会成为工具名的一部分,如 sales")
kind: str
host: str | None = None
port: int | None = None
database: str | None = None
username: str | None = None
password: str | None = None
options: dict[str, Any] = Field(default_factory=dict)
readonly: bool = True
description: str = ""
enabled: bool = True
class DataSourcePatch(BaseModel):
kind: str | None = None
host: str | None = None
port: int | None = None
database: str | None = None
username: str | None = None
password: str | None = None # 空串=清空;不传=不动
options: dict[str, Any] | None = None
readonly: bool | None = None
description: str | None = None
enabled: bool | None = None
class DataSourceOut(BaseModel):
id: str
name: str
kind: str
host: str | None
port: int | None
database: str | None
username: str | None
options: dict[str, Any]
readonly: bool
description: str
enabled: bool
password_masked: str = ""
has_password: bool = False
table_count: int = 0
schema_synced_at: str | None = None
tools: list[str] = Field(default_factory=list)
def _to_out(row: DataSource) -> DataSourceOut:
tables = (row.schema_cache or {}).get("tables") or {}
return DataSourceOut(
id=row.id, name=row.name, kind=row.kind, host=row.host, port=row.port,
database=row.database, username=row.username, options=row.options or {},
readonly=row.readonly, description=row.description, enabled=row.enabled,
password_masked=mask(row.password),
has_password=bool(row.password),
table_count=len(tables),
schema_synced_at=row.schema_synced_at.isoformat() if row.schema_synced_at else None,
tools=tool_names(row),
)
async def _get_or_404(session: AsyncSession, source_id: str) -> DataSource:
row = await session.get(DataSource, source_id)
if not row:
raise HTTPException(404, "数据源不存在")
return row
@router.get("", response_model=list[DataSourceOut])
async def list_sources(session: AsyncSession = Depends(get_session)) -> list[DataSourceOut]:
rows = (await session.execute(select(DataSource).order_by(DataSource.name))).scalars()
return [_to_out(r) for r in rows]
@router.get("/kinds")
async def list_kinds() -> dict[str, Any]:
"""支持哪些数据库,以及各自需要填什么——前端表单据此渲染。"""
return {
"kinds": [
{"value": "mysql", "label": "MySQL / MariaDB", "default_port": 3306,
"needs": ["host", "database", "username", "password"],
"hint": "options 可填 charset(默认 utf8mb4)"},
{"value": "postgres", "label": "PostgreSQL", "default_port": 5432,
"needs": ["host", "database", "username", "password"], "hint": ""},
{"value": "oracle", "label": "Oracle", "default_port": 1521,
"needs": ["host", "username", "password"],
"hint": "库名填 service_name,或在 options 里给 service_name / sid。"
"驱动走 thin 模式,不需要装 Instant Client"},
{"value": "sqlite", "label": "SQLite(文件)", "default_port": None,
"needs": ["database"], "hint": "库名填数据库文件的绝对路径"},
],
"supported": list(SUPPORTED_KINDS),
}
@router.post("", response_model=DataSourceOut, status_code=201)
async def create_source(
payload: DataSourceIn, session: AsyncSession = Depends(get_session)
) -> DataSourceOut:
if payload.kind not in SUPPORTED_KINDS:
raise HTTPException(400, f"不支持的类型 {payload.kind},支持:{', '.join(SUPPORTED_KINDS)}")
exists = (await session.execute(
select(DataSource).where(DataSource.name == payload.name)
)).scalar_one_or_none()
if exists:
raise HTTPException(409, f"已经有叫 {payload.name} 的数据源了")
row = DataSource(
**payload.model_dump(exclude={"password"}),
password=encrypt(payload.password),
)
session.add(row)
await session.commit()
await session.refresh(row)
return _to_out(row)
@router.patch("/{source_id}", response_model=DataSourceOut)
async def update_source(
source_id: str, payload: DataSourcePatch, session: AsyncSession = Depends(get_session)
) -> DataSourceOut:
row = await _get_or_404(session, source_id)
data = payload.model_dump(exclude_unset=True)
if "password" in data:
# 空串表示清空,非空表示换新的
data["password"] = encrypt(data["password"]) if data["password"] else None
for key, value in data.items():
setattr(row, key, value)
await session.commit()
await session.refresh(row)
# 连接参数可能变了,旧连接池不能再用——否则改完密码还在用旧连接,很难排查
await engines.invalidate(source_id)
return _to_out(row)
@router.delete("/{source_id}", status_code=204)
async def delete_source(source_id: str, session: AsyncSession = Depends(get_session)) -> None:
row = await _get_or_404(session, source_id)
await session.delete(row)
await session.commit()
await engines.invalidate(source_id)
@router.post("/{source_id}/test")
async def test_source(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
"""测连接。失败时把驱动的原始报错带出来——这类问题九成靠错误信息定位。"""
row = await _get_or_404(session, source_id)
return await test_connection(row)
@router.post("/{source_id}/introspect", response_model=DataSourceOut)
async def introspect_source(
source_id: str, schema: str | None = None, session: AsyncSession = Depends(get_session)
) -> DataSourceOut:
"""探查结构并缓存。显式动作,不做后台轮询——生产库不该被实验工具定时扫。"""
row = await _get_or_404(session, source_id)
try:
row.schema_cache = await introspect_mod.introspect(row, schema=schema)
except Exception as e: # noqa: BLE001
raise HTTPException(400, f"探查失败:{type(e).__name__}: {e}") from e
row.schema_synced_at = datetime.now(timezone.utc)
await session.commit()
await session.refresh(row)
return _to_out(row)
@router.get("/{source_id}/schema")
async def get_schema(source_id: str, table: str | None = None,
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
row = await _get_or_404(session, source_id)
if table:
return {"table": table, "detail": introspect_mod.describe_table(row, table)}
return {
"tables": introspect_mod.table_names(row),
"summary": introspect_mod.summary(row),
"synced_at": row.schema_synced_at.isoformat() if row.schema_synced_at else None,
}