mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 04:11:58 +00:00
39e7a5b014
阶段①。产品原先能取数的只有 http_request 和沙箱里自己写代码——面对"分析上季度
华东区销量下滑"这类需求,用户得自己知道数据在哪张表、字段叫什么,再手写 SQL 塞进
代码节点。最难的部分留给了用户。
## 连接层(app/data/)
方言差异全收敛在 engine.py 一处:Oracle 要 service_name/sid、MySQL 要 charset、
各家默认端口不同。三个驱动都是 asyncio 原生且免装客户端——oracledb 走 thin 模式,
不需要 Instant Client。密码复用 app/core/crypto 的 Fernet,和 Provider 同一套;
build_url 默认把密码替换成星号,日志和错误信息里用这个版本。
introspect.py 产出两种粒度:summary()(表名+代表字段,几百 token,给 Copilot 的
system prompt 用)和 describe_table()(单表完整字段,按需取)。分两种是因为一个
中等业务库全量塞进 prompt 轻松几万 token,既贵又会把真正的需求淹掉。
## 守卫(app/data/guard.py)
这是唯一不能出错的地方:SQL 由模型生成,模型会写出什么事前无法穷举,防线不能
建立在"它应该不会这么干"上。三层:
- 只读判定用**白名单**而非黑名单——黑名单永远漏,MERGE/CALL/GRANT/各家私货写不完
- 多语句在到达驱动前就断掉,不赌驱动默认只跑第一条(MySQL 开了
CLIENT_MULTI_STATEMENTS 就会全跑)
- 行数/字节/超时三重上限,一条 SELECT * 不该把内存和模型上下文一起打爆
刻意不引 sqlparse:完整解析 SQL 方言是无底洞,而这里的判断很窄——"首关键字是
什么"和"有没有第二条语句"。窄问题用窄解法。难点是不能误杀:`SELECT ';'` 只是
一条语句,简单 split(";") 会把它切两半然后拒掉,那比漏杀更让人无从下手。
## 工具(app/tools/datasource.py)
每个数据源动态生成一对 db_query__<源> / db_schema__<源>,而不是一个通用工具带
datasource 参数:工具名自带语义,模型不会把查销售库的 SQL 发给财务库;description
里直接塞表清单,省掉"先问有哪些表"那一轮。接进 registry.build_tools 作为第四来源
(前三个是内置/MCP/自定义)。
查询结果落 put_json(kind="query_snapshot"),把 SQL 原文 + 结果集 + 数据源一起存,
出具体系的数字回指能下钻到"这个数是哪条 SQL 查出来的"。
## 实测
真实 SQLite 业务库(600 行订单,埋了华东 6 月断崖的信号)+ 真实模型:
agent 8 轮自主查询——列表 → 查三张表结构 → 探数据时间范围 → 查地区维度 →
按月汇总 → 按地区+月份下钻,准确定位到华东 6 月从 1187 件跌到 354 件。
只读守卫在 agent 路径上同样生效(UPDATE/DROP/多语句全被拒且数据未变)。
测试 55 项:守卫 30 项(含"字符串里的分号不能误杀")、连接与探查 15 项、
URL 构造 6 项、Copilot 流式 4 项。
207 lines
7.8 KiB
Python
207 lines
7.8 KiB
Python
"""数据源的增删改查、连接测试、结构探查。
|
||
|
||
密码的处理规则和 Provider 完全一致(app/api/settings.py):入库加密、
|
||
出站只给掩码、PATCH 时不传表示不动、传空串表示清空。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime, timezone
|
||
from typing import Any
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException
|
||
from pydantic import BaseModel, Field
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.crypto import encrypt, mask
|
||
from app.data import introspect as introspect_mod
|
||
from app.data.engine import SUPPORTED_KINDS, engines, test_connection
|
||
from app.db.base import get_session
|
||
from app.db.models import DataSource
|
||
from app.tools.datasource import tool_names
|
||
|
||
router = APIRouter(prefix="/api/datasources", tags=["datasources"])
|
||
|
||
# 名字会成为工具名的一部分(db_query__<name>),而模型的 function name 只认
|
||
# ASCII。所以这里收紧成 slug,中文说明放 description 字段。
|
||
_NAME_PATTERN = r"^[a-z][a-z0-9_]{0,40}$"
|
||
|
||
|
||
class DataSourceIn(BaseModel):
|
||
name: str = Field(pattern=_NAME_PATTERN,
|
||
description="英文标识,会成为工具名的一部分,如 sales")
|
||
kind: str
|
||
host: str | None = None
|
||
port: int | None = None
|
||
database: str | None = None
|
||
username: str | None = None
|
||
password: str | None = None
|
||
options: dict[str, Any] = Field(default_factory=dict)
|
||
readonly: bool = True
|
||
description: str = ""
|
||
enabled: bool = True
|
||
|
||
|
||
class DataSourcePatch(BaseModel):
|
||
kind: str | None = None
|
||
host: str | None = None
|
||
port: int | None = None
|
||
database: str | None = None
|
||
username: str | None = None
|
||
password: str | None = None # 空串=清空;不传=不动
|
||
options: dict[str, Any] | None = None
|
||
readonly: bool | None = None
|
||
description: str | None = None
|
||
enabled: bool | None = None
|
||
|
||
|
||
class DataSourceOut(BaseModel):
|
||
id: str
|
||
name: str
|
||
kind: str
|
||
host: str | None
|
||
port: int | None
|
||
database: str | None
|
||
username: str | None
|
||
options: dict[str, Any]
|
||
readonly: bool
|
||
description: str
|
||
enabled: bool
|
||
password_masked: str = ""
|
||
has_password: bool = False
|
||
table_count: int = 0
|
||
schema_synced_at: str | None = None
|
||
tools: list[str] = Field(default_factory=list)
|
||
|
||
|
||
def _to_out(row: DataSource) -> DataSourceOut:
|
||
tables = (row.schema_cache or {}).get("tables") or {}
|
||
return DataSourceOut(
|
||
id=row.id, name=row.name, kind=row.kind, host=row.host, port=row.port,
|
||
database=row.database, username=row.username, options=row.options or {},
|
||
readonly=row.readonly, description=row.description, enabled=row.enabled,
|
||
password_masked=mask(row.password),
|
||
has_password=bool(row.password),
|
||
table_count=len(tables),
|
||
schema_synced_at=row.schema_synced_at.isoformat() if row.schema_synced_at else None,
|
||
tools=tool_names(row),
|
||
)
|
||
|
||
|
||
async def _get_or_404(session: AsyncSession, source_id: str) -> DataSource:
|
||
row = await session.get(DataSource, source_id)
|
||
if not row:
|
||
raise HTTPException(404, "数据源不存在")
|
||
return row
|
||
|
||
|
||
@router.get("", response_model=list[DataSourceOut])
|
||
async def list_sources(session: AsyncSession = Depends(get_session)) -> list[DataSourceOut]:
|
||
rows = (await session.execute(select(DataSource).order_by(DataSource.name))).scalars()
|
||
return [_to_out(r) for r in rows]
|
||
|
||
|
||
@router.get("/kinds")
|
||
async def list_kinds() -> dict[str, Any]:
|
||
"""支持哪些数据库,以及各自需要填什么——前端表单据此渲染。"""
|
||
return {
|
||
"kinds": [
|
||
{"value": "mysql", "label": "MySQL / MariaDB", "default_port": 3306,
|
||
"needs": ["host", "database", "username", "password"],
|
||
"hint": "options 可填 charset(默认 utf8mb4)"},
|
||
{"value": "postgres", "label": "PostgreSQL", "default_port": 5432,
|
||
"needs": ["host", "database", "username", "password"], "hint": ""},
|
||
{"value": "oracle", "label": "Oracle", "default_port": 1521,
|
||
"needs": ["host", "username", "password"],
|
||
"hint": "库名填 service_name,或在 options 里给 service_name / sid。"
|
||
"驱动走 thin 模式,不需要装 Instant Client"},
|
||
{"value": "sqlite", "label": "SQLite(文件)", "default_port": None,
|
||
"needs": ["database"], "hint": "库名填数据库文件的绝对路径"},
|
||
],
|
||
"supported": list(SUPPORTED_KINDS),
|
||
}
|
||
|
||
|
||
@router.post("", response_model=DataSourceOut, status_code=201)
|
||
async def create_source(
|
||
payload: DataSourceIn, session: AsyncSession = Depends(get_session)
|
||
) -> DataSourceOut:
|
||
if payload.kind not in SUPPORTED_KINDS:
|
||
raise HTTPException(400, f"不支持的类型 {payload.kind},支持:{', '.join(SUPPORTED_KINDS)}")
|
||
exists = (await session.execute(
|
||
select(DataSource).where(DataSource.name == payload.name)
|
||
)).scalar_one_or_none()
|
||
if exists:
|
||
raise HTTPException(409, f"已经有叫 {payload.name} 的数据源了")
|
||
|
||
row = DataSource(
|
||
**payload.model_dump(exclude={"password"}),
|
||
password=encrypt(payload.password),
|
||
)
|
||
session.add(row)
|
||
await session.commit()
|
||
await session.refresh(row)
|
||
return _to_out(row)
|
||
|
||
|
||
@router.patch("/{source_id}", response_model=DataSourceOut)
|
||
async def update_source(
|
||
source_id: str, payload: DataSourcePatch, session: AsyncSession = Depends(get_session)
|
||
) -> DataSourceOut:
|
||
row = await _get_or_404(session, source_id)
|
||
data = payload.model_dump(exclude_unset=True)
|
||
if "password" in data:
|
||
# 空串表示清空,非空表示换新的
|
||
data["password"] = encrypt(data["password"]) if data["password"] else None
|
||
for key, value in data.items():
|
||
setattr(row, key, value)
|
||
await session.commit()
|
||
await session.refresh(row)
|
||
# 连接参数可能变了,旧连接池不能再用——否则改完密码还在用旧连接,很难排查
|
||
await engines.invalidate(source_id)
|
||
return _to_out(row)
|
||
|
||
|
||
@router.delete("/{source_id}", status_code=204)
|
||
async def delete_source(source_id: str, session: AsyncSession = Depends(get_session)) -> None:
|
||
row = await _get_or_404(session, source_id)
|
||
await session.delete(row)
|
||
await session.commit()
|
||
await engines.invalidate(source_id)
|
||
|
||
|
||
@router.post("/{source_id}/test")
|
||
async def test_source(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||
"""测连接。失败时把驱动的原始报错带出来——这类问题九成靠错误信息定位。"""
|
||
row = await _get_or_404(session, source_id)
|
||
return await test_connection(row)
|
||
|
||
|
||
@router.post("/{source_id}/introspect", response_model=DataSourceOut)
|
||
async def introspect_source(
|
||
source_id: str, schema: str | None = None, session: AsyncSession = Depends(get_session)
|
||
) -> DataSourceOut:
|
||
"""探查结构并缓存。显式动作,不做后台轮询——生产库不该被实验工具定时扫。"""
|
||
row = await _get_or_404(session, source_id)
|
||
try:
|
||
row.schema_cache = await introspect_mod.introspect(row, schema=schema)
|
||
except Exception as e: # noqa: BLE001
|
||
raise HTTPException(400, f"探查失败:{type(e).__name__}: {e}") from e
|
||
row.schema_synced_at = datetime.now(timezone.utc)
|
||
await session.commit()
|
||
await session.refresh(row)
|
||
return _to_out(row)
|
||
|
||
|
||
@router.get("/{source_id}/schema")
|
||
async def get_schema(source_id: str, table: str | None = None,
|
||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||
row = await _get_or_404(session, source_id)
|
||
if table:
|
||
return {"table": table, "detail": introspect_mod.describe_table(row, table)}
|
||
return {
|
||
"tables": introspect_mod.table_names(row),
|
||
"summary": introspect_mod.summary(row),
|
||
"synced_at": row.schema_synced_at.isoformat() if row.schema_synced_at else None,
|
||
}
|