mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 06:51:59 +00:00
d071bf119f
一、表结构、实体与引文
- 数据源工具查库时顺带存一份表结构快照(schema_snapshot),和查询快照一起进台账、进封存;
台账里的查询记下 SQL 用到的表
- 报告里的表名、字段名可以点:反引号里的名字一律核对,正文里的裸名字只在明显是标识符时才自动链接,
常见英文单词不链接;点开看到它出现在哪次查询、类型、所属的表,快照被截断时写明
- 找不到的名字标成可疑实体:在快照、SQL、结果列里都没有的记「可能是编造的名字」并给出最接近的 3 个;
快照被截断、核对不了的记「核对不了」,不冤枉成编造
- 检索引文 [[q:K1|原文]]:原文在检索快照的命中片段里逐字出现(空白归一化后)才算有出处,点开高亮原文位置
- 自动链接的名字不算句子的引用,只有写作者写的引用才算
二、结论句与出具
- 报告节点新增 claims:off(默认,保持原样)或 require_citation(没有引用的结论句计入缺口)
- 受管级别的正式运行里,可疑实体和没有引用的结论句按「有缺口」降档,不拦截;探索运行和已发布级别只标注
- 契约的 claims 和报告节点的设置取更严的那个;受管正式运行里写 ignore 也按降档处理,并给出警告
- 出具横幅除了数字,也写出可疑实体和没有引用的结论句
三、受管门禁 G1–G5(只看图和配置)
- G1 带契约的出口,成果字段的文字只能来自报告撰写节点
- G2 模型写的文字不绕过报告撰写节点直接流进出口
- G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation
- G4 口径卡的输入不来自计算角色的沙箱代码
- G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields
- 受管级别是错误、挡住发布;已发布级别只给警告;已经发布的版本照常运行,下次发布时按新规则检查
- 每条规则都接进发布前自动修复(一键修复 / 给选项 / 交给 Copilot);禁止的改动补上关 cite_fields、
删 output_schema、放宽契约 claims;Copilot 兜底不许自己把代码节点标成 source、不许放宽报告节点
- 发布弹窗里选「交给 Copilot」时明确写「正在交给 Copilot」
四、旧运行与审计
- 没有契约的旧答案:数字和已封存的查询单元格、口径卡值按数值比对,每个数最多 3 个候选,
默认折叠,写明「猜测的来源,不能当证据」
- 有契约的旧运行:候选只从已封存的口径卡里找,人为插进工件表的行不当证据
- 证据审计:GET /api/runs/{id}/evidence/audit 按「无证据 / 可疑实体 / 有出处 / 旧运行猜测」分组,
可导出 JSON 和 CSV(带封存状态);记录页和画布上的报告节点显示「有出处 N · 无证据 M」
五、SQL 守卫修正(安全)
- 去注释改成单遍扫描,认得字符串和各方言的引号(单引号转义、MySQL 反斜杠与 #、PostgreSQL 的 $$ 与嵌套注释、
Oracle 的 q'[…]'、SQLite 的方括号);以前字符串里的 -- 会被当成注释,SELECT '--'; DROP … 这类写法能骗过多语句拦截
- 看不到服务器设置时每种读法都扫一遍,读法之间切分不一致就拒绝,任何一种读法是写操作就按写操作处理
- 执行查询和工具危险判定都按数据源的方言检查
六、其他
- 修掉检查脚本里画布检查的一处竞态,并行估时按实测更新
647 lines
27 KiB
Python
647 lines
27 KiB
Python
"""数据源连接层。各家数据库的方言差异全部收敛在这个文件里。
|
||
|
||
上层(工具、探查、API)只认 DataSource 这个模型,不需要知道 Oracle 要 service_name、
|
||
MySQL 要 charset、asyncpg 不认 sslmode 这些事。哪天要加一种库,改这里一处。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import re
|
||
import time
|
||
from contextlib import asynccontextmanager
|
||
from dataclasses import dataclass
|
||
from typing import Any, AsyncIterator, Awaitable, Callable
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
|
||
from sqlalchemy import text
|
||
|
||
from app.core.crypto import decrypt
|
||
from app.data import guard
|
||
from app.data.guard import QueryLimits, SqlRejected, check, is_write
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 各方言的 SQLAlchemy 驱动。都选 asyncio 原生驱动,没有一个需要装数据库客户端:
|
||
# oracledb 走 thin 模式(纯 Python 协议实现),省掉了 Oracle Instant Client 这个
|
||
# 历来最劝退的一步。
|
||
_DRIVERS = {
|
||
"mysql": "mysql+aiomysql",
|
||
"mariadb": "mysql+aiomysql",
|
||
"postgres": "postgresql+asyncpg",
|
||
"postgresql": "postgresql+asyncpg",
|
||
"oracle": "oracle+oracledb",
|
||
"sqlite": "sqlite+aiosqlite",
|
||
}
|
||
|
||
_DEFAULT_PORTS = {
|
||
"mysql": 3306, "mariadb": 3306,
|
||
"postgres": 5432, "postgresql": 5432,
|
||
"oracle": 1521,
|
||
}
|
||
|
||
SUPPORTED_KINDS = tuple(sorted(set(_DRIVERS)))
|
||
|
||
#: 数据源 options 里按源配置的查询时限(秒)。表单上叫「查询时限」
|
||
QUERY_TIMEOUT_OPTION = "query_timeout_s"
|
||
#: 按源配置时能填的范围。再长的查询不该让一个 agent 干等:该做成离线任务了
|
||
MIN_QUERY_TIMEOUT_S = 1
|
||
MAX_QUERY_TIMEOUT_S = 600
|
||
|
||
#: 数据源 options 里要遮罩的列(列名列表,或者逗号分隔的一段文字)。证据面板展示原始行时,
|
||
#: 这些列的值一律换成「已遮罩」。在有身份体系之前,遮罩只减少暴露,不是安全边界:
|
||
#: 完整快照仍能按工件 id 取到,SQL 里给列起个别名也能绕开
|
||
MASK_COLUMNS_OPTION = "mask_columns"
|
||
|
||
# options 里这些 key 是 AgentLab 自己的配置,不是驱动参数,拼 URL 时要摘掉。
|
||
# schema:探查哪个 schema(企业库里只读账号名下常常什么都没有,数据在别处)
|
||
# query_timeout_s:查询时限,由数据层按语句下发给数据库,见 _server_deadline
|
||
# mask_columns:证据面板遮罩的列,驱动不认识它,拼进连接串会被当成未知参数拒掉
|
||
_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION})
|
||
|
||
_MASK_SPLIT = re.compile(r"[,,、;;\n]+")
|
||
|
||
|
||
@dataclass
|
||
class QueryResult:
|
||
columns: list[str]
|
||
rows: list[list[Any]]
|
||
row_count: int
|
||
truncated: bool
|
||
elapsed_ms: int
|
||
sql: str
|
||
#: 列名 → number / text / date / datetime / time / boolean。拿不准的列(全是空值、类型混着)不在里面
|
||
column_types: dict[str, str] | None = None
|
||
|
||
def to_payload(self) -> dict[str, Any]:
|
||
payload: dict[str, Any] = {
|
||
"columns": self.columns,
|
||
"rows": self.rows,
|
||
"row_count": self.row_count,
|
||
"truncated": self.truncated,
|
||
"elapsed_ms": self.elapsed_ms,
|
||
"sql": self.sql,
|
||
}
|
||
if self.column_types:
|
||
payload["column_types"] = self.column_types
|
||
return payload
|
||
|
||
|
||
def build_url(source: Any, *, reveal: bool = False) -> str:
|
||
"""拼连接串。reveal=False 时把密码换成星号——日志和错误信息里用这个。
|
||
|
||
密码永远不该出现在日志、事件流、或者返回给前端的任何地方。默认遮掉,
|
||
真要连库时才显式 reveal=True。
|
||
"""
|
||
kind = (source.kind or "").lower()
|
||
driver = _DRIVERS.get(kind)
|
||
if not driver:
|
||
raise ValueError(f"不支持的数据库类型:{source.kind}(支持 {', '.join(SUPPORTED_KINDS)})")
|
||
|
||
options: dict[str, Any] = dict(source.options or {})
|
||
# options 混着两类东西:驱动参数(charset、service_name…)和我们自己的配置
|
||
# (schema 指探查哪个 schema)。后者不能拼进连接串,否则驱动会把它当成
|
||
# 未知的连接参数直接拒掉。
|
||
for key in _NON_DRIVER_OPTIONS:
|
||
options.pop(key, None)
|
||
|
||
if kind == "sqlite":
|
||
# SQLite 没有主机和账号,database 就是文件路径
|
||
return f"{driver}:///{source.database or ':memory:'}"
|
||
|
||
password = decrypt(source.password) or "" if reveal else "***"
|
||
from urllib.parse import quote_plus
|
||
|
||
user = quote_plus(source.username or "")
|
||
pw = quote_plus(password) if reveal else password
|
||
auth = f"{user}:{pw}@" if user else ""
|
||
port = source.port or _DEFAULT_PORTS.get(kind)
|
||
host = f"{source.host or 'localhost'}:{port}" if port else (source.host or "localhost")
|
||
|
||
if kind == "oracle":
|
||
# Oracle 用 service_name 或 sid,不是"数据库名"。这是最常见的配错点,
|
||
# 所以两种都认,并且在都没填时给出明确提示而不是让驱动报晦涩的 ORA 错误。
|
||
service = options.pop("service_name", None) or source.database
|
||
sid = options.pop("sid", None)
|
||
if not service and not sid:
|
||
raise ValueError("Oracle 需要在 options 里给 service_name 或 sid(也可以填在库名里)")
|
||
tail = f"?service_name={service}" if service else f"?sid={sid}"
|
||
query = "&".join(f"{k}={v}" for k, v in options.items())
|
||
return f"{driver}://{auth}{host}/{tail}{('&' + query) if query else ''}"
|
||
|
||
query = "&".join(f"{k}={v}" for k, v in options.items())
|
||
return f"{driver}://{auth}{host}/{source.database or ''}{('?' + query) if query else ''}"
|
||
|
||
|
||
def engine_args(source: Any) -> tuple[str, dict[str, Any]]:
|
||
"""真正连库用的连接串和额外参数。只读源在这里拿到**连接层**的只读。
|
||
|
||
守卫按关键字判定,而模型能写出什么 SQL 事前无法穷举——`WITH … DELETE`
|
||
就曾从首关键字白名单下面钻过去,在 SQLite 上真把数据删了(pysqlite 只在
|
||
INSERT/UPDATE/DELETE 开头的语句前隐式开事务,WITH 开头的直接自动提交)。
|
||
所以只读不能只靠猜,连接本身就得写不进去:
|
||
|
||
- SQLite:以 mode=ro 打开文件,写操作由 SQLite 自己拒绝。
|
||
- PostgreSQL:会话的默认事务只读。能关掉它的 SET / set_config() 守卫都拦。
|
||
- MySQL / MariaDB:会话事务只读。能关掉它的只有 SET,守卫拦。
|
||
- Oracle:没有会话级只读开关,只剩守卫这一道——只读数据源请配只读账号。
|
||
"""
|
||
url = build_url(source, reveal=True)
|
||
if not source.readonly:
|
||
return url, {}
|
||
kind = (source.kind or "").lower()
|
||
if kind == "sqlite":
|
||
path = source.database or ""
|
||
if path and path != ":memory:":
|
||
from urllib.parse import quote
|
||
url = f"{_DRIVERS['sqlite']}:///file:{quote(path)}?mode=ro&uri=true"
|
||
return url, {}
|
||
if kind in ("postgres", "postgresql"):
|
||
return url, {"connect_args": {"server_settings": {"default_transaction_read_only": "on"}}}
|
||
if kind in ("mysql", "mariadb"):
|
||
return url, {"connect_args": {"init_command": "SET SESSION TRANSACTION READ ONLY"}}
|
||
return url, {}
|
||
|
||
|
||
def _timeout_value(value: Any) -> float | None:
|
||
"""options 里的查询时限:表单存的是字符串,也认数字。填得不对返回 None。"""
|
||
if value is None or isinstance(value, bool):
|
||
return None
|
||
try:
|
||
seconds = float(str(value).strip())
|
||
except ValueError:
|
||
return None
|
||
if not MIN_QUERY_TIMEOUT_S <= seconds <= MAX_QUERY_TIMEOUT_S:
|
||
return None
|
||
return int(seconds) if seconds.is_integer() else seconds
|
||
|
||
|
||
def query_timeout_problem(options: dict[str, Any] | None) -> str | None:
|
||
"""保存数据源时查一下查询时限填得对不对;没填、或者填对了返回 None。"""
|
||
value = (options or {}).get(QUERY_TIMEOUT_OPTION)
|
||
if value is None or str(value).strip() == "":
|
||
return None
|
||
if _timeout_value(value) is None:
|
||
return (f"查询时限要填 {MIN_QUERY_TIMEOUT_S} 到 {MAX_QUERY_TIMEOUT_S} 之间的秒数,"
|
||
f"比如 60;现在填的是「{value}」")
|
||
return None
|
||
|
||
|
||
def masked_columns(options: dict[str, Any] | None) -> list[str]:
|
||
"""options.mask_columns → 列名列表(去空白、去重,保持顺序)。列表和「a, b」这样的文字都认。"""
|
||
raw = (options or {}).get(MASK_COLUMNS_OPTION)
|
||
items = _MASK_SPLIT.split(raw) if isinstance(raw, str) else raw if isinstance(raw, list) else []
|
||
return list(dict.fromkeys(str(c).strip() for c in items if isinstance(c, (str, int)) and str(c).strip()))
|
||
|
||
|
||
def mask_columns_problem(options: dict[str, Any] | None) -> str | None:
|
||
"""保存数据源时查一下遮罩列填得对不对;没填、或者填对了返回 None。"""
|
||
raw = (options or {}).get(MASK_COLUMNS_OPTION)
|
||
if raw is None or raw == "" or raw == []:
|
||
return None
|
||
if isinstance(raw, str) or (isinstance(raw, list) and all(isinstance(c, str) for c in raw)):
|
||
return None
|
||
return f"遮罩的列要填列名,用逗号分开,比如「phone, email」;现在填的是「{raw}」"
|
||
|
||
|
||
def query_timeout(source: Any) -> float:
|
||
"""这个数据源上一条查询最多跑多久(秒)。没配、或者库里存着的值不对,用缺省。
|
||
|
||
缺省按模块属性现取:只有 guard.QueryLimits 这一处定义。
|
||
"""
|
||
value = _timeout_value((source.options or {}).get(QUERY_TIMEOUT_OPTION))
|
||
return value if value is not None else guard.QueryLimits().timeout_seconds
|
||
|
||
|
||
def _seconds(value: float) -> str:
|
||
return str(int(value)) if float(value).is_integer() else f"{value:g}"
|
||
|
||
|
||
class EngineCache:
|
||
"""按数据源 id 缓存 engine。连接池的建立不便宜,不该每次查询都重来一遍。
|
||
|
||
配置变了要显式 invalidate——改了密码却还在用旧连接,排查起来很费神;
|
||
改了只读开关却还在用旧连接,只读就形同虚设。
|
||
"""
|
||
|
||
def __init__(self) -> None:
|
||
self._engines: dict[str, AsyncEngine] = {}
|
||
self._lock = asyncio.Lock()
|
||
|
||
async def get(self, source: Any) -> AsyncEngine:
|
||
key = str(source.id)
|
||
cached = self._engines.get(key)
|
||
if cached is not None:
|
||
return cached
|
||
async with self._lock:
|
||
if key in self._engines:
|
||
return self._engines[key]
|
||
url, extra = engine_args(source)
|
||
engine = create_async_engine(
|
||
url,
|
||
pool_size=3,
|
||
max_overflow=2,
|
||
pool_pre_ping=True, # 长时间空闲后连接会被数据库掐掉,先探活再用
|
||
pool_recycle=1800,
|
||
**extra,
|
||
)
|
||
self._engines[key] = engine
|
||
return engine
|
||
|
||
async def invalidate(self, source_id: str) -> None:
|
||
engine = self._engines.pop(str(source_id), None)
|
||
if engine is not None:
|
||
await engine.dispose()
|
||
|
||
async def close(self) -> None:
|
||
for engine in list(self._engines.values()):
|
||
await engine.dispose()
|
||
self._engines.clear()
|
||
|
||
|
||
engines = EngineCache()
|
||
|
||
|
||
async def test_connection(source: Any) -> dict[str, Any]:
|
||
"""测连接。失败时把驱动的原始错误带出来——这类问题九成靠错误信息定位。"""
|
||
started = time.perf_counter()
|
||
try:
|
||
engine = await engines.get(source)
|
||
async with engine.connect() as conn:
|
||
probe = "SELECT 1 FROM DUAL" if (source.kind or "").lower() == "oracle" else "SELECT 1"
|
||
await conn.execute(text(probe))
|
||
return {
|
||
"ok": True,
|
||
"elapsed_ms": int((time.perf_counter() - started) * 1000),
|
||
"url": build_url(source), # 遮掉密码的版本
|
||
}
|
||
except Exception as e: # noqa: BLE001
|
||
return {
|
||
"ok": False,
|
||
"error": f"{type(e).__name__}: {e}",
|
||
"elapsed_ms": int((time.perf_counter() - started) * 1000),
|
||
"url": build_url(source),
|
||
}
|
||
|
||
|
||
#: 数据库按时限停下语句之后,数据层自己的兜底再多等多久。要比引擎的宽限
|
||
#: (toolcalls.GRACE_S)短:数据库报上来的「超过 Ns 被中断」才是给人看的那句,
|
||
#: 引擎那边只在数据层也卡住时才出面
|
||
BACKSTOP_S = 0.5
|
||
|
||
#: SQLite 每执行多少条虚拟机指令问一次进度回调。回调只比一次时间,几毫秒一问足够准
|
||
_SQLITE_STEPS = 10_000
|
||
|
||
#: 各家数据库按时限停下语句时的报错。认出来就说「超时」,而不是把驱动原文当成
|
||
#: SQL 写错了交给模型——它会去改一条本来没错的 SQL。
|
||
#:
|
||
#: 先认错误码:服务器的报错文字会跟着它的语言设置走,中文环境的 PostgreSQL 报的是
|
||
#: 「由于语句执行超时,正在取消查询命令」,按英文原文找一个也认不出来
|
||
_PG_CANCELED = "57014" # PostgreSQL query_canceled(statement_timeout 到点)
|
||
_MYSQL_TIMEOUT_CODES = frozenset({3024, # MySQL:max_execution_time 到点
|
||
1969}) # MariaDB:max_statement_time 到点
|
||
_TIMEOUT_MARKERS = (
|
||
"statement timeout", # PostgreSQL(英文环境)
|
||
"maximum statement execution time", # MySQL 3024
|
||
"max_statement_time", # MariaDB 1969
|
||
"dpy-4024", "dpi-1067", "call timeout", # python-oracledb:call timeout of N ms exceeded(驱动自己的话,不随服务器语言变)
|
||
)
|
||
|
||
|
||
def _causes(e: BaseException) -> list[BaseException]:
|
||
"""异常连同它包着的原始驱动异常。SQLAlchemy 包一层(.orig),适配器再包一层(__cause__),
|
||
有的路径(服务端游标取行)又原样抛出驱动异常,所以几条链都走一遍。"""
|
||
out: list[BaseException] = []
|
||
todo: list[Any] = [e]
|
||
while todo and len(out) < 8:
|
||
err = todo.pop(0)
|
||
if isinstance(err, BaseException) and err not in out:
|
||
out.append(err)
|
||
todo += [getattr(err, "orig", None), err.__cause__, err.__context__]
|
||
return out
|
||
|
||
|
||
class _Deadline:
|
||
"""一条语句的时限。数据库那边按它停下;停下时报的错据此认成超时。
|
||
|
||
从取连接之前起算,和后端兜底同一个起点:取连接花掉的也算在时限里,数据库只拿到
|
||
剩下的那段(remaining_ms)。否则取连接一慢,兜底就抢在数据库前面取消语句。
|
||
"""
|
||
|
||
def __init__(self, seconds: float) -> None:
|
||
self.seconds = seconds
|
||
self.ends = time.monotonic() + seconds
|
||
#: SQLite 的进度回调真的掐过
|
||
self.fired = False
|
||
#: 语句已经结束。回调这时要是还挂在连接上(摘除没成功),也不能再掐别人的语句
|
||
self.over = False
|
||
|
||
def remaining_ms(self) -> int:
|
||
return max(1, int((self.ends - time.monotonic()) * 1000))
|
||
|
||
def remaining_seconds(self) -> float:
|
||
return max(0.001, round(self.ends - time.monotonic(), 3))
|
||
|
||
def expired(self) -> bool:
|
||
return time.monotonic() >= self.ends
|
||
|
||
def sqlite_progress(self) -> int:
|
||
# 在 SQLite 执行语句的那个线程里被调用;返回非零,SQLite 就中止当前语句
|
||
if not self.over and time.monotonic() >= self.ends:
|
||
self.fired = True
|
||
return 1
|
||
return 0
|
||
|
||
def stopped(self, e: BaseException) -> bool:
|
||
"""这个报错是不是数据库按时限停下语句造成的。"""
|
||
if self.fired:
|
||
return True
|
||
for err in _causes(e):
|
||
if _PG_CANCELED in (getattr(err, "sqlstate", None), getattr(err, "pgcode", None)):
|
||
return True
|
||
args = getattr(err, "args", None) or ()
|
||
if args and isinstance(args[0], int) and args[0] in _MYSQL_TIMEOUT_CODES:
|
||
return True
|
||
low = str(err).lower()
|
||
if any(marker in low for marker in _TIMEOUT_MARKERS):
|
||
return True
|
||
# 各家的措辞不一(interrupted / canceled,或者干脆只在类名里);
|
||
# 过了时限才报出来的,就是时限掐的
|
||
if self.expired() and any(w in f"{type(err).__name__} {low}".lower()
|
||
for w in ("interrupt", "cancel")):
|
||
return True
|
||
return False
|
||
|
||
|
||
async def _driver(conn: AsyncConnection) -> Any:
|
||
return (await conn.get_raw_connection()).driver_connection
|
||
|
||
|
||
@asynccontextmanager
|
||
async def _server_deadline(conn: AsyncConnection, kind: str, deadline: _Deadline) -> AsyncIterator[None]:
|
||
"""让数据库自己在时限处停下这条语句。
|
||
|
||
只靠后端这边 wait_for 放弃等待是不够的:数据库那边的语句照样在跑,占着连接池
|
||
里的一个连接直到跑完(Oracle 实测多占了 60 秒);aiosqlite 更糟,语句在线程里
|
||
跑,取消之后关连接要排在它后面,一条不收敛的查询能让调用永远回不来。
|
||
|
||
会话级的设置用完要恢复:连接会回到池子里,探查结构这类长操作也用它。
|
||
下发失败(老版本的 MySQL 没有这个变量)不挡查询,只剩后端这边的兜底。
|
||
"""
|
||
if kind in ("postgres", "postgresql"):
|
||
# SET LOCAL 只管这一个事务:连接还回池子时回滚,设置随之作废
|
||
await conn.execute(text(f"SET LOCAL statement_timeout = {deadline.remaining_ms()}"))
|
||
yield
|
||
return
|
||
|
||
if kind in ("mysql", "mariadb"):
|
||
# MySQL 的 max_execution_time 以毫秒计、只管 SELECT;MariaDB 没有它,
|
||
# 用 max_statement_time(秒)。按连上的服务器认,不按表单里选的类型
|
||
if getattr(conn.dialect, "is_mariadb", False):
|
||
var, value = "max_statement_time", _seconds(deadline.remaining_seconds())
|
||
else:
|
||
var, value = "max_execution_time", str(deadline.remaining_ms())
|
||
try:
|
||
await conn.execute(text(f"SET SESSION {var} = {value}"))
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("MySQL 语句时限没下发成功(%s),只剩后端兜底", e)
|
||
yield
|
||
return
|
||
cancelled = False
|
||
try:
|
||
yield
|
||
except asyncio.CancelledError:
|
||
# 连接还卡在那条语句上,这时再发一条只会排在它后面
|
||
cancelled = True
|
||
raise
|
||
finally:
|
||
if not cancelled:
|
||
try:
|
||
await conn.execute(text(f"SET SESSION {var} = DEFAULT"))
|
||
except Exception: # noqa: BLE001 - 连接已经坏了的话,连接池会把它作废
|
||
pass
|
||
return
|
||
|
||
if kind == "oracle":
|
||
driver = await _driver(conn)
|
||
before = getattr(driver, "call_timeout", 0)
|
||
try:
|
||
driver.call_timeout = deadline.remaining_ms()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("Oracle call_timeout 没设上(%s),只剩后端兜底", e)
|
||
yield
|
||
return
|
||
try:
|
||
yield
|
||
finally:
|
||
try:
|
||
driver.call_timeout = before
|
||
except Exception: # noqa: BLE001 - 超时后连接可能已经被驱动关掉
|
||
pass
|
||
return
|
||
|
||
if kind == "sqlite":
|
||
driver = await _driver(conn)
|
||
await driver.set_progress_handler(deadline.sqlite_progress, _SQLITE_STEPS)
|
||
try:
|
||
yield
|
||
finally:
|
||
deadline.over = True
|
||
try:
|
||
await driver.set_progress_handler(None, 0)
|
||
except Exception: # noqa: BLE001 - over 已经让它不再掐人
|
||
pass
|
||
return
|
||
|
||
yield
|
||
|
||
|
||
async def _bounded(
|
||
opener: Callable[[], Any], kind: str, limits: QueryLimits, over: str,
|
||
work: Callable[[AsyncConnection], Awaitable[Any]],
|
||
) -> Any:
|
||
"""在 opener() 开的连接上跑 work,时限交给数据库执行,后端这边再留一道兜底。
|
||
|
||
数据库按时限停下语句时报的错,翻成 over 这句话:「超过 Ns 被中断」。
|
||
两道时限从同一刻(取连接之前)算起,数据库那道先到,兜底晚 BACKSTOP_S。
|
||
"""
|
||
deadline: _Deadline | None = None
|
||
|
||
async def _exec() -> Any:
|
||
nonlocal deadline
|
||
deadline = _Deadline(limits.timeout_seconds)
|
||
async with opener() as conn:
|
||
# 光等连接就把时限用完了:语句不必再发,发出去也只剩 1ms
|
||
if deadline.expired():
|
||
raise SqlRejected(over)
|
||
async with _server_deadline(conn, kind, deadline):
|
||
return await work(conn)
|
||
|
||
try:
|
||
return await asyncio.wait_for(_exec(), timeout=limits.timeout_seconds + BACKSTOP_S)
|
||
except asyncio.TimeoutError as e:
|
||
raise SqlRejected(over) from e
|
||
except Exception as e:
|
||
if deadline is not None and deadline.stopped(e):
|
||
raise SqlRejected(over) from e
|
||
raise
|
||
|
||
|
||
async def run_query(
|
||
source: Any, sql: str, *, limits: QueryLimits | None = None
|
||
) -> QueryResult:
|
||
"""执行一条查询。守卫先过,再谈执行。
|
||
|
||
注意执行的是 guard.check 的返回值而不是原始输入——守卫会把尾分号之类
|
||
规范掉,执行原文等于绕过了守卫。
|
||
|
||
时限没指定时按数据源自己的配置(query_timeout),由数据库执行(_server_deadline)。
|
||
"""
|
||
limits = limits or QueryLimits(timeout_seconds=query_timeout(source))
|
||
kind = (source.kind or "").lower()
|
||
# 按数据源的方言扫:MySQL 的反斜杠转义、PG 的 $$ 字符串,守卫要和数据库读得一样
|
||
statement = check(sql, readonly=bool(source.readonly), source_name=source.name, dialect=kind)
|
||
|
||
engine = await engines.get(source)
|
||
started = time.perf_counter()
|
||
|
||
if not source.readonly and is_write(statement, dialect=kind):
|
||
return await _run_write(engine, kind, statement, limits, started)
|
||
|
||
async def _read(conn: AsyncConnection) -> tuple[list[str], list[list[Any]], bool, dict[str, str]]:
|
||
cursor = await conn.stream(text(statement))
|
||
columns = list(cursor.keys())
|
||
rows: list[list[Any]] = []
|
||
seen = _TypeTally(len(columns))
|
||
truncated = False
|
||
size = 0
|
||
async for row in cursor:
|
||
# 逐行累加,边收边判上限:一次性 fetchall 一个亿级表就晚了
|
||
seen.add(row)
|
||
values = [_jsonable(v) for v in row]
|
||
rows.append(values)
|
||
size += len(json.dumps(values, ensure_ascii=False, default=str))
|
||
if len(rows) >= limits.max_rows or size >= limits.max_bytes:
|
||
truncated = True
|
||
break
|
||
return columns, rows, truncated, seen.types(columns)
|
||
|
||
columns, rows, truncated, types = await _bounded(
|
||
engine.connect, kind, limits,
|
||
f"查询超过 {_seconds(limits.timeout_seconds)}s 被中断。加上 WHERE 条件或 LIMIT 缩小范围。",
|
||
_read,
|
||
)
|
||
return QueryResult(
|
||
columns=columns,
|
||
rows=rows,
|
||
row_count=len(rows),
|
||
truncated=truncated,
|
||
elapsed_ms=int((time.perf_counter() - started) * 1000),
|
||
sql=statement,
|
||
column_types=types,
|
||
)
|
||
|
||
|
||
class _TypeTally:
|
||
"""边读边记每列见过的值类型。只看驱动交回的原始值:_jsonable 之后 Decimal 就成了字符串,
|
||
小数位为 0 的 DECIMAL("45678")和文本列的 "2026" 再也分不开。"""
|
||
|
||
def __init__(self, width: int) -> None:
|
||
self._kinds: list[set[str]] = [set() for _ in range(width)]
|
||
|
||
def add(self, row: Any) -> None:
|
||
for kinds, value in zip(self._kinds, row):
|
||
if value is not None:
|
||
kinds.add(_value_kind(value))
|
||
|
||
def types(self, columns: list[str]) -> dict[str, str]:
|
||
return {str(name): kind for name, kinds in zip(columns, self._kinds)
|
||
if len(kinds) == 1 and (kind := next(iter(kinds))) != "other"}
|
||
|
||
|
||
def _value_kind(value: Any) -> str:
|
||
from datetime import date, datetime, time as dtime
|
||
from decimal import Decimal
|
||
|
||
if isinstance(value, bool):
|
||
return "boolean"
|
||
if isinstance(value, (int, float, Decimal)):
|
||
return "number"
|
||
if isinstance(value, datetime):
|
||
return "datetime"
|
||
if isinstance(value, date):
|
||
return "date"
|
||
if isinstance(value, dtime):
|
||
return "time"
|
||
return "text" if isinstance(value, str) else "other"
|
||
|
||
|
||
def column_types(columns: list[str], rows: list[Any]) -> dict[str, str]:
|
||
"""一批原始行里每列的类型:number / text / date / datetime / time / boolean。
|
||
|
||
全是空值、类型混着(同一列里有数也有文本)的列不记——拿不准就不说,下游照旧按值猜。
|
||
"""
|
||
tally = _TypeTally(len(columns))
|
||
for row in rows:
|
||
tally.add(row)
|
||
return tally.types(columns)
|
||
|
||
|
||
async def _run_write(
|
||
engine: AsyncEngine, kind: str, statement: str, limits: QueryLimits, started: float
|
||
) -> QueryResult:
|
||
"""可写源上的写操作:真的提交。
|
||
|
||
以前写和查询走同一条 stream 路径:拿不到结果集就抛 ResourceClosedError,
|
||
连接关闭时再回滚——可写源上的写从来没落过库,调用方只看到一个报错。
|
||
能走到这里的写都已经过了审批(registry.call_is_dangerous → 各节点的审批关卡),
|
||
只读源上的写在 check 里就被拒了。
|
||
"""
|
||
|
||
async def _write(conn: AsyncConnection) -> tuple[list[str], list[list[Any]]]:
|
||
result = await conn.execute(text(statement))
|
||
if result.returns_rows: # RETURNING、PG 的数据修改 CTE
|
||
return (list(result.keys()),
|
||
[[_jsonable(v) for v in row] for row in result.fetchmany(limits.max_rows)])
|
||
return ["affected_rows"], [[result.rowcount]]
|
||
|
||
# engine.begin():正常退出提交,出错(包括数据库按时限停下)回滚
|
||
columns, rows = await _bounded(
|
||
engine.begin, kind, limits,
|
||
f"写操作超过 {_seconds(limits.timeout_seconds)}s 被中断,已回滚。", _write,
|
||
)
|
||
return QueryResult(
|
||
columns=columns,
|
||
rows=rows,
|
||
row_count=len(rows),
|
||
truncated=len(rows) >= limits.max_rows,
|
||
elapsed_ms=int((time.perf_counter() - started) * 1000),
|
||
sql=statement,
|
||
)
|
||
|
||
|
||
def _jsonable(value: Any) -> Any:
|
||
"""把驱动返回的类型转成能进 JSON 的形态。
|
||
|
||
Decimal、date、datetime、bytes 这些直接 json.dumps 会炸,而它们在真实业务
|
||
数据里遍地都是——金额是 Decimal、时间是 datetime,不处理等于不可用。
|
||
"""
|
||
from datetime import date, datetime, time as dtime
|
||
from decimal import Decimal
|
||
|
||
if value is None or isinstance(value, (str, int, float, bool)):
|
||
return value
|
||
if isinstance(value, Decimal):
|
||
# 转 float 会丢精度,而金额恰恰不能丢。字符串保真,下游要算再自己转。
|
||
return str(value)
|
||
if isinstance(value, (datetime, date, dtime)):
|
||
return value.isoformat()
|
||
if isinstance(value, (bytes, bytearray, memoryview)):
|
||
raw = bytes(value)
|
||
try:
|
||
return raw.decode()
|
||
except UnicodeDecodeError:
|
||
return f"<binary {len(raw)} bytes>"
|
||
return str(value)
|