mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 13:02:01 +00:00
加直连地址 AGENTLAB_DATABASE_DIRECT_URL 和 PgBouncer 事务模式开关 AGENTLAB_DB_PGBOUNCER(用户拍板):依赖会话状态的连接——收发进程间通知的那一条(LISTEN)、跨进程的锁(会话级咨询锁,另建一个直连的小池子)、断点库(缺省)——走直连地址,普通查询走 AGENTLAB_DATABASE_URL,不设就全用同一个;开关打开时普通地址不用客户端连接池、关掉 SQLAlchemy 和 asyncpg 的预备语句缓存、预备语句用随机名字(照 SQLAlchemy 文档),事务级咨询锁照常成立;开关打开却没给直连地址时拒绝启动。README 写三种部署(直连、PgBouncer 会话模式、事务模式加直连地址)和要配的 server_reset_query,照实说没在真的 PgBouncer 上跑过。补测试:地址解析和规范化、断点库缺省走直连、事务模式关掉的几样、LISTEN 和跨进程锁走直连、事务模式没给直连地址拒绝启动
This commit is contained in:
+66
-10
@@ -188,12 +188,45 @@ def pg_json_dumps(value: Any) -> str:
|
||||
APPLICATION_NAME = "agentlab"
|
||||
|
||||
|
||||
def make_engine(url: str | None = None, *, application_name: str = APPLICATION_NAME) -> AsyncEngine:
|
||||
def pg_engine_kwargs(*, application_name: str = APPLICATION_NAME, pgbouncer: bool = False,
|
||||
pool_size: int | None = None) -> dict[str, Any]:
|
||||
"""PostgreSQL 引擎的连接池和连接参数。pgbouncer:这个地址是 PgBouncer 的事务模式(AGENTLAB_DB_PGBOUNCER)——
|
||||
|
||||
- 不用客户端的连接池(NullPool,连接池交给 PgBouncer;SQLAlchemy 的说明里同样这么建议);
|
||||
- SQLAlchemy 的预备语句缓存(prepared_statement_cache_size)和 asyncpg 自己的(statement_cache_size)都关掉:事务模式下同一个
|
||||
客户端连接的下一条语句可能落在另一条服务端连接上,缓存着的预备语句在那边不存在;
|
||||
- asyncpg 每条语句仍要先预备再执行(SQLAlchemy 的 asyncpg 方言一律这么做,同一个事务里,落在同一条服务端连接上),名字用随机的
|
||||
(prepared_statement_name_func),不和别的客户端在同一条服务端连接上留下的撞名。PgBouncer 要配 server_reset_query=DISCARD ALL
|
||||
(且 server_reset_query_always=1)收掉留下的预备语句,或用 1.21 以上开 max_prepared_statements。
|
||||
事务级的咨询锁(pg_advisory_xact_lock,发名额、同源队、建表时用)在事务模式下照常成立:取锁和放锁都在同一个事务里。"""
|
||||
connect_args: dict[str, Any] = {
|
||||
# 会话时区定成 UTC:时间列是不带时区的 UTC(UTCDateTime),库里 now() 之类的也按 UTC 算
|
||||
"server_settings": {"application_name": application_name, "timezone": "UTC"}}
|
||||
size = settings.db_pool_size if pool_size is None else pool_size
|
||||
if pgbouncer:
|
||||
connect_args.update(statement_cache_size=0, prepared_statement_cache_size=0,
|
||||
prepared_statement_name_func=_unique_statement_name)
|
||||
return {"poolclass": NullPool, "connect_args": connect_args}
|
||||
if size <= 0:
|
||||
return {"poolclass": NullPool, "connect_args": connect_args}
|
||||
return {"pool_size": size, "max_overflow": max(settings.db_max_overflow, 0), "pool_timeout": settings.db_pool_timeout,
|
||||
"pool_recycle": settings.db_pool_recycle, "connect_args": connect_args}
|
||||
|
||||
|
||||
def _unique_statement_name() -> str:
|
||||
return f"__agentlab_{uuid.uuid4().hex}__"
|
||||
|
||||
|
||||
def make_engine(url: str | None = None, *, application_name: str = APPLICATION_NAME, pgbouncer: bool | None = None,
|
||||
pool_size: int | None = None) -> AsyncEngine:
|
||||
"""应用库的引擎。SQLite:WAL、外键、忙等,单个文件;PostgreSQL:连接池、方言调整(db/postgres.py)。
|
||||
|
||||
copy-to-postgres 建目标库的引擎也走这里,两边的列类型、排序规则是同一套。
|
||||
"""
|
||||
url = url or settings.db_url
|
||||
if pgbouncer is None:
|
||||
# 只有应用库的普通地址按 AGENTLAB_DB_PGBOUNCER 处理:直连地址、copy-to-postgres 给的目标库照常
|
||||
pgbouncer = bool(settings.db_pgbouncer) and url == settings.database_url
|
||||
if url.startswith("sqlite"):
|
||||
# pool_pre_ping:取连接前先探一下活。取消异常打进 SQLAlchemy 的清理流程时
|
||||
# (_close_connection 记完日志会把 CancelledError 原样抛出,跳过了把连接记录
|
||||
@@ -206,15 +239,9 @@ def make_engine(url: str | None = None, *, application_name: str = APPLICATION_N
|
||||
from app.db import postgres
|
||||
|
||||
_require_postgres_driver("asyncpg")
|
||||
pool: dict[str, Any] = (
|
||||
{"poolclass": NullPool} if settings.db_pool_size <= 0 else {
|
||||
"pool_size": settings.db_pool_size, "max_overflow": max(settings.db_max_overflow, 0),
|
||||
"pool_timeout": settings.db_pool_timeout, "pool_recycle": settings.db_pool_recycle})
|
||||
made = create_async_engine(
|
||||
url, echo=False, future=True, pool_pre_ping=True, json_serializer=pg_json_dumps,
|
||||
# 会话时区定成 UTC:时间列是不带时区的 UTC(UTCDateTime),库里 now() 之类的也按 UTC 算
|
||||
connect_args={"server_settings": {"application_name": application_name, "timezone": "UTC"}},
|
||||
**pool)
|
||||
made = create_async_engine(url, echo=False, future=True, pool_pre_ping=True, json_serializer=pg_json_dumps,
|
||||
**pg_engine_kwargs(application_name=application_name, pgbouncer=pgbouncer,
|
||||
pool_size=pool_size))
|
||||
postgres.install(made.sync_engine.dialect)
|
||||
event.listen(made.sync_engine, "before_cursor_execute", postgres.strip_nul, retval=True)
|
||||
return made
|
||||
@@ -250,6 +277,35 @@ engine = make_engine()
|
||||
|
||||
SessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
|
||||
|
||||
_session_engine: AsyncEngine | None = None
|
||||
|
||||
|
||||
def session_engine() -> AsyncEngine:
|
||||
"""依赖会话状态的连接用的引擎(跨进程锁的会话级咨询锁,app/cluster/mutex):设了 AGENTLAB_DATABASE_DIRECT_URL 时是直连那个库的
|
||||
一个小池子,否则就是应用库的引擎。"""
|
||||
global _session_engine
|
||||
if not settings.database_direct_url or settings.database_direct_url == settings.database_url:
|
||||
return engine
|
||||
if _session_engine is None:
|
||||
_session_engine = make_engine(settings.database_direct_url, pgbouncer=False, pool_size=max(2, min(settings.db_pool_size or 5, 10)))
|
||||
return _session_engine
|
||||
|
||||
|
||||
async def dispose_session_engine() -> None:
|
||||
"""关停时关掉直连那个小池子(没另建就什么都不做)。"""
|
||||
global _session_engine
|
||||
if _session_engine is not None:
|
||||
await _session_engine.dispose()
|
||||
_session_engine = None
|
||||
|
||||
|
||||
def check_pgbouncer() -> None:
|
||||
"""AGENTLAB_DB_PGBOUNCER=1 却没给直连地址:拒绝启动(事务模式下 LISTEN、会话级咨询锁不成立,跨进程的锁会锁不住、放不掉)。"""
|
||||
if settings.db_pgbouncer and settings.db_dialect == "postgresql" and not settings.database_direct_url:
|
||||
raise RuntimeError("AGENTLAB_DB_PGBOUNCER=1(应用库地址是 PgBouncer 的事务模式)时要另给 AGENTLAB_DATABASE_DIRECT_URL:"
|
||||
"进程间通知(LISTEN)、跨进程的锁(会话级咨询锁)、断点库要直连 PostgreSQL(或 PgBouncer 的会话模式)。"
|
||||
"见 README「用 PostgreSQL」里的三种部署")
|
||||
|
||||
|
||||
async def get_session() -> AsyncIterator[AsyncSession]:
|
||||
"""FastAPI 依赖注入用的会话。"""
|
||||
|
||||
Reference in New Issue
Block a user