应用库可移植到 PostgreSQL:AGENTLAB_DATABASE_URL 设成 postgresql+asyncpg://… 时整个应用库用 PostgreSQL,断点用官方的 langgraph-checkpoint-postgres(同一个库,AGENTLAB_CHECKPOINT_URL 可另给);不设时照旧是数据目录里的 SQLite,行为不变

- 配置:database_url、checkpoint_url(启动时校验写法,只认 PostgreSQL)、连接池四项;启动日志写用的是哪种库(主机、库名、用户,不写密码)
- 引擎(db/base.make_engine):SQLite 照旧 WAL、外键、忙等;PostgreSQL 用连接池、探活、会话时区 UTC,装上方言调整(db/postgres.py):建表时 String(n) 是不限长的 VARCHAR(SQLite 从不查长度,导过来的存量也可能超长)、ORDER BY 里可空列的 NULL 位置照 SQLite 补写(升序在前、降序在后,窗口函数里同样);Integer 列在 PostgreSQL 上是 BIGINT(列挂到表上时换成 with_variant,绑定参数跟着列按 BIGINT 转换);JSON 列两边都存原文(PostgreSQL 的 json,不用 JSONB:JSONB 重排对象里的键,快照的列序、表的先后读回来就变了);JSON 里的 NaN 写成 null、NUL 去掉(PostgreSQL 不收)
- 迁移按方言:已有的列和索引用 inspector 认;PostgreSQL 上补列的类型取模型里这一列的、缺省值取迁移表里的(布尔 0/1 换成 false/true),ADD COLUMN IF NOT EXISTS;去重语句里的布尔按方言写;部分唯一索引补 postgresql_where;init_db 在 PostgreSQL 上拿咨询锁(几个进程同时启动不撞)
- 应用库上的 SQLite 专用写法改成按方言编译的表达式(db/portable.py):json_extract 的十几处(口径召回、出具、对账、个人应用、幂等键、访问控制、数据源变化通知、剖析任务、待办超时)换成 json_eq / json_flag / json_text,json_type 换成 json_is_string(两种库同一套带类型的比较);能力登记的 sqlite insert 换成按会话方言的 insert_ignore;能力使用次数的 json_each 按方言分支;附件「没绑记录」的条件不靠 json 的等号;按报错文字认唯一约束的地方换成 unique_violation(按表和列认,两种库的报错都认得)
- 断点(engine/checkpoints.py):PostgreSQL 时开 psycopg 连接池、在咨询锁里 setup;runner 只认 BaseCheckpointSaver,读断点仍只走 saver 的公开接口
- 依赖:新 extra postgres(asyncpg、psycopg[binary] 3、psycopg-pool、langgraph-checkpoint-postgres 3.1.2);langgraph-checkpoint 卡在 <4.3(断点格式就在这个包里,postgres 包新版要 4.3,不动已装的 4.2.0);锁文件补上新包
- 运行收尾:记 action.unknown、安排核实做完才把运行摘掉(以前先摘掉,等「不在跑了」再读事件的一方在库慢时读不到这一条)
- 测试:AGENTLAB_TEST_DATABASE_URL 时跑在 PostgreSQL 上(每个 xdist 进程建自己的库、按事件循环分的连接池 tests/pgpool.py),不设时盖掉环境里的数据库地址照旧用 SQLite;scripts/test-postgres.sh 起临时的 postgres:16 容器跑全量;sqlite_only 标记;另开连接改库的几个竞态测试、触发器测试改成两种库都能跑(tests/appdb.py);新增 test_db_portable 守着方言调整、JSON 键序、超长字符串、大整数、NULL 排序、时区、PostgreSQL 老库迁移
This commit is contained in:
ThearchyHelios
2026-10-19 09:52:09 +08:00
parent c79f4e8bf7
commit 2e7d6103ea
39 changed files with 1522 additions and 155 deletions
+2 -1
View File
@@ -62,6 +62,7 @@ def hidden_runs(principal: Any) -> Any:
from app.db.models import Run
from app.db.models_apps import App
from app.db.models_blueprints import Blueprint
from app.db.portable import json_text
if principal.mode == "off":
return false()
@@ -72,7 +73,7 @@ def hidden_runs(principal: Any) -> Any:
kind = func.coalesce(Run.origin_kind, "")
return or_(
and_(kind == "personal_app", or_(Run.app_id.is_(None), Run.app_id.not_in(mine))),
and_(kind == "trial", func.coalesce(func.json_extract(Run.origin, "$.blueprint_id"), "").in_(others)),
and_(kind == "trial", func.coalesce(json_text(Run.origin, "blueprint_id"), "").in_(others)),
)
+4 -3
View File
@@ -39,7 +39,7 @@ from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Any, Awaitable, Callable
from sqlalchemy import delete, func, select
from sqlalchemy import delete, select
from starlette.requests import HTTPConnection
from app.api.coded import CodedHTTPException
@@ -535,12 +535,13 @@ async def started_before(principal: Any, idem_key: str) -> str | None:
"""拥有者按这个幂等键用密钥发起过的实例(来源里记着 idempotency_key):处理到一半服务停了、同一个键重新认领时交回它。"""
from app.db.base import SessionLocal
from app.db.models import Run
from app.db.portable import json_eq
async with SessionLocal() as session:
return await session.scalar(select(Run.id).where(
Run.origin_kind == "app_entry", Run.started_by == principal.username,
func.json_extract(Run.origin, "$.idempotency_key") == idem_key,
func.json_extract(Run.origin, "$.via") == "key").order_by(Run.created_at.desc()).limit(1))
json_eq(Run.origin, "idempotency_key", idem_key),
json_eq(Run.origin, "via", "key")).order_by(Run.created_at.desc()).limit(1))
# ---------------------------------------------------------------- 请求记录(A6)
+2 -1
View File
@@ -416,9 +416,10 @@ async def refresh(session: AsyncSession, app: Any, *, principal: Any) -> tuple[A
def _runs_of(app_id: str, graph_hash: str) -> Any:
from app.db.models import Run
from app.db.portable import json_eq
return select(Run).where(Run.app_id == app_id, Run.origin_kind == ORIGIN,
func.json_extract(Run.origin, "$.graph_hash") == graph_hash)
json_eq(Run.origin, "graph_hash", graph_hash))
async def latest_refresh(session: AsyncSession, app_id: str, graph_hash: str, *, active_only: bool = False) -> Any:
+5 -4
View File
@@ -127,10 +127,9 @@ async def _discover(session: AsyncSession) -> dict[str, dict[str, Any]]:
async def sync_all(session: AsyncSession) -> int:
"""把内置工具、数据源、自定义工具、MCP 同步进 capabilities。返回变动的行数。启动时和各对象增删改时调。"""
from sqlalchemy.dialects.sqlite import insert
from app.db.base import utcnow
from app.db.models import Capability
from app.db.portable import insert_ignore
found = await _discover(session)
existing = {row.name: row for row in (await session.execute(select(Capability))).scalars()}
@@ -169,8 +168,10 @@ async def sync_all(session: AsyncSession) -> int:
row.synced_at = now
changed += 1
if fresh:
# 两个请求同时同步(列表页和发起运行撞在一起):后到的那条插入什么都不做,不报主键冲突。不加锁(单进程 + SQLite)
done = await session.execute(insert(Capability).values(fresh).on_conflict_do_nothing(index_elements=["name"]))
# 两个请求同时同步(列表页和发起运行撞在一起):后到的那条插入什么都不做,不报主键冲突。不加锁:两种库都是同一条
# INSERT … ON CONFLICT DO NOTHING,多个进程同时同步也一样
done = await session.execute(
insert_ignore(session, Capability).values(fresh).on_conflict_do_nothing(index_elements=["name"]))
changed += done.rowcount or 0
if changed or fresh:
await session.commit()
+2 -1
View File
@@ -98,6 +98,7 @@ async def recent_questions(session: AsyncSession, principal: Any, *, limit: int)
"""最近问过的问题,从新到旧,最多 limit 条(取法见模块开头)。principal 用来去掉这个人读不了的只给拥有者的运行。"""
from app.auth.access import hidden_runs
from app.db.models import Conversation, ConversationTurn, Run
from app.db.portable import json_is_string
from app.engine.origin import counted_runs
hidden = hidden_runs(principal)
@@ -112,7 +113,7 @@ async def recent_questions(session: AsyncSession, principal: Any, *, limit: int)
runs = (await session.execute(
select(Run.id, Run.input, Run.workflow_name, Run.created_at, Run.origin_kind)
.where(Run.run_class != "formal", func.coalesce(Run.origin_kind, "") != "chat", counted_runs(), not_(hidden),
func.json_type(Run.input, "$.question") == "text")
json_is_string(Run.input, "question"))
.order_by(Run.created_at.desc()).limit(limit)
)).all()
out = [Question(text=question.strip(), at=at, run_id=run_id, turn_id=turn_id, workflow=workflow, origin="chat")
+47 -2
View File
@@ -47,6 +47,17 @@ class Settings(BaseSettings):
# --- 存储 ---
data_dir: Path = PROJECT_DIR / "data"
# 应用自己的库(工作流、运行、事件、审批……)。不设:数据目录里的 SQLite(agentlab.db,断点在 checkpoints.db),
# 单机单进程。设成 postgresql+asyncpg://用户:密码@主机:端口/库名(postgresql:// 也认)时整个应用库用 PostgreSQL,
# LangGraph 的断点也放进同一个库;checkpoint_url 另给就放那里。数据目录照旧要有:工件、上传表格、沙箱等仍是文件
database_url: str = ""
checkpoint_url: str = ""
# PostgreSQL 的连接池(SQLite 不用这几项)。pool_size=0 表示不用连接池、每次新建连接(前面有 PgBouncer 时用,
# 测试里也这么用:每个测试一个事件循环,asyncpg 的连接不能跨循环)
db_pool_size: int = 10
db_max_overflow: int = 20
db_pool_timeout: int = 30
db_pool_recycle: int = 1800
# --- 前端构建产物(生产部署用)---
# 设了就由服务端一并托管页面(见 app/web.py),页面和接口同源、只起一个进程。
@@ -195,6 +206,23 @@ class Settings(BaseSettings):
return True
return value
@field_validator("database_url", "checkpoint_url", mode="before")
@classmethod
def _valid_postgres_url(cls, value: object, info: Any) -> object:
# 写错了启动时就报出来,别等到第一次查库。只认 PostgreSQL:SQLite 就是不设
if value is None or (isinstance(value, str) and not value.strip()):
return ""
if not isinstance(value, str):
return value
from app.db.urls import normalize_database_url, normalize_checkpoint_url
name = f"AGENTLAB_{(info.field_name or '').upper()}"
normalize = normalize_database_url if info.field_name == "database_url" else normalize_checkpoint_url
try:
return normalize(value.strip())
except ValueError as e:
raise ValueError(f"{name}:{e}") from None
@field_validator("web_dist", mode="before")
@classmethod
def _blank_is_unset(cls, value: object) -> object:
@@ -223,13 +251,30 @@ class Settings(BaseSettings):
@property
def db_url(self) -> str:
return f"sqlite+aiosqlite:///{self.db_path}"
"""应用库的 SQLAlchemy 地址:设了 database_url 用它(已规范成 postgresql+asyncpg://),否则数据目录里的 SQLite。"""
return self.database_url or f"sqlite+aiosqlite:///{self.db_path}"
@property
def db_dialect(self) -> str:
""""sqlite" 或 "postgresql"。"""
return "postgresql" if self.database_url else "sqlite"
@property
def checkpoint_path(self) -> Path:
"""LangGraph checkpointer 用独立的库文件,避免和业务表抢写锁。"""
"""LangGraph checkpointer 用独立的库文件,避免和业务表抢写锁。只在 SQLite 时用。"""
return self.data_dir / "checkpoints.db"
@property
def checkpoint_conninfo(self) -> str | None:
"""PostgreSQL 断点库的 libpq 连接串(psycopg 用):checkpoint_url 优先,否则由 database_url 换算。SQLite 时 None。"""
if self.checkpoint_url:
return self.checkpoint_url
if self.database_url:
from app.db.urls import libpq_conninfo
return libpq_conninfo(self.database_url)
return None
@property
def workspace_dir(self) -> Path:
"""沙箱与文件工具的可写根目录,一切文件访问都被限制在这里面。"""
+12 -4
View File
@@ -144,10 +144,18 @@ def normalize_fields(existing: Mapping[str, Any] | None, submitted: Any, *, at:
async def run_counts(session: AsyncSession) -> dict[str, int]:
"""每项能力被几次运行用到:{能力名: 运行次数},没用过的不出现。按运行发起时记下的能力治理快照数
(Run.options["capabilities"],一次运行里一项能力只记一次)。只用 SQLite 的 json_each,一次查询。"""
rows = await session.execute(text(
"SELECT j.key, COUNT(*) FROM runs, json_each(runs.options, '$.capabilities') AS j "
"WHERE json_valid(runs.options) GROUP BY j.key"))
(Run.options["capabilities"],一次运行里一项能力只记一次)。一次查询:两种库都用 json_each(PostgreSQL 的碰到不是
对象的值会报错,先按类型挑出来)。"""
from app.db.portable import dialect_of
if dialect_of(session) == "postgresql":
sql = ("SELECT j.key, COUNT(*) FROM runs CROSS JOIN LATERAL json_each(CASE WHEN "
"json_typeof(runs.options -> 'capabilities') = 'object' THEN runs.options -> 'capabilities' "
"ELSE '{}'::json END) AS j GROUP BY j.key")
else:
sql = ("SELECT j.key, COUNT(*) FROM runs, json_each(runs.options, '$.capabilities') AS j "
"WHERE json_valid(runs.options) GROUP BY j.key")
rows = await session.execute(text(sql))
return {str(name): int(count) for name, count in rows.all()}
+3 -2
View File
@@ -48,7 +48,7 @@ import logging
import time
from typing import Any
from sqlalchemy import case, delete, func, or_, select, update
from sqlalchemy import case, delete, func, not_, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
@@ -60,6 +60,7 @@ from app.data.catalog_profile import (
)
from app.data.engine import SnapshotTampered
from app.db.base import SessionLocal, new_id, utcnow
from app.db.portable import json_flag
from app.db.models import CatalogDraftJob, CatalogProfileJob, CatalogProfileTask, DataSource
logger = logging.getLogger(__name__)
@@ -345,7 +346,7 @@ _SHOWN = (case((CatalogProfileTask.started_at.is_(None), 1), else_=0), *_NEWEST)
#: 只做了取值观察的那一项(observe 范围、maintenance 范围里只到期观察的表)在逐表一行的 summary 里带这个标记:它不是一次
#: 剖析,不算「上次剖析」——不然表清单的「上次剖析」、给助手的剖析事实(行数、日期范围)会被一次只记了行数的观察顶掉
OBSERVE_ONLY = "observe_only"
_PROFILED = func.coalesce(func.json_extract(CatalogProfileTask.summary, f"$.{OBSERVE_ONLY}"), 0) == 0
_PROFILED = not_(json_flag(CatalogProfileTask.summary, OBSERVE_ONLY))
def _observe_only(checks: list[Any] | None) -> bool:
+2 -1
View File
@@ -90,6 +90,7 @@ from app.data.recipe_types import (
from app.data.xlsx_scan import WorkbookScan
from app.db.base import new_id, utcnow
from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableBuild, TableImport, TableRecipe
from app.db.portable import unique_violation
logger = logging.getLogger(__name__)
@@ -2747,7 +2748,7 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial
raise _publish_refused(e) from e
except IntegrityError as e:
await _back_to_drafting(session, staging_id)
if "data_sources.name" in str(e.orig if e.orig is not None else e):
if unique_violation(e, "data_sources", "name"):
raise ImportRefused(409, "name_taken", f"名称「{name}」已被占用,请放弃这次导入后换一个名称") from e
raise ImportRefused(500, "publish_failed", "导入失败:写入版本记录时出错,当前版本未受影响") from e
except BaseException:
+2 -1
View File
@@ -37,6 +37,7 @@ from app.data import schema_diff as sd
from app.data.names import name_key
from app.db.base import utcnow
from app.db.models_sources import SourceChange
from app.db.portable import json_eq
logger = logging.getLogger(__name__)
@@ -337,7 +338,7 @@ async def notify_after_job(source_id: str, job_id: str) -> int:
n = int((await session.execute(select(func.count()).select_from(SourceChange).where(
SourceChange.source_id == source_id, SourceChange.severity == "action",
SourceChange.status == "open",
func.json_extract(SourceChange.detected_by, "$.job_id") == job_id))).scalar() or 0)
json_eq(SourceChange.detected_by, "job_id", job_id)))).scalar() or 0)
if n:
from app.notify import service
+5 -2
View File
@@ -540,10 +540,13 @@ _DB_SIDE_FILES = ("", "-wal", "-shm", "-journal")
def _app_db_files() -> list[Path]:
"""应用自己的库:元数据库、运行检查点库,以及实际连着的元数据库(测试里和 data_dir 下的不是同一个)。"""
"""应用自己的库:元数据库、运行检查点库,以及实际连着的元数据库(测试里和 data_dir 下的不是同一个)。
应用库是 PostgreSQL 时,数据目录里的 agentlab.db、checkpoints.db 照样算:导入 PostgreSQL 之前的那份还留在那里,
里面是同一批元数据。连着的库这时是库名不是文件,不算进来。"""
dbs = [settings.db_path, settings.checkpoint_path]
try:
live = db_base.engine.url.database
live = db_base.engine.url.database if db_base.engine.dialect.name == "sqlite" else None
except Exception: # noqa: BLE001 - 取不到就只按配置里的路径判断
live = None
if live and live != ":memory:":
+194 -37
View File
@@ -1,16 +1,23 @@
from __future__ import annotations
import json
import logging
import math
import re
import uuid
from datetime import datetime, timezone
from typing import Any, AsyncIterator
from sqlalchemy import JSON, DateTime, MetaData, event
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy import JSON, BigInteger, Column, DateTime, Integer, MetaData, event
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from sqlalchemy.pool import NullPool
from sqlalchemy.types import TypeDecorator
from app.core.config import settings
logger = logging.getLogger(__name__)
NAMING_CONVENTION = {
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
@@ -57,11 +64,31 @@ class UTCDateTime(TypeDecorator):
return value.astimezone(timezone.utc)
#: 应用库的 JSON 列:两种库上都是存原文的 JSON(PostgreSQL 的 json,不是 JSONB)。JSONB 会把对象里的键重新排序:
#: 应用里不少地方按存进去的键的先后出东西(快照的列、表的先后、复核结论各段的先后),换成 JSONB 读回来就变了样,
#: 测试当场抓到好几处。json 的代价是没有相等运算符:== 比较、DISTINCT、GROUP BY、ORDER BY 不能直接作用在整列上
#: (应用里没有这种写法,test_db_portable 守着);按里面某个键筛选走 db/portable 的 json_eq 等,只把取出来的那一小段
#: 转成 JSONB 比较
PortableJSON = JSON()
class Base(DeclarativeBase):
metadata = MetaData(naming_convention=NAMING_CONVENTION)
# 图定义、节点配置、事件负载都是任意形状的 JSON,统一映射到 SQLite 的 JSON 列
type_annotation_map = {dict[str, Any]: JSON, list[Any]: JSON, datetime: UTCDateTime}
# 图定义、节点配置、事件负载都是任意形状的 JSON,统一映射到 JSON 列(见 PortableJSON)
type_annotation_map = {dict[str, Any]: PortableJSON, list[Any]: PortableJSON, datetime: UTCDateTime}
@event.listens_for(Column, "after_parent_attach")
def _portable_integer(column: Column, table: Any) -> None:
"""应用库的 Integer 列在 PostgreSQL 上换成 BIGINT(SQLite 照旧是 INTEGER,主键照旧是 rowid)。
SQLite 的 INTEGER 是 64 位,PostgreSQL 的只有 32 位:行数估计、字节数、累计的计数都可能过 21 亿。在列挂到表上
的那一刻换(定义模型时,任何查询之前):建表是 BIGINT(自增主键是 BIGSERIAL),和这一列比较、写进这一列的绑定
参数也按 BIGINT 转换。只管挂在应用 metadata 上、类型正好是 Integer 的列;业务数据源那边的表不受影响。"""
if getattr(table, "metadata", None) is Base.metadata and type(column.type) is Integer \
and not column.type._variant_mapping:
column.type = Integer().with_variant(BigInteger(), "postgresql")
class TimestampMixin:
@@ -101,16 +128,10 @@ def _stop_aiosqlite_once() -> None:
aiosqlite.Connection.stop = stop # type: ignore[method-assign]
# 应用库是 PostgreSQL 时也照样打:上传表格、手工登记的 SQLite 业务库(data/engine)还是走 aiosqlite 的连接池
_stop_aiosqlite_once()
# pool_pre_ping:取连接前先探一下活。取消异常打进 SQLAlchemy 的清理流程时
# (_close_connection 记完日志会把 CancelledError 原样抛出,跳过了把连接记录
# 清空的那一行),已经关掉的连接会留在池里,下一个取到它的人拿到的是
# "no active connection"。探活失败会被当成断开,换一条新的——SQLite 本地一次
# SELECT 1 的代价,换池子不被污染。
engine = create_async_engine(settings.db_url, echo=False, future=True, pool_pre_ping=True)
@event.listens_for(engine.sync_engine, "connect")
def _set_sqlite_pragma(dbapi_conn, _record): # pragma: no cover - 驱动回调
"""WAL + 外键约束。WAL 让读写不互相阻塞,事件写入和前端轮询能并行。"""
cur = dbapi_conn.cursor()
@@ -120,6 +141,92 @@ def _set_sqlite_pragma(dbapi_conn, _record): # pragma: no cover - 驱动回调
cur.close()
def _finite(value: Any) -> Any:
if isinstance(value, float):
return value if math.isfinite(value) else None
if isinstance(value, dict):
return {k: _finite(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [_finite(v) for v in value]
return value
def _no_nul(value: Any) -> Any:
if isinstance(value, str):
return value.replace("\x00", "")
if isinstance(value, dict):
return {_no_nul(k): _no_nul(v) for k, v in value.items()}
if isinstance(value, list):
return [_no_nul(v) for v in value]
return value
def pg_json_dumps(value: Any) -> str:
"""PostgreSQL 上写 JSON 列。它不收 NaN / Infinity(SQLite 照存);字符串里的 \\u0000 json 列收,但一转成 JSONB
(按键筛选时,db/portable)、一取成文本就报错,那一整条查询跟着失败。
非有限的浮点写成 null(同浏览器的 JSON.stringify),字符串里的 NUL 字符去掉;平常的值一次 dumps 就过,
只有碰上了才多走一遍。"""
try:
text = json.dumps(value, allow_nan=False)
except ValueError:
text = json.dumps(_finite(value), allow_nan=False)
if "\\u0000" in text:
text = json.dumps(_no_nul(json.loads(text)), allow_nan=False)
return text
#: 应用连 PostgreSQL 时报的 application_name。copy-to-postgres 按它认目标库上还有没有服务连着(它自己报的是 agentlab-copy)
APPLICATION_NAME = "agentlab"
def make_engine(url: str | None = None, *, application_name: str = APPLICATION_NAME) -> AsyncEngine:
"""应用库的引擎。SQLite:WAL、外键、忙等,单个文件;PostgreSQL:连接池、方言调整(db/postgres.py)。
copy-to-postgres 建目标库的引擎也走这里,两边的列类型、排序规则是同一套。
"""
url = url or settings.db_url
if url.startswith("sqlite"):
# pool_pre_ping:取连接前先探一下活。取消异常打进 SQLAlchemy 的清理流程时
# (_close_connection 记完日志会把 CancelledError 原样抛出,跳过了把连接记录
# 清空的那一行),已经关掉的连接会留在池里,下一个取到它的人拿到的是
# "no active connection"。探活失败会被当成断开,换一条新的——SQLite 本地一次
# SELECT 1 的代价,换池子不被污染。
made = create_async_engine(url, echo=False, future=True, pool_pre_ping=True)
event.listen(made.sync_engine, "connect", _set_sqlite_pragma)
return made
from app.db import postgres
pool: dict[str, Any] = (
{"poolclass": NullPool} if settings.db_pool_size <= 0 else {
"pool_size": settings.db_pool_size, "max_overflow": max(settings.db_max_overflow, 0),
"pool_timeout": settings.db_pool_timeout, "pool_recycle": settings.db_pool_recycle})
made = create_async_engine(
url, echo=False, future=True, pool_pre_ping=True, json_serializer=pg_json_dumps,
# 会话时区定成 UTC:时间列是不带时区的 UTC(UTCDateTime),库里 now() 之类的也按 UTC 算
connect_args={"server_settings": {"application_name": application_name, "timezone": "UTC"}},
**pool)
postgres.install(made.sync_engine.dialect)
return made
def describe_database() -> str:
"""启动日志里的一句:用的是哪种库、在哪(不写密码)。"""
from app.db.urls import describe
if settings.checkpoint_url:
points = f"断点在 PostgreSQL {describe(settings.checkpoint_url)}"
elif settings.db_dialect == "sqlite":
points = f"断点 {settings.checkpoint_path}"
else:
points = "断点在同一个库"
if settings.db_dialect == "sqlite":
return f"SQLite({settings.db_path},{points})"
return f"PostgreSQL({describe(settings.db_url)},{points})"
engine = make_engine()
SessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
@@ -129,8 +236,10 @@ async def get_session() -> AsyncIterator[AsyncSession]:
yield session
# 轻量迁移:create_all 只建新表,不给已有表加列。SQLite 的 ADD COLUMN
# 便宜且带默认值回填,够用;等真需要改列类型再考虑 alembic。
# 轻量迁移:create_all 只建新表,不给已有表加列。ADD COLUMN 便宜且带默认值回填,够用;等真需要改列类型再考虑 alembic。
# 第三项照 SQLite 写(类型和缺省值的字面量)。PostgreSQL 上类型取模型里这一列的(按应用库的方言调整编译:String(n)
# 是 VARCHAR、Integer 是 BIGINT、JSON 是 json、时间是 TIMESTAMP),缺省值取这里写的,布尔的 0 / 1 换成 false / true
# (_column_ddl)。所以这里加的每一列模型里都得有:test_db_portable 守着
_COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [
("runs", "run_class", "TEXT DEFAULT 'exploratory'"),
("runs", "version", "INTEGER"),
@@ -274,44 +383,92 @@ _INDEX_MIGRATIONS: list[tuple[str, str, str | tuple[str, ...]]] = [
#: 迁移补建的唯一索引:(索引名, 表, 列, 部分索引的条件, 建之前的去重语句)。只在索引还不存在时先去重再建——存量库里可能
#: 已经有并发留下的重复行,直接建会失败。去重的规矩写在各自的语句里:多出的钩子行删掉(只留最早的那一行,它的密钥照旧
#: 能用),多出的助手对话归档(不删,历史还在)
#: 能用),多出的助手对话归档(不删,历史还在)。条件和语句里的 {false} / {true} 按方言换成布尔的字面量(SQLite 是 0 / 1)
_UNIQUE_INDEX_MIGRATIONS: list[tuple[str, str, tuple[str, ...], str | None, str]] = [
("ux_app_hooks_app_name", "app_hooks", ("app_id", "name"), None,
"DELETE FROM app_hooks WHERE id NOT IN (SELECT id FROM (SELECT id, ROW_NUMBER() OVER "
"(PARTITION BY app_id, name ORDER BY created_at, id) AS rn FROM app_hooks) WHERE rn = 1)"),
("ux_conversations_app_owner", "conversations", ("app_id", "created_by"), "kind = 'app' AND archived = 0",
"UPDATE conversations SET archived = 1 WHERE kind = 'app' AND archived = 0 AND created_by IS NOT NULL "
"(PARTITION BY app_id, name ORDER BY created_at, id) AS rn FROM app_hooks) AS ranked WHERE rn = 1)"),
("ux_conversations_app_owner", "conversations", ("app_id", "created_by"), "kind = 'app' AND archived = {false}",
"UPDATE conversations SET archived = {true} WHERE kind = 'app' AND archived = {false} AND created_by IS NOT NULL "
"AND id NOT IN (SELECT id FROM (SELECT id, ROW_NUMBER() OVER (PARTITION BY app_id, created_by "
"ORDER BY last_active_at DESC, id) AS rn FROM conversations WHERE kind = 'app' AND archived = 0 "
"AND created_by IS NOT NULL) WHERE rn = 1)"),
"ORDER BY last_active_at DESC, id) AS rn FROM conversations WHERE kind = 'app' AND archived = {false} "
"AND created_by IS NOT NULL) AS ranked WHERE rn = 1)"),
]
#: 布尔字面量:SQLite 的布尔列存 0 / 1
_BOOL_LITERALS = {"sqlite": {"false": "0", "true": "1"}, "postgresql": {"false": "false", "true": "true"}}
_DEFAULT = re.compile(r"\s+DEFAULT\s+(.+)$", re.I | re.S)
def _column_ddl(dialect: Any, table: str, column: str, ddl: str) -> str:
"""ADD COLUMN 里列名后面那一段。SQLite 原样用迁移表里写的;PostgreSQL 类型取模型里这一列的,缺省值取迁移表里的。"""
if dialect.name == "sqlite":
return ddl
from sqlalchemy import Boolean
from app.db.postgres import AppTypeCompiler
col = Base.metadata.tables[table].c[column]
out = AppTypeCompiler(dialect).process(col.type, type_expression=col)
if match := _DEFAULT.search(ddl):
default = match.group(1).strip()
if isinstance(col.type, Boolean):
default = {"0": "false", "1": "true"}.get(default, default)
out += f" DEFAULT {default}"
return out
def _migrate(conn) -> None: # pragma: no cover - 同步回调
from sqlalchemy import text
"""给老库补列、补索引。已有的列和索引按 inspector 认(两种库同一套),再跑一遍什么都不做。"""
from sqlalchemy import inspect, text
dialect = conn.dialect
literals = _BOOL_LITERALS.get(dialect.name, _BOOL_LITERALS["postgresql"])
inspector = inspect(conn)
tables = set(inspector.get_table_names())
columns = {t: {c["name"] for c in inspector.get_columns(t)} for t in tables}
# IF NOT EXISTS:PostgreSQL 上多个进程同时启动,各自的检查和补列之间别人可能已经补上了(init_db 另外拿着锁)
add = "ADD COLUMN IF NOT EXISTS" if dialect.name == "postgresql" else "ADD COLUMN"
for table, column, ddl in _COLUMN_MIGRATIONS:
cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
cols = columns.get(table)
if cols and column not in cols:
conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}"))
for name, table, columns in _INDEX_MIGRATIONS:
wanted = (columns,) if isinstance(columns, str) else columns
cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
conn.execute(text(f"ALTER TABLE {table} {add} {column} {_column_ddl(dialect, table, column, ddl)}"))
cols.add(column)
for name, table, wanted in _INDEX_MIGRATIONS:
wanted = (wanted,) if isinstance(wanted, str) else wanted
cols = columns.get(table) or set()
if all(c in cols for c in wanted):
conn.execute(text(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({', '.join(wanted)})"))
for name, table, columns, where, dedupe in _UNIQUE_INDEX_MIGRATIONS:
cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))}
exists = conn.execute(text("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = :n"), {"n": name}).first()
if exists or not all(c in cols for c in columns):
for name, table, wanted, where, dedupe in _UNIQUE_INDEX_MIGRATIONS:
cols = columns.get(table) or set()
if not all(c in cols for c in wanted):
continue
conn.execute(text(dedupe))
conn.execute(text(f"CREATE UNIQUE INDEX {name} ON {table} ({', '.join(columns)})"
+ (f" WHERE {where}" if where else "")))
if any(ix["name"] == name for ix in inspect(conn).get_indexes(table)):
continue
conn.execute(text(dedupe.format(**literals)))
conn.execute(text(f"CREATE UNIQUE INDEX {name} ON {table} ({', '.join(wanted)})"
+ (f" WHERE {where.format(**literals)}" if where else "")))
#: init_db 在 PostgreSQL 上拿的会话级咨询锁的号:几个进程同时启动时,建表和迁移一个一个来(同一个号别处别用)
_INIT_LOCK = 0x4167656E744C6162 # "AgentLab"
async def create_schema(target: AsyncEngine) -> None:
"""建表、补列补索引(create_all 只建新表,老库的新列靠 _migrate)。两种库同一套;copy-to-postgres 建目标库也用它。"""
from app.db import models # noqa: F401 确保模型已注册到 metadata
async with target.begin() as conn:
if conn.dialect.name == "postgresql":
# 事务结束自动放锁。CREATE TABLE IF NOT EXISTS 在 PostgreSQL 上并发执行也会撞(系统表的唯一约束)
from sqlalchemy import text
await conn.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": _INIT_LOCK})
await conn.run_sync(Base.metadata.create_all)
await conn.run_sync(_migrate)
async def init_db() -> None:
from app.db import models # noqa: F401 确保模型已注册到 metadata
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await conn.run_sync(_migrate)
logger.info("应用库:%s", describe_database())
await create_schema(engine)
+6 -3
View File
@@ -362,7 +362,8 @@ class CatalogDraftJob(Base, TimestampMixin):
__table_args__ = (
Index("ix_catalog_draft_jobs_source_created", "source_id", "created_at"),
Index("ux_catalog_draft_jobs_active", "source_id", unique=True,
sqlite_where=text("status IN ('queued', 'running')")),
sqlite_where=text("status IN ('queued', 'running')"),
postgresql_where=text("status IN ('queued', 'running')")),
)
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
@@ -450,7 +451,8 @@ class CatalogProfileJob(Base, TimestampMixin):
__table_args__ = (
Index("ix_catalog_profile_jobs_source_created", "source_id", "created_at"),
Index("ux_catalog_profile_jobs_active", "source_id", unique=True,
sqlite_where=text("status IN ('queued', 'running')")),
sqlite_where=text("status IN ('queued', 'running')"),
postgresql_where=text("status IN ('queued', 'running')")),
)
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
@@ -904,7 +906,8 @@ class Conversation(Base, TimestampMixin):
# 一个人在一个应用里只有一段(未归档的)助手对话(D-e)。存量库由 db/base.py 的 _UNIQUE_INDEX_MIGRATIONS 补建。
# created_by 为空(关登录又没填署名)的不受约束:SQLite 的唯一索引里 NULL 互不相等,那种情况靠接口里的进程内锁
__table_args__ = (Index("ux_conversations_app_owner", "app_id", "created_by", unique=True,
sqlite_where=text("kind = 'app' AND archived = 0")),)
sqlite_where=text("kind = 'app' AND archived = 0"),
postgresql_where=text("kind = 'app' AND archived = false")),)
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
title: Mapped[str] = mapped_column(String(200), default="")
+6 -5
View File
@@ -19,10 +19,10 @@ from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import JSON, Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint, text
from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint, text
from sqlalchemy.orm import Mapped, mapped_column
from app.db.base import Base, UTCDateTime, new_id, utcnow
from app.db.base import Base, PortableJSON, UTCDateTime, new_id, utcnow
class SourceObject(Base):
@@ -92,7 +92,8 @@ class IntrospectJob(Base):
__table_args__ = (
Index("ix_introspect_jobs_source_created", "source_id", "created_at"),
Index("ux_introspect_jobs_active", "source_id", unique=True,
sqlite_where=text("status IN ('queued', 'running')")),
sqlite_where=text("status IN ('queued', 'running')"),
postgresql_where=text("status IN ('queued', 'running')")),
)
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
@@ -219,8 +220,8 @@ class SourceChange(Base):
seen_count: Mapped[int] = mapped_column(Integer, default=1)
detected_by: Mapped[dict[str, Any]] = mapped_column(default=dict)
basis: Mapped[dict[str, Any] | None] = mapped_column(default=None)
before: Mapped[Any] = mapped_column(JSON, nullable=True, default=None)
after: Mapped[Any] = mapped_column(JSON, nullable=True, default=None)
before: Mapped[Any] = mapped_column(PortableJSON, nullable=True, default=None)
after: Mapped[Any] = mapped_column(PortableJSON, nullable=True, default=None)
affects: Mapped[dict[str, Any] | None] = mapped_column(default=None)
struct_version_before: Mapped[int | None] = mapped_column(Integer, default=None)
struct_version_after: Mapped[int | None] = mapped_column(Integer, default=None)
+235
View File
@@ -0,0 +1,235 @@
"""应用库上 SQLite、PostgreSQL 写法不一样的那几样,收成按方言编译的表达式。
只给应用库(agentlab.db / AGENTLAB_DATABASE_URL)用;发给业务数据源的 SQL 不走这里。
**按 JSON 列里的键筛选。** SQLite 用 json_extract:取出来是 SQL 标量(字符串是文本、数字是数字、true 是 1、
没有这个键或值为 null 是 NULL),和 Python 值比是带类型的比较(数字 3 不等于字符串 "3")。PostgreSQL 的 JSON 列是
存原文的 json(db/base.PortableJSON),用 -> / ->> 取,要比较的只把取出来的那一小段转成 JSONB:
- json_eq:PostgreSQL 上按 JSONB 相等比(`(col -> 'k')::jsonb = '3'::jsonb`),同样带类型,和 SQLite 一致;
- json_text:只要文本(字符串原样、数字是它的写法),PostgreSQL 上是 ->>;
- json_flag:标记位「真」(true、非零的数):没有这个键、null、false、0 都算假;
- json_is_string:这个键的值是不是字符串(SQLite 的 json_type 叫 text,PostgreSQL 的 jsonb_typeof 叫 string)。
键名只收标识符(字母、数字、下划线),直接写进 SQL:键都是代码里的常量,写进去语句缓存也按它区分。
**插入,撞了唯一键就什么都不做。** insert_ignore 按会话连着的库挑 sqlite / postgresql 的 insert,两边的
on_conflict_do_nothing 是同一个接口。
**撞的是哪个唯一约束。** unique_violation 认 IntegrityError:SQLite 的报错是「UNIQUE constraint failed: 表.列」,
PostgreSQL(asyncpg)的是 23505,表名、DETAIL 里的「Key (列)=」分别带着。不靠约束名(命名规则改了也认得出)。
"""
from __future__ import annotations
import json
import re
from typing import Any
from sqlalchemy import Boolean, String, bindparam, literal_column
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.functions import FunctionElement
from sqlalchemy.types import TypeDecorator
_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
def _key(name: str) -> Any:
if not _KEY.match(name):
raise ValueError(f"JSON 键名只能是标识符:{name!r}")
return literal_column(name)
class _JsonScalar(TypeDecorator):
"""json_eq 右边的值:SQLite 原样绑定(和 json_extract 取出的标量比),PostgreSQL 绑定成 JSON 文本再转 JSONB。"""
impl = String
cache_ok = True
def process_bind_param(self, value: Any, dialect: Any) -> Any:
if dialect.name == "postgresql":
return json.dumps(value, ensure_ascii=False)
return value
class json_text(FunctionElement[str]):
"""JSON 列顶层键的值,当文本用。"""
type = String()
inherit_cache = True
name = "json_text"
def __init__(self, column: Any, key: str) -> None:
super().__init__(column, _key(key))
class _JsonEq(FunctionElement[bool]):
type = Boolean()
inherit_cache = True
name = "json_eq"
def __init__(self, column: Any, key: str, value: Any) -> None:
super().__init__(column, _key(key), bindparam(None, value, type_=_JsonScalar()))
class _JsonNull(FunctionElement[bool]):
type = Boolean()
inherit_cache = True
name = "json_null"
def __init__(self, column: Any, key: str) -> None:
super().__init__(column, _key(key))
def json_eq(column: Any, key: str, value: Any) -> Any:
"""JSON 列顶层键的值等于 value(带类型的比较,见模块说明)。value 为 None 时是「没有这个键或值为 null」。
None 用另一个表达式:语句缓存不按绑定参数的值区分,同一个表达式不能因为值是 None 编译成两样。"""
return _JsonNull(column, key) if value is None else _JsonEq(column, key, value)
class json_flag(FunctionElement[bool]):
"""JSON 列顶层键是「真」的标记位:true 或非零的数(字符串也算真,同 SQLite)。没有这个键、null、false、0 是假。"""
type = Boolean()
inherit_cache = True
name = "json_flag"
def __init__(self, column: Any, key: str) -> None:
super().__init__(column, _key(key))
class json_is_string(FunctionElement[bool]):
"""JSON 列顶层键的值是字符串。"""
type = Boolean()
inherit_cache = True
name = "json_is_string"
def __init__(self, column: Any, key: str) -> None:
super().__init__(column, _key(key))
def _parts(element: Any, compiler: Any, **kw: Any) -> tuple[str, str, list[Any]]:
column, key, *rest = list(element.clauses)
return compiler.process(column, **kw), key.name, rest
# ---- SQLite(缺省)----
@compiles(json_text)
def _json_text_sqlite(element: Any, compiler: Any, **kw: Any) -> str:
col, key, _ = _parts(element, compiler, **kw)
return f"json_extract({col}, '$.{key}')"
@compiles(_JsonEq)
def _json_eq_sqlite(element: Any, compiler: Any, **kw: Any) -> str:
col, key, (value,) = _parts(element, compiler, **kw)
return f"(json_extract({col}, '$.{key}') = {compiler.process(value, **kw)})"
@compiles(_JsonNull)
def _json_null_sqlite(element: Any, compiler: Any, **kw: Any) -> str:
col, key, _ = _parts(element, compiler, **kw)
return f"(json_extract({col}, '$.{key}') IS NULL)"
@compiles(json_flag)
def _json_flag_sqlite(element: Any, compiler: Any, **kw: Any) -> str:
col, key, _ = _parts(element, compiler, **kw)
return f"(coalesce(json_extract({col}, '$.{key}'), 0) != 0)"
@compiles(json_is_string)
def _json_is_string_sqlite(element: Any, compiler: Any, **kw: Any) -> str:
col, key, _ = _parts(element, compiler, **kw)
return f"(json_type({col}, '$.{key}') = 'text')"
# ---- PostgreSQL(应用库的 JSON 列是 json)----
def _got(col: str, key: str) -> str:
"""取出的那一段转成 JSONB:json 没有相等运算符。列本身是 JSONB 也照样成立。"""
return f"CAST(({col} -> '{key}') AS JSONB)"
@compiles(json_text, "postgresql")
def _json_text_pg(element: Any, compiler: Any, **kw: Any) -> str:
col, key, _ = _parts(element, compiler, **kw)
return f"({col} ->> '{key}')"
@compiles(_JsonEq, "postgresql")
def _json_eq_pg(element: Any, compiler: Any, **kw: Any) -> str:
col, key, (value,) = _parts(element, compiler, **kw)
return f"({_got(col, key)} = CAST({compiler.process(value, **kw)} AS JSONB))"
@compiles(_JsonNull, "postgresql")
def _json_null_pg(element: Any, compiler: Any, **kw: Any) -> str:
# json_extract 对「没有这个键」和「值为 null」都给 NULL;-> 对后者给的是 JSON 的 null
col, key, _ = _parts(element, compiler, **kw)
return f"(coalesce({_got(col, key)}, 'null'::jsonb) = 'null'::jsonb)"
@compiles(json_flag, "postgresql")
def _json_flag_pg(element: Any, compiler: Any, **kw: Any) -> str:
col, key, _ = _parts(element, compiler, **kw)
return f"(coalesce({_got(col, key)}, 'null'::jsonb) NOT IN ('null'::jsonb, 'false'::jsonb, '0'::jsonb))"
@compiles(json_is_string, "postgresql")
def _json_is_string_pg(element: Any, compiler: Any, **kw: Any) -> str:
col, key, _ = _parts(element, compiler, **kw)
return f"(jsonb_typeof({_got(col, key)}) = 'string')"
# ---- 插入,撞了唯一键就跳过 ----
def dialect_of(session: Any) -> str:
"""会话(AsyncSession / Session)连着的库的方言名。"""
return session.get_bind().dialect.name
def insert_ignore(session: Any, table: Any) -> Any:
"""按会话连着的库挑方言的 insert,调用方接着 .values(...).on_conflict_do_nothing(index_elements=[...])。"""
if dialect_of(session) == "postgresql":
from sqlalchemy.dialects.postgresql import insert
else:
from sqlalchemy.dialects.sqlite import insert
return insert(table)
# ---- 撞的是哪个唯一约束 ----
_SQLITE_UNIQUE = "UNIQUE constraint failed:"
_PG_KEY = re.compile(r"Key \(([^)]*)\)=")
def _driver_error(error: BaseException) -> Any:
"""SQLAlchemy 的异常 → asyncpg 自己的异常(在 orig 的 __cause__ 上;orig 只抄了 sqlstate,没有表名和 DETAIL)。"""
seen: Any = getattr(error, "orig", None) or error
for _ in range(4):
if getattr(seen, "sqlstate", None) and hasattr(seen, "table_name"):
return seen
seen = getattr(seen, "__cause__", None)
if seen is None:
break
return None
def unique_violation(error: BaseException, table: str, *columns: str) -> bool:
"""这个 IntegrityError 是不是撞了 table 上正好这几列的唯一约束(主键、唯一索引都算)。"""
text = str(getattr(error, "orig", None) or error)
if _SQLITE_UNIQUE in text:
failed = text.split(_SQLITE_UNIQUE, 1)[1].splitlines()[0]
return {part.strip() for part in failed.split(",")} == {f"{table}.{c}" for c in columns}
cause = _driver_error(error)
if cause is None or getattr(cause, "sqlstate", None) != "23505" or getattr(cause, "table_name", None) != table:
return False
match = _PG_KEY.search(getattr(cause, "detail", None) or "")
return bool(match) and [c.strip().strip('"') for c in match.group(1).split(",")] == list(columns)
+76
View File
@@ -0,0 +1,76 @@
"""应用库是 PostgreSQL 时,让它在 SQLite 上写成的那些假设下照样成立。
只装在应用库那个引擎的方言实例上(db/base.make_engine),业务数据源的 PostgreSQL 引擎不受影响。
**建表的列类型。** 应用是在 SQLite 上写成的:SQLite 不查 VARCHAR 的长度,INTEGER 是 64 位。PostgreSQL 上
VARCHAR(n) 超长直接报错、INTEGER 只有 32 位。模型里的 String(n) 是写给人看的「大致多长」,从来没有被库约束过:
逐列核对只核得到测试走到的值,漏一处就是线上的写入失败,从 SQLite 导过来的存量数据里也可能已经有超长的。所以
应用库上 String(n) 建成不限长的 VARCHAR(PostgreSQL 里 varchar、varchar(n)、text 存储和性能一样,限长只多一次
检查)。CAST(x AS VARCHAR(n)) 这种查询里的写法不动——那是截断的意思。Integer 列在 PostgreSQL 上是 BIGINT:
那是在模型这一层换的(db/base._portable_integer),绑定参数的类型转换跟着列走,大数不在半路溢出。
**ORDER BY 里 NULL 的位置。** SQLite 升序 NULL 在前、降序 NULL 在后,PostgreSQL 正好相反。应用里两百多处排序,
可空的列(开始时间、到期时间、发布时间……)排出来的先后要和 SQLite 一致:可空列上没写 nulls_first / nulls_last
的 ASC 补 NULLS FIRST、DESC 补 NULLS LAST,ORDER BY 里直接写列名(隐含 ASC)的同样补上。不可空的列不动——
补了会让索引用不上,又没有 NULL 可排。窗口函数 OVER (ORDER BY …) 里同样补。原生 text() 里写的 ORDER BY 不经过
这里,要自己写明(db/base 的去重语句里就写了)。
"""
from __future__ import annotations
from typing import Any
from sqlalchemy import Column
from sqlalchemy.dialects.postgresql.asyncpg import PGCompiler_asyncpg
from sqlalchemy.dialects.postgresql.base import PGTypeCompiler
from sqlalchemy.sql import operators
from sqlalchemy.sql.elements import ClauseList, UnaryExpression
#: 外层已经写了 nulls_first / nulls_last:里面那一层 ASC / DESC 不再补
_EXPLICIT = "_agentlab_explicit_nulls"
def _nullable(element: Any) -> bool:
return isinstance(element, Column) and bool(element.nullable) and not element.primary_key
class AppTypeCompiler(PGTypeCompiler):
def visit_string(self, type_: Any, **kw: Any) -> str:
if type_.length and isinstance(kw.get("type_expression"), Column):
return "VARCHAR"
return super().visit_string(type_, **kw)
def _ordered(clauses: Any) -> list[Any]:
"""ORDER BY 里直接写的可空列补成 ASC(visit_unary 再给它补 NULLS FIRST)。"""
return [c.asc() if _nullable(c) else c for c in clauses]
class AppCompiler(PGCompiler_asyncpg):
def order_by_clause(self, select: Any, **kw: Any) -> str:
order_by = self._generate_delimited_list(_ordered(select._order_by_clauses), ", ", **kw)
return " ORDER BY " + order_by if order_by else ""
def visit_over(self, over: Any, **kw: Any) -> str:
clauses = list(over.order_by) if over.order_by is not None else []
if any(_nullable(c) for c in clauses):
# 在副本上换:语句对象可能被缓存、被别处共用,不能改原件
over = over._clone()
over.order_by = ClauseList(*_ordered(clauses))
return super().visit_over(over, **kw)
def visit_unary(self, unary: UnaryExpression, **kw: Any) -> str:
modifier = unary.modifier
if modifier in (operators.nulls_first_op, operators.nulls_last_op):
return super().visit_unary(unary, **{**kw, _EXPLICIT: True})
explicit = kw.pop(_EXPLICIT, False)
text = super().visit_unary(unary, **kw)
if not explicit and modifier in (operators.asc_op, operators.desc_op) and _nullable(unary.element):
text += " NULLS FIRST" if modifier is operators.asc_op else " NULLS LAST"
return text
def install(dialect: Any) -> None:
"""装到应用库引擎的方言实例上。"""
dialect.statement_compiler = AppCompiler
dialect.type_compiler_instance = dialect.type_compiler = AppTypeCompiler(dialect)
+57
View File
@@ -0,0 +1,57 @@
"""应用库、断点库的地址:规范写法、换算成 psycopg 用的连接串、写进日志的样子(不带密码)。
不依赖 app.core.config:配置的校验器要调这里,这里再导 config 就成了循环导入。
"""
from __future__ import annotations
from sqlalchemy.engine import URL, make_url
from sqlalchemy.exc import ArgumentError
#: 应用库认的写法。asyncpg 是 SQLAlchemy 这边的驱动;postgresql:// / postgres:// 是常见的通用写法,一律换成 asyncpg
_APP_SCHEMES = {"postgresql+asyncpg", "postgresql", "postgres"}
#: 断点库认的写法:psycopg 只认 libpq 的 postgresql:// / postgres://;带驱动名的 SQLAlchemy 写法也收,去掉驱动名
_CHECKPOINT_SCHEMES = {"postgresql", "postgres", "postgresql+psycopg", "postgresql+asyncpg"}
#: asyncpg 和 libpq 对 SSL 的参数名不一样:asyncpg 是 ssl=require,libpq 是 sslmode=require(取值同一套)
_ASYNCPG_TO_LIBPQ = {"ssl": "sslmode"}
def _parse(raw: str, schemes: set[str], example: str) -> URL:
try:
url = make_url(raw)
except ArgumentError:
raise ValueError(f"认不出这个地址,应当写成 {example}") from None
if url.drivername not in schemes:
raise ValueError(f"只认 PostgreSQL({example}),不认 {url.drivername}://;用 SQLite 就不要设这一项")
if not url.database:
raise ValueError(f"地址里要写库名,例如 {example}")
return url
def normalize_database_url(raw: str) -> str:
"""AGENTLAB_DATABASE_URL → postgresql+asyncpg://…(密码原样保留)。认不出的写法抛 ValueError。"""
url = _parse(raw, _APP_SCHEMES, "postgresql+asyncpg://用户:密码@主机:5432/库名")
return url.set(drivername="postgresql+asyncpg").render_as_string(hide_password=False)
def normalize_checkpoint_url(raw: str) -> str:
"""AGENTLAB_CHECKPOINT_URL → libpq 认的 postgresql://…。认不出的写法抛 ValueError。"""
url = _parse(raw, _CHECKPOINT_SCHEMES, "postgresql://用户:密码@主机:5432/库名")
return libpq_conninfo(url.render_as_string(hide_password=False))
def libpq_conninfo(database_url: str) -> str:
"""SQLAlchemy 的地址 → psycopg 的连接串:去掉驱动名,asyncpg 写法的参数换成 libpq 的。"""
url = make_url(database_url)
query = {_ASYNCPG_TO_LIBPQ.get(k, k): v for k, v in url.query.items()}
return url.set(drivername="postgresql", query=query).render_as_string(hide_password=False)
def describe(database_url: str) -> str:
"""写进日志的样子:主机、端口、库名、用户,不写密码。"""
url = make_url(database_url)
host = url.host or "本机套接字"
port = f":{url.port}" if url.port else ""
user = f",用户 {url.username}" if url.username else ""
return f"{host}{port}/{url.database}{user}"
+87
View File
@@ -0,0 +1,87 @@
"""LangGraph 的断点库:应用库是 SQLite 时用 AsyncSqliteSaver(数据目录里的 checkpoints.db,和业务表分开免得抢写锁),
是 PostgreSQL 时用官方的 AsyncPostgresSaver(psycopg 3 + 连接池),放在同一个库里,AGENTLAB_CHECKPOINT_URL 另给就放那里。
它建的四张表(checkpoints、checkpoint_blobs、checkpoint_writes、checkpoint_migrations)和应用的表不重名
(test_db_portable 守着),所以同一个库、同一个 schema 就行。
应用里读断点一律走 saver 的公开接口(aget_state / aget_state_history / aget_tuple / alist),不直接查这几张表:
两种库的表结构完全不一样(SQLite 整份断点是一个 msgpack,PostgreSQL 拆成 JSONB 加 blob)。
"""
from __future__ import annotations
import contextlib
from collections.abc import AsyncIterator
from pathlib import Path
from typing import Any
from langgraph.checkpoint.base import BaseCheckpointSaver
from app.core.config import settings
#: 建断点表时拿的会话级咨询锁的号:几个进程同时启动,各自的 setup(建表、记迁移版本)一个一个来
_SETUP_LOCK = 0x4167656E74436B70 # "AgentCkp"
def _pool_size() -> int:
# 运行里每个超步写一次断点、每个任务写一次 pending writes,都是一来一回的短操作。并发运行多时多给几条
return max(4, min(settings.max_concurrent_runs, 20))
@contextlib.asynccontextmanager
async def open_postgres_saver(conninfo: str, *, max_size: int | None = None,
application_name: str = "agentlab") -> AsyncIterator[BaseCheckpointSaver]:
"""PostgreSQL 的断点库:开连接池、建表(幂等),退出时关池子。application_name 同应用库(db/base.APPLICATION_NAME)。"""
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool
# 连接参数照 AsyncPostgresSaver.from_conn_string:自动提交(它不自己管事务)、语句第一次执行就预备、行按 dict 取
pool = AsyncConnectionPool(
conninfo, min_size=1, max_size=max_size or _pool_size(), open=False,
kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row,
"application_name": application_name},
check=AsyncConnectionPool.check_connection, name="agentlab-checkpoints")
await pool.open(wait=True)
try:
saver = AsyncPostgresSaver(pool) # type: ignore[arg-type]
async with pool.connection() as conn:
await conn.execute("SELECT pg_advisory_lock(%s)", (_SETUP_LOCK,))
try:
await saver.setup()
finally:
await conn.execute("SELECT pg_advisory_unlock(%s)", (_SETUP_LOCK,))
yield saver
finally:
await pool.close()
@contextlib.asynccontextmanager
async def open_sqlite_saver(path: Path | str) -> AsyncIterator[BaseCheckpointSaver]:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
async with AsyncSqliteSaver.from_conn_string(str(path)) as saver:
yield saver
def open_saver() -> contextlib.AbstractAsyncContextManager[BaseCheckpointSaver]:
"""按配置开这个进程的断点库。"""
conninfo = settings.checkpoint_conninfo
if conninfo:
return open_postgres_saver(conninfo)
return open_sqlite_saver(settings.checkpoint_path)
def kind() -> str:
""""sqlite" 或 "postgresql":断点库放在哪种库里。"""
return "postgresql" if settings.checkpoint_conninfo else "sqlite"
async def count(saver: Any, thread_id: str | None = None) -> tuple[int, int]:
"""(断点条数, pending writes 条数)。thread_id 为空数全部。copy-to-postgres 核对用,走公开接口。"""
config = {"configurable": {"thread_id": thread_id}} if thread_id else None
checkpoints = writes = 0
async for tup in saver.alist(config):
checkpoints += 1
writes += len(tup.pending_writes or ())
return checkpoints, writes
+16 -11
View File
@@ -10,7 +10,7 @@ from collections.abc import Callable, Sequence
from datetime import datetime, timezone
from typing import Any
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from langgraph.checkpoint.base import BaseCheckpointSaver
from langgraph.errors import GraphRecursionError
from langgraph.types import Command
from sqlalchemy import case, func, select, update
@@ -71,7 +71,7 @@ class RunManager:
def __init__(self) -> None:
self._tasks: dict[str, asyncio.Task[None]] = {}
self._checkpointer: AsyncSqliteSaver | None = None
self._checkpointer: BaseCheckpointSaver | None = None
self._cm: Any = None
self._semaphore = asyncio.Semaphore(settings.max_concurrent_runs)
#: 正在关停。此时收到的取消来自关停而不是用户,运行要记成可恢复的中断
@@ -86,7 +86,10 @@ class RunManager:
async def setup(self) -> None:
self._closing = False
settings.ensure_dirs()
self._cm = AsyncSqliteSaver.from_conn_string(str(settings.checkpoint_path))
# SQLite:数据目录里的 checkpoints.db;PostgreSQL:同一个库(或 AGENTLAB_CHECKPOINT_URL),见 engine/checkpoints.py
from app.engine.checkpoints import open_saver
self._cm = open_saver()
self._checkpointer = await self._cm.__aenter__()
# 上次进程被强杀时留下的记录。running 和 queued 要分开处理:
# running 至少跑过一步、有 checkpoint,可以续;queued 还在排队,
@@ -163,7 +166,7 @@ class RunManager:
await self._cm.__aexit__(None, None, None)
@property
def checkpointer(self) -> AsyncSqliteSaver:
def checkpointer(self) -> BaseCheckpointSaver:
if self._checkpointer is None:
raise RuntimeError("RunManager 还没初始化")
return self._checkpointer
@@ -889,6 +892,15 @@ class RunManager:
try:
await self._finalize(run_id, status, error, final_state, elapsed,
terminal=terminal, error_node=error_node, end_reason=end_reason)
# 记 action.unknown、安排核实也算收尾:做完才把自己摘掉。以前摘掉之后才做,等运行「不在跑了」
# 再去读事件的一方(接口、测试)有一小段读不到这一条;库慢一点(PostgreSQL 满载时)就读不到
if pending_action is not None:
from app.process.actions import on_unknown
try:
await on_unknown(pending_action)
except Exception: # noqa: BLE001 - 运行已经收成挂起;没安排上的核实,下次启动时 actions.startup 补上
logger.exception("run %s 的动作 %s 结果未知,安排核实时出错", run_id, pending_action)
finally:
# 收尾被打断(关停时的取消、库连接出错)也得把自己摘掉。以前写在
# _finalize 后面,一被打断就漏掉,留下的任务让下一次 shutdown 的
@@ -896,13 +908,6 @@ class RunManager:
self._finalizing.discard(run_id)
self._tasks.pop(run_id, None)
self._stopped_by.pop(run_id, None)
if pending_action is not None:
from app.process.actions import on_unknown
try:
await on_unknown(pending_action)
except Exception: # noqa: BLE001 - 运行已经收成挂起;没安排上的核实,下次启动时 actions.startup 补上
logger.exception("run %s 的动作 %s 结果未知,安排核实时出错", run_id, pending_action)
# 运行到终态就把沙箱会话收掉。thread_id 每个 run 都是新的,
# 不收的话每跑一次带代码节点的图就多一台常驻 microVM(几百 MB),
# 而且没有任何自动回收路径会碰它——闲置回收只在"下次有人执行
+1 -1
View File
@@ -294,7 +294,7 @@ def accept_problem(setting: dict[str, Any] | None, *, name: str, mime: str, size
# --------------------------------------------------------------------------
#: 还没绑到记录上:JSON 列显式写过 None 的存的是 JSON 的 null,不是 SQL 的 NULL,两种都算
_NO_RECORD = text("(files.bound_record IS NULL OR files.bound_record = 'null')")
_NO_RECORD = text("(files.bound_record IS NULL OR CAST(files.bound_record AS TEXT) = 'null')")
def file_fields(form: Any) -> list[str]:
+3 -2
View File
@@ -44,6 +44,7 @@ from sqlalchemy import func, select
from app.db.base import SessionLocal
from app.db.models import Run, Workflow
from app.db.portable import json_eq
logger = logging.getLogger(__name__)
@@ -150,9 +151,9 @@ async def issued_today(session: Any) -> int:
async def issued_runs(session: Any, run_id: str) -> list[Run]:
"""这次运行按口径出具过的正式运行(origin.source_run 是它),新的在前。origin_kind 有索引,先按它筛再看 origin
(json_extract,同 auth/keys 认幂等键的写法)。"""
(db/portable.json_eq,同 auth/keys 认幂等键的写法)。"""
rows = (await session.execute(select(Run).where(
Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.source_run") == run_id)
Run.origin_kind == ORIGIN, json_eq(Run.origin, "source_run", run_id))
.order_by(Run.created_at.desc()))).scalars()
return list(rows)
+13 -14
View File
@@ -66,11 +66,12 @@ from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import func, select
from sqlalchemy import func, not_, select
from app.core.events import EventType
from app.db.base import SessionLocal, utcnow
from app.db.models import Run, RunEvent, Workflow
from app.db.portable import json_eq, json_flag, json_text
from app.ledger import uses as ledger_uses
logger = logging.getLogger(__name__)
@@ -225,10 +226,9 @@ def source_cards(graph: Any) -> list[str]:
async def recall_runs(session: Any, workflow_id: str, new_version: int | None = None) -> list[Run]:
"""以 W(新版本 N)发起过的召回运行(试算的不算),新的在前。origin_kind 有索引,先按它筛再看 origin。"""
stmt = select(Run).where(Run.origin_kind == ORIGIN,
func.json_extract(Run.origin, "$.workflow_id") == workflow_id,
func.coalesce(func.json_extract(Run.origin, "$.probe"), 0) == 0)
json_eq(Run.origin, "workflow_id", workflow_id), not_(json_flag(Run.origin, "probe")))
if new_version is not None:
stmt = stmt.where(func.json_extract(Run.origin, "$.to_version") == int(new_version))
stmt = stmt.where(json_eq(Run.origin, "to_version", int(new_version)))
return list((await session.execute(stmt.order_by(Run.created_at.desc()))).scalars())
@@ -567,9 +567,8 @@ async def version_recall_counts(session: Any, workflow_id: str, version: int) ->
state, _ = await outcome_of(session, run)
changed += state == "changed"
as_old = int(await session.scalar(select(func.count()).select_from(Run).where(
Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.workflow_id") == workflow_id,
func.json_extract(Run.origin, "$.from_version") == int(version),
func.coalesce(func.json_extract(Run.origin, "$.probe"), 0) == 0)) or 0)
Run.origin_kind == ORIGIN, json_eq(Run.origin, "workflow_id", workflow_id),
json_eq(Run.origin, "from_version", int(version)), not_(json_flag(Run.origin, "probe")))) or 0)
return {"groups": len(as_new), "changed": changed, "recalled_from": as_old}
@@ -986,7 +985,7 @@ async def answer_marks(session: Any, run_id: str, principal: Any) -> dict[str, A
select(Task, Run.origin).join(Run, Run.id == Task.run_id)
.where(Task.kind == TASK_KIND, Task.status == "pending",
(Task.run_id == run_id) | ((Run.origin_kind == ORIGIN)
& (func.instr(func.json_extract(Run.origin, "$.uses"), run_id) > 0)))
& json_text(Run.origin, "uses").contains(run_id, autoescape=True)))
.order_by(Task.created_at, Task.id))).all()
mine = await _assignments(session, [t.id for t, _ in candidates])
tasks: list[dict[str, Any]] = []
@@ -1134,8 +1133,8 @@ async def _explore_snapshots(session: Any, source: Any, tables: Iterable[str], *
stmt = (select(Artifact.id, Artifact.meta, Run.id, Run.finished_at).join(Run, Run.id == Artifact.run_id)
.where(Artifact.kind == QUERY_SNAPSHOT_KIND, Run.run_class == "exploratory", Run.status == "succeeded",
Run.manifest_hash.is_not(None), counted_runs(),
or_(func.json_extract(Artifact.meta, "$.source_id") == source.id,
func.json_extract(Artifact.meta, "$.source") == source.name))
or_(json_eq(Artifact.meta, "source_id", source.id),
json_eq(Artifact.meta, "source", source.name)))
.order_by(Run.finished_at.desc(), Run.id, Artifact.id))
cut = _aware(before)
offset = 0
@@ -1473,10 +1472,10 @@ async def _probe_calibers(change: Mapping[str, Any]) -> list[str]:
continue
async with SessionLocal() as session:
done = await session.scalar(select(func.count()).select_from(Run).where(
Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.probe") == 1,
func.json_extract(Run.origin, "$.workflow_id") == caliber.workflow_id,
func.json_extract(Run.origin, "$.version") == caliber.version,
func.json_extract(Run.origin, "$.change") == change.get("id")))
Run.origin_kind == ORIGIN, json_flag(Run.origin, "probe"),
json_eq(Run.origin, "workflow_id", caliber.workflow_id),
json_eq(Run.origin, "version", caliber.version),
json_eq(Run.origin, "change", change.get("id"))))
if done:
continue
latest = (await session.execute(select(Run.input).where(
+2 -2
View File
@@ -143,10 +143,10 @@ def actions_of(content: dict[str, Any], names: dict[str, str], hide: Any = None)
async def reconciled_runs(session: Any, run_id: str) -> list[Run]:
"""对这次运行对过的账(origin.source_run 是它),新的在前。"""
from sqlalchemy import func
from app.db.portable import json_eq
rows = (await session.execute(select(Run).where(
Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.source_run") == run_id)
Run.origin_kind == ORIGIN, json_eq(Run.origin, "source_run", run_id))
.order_by(Run.created_at.desc()))).scalars()
return list(rows)
+2 -1
View File
@@ -35,6 +35,7 @@ from app.db.base import SessionLocal, utcnow
from app.db.models import (
App, Approval, AppVersion, AuditEvent, Group, GroupMember, Run, Task, TaskAssignment, User,
)
from app.db.portable import json_text
from app.engine.lint_process import normalize_policy, parse_duration
from app.process.events import record
@@ -913,7 +914,7 @@ async def tick(now: datetime | None = None) -> list[str]:
"""调度循环每一拍:提醒、升级、超时驳回(永不自动批准)。交回这一拍处理过的待办 id(生效可查:办了哪几条,
不只是几条;测试按自己造的那条断言,不受同一个库里别的待办影响)。试运行的待办不处理(只显示时限)。"""
now = now or utcnow()
action = func.json_extract(Task.on_timeout, "$.action")
action = json_text(Task.on_timeout, "action")
trial = select(Run.id).where(Run.origin_kind == TRIAL_ORIGIN)
async with SessionLocal() as session:
due = list((await session.execute(select(Task.id).where(