应用库可移植到 PostgreSQL:AGENTLAB_DATABASE_URL 设成 postgresql+asyncpg://… 时整个应用库用 PostgreSQL,断点用官方的 langgraph-checkpoint-postgres(同一个库,AGENTLAB_CHECKPOINT_URL 可另给);不设时照旧是数据目录里的 SQLite,行为不变

- 配置:database_url、checkpoint_url(启动时校验写法,只认 PostgreSQL)、连接池四项;启动日志写用的是哪种库(主机、库名、用户,不写密码)
- 引擎(db/base.make_engine):SQLite 照旧 WAL、外键、忙等;PostgreSQL 用连接池、探活、会话时区 UTC,装上方言调整(db/postgres.py):建表时 String(n) 是不限长的 VARCHAR(SQLite 从不查长度,导过来的存量也可能超长)、ORDER BY 里可空列的 NULL 位置照 SQLite 补写(升序在前、降序在后,窗口函数里同样);Integer 列在 PostgreSQL 上是 BIGINT(列挂到表上时换成 with_variant,绑定参数跟着列按 BIGINT 转换);JSON 列两边都存原文(PostgreSQL 的 json,不用 JSONB:JSONB 重排对象里的键,快照的列序、表的先后读回来就变了);JSON 里的 NaN 写成 null、NUL 去掉(PostgreSQL 不收)
- 迁移按方言:已有的列和索引用 inspector 认;PostgreSQL 上补列的类型取模型里这一列的、缺省值取迁移表里的(布尔 0/1 换成 false/true),ADD COLUMN IF NOT EXISTS;去重语句里的布尔按方言写;部分唯一索引补 postgresql_where;init_db 在 PostgreSQL 上拿咨询锁(几个进程同时启动不撞)
- 应用库上的 SQLite 专用写法改成按方言编译的表达式(db/portable.py):json_extract 的十几处(口径召回、出具、对账、个人应用、幂等键、访问控制、数据源变化通知、剖析任务、待办超时)换成 json_eq / json_flag / json_text,json_type 换成 json_is_string(两种库同一套带类型的比较);能力登记的 sqlite insert 换成按会话方言的 insert_ignore;能力使用次数的 json_each 按方言分支;附件「没绑记录」的条件不靠 json 的等号;按报错文字认唯一约束的地方换成 unique_violation(按表和列认,两种库的报错都认得)
- 断点(engine/checkpoints.py):PostgreSQL 时开 psycopg 连接池、在咨询锁里 setup;runner 只认 BaseCheckpointSaver,读断点仍只走 saver 的公开接口
- 依赖:新 extra postgres(asyncpg、psycopg[binary] 3、psycopg-pool、langgraph-checkpoint-postgres 3.1.2);langgraph-checkpoint 卡在 <4.3(断点格式就在这个包里,postgres 包新版要 4.3,不动已装的 4.2.0);锁文件补上新包
- 运行收尾:记 action.unknown、安排核实做完才把运行摘掉(以前先摘掉,等「不在跑了」再读事件的一方在库慢时读不到这一条)
- 测试:AGENTLAB_TEST_DATABASE_URL 时跑在 PostgreSQL 上(每个 xdist 进程建自己的库、按事件循环分的连接池 tests/pgpool.py),不设时盖掉环境里的数据库地址照旧用 SQLite;scripts/test-postgres.sh 起临时的 postgres:16 容器跑全量;sqlite_only 标记;另开连接改库的几个竞态测试、触发器测试改成两种库都能跑(tests/appdb.py);新增 test_db_portable 守着方言调整、JSON 键序、超长字符串、大整数、NULL 排序、时区、PostgreSQL 老库迁移
This commit is contained in:
ThearchyHelios
2026-10-19 09:52:09 +08:00
parent c79f4e8bf7
commit 2e7d6103ea
39 changed files with 1522 additions and 155 deletions
+77
View File
@@ -0,0 +1,77 @@
"""测试里另开一条连接直接改应用库:模拟别处(另一个进程、恢复、继续运行)抢在中间提交。
应用库是 SQLite 时用 sqlite3 连数据目录里的 agentlab.db(同以前各处的写法);是 PostgreSQL 时(AGENTLAB_TEST_DATABASE_URL,
见 conftest)用 psycopg 连同一个库。SQL 一律用 ? 占位,两种库写成同一句;锁等 5 秒(同以前 sqlite3 的 timeout=5):
另一条连接要等的锁正好被调用方拿着的话,等满报错,不会把测试卡死。
"""
from __future__ import annotations
import re
from typing import Any
from app.core.config import settings
def is_postgres() -> bool:
return settings.db_dialect == "postgresql"
def _pg_conninfo() -> str:
from app.db.urls import libpq_conninfo
return libpq_conninfo(settings.db_url)
def execute(*statements: str | tuple[str, tuple[Any, ...]]) -> None:
"""在另一条连接上依次执行、一起提交。每一项是 SQL,或者 (SQL, 参数)。"""
items = [(s, ()) if isinstance(s, str) else (s[0], tuple(s[1])) for s in statements]
if is_postgres():
import psycopg
with psycopg.connect(_pg_conninfo(), options="-c lock_timeout=5000") as conn:
for sql, params in items:
conn.execute(re.sub(r"\?", "%s", sql), params)
conn.commit()
return
import sqlite3
db = sqlite3.connect(settings.db_path, timeout=5)
try:
for sql, params in items:
db.execute(sql, params)
db.commit()
finally:
db.close()
def query(sql: str, *params: Any) -> list[tuple[Any, ...]]:
"""在另一条连接上读:看别人这时提交了什么(不是调用方事务里的)。"""
if is_postgres():
import psycopg
with psycopg.connect(_pg_conninfo(), options="-c lock_timeout=5000") as conn:
return [tuple(r) for r in conn.execute(re.sub(r"\?", "%s", sql), params).fetchall()]
import sqlite3
db = sqlite3.connect(settings.db_path, timeout=5)
try:
return [tuple(r) for r in db.execute(sql, params).fetchall()]
finally:
db.close()
def refuse_inserts(table: str, condition: str, *, name: str) -> tuple[list[str], list[str]]:
"""建一个触发器:往 table 插入满足 condition(用 NEW.列 写)的行时报错,报的是完整性错误(IntegrityError),同 SQLite
的 RAISE(ABORT)。返回 (建的语句, 删的语句),各是一串语句,逐条在应用库的连接上执行。
PostgreSQL 要先建一个触发器函数(plpgsql)再挂上去,错误码给 23000(integrity_constraint_violation)。"""
if is_postgres():
create = [f"CREATE FUNCTION {name}_fn() RETURNS trigger AS $$ BEGIN IF {condition} THEN "
f"RAISE EXCEPTION 'refused by test' USING ERRCODE = 'integrity_constraint_violation'; END IF; "
f"RETURN NEW; END $$ LANGUAGE plpgsql",
f"CREATE TRIGGER {name} BEFORE INSERT ON {table} FOR EACH ROW EXECUTE FUNCTION {name}_fn()"]
drop = [f"DROP TRIGGER IF EXISTS {name} ON {table}", f"DROP FUNCTION IF EXISTS {name}_fn()"]
return create, drop
return ([f"CREATE TRIGGER {name} BEFORE INSERT ON {table} WHEN {condition} "
f"BEGIN SELECT RAISE(ABORT, 'refused by test'); END"], [f"DROP TRIGGER IF EXISTS {name}"])
+58 -1
View File
@@ -12,12 +12,58 @@ app.db.base 导入时就按 db_url 建好了。conftest 早于测试模块加载
from __future__ import annotations
import atexit
import os
import tempfile
from pathlib import Path
_TMP = Path(tempfile.mkdtemp(prefix="agentlab-tests-"))
os.environ["AGENTLAB_DATA_DIR"] = str(_TMP)
def _postgres_database(admin_url: str) -> str:
"""AGENTLAB_TEST_DATABASE_URL 给的是管理用的库:每个 xdist 进程在同一台服务器上建一个自己的空库(库名加进程号后缀,
已有的同名库先删掉),进程退出时删掉。返回这个库的地址。scripts/test-postgres.sh 起临时容器、设好这个变量。"""
import psycopg
from sqlalchemy.engine import make_url
from app.db.urls import libpq_conninfo, normalize_database_url
url = make_url(normalize_database_url(admin_url))
name = f"{url.database}_{os.environ.get('PYTEST_XDIST_WORKER', 'main')}"
admin = libpq_conninfo(url.render_as_string(hide_password=False))
def run(sql: str) -> None:
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql)
run(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)')
run(f'CREATE DATABASE "{name}"')
def drop() -> None:
try:
run(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)')
except Exception: # noqa: BLE001 - 服务器先走了(容器已删)就算了
pass
atexit.register(drop)
return url.set(database=name).render_as_string(hide_password=False)
# 应用库:缺省数据目录里的 SQLite。设了 AGENTLAB_TEST_DATABASE_URL 就跑在 PostgreSQL 上。不管哪种,都盖掉环境和 .env
# 里可能有的 AGENTLAB_DATABASE_URL / AGENTLAB_CHECKPOINT_URL:测试绝不能连到真在用的库上
_TEST_PG = os.environ.get("AGENTLAB_TEST_DATABASE_URL", "").strip()
os.environ["AGENTLAB_CHECKPOINT_URL"] = ""
os.environ["AGENTLAB_DATABASE_URL"] = _postgres_database(_TEST_PG) if _TEST_PG else ""
if _TEST_PG:
# 每个测试一个事件循环(pytest-asyncio 的缺省),asyncpg 的连接不能跨循环用:引擎先按不用连接池建,
# 再换成按事件循环分的池子(tests/pgpool.py),一个测试里的查库照样复用连接
os.environ["AGENTLAB_DB_POOL_SIZE"] = "0"
from app.db.base import engine as _app_engine # noqa: E402
from tests.pgpool import install as _loop_local_pool # noqa: E402
_loop_local_pool(_app_engine)
# 别把真实的 key 带进测试环境
os.environ.setdefault("AGENTLAB_SECRET_KEY", "test-only-not-a-real-key")
# 定时任务的调度循环不在测试里起:走 lifespan 的测试不该到点替人发起运行。要测调度的直接调它
@@ -42,7 +88,18 @@ def _prepare_schema():
def pytest_report_header(config) -> str:
return f"测试数据目录:{_TMP}"
where = "PostgreSQL(AGENTLAB_TEST_DATABASE_URL)" if _TEST_PG else "SQLite"
return f"测试数据目录:{_TMP};应用库:{where}"
def pytest_collection_modifyitems(config, items) -> None:
"""标了 sqlite_only 的测试只在应用库是 SQLite 时跑:它们断言的是 SQLite 自己的东西(数据目录里的库文件、PRAGMA)。"""
if not _TEST_PG:
return
skip = pytest.mark.skip(reason="只对 SQLite 应用库成立")
for item in items:
if "sqlite_only" in item.keywords:
item.add_marker(skip)
@pytest.fixture(scope="session")
+123
View File
@@ -0,0 +1,123 @@
"""应用库是 PostgreSQL 时测试用的连接池:每个事件循环一个池子,连接只给建它的那个循环用。
pytest-asyncio 每个测试一个事件循环,TestClient 在自己的线程里另起一个;asyncpg 的连接绑在建它的循环上,换个循环用
就报「attached to a different loop」。不用连接池(每次查库新建连接)能跑,但一次运行几十次查库,每次多几毫秒,
合起来让计时类的测试(审批收尾不计时长、反查的时限)在满载时越界——生产上有连接池,没有这笔开销。
这里:连接按建它的循环归池子,取的时候只从当前循环的池子里拿,还回来放回它自己的池子;循环关掉之前(测试结束、
TestClient 退出)把它池子里的连接在它自己的循环里关掉。只在 conftest 里、AGENTLAB_TEST_DATABASE_URL 时装上,
生产代码不用它。
"""
from __future__ import annotations
import asyncio
import contextlib
import threading
import weakref
from typing import Any
from sqlalchemy.pool import Pool
_LOOP = "agentlab_loop"
class LoopLocalPool(Pool):
_is_asyncio = True
def __init__(self, creator: Any, **kw: Any) -> None:
super().__init__(creator, **kw)
self._idle: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, list[Any]] = weakref.WeakKeyDictionary()
self._lock = threading.Lock()
@classmethod
def replacing(cls, pool: Pool) -> LoopLocalPool:
"""换掉引擎原来的池子:同一个建连接的函数、同一套连接事件(方言的 on_connect 挂在上面)。"""
return cls(pool._creator, recycle=pool._recycle, echo=pool.echo, logging_name=pool._orig_logging_name,
reset_on_return=pool._reset_on_return, pre_ping=False, _dispatch=pool.dispatch,
dialect=pool._dialect)
def recreate(self) -> LoopLocalPool:
return LoopLocalPool.replacing(self)
def _bucket(self, loop: asyncio.AbstractEventLoop) -> list[Any]:
with self._lock:
return self._idle.setdefault(loop, [])
def _do_get(self) -> Any:
bucket = self._bucket(asyncio.get_running_loop())
with self._lock:
if bucket:
return bucket.pop()
record = self._create_connection()
record.info[_LOOP] = weakref.ref(asyncio.get_running_loop())
return record
def _do_return_conn(self, record: Any) -> None:
ref = record.info.get(_LOOP)
loop = ref() if ref is not None else None
if loop is None or loop.is_closed():
with contextlib.suppress(Exception):
record.close()
return
bucket = self._bucket(loop)
with self._lock:
bucket.append(record)
def take(self, loop: asyncio.AbstractEventLoop) -> list[Any]:
"""这个循环池子里空闲的连接,取出来(交给调用方在这个循环里关掉)。"""
with self._lock:
return self._idle.pop(loop, [])
def dispose(self) -> None:
"""只关当前循环的:别的循环的连接不能在这里关。没有正在跑的循环时什么都不做(各循环关掉时自己收拾)。"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return
for record in self.take(loop):
with contextlib.suppress(Exception):
record.close()
def status(self) -> str:
with self._lock:
idle = sum(len(v) for v in self._idle.values())
return f"LoopLocalPool id:{id(self)} idle:{idle} loops:{len(self._idle)}"
def install(engine: Any) -> LoopLocalPool:
"""把应用库引擎的池子换成 LoopLocalPool,并让每个事件循环关掉之前先关它自己的连接。"""
from sqlalchemy.util import greenlet_spawn
pool = LoopLocalPool.replacing(engine.sync_engine.pool)
engine.sync_engine.pool = pool
async def close_mine(records: list[Any]) -> None:
def run() -> None:
for record in records:
with contextlib.suppress(Exception):
record.close()
await greenlet_spawn(run)
base = asyncio.get_event_loop_policy()
class Policy(type(base)): # type: ignore[misc]
def new_event_loop(self) -> asyncio.AbstractEventLoop:
loop = super().new_event_loop()
close = loop.close
def closing() -> None:
current = engine.sync_engine.pool
if isinstance(current, LoopLocalPool) and not loop.is_closed() and not loop.is_running():
records = current.take(loop)
if records:
with contextlib.suppress(Exception):
loop.run_until_complete(close_mine(records))
close()
loop.close = closing # type: ignore[method-assign]
return loop
asyncio.set_event_loop_policy(Policy())
return pool
+8 -1
View File
@@ -556,9 +556,16 @@ async def test_startup_locks_legacy_sources_registered_under_other_spellings(cli
def _app_db() -> str:
"""测试进程实际连着的元数据库(conftest 建的那个)。data_dir 被夹具换过,和 settings.db_path 不是同一个。"""
"""测试进程实际连着的元数据库(conftest 建的那个)。data_dir 被夹具换过,和 settings.db_path 不是同一个。
应用库是 PostgreSQL 时没有连着的库文件:拿数据目录里的 agentlab.db 顶上(导入 PostgreSQL 之前留下的那一份,
同一批元数据,照样不许登记),先建出这个文件,硬链接、大小写这些变体才有东西可指。"""
from app.db import base as db_base
if db_base.engine.dialect.name != "sqlite":
settings.db_path.parent.mkdir(parents=True, exist_ok=True)
settings.db_path.touch()
return str(settings.db_path)
return str(db_base.engine.url.database)
+357
View File
@@ -0,0 +1,357 @@
"""应用库在 SQLite、PostgreSQL 上行为一致(db/portable.py、db/postgres.py、db/base.py 的迁移)。
连着库的那几项在两种库上各跑一遍(scripts/test-postgres.sh 跑 PostgreSQL),断言同一套结果;只看编译结果的不连库;
PostgreSQL 老库迁移另建一个库(只在 AGENTLAB_TEST_DATABASE_URL 时跑)。全是虚构数据。
"""
from __future__ import annotations
import os
import re
import uuid
import pytest
from sqlalchemy import func, select, text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.schema import CreateTable
from app.db import base as db_base
from app.db.base import Base, SessionLocal
from app.db.models import Run, TableRecipe
from app.db.portable import json_eq, json_flag, json_is_string, json_text, unique_violation
PG = bool(os.environ.get("AGENTLAB_TEST_DATABASE_URL", "").strip())
def _pg_dialect():
from sqlalchemy.dialects.postgresql.asyncpg import dialect
from app.db import postgres
made = dialect()
postgres.install(made)
return made
# --------------------------------------------------------------------------
# 守着的约定
# --------------------------------------------------------------------------
def test_every_migrated_column_is_in_the_models():
"""PostgreSQL 上补列的类型取模型里这一列的(db/base._column_ddl):迁移表里的每一列模型里都得有。"""
missing = [(t, c) for t, c, _ in db_base._COLUMN_MIGRATIONS
if t not in Base.metadata.tables or c not in Base.metadata.tables[t].c]
assert missing == []
def test_checkpoint_tables_do_not_clash_with_app_tables():
"""断点和应用表放在同一个库、同一个 schema:两边的表名不能撞。名字从两个 saver 自己的建表语句里取。"""
import inspect as pyinspect
from langgraph.checkpoint.postgres.base import BasePostgresSaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
created = set(re.findall(r"CREATE TABLE IF NOT EXISTS (\w+)", " ".join(BasePostgresSaver.MIGRATIONS)))
created |= set(re.findall(r"CREATE TABLE IF NOT EXISTS (\w+)", pyinspect.getsource(AsyncSqliteSaver.setup)))
assert {"checkpoints", "checkpoint_blobs", "checkpoint_writes", "checkpoint_migrations"} <= created
assert created & set(Base.metadata.tables) == set()
def test_postgres_columns_are_unbounded_varchar_bigint_and_json():
"""应用库上 String(n) 建成不限长的 VARCHAR、Integer 是 BIGINT(自增主键 BIGSERIAL)、JSON 是 json(不是 JSONB:
JSONB 会重排对象里的键)、时间是不带时区的 TIMESTAMP。SQLite 上的建表语句不变。"""
from sqlalchemy.dialects.sqlite.aiosqlite import dialect as sqlite_dialect
runs = Base.metadata.tables["runs"]
pg = CreateTable(runs).compile(dialect=_pg_dialect()).string
assert "id VARCHAR NOT NULL" in pg and "VARCHAR(" not in pg
assert "last_seq BIGINT" in pg and "INTEGER" not in pg
assert "input JSON NOT NULL" in pg and "JSONB" not in pg
assert "created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL" in pg
changes = CreateTable(Base.metadata.tables["source_changes"]).compile(dialect=_pg_dialect()).string
assert "seq BIGSERIAL NOT NULL" in changes
lite = CreateTable(runs).compile(dialect=sqlite_dialect()).string
assert "id VARCHAR(32) NOT NULL" in lite and "last_seq INTEGER" in lite and "input JSON NOT NULL" in lite
def test_postgres_casts_keep_their_length():
"""CAST(x AS VARCHAR(n)) 是截断的意思,照旧写长度;只有建表时的列不限长。"""
from sqlalchemy import String, cast
sql = str(select(cast(Run.workflow_name, String(4))).compile(dialect=_pg_dialect()))
assert "CAST(runs.workflow_name AS VARCHAR(4))" in sql
def test_postgres_orders_nulls_like_sqlite():
"""SQLite 升序 NULL 在前、降序 NULL 在后:PostgreSQL 上可空列照这个补写,不可空的列、写明了的不动。"""
from app.db.models import Task
sql = str(select(Run.id, func.row_number().over(order_by=Run.started_at).label("n"))
.order_by(Run.started_at, Run.finished_at.desc(), Run.created_at.desc(),
Task.due_at.desc().nulls_first(), Task.remind_at.asc().nulls_last())
.compile(dialect=_pg_dialect()))
assert "OVER (ORDER BY runs.started_at ASC NULLS FIRST)" in sql
order = sql.split("ORDER BY", 2)[2]
assert "runs.started_at ASC NULLS FIRST" in order
assert "runs.finished_at DESC NULLS LAST" in order
assert "runs.created_at DESC," in order # 不可空:不补
assert "tasks.due_at DESC NULLS FIRST" in order and "NULLS FIRST NULLS" not in order
assert "tasks.remind_at ASC NULLS LAST" in order
def test_postgres_integer_binds_are_bigint():
"""和 Integer 列比较、写进 Integer 列的参数按 BIGINT 转换,超过 21 亿的值不在半路溢出。"""
sql = str(select(Run.id).where(Run.last_seq > 5).compile(dialect=_pg_dialect()))
assert "runs.last_seq > $1::BIGINT" in sql
# --------------------------------------------------------------------------
# 连着库:两种库同一套结果
# --------------------------------------------------------------------------
async def _runs(origins: list[dict | None]) -> tuple[str, list[str]]:
tag = uuid.uuid4().hex[:12]
ids = []
async with SessionLocal() as session:
for i, origin in enumerate(origins):
row = Run(workflow_name=f"便携-{tag}-{i}", origin=origin, input={"question": "虚构的问题"} if i % 2 else
{"question": 7})
session.add(row)
await session.flush()
ids.append(row.id)
await session.commit()
return tag, ids
async def _match(tag: str, *where) -> list[int]:
async with SessionLocal() as session:
names = (await session.execute(select(Run.workflow_name).where(
Run.workflow_name.like(f"便携-{tag}-%"), *where))).scalars().all()
return sorted(int(n.rsplit("-", 1)[1]) for n in names)
async def test_json_helpers_match_the_same_rows_on_both_databases():
tag, _ = await _runs([
{"v": 3, "s": "3", "flag": True, "k": "甲"}, # 0
{"v": "3", "s": 3, "flag": 1, "k": None}, # 1
{"v": 4, "flag": 0, "k": "乙", "uses": ["r-1", "r-22"]}, # 2
{"flag": False}, # 3
{}, # 4
None, # 5(JSON 的 null)
])
# 带类型的相等:数字 3 不等于字符串 "3"
assert await _match(tag, json_eq(Run.origin, "v", 3)) == [0]
assert await _match(tag, json_eq(Run.origin, "v", "3")) == [1]
assert await _match(tag, json_eq(Run.origin, "s", "3")) == [0]
# None:没有这个键、或者值是 null
assert await _match(tag, json_eq(Run.origin, "k", None)) == [1, 3, 4, 5]
# 标记位:true 和非零的数算真
assert await _match(tag, json_flag(Run.origin, "flag")) == [0, 1]
from sqlalchemy import not_
assert await _match(tag, not_(json_flag(Run.origin, "flag"))) == [2, 3, 4, 5]
# 文本
assert await _match(tag, json_text(Run.origin, "k") == "乙") == [2]
assert await _match(tag, func.coalesce(json_text(Run.origin, "k"), "").in_(["甲", ""])) == [0, 1, 3, 4, 5]
assert await _match(tag, json_text(Run.origin, "uses").contains("r-22", autoescape=True)) == [2]
# 是不是字符串:input.question 偶数行是数字、奇数行是字符串
assert await _match(tag, json_is_string(Run.input, "question")) == [1, 3, 5]
async def test_json_columns_keep_the_order_of_keys():
"""JSON 列存原文:对象里键的先后读回来不变(JSONB 会重排,应用里按键的先后出列名、出段落的地方就乱了)。"""
keys = ["单号", "金额", "电话", "b", "a", "aa"]
async with SessionLocal() as session:
row = Run(workflow_name="键序", origin={k: i for i, k in enumerate(keys)})
session.add(row)
await session.commit()
rid = row.id
async with SessionLocal() as session:
got = (await session.execute(select(Run.origin).where(Run.id == rid))).scalar_one()
assert list(got) == keys
async def test_json_null_and_sql_null_stay_apart():
"""JSON 列写 None 存的是 JSON 的 null(不是 SQL 的 NULL),两种库一样:按 IS NULL 筛不到它。"""
async with SessionLocal() as session:
row = Run(workflow_name="空值", origin=None)
session.add(row)
await session.commit()
rid = row.id
async with SessionLocal() as session:
is_sql_null = (await session.execute(select(Run.origin.is_(None)).where(Run.id == rid))).scalar_one()
raw = (await session.execute(text("SELECT CAST(origin AS TEXT) FROM runs WHERE id = :i"), {"i": rid})).scalar_one()
assert is_sql_null is False and raw == "null"
async def test_long_strings_and_big_integers_are_kept():
"""String(n) 超长、Integer 过 21 亿:SQLite 照存,PostgreSQL 上也照存(列不限长、是 BIGINT)。"""
long_name = "长" * 500 # workflow_name 是 String(200)
big = 5_000_000_000
async with SessionLocal() as session:
row = Run(workflow_name=long_name, last_seq=big)
session.add(row)
await session.commit()
rid = row.id
async with SessionLocal() as session:
got = (await session.execute(select(Run.workflow_name, Run.last_seq).where(
Run.id == rid, Run.last_seq > big - 1))).one()
assert tuple(got) == (long_name, big)
async def test_non_finite_floats_in_json_are_written():
"""JSON 里的 NaN:SQLite 照存;PostgreSQL 的 json 不收,写成 null(同浏览器的 JSON.stringify),不让整条写入失败。"""
async with SessionLocal() as session:
row = Run(workflow_name="非有限", origin={"x": float("nan"), "y": 1.5, "s": "a\x00b"})
session.add(row)
await session.commit()
rid = row.id
async with SessionLocal() as session:
got = (await session.execute(select(Run.origin).where(Run.id == rid))).scalar_one()
assert got["y"] == 1.5
if db_base.engine.dialect.name == "postgresql":
assert got["x"] is None and got["s"] == "ab"
else:
assert got["x"] != got["x"] and got["s"] == "a\x00b" # NaN、NUL 原样
async def test_unique_violation_names_the_table_and_columns():
sid = uuid.uuid4().hex
async with SessionLocal() as session:
session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256="c" * 64))
await session.commit()
session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256="c" * 64))
with pytest.raises(IntegrityError) as caught:
await session.commit()
assert unique_violation(caught.value, "table_recipes", "source_id", "seq")
assert not unique_violation(caught.value, "table_recipes", "source_id")
assert not unique_violation(caught.value, "data_sources", "source_id", "seq")
async def test_order_by_puts_nulls_where_sqlite_does():
tag = uuid.uuid4().hex[:12]
from datetime import datetime, timezone
async with SessionLocal() as session:
for i, started in enumerate([datetime(2026, 1, 2, tzinfo=timezone.utc), None,
datetime(2026, 1, 1, tzinfo=timezone.utc)]):
session.add(Run(workflow_name=f"排序-{tag}-{i}", started_at=started))
await session.commit()
async with SessionLocal() as session:
where = Run.workflow_name.like(f"排序-{tag}-%")
up = (await session.execute(select(Run.workflow_name).where(where).order_by(Run.started_at))).scalars().all()
down = (await session.execute(select(Run.workflow_name).where(where)
.order_by(Run.started_at.desc()))).scalars().all()
assert [n[-1] for n in up] == ["1", "2", "0"] # 升序:NULL 在前
assert [n[-1] for n in down] == ["0", "2", "1"] # 降序:NULL 在后
async def test_timestamps_read_back_as_aware_utc():
from datetime import datetime, timedelta, timezone
local = datetime(2026, 3, 4, 9, 30, tzinfo=timezone(timedelta(hours=8)))
async with SessionLocal() as session:
row = Run(workflow_name="时区", started_at=local)
session.add(row)
await session.commit()
rid = row.id
async with SessionLocal() as session:
got = (await session.execute(select(Run.started_at).where(Run.id == rid, Run.started_at == local))).scalar_one()
assert got == local and got.tzinfo == timezone.utc and got.hour == 1
# --------------------------------------------------------------------------
# PostgreSQL 上的老库迁移
# --------------------------------------------------------------------------
@pytest.fixture
def scratch_pg():
"""同一台服务器上另建一个空库,给迁移测试造「老版本建的库」。只在 PostgreSQL 时有。"""
if not PG:
pytest.skip("要 AGENTLAB_TEST_DATABASE_URL(scripts/test-postgres.sh)")
import psycopg
from sqlalchemy import create_engine
from sqlalchemy.engine import make_url
from app.db.urls import libpq_conninfo
admin = make_url(db_base.settings.db_url)
name = f"{admin.database}_mig_{uuid.uuid4().hex[:8]}"
conninfo = libpq_conninfo(admin.render_as_string(hide_password=False))
with psycopg.connect(conninfo, autocommit=True) as conn:
conn.execute(f'CREATE DATABASE "{name}"')
url = admin.set(drivername="postgresql+psycopg", database=name)
engine = create_engine(url)
try:
yield engine
finally:
engine.dispose()
with psycopg.connect(conninfo, autocommit=True) as conn:
conn.execute(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)')
def _columns(conn, table: str) -> dict[str, tuple[str, str | None]]:
rows = conn.execute(text("SELECT column_name, data_type, column_default FROM information_schema.columns "
"WHERE table_name = :t"), {"t": table}).all()
return {r[0]: (r[1], r[2]) for r in rows}
def test_old_postgres_databases_get_the_missing_columns_and_indexes(scratch_pg):
"""老版本建的 PostgreSQL 库(少几列、少索引、有并发留下的重复行):补上的列类型按模型的、缺省值回填,布尔的缺省是
false 不是 0,索引补上,唯一索引建之前先去重;再跑一遍什么都不做。"""
with scratch_pg.begin() as conn:
conn.execute(text("CREATE TABLE runs (id VARCHAR PRIMARY KEY, workflow_id VARCHAR, version BIGINT, "
"status VARCHAR, last_seq BIGINT)"))
conn.execute(text("INSERT INTO runs (id, status, last_seq) VALUES ('r1', 'succeeded', 9)"))
conn.execute(text("CREATE TABLE schedules (id VARCHAR PRIMARY KEY, name VARCHAR)"))
conn.execute(text("INSERT INTO schedules (id, name) VALUES ('s1', '虚构的定时')"))
conn.execute(text("CREATE TABLE conversations (id VARCHAR PRIMARY KEY, kind VARCHAR, archived BOOLEAN, "
"workflow_id VARCHAR, last_active_at TIMESTAMP)"))
for cid, at in (("c1", "2026-01-01"), ("c2", "2026-01-02")):
conn.execute(text("INSERT INTO conversations (id, kind, archived, last_active_at) "
"VALUES (:i, 'app', false, :at)"), {"i": cid, "at": at})
db_base._migrate(conn)
db_base._migrate(conn)
runs = _columns(conn, "runs")
assert runs["options"][0] == "json" and runs["options"][1].startswith("'{}'")
assert runs["schedule_id"][0] == "character varying" and runs["manifest_seq"][0] == "bigint"
assert runs["started_by"][0] == "character varying"
assert conn.execute(text("SELECT CAST(options AS TEXT), run_class FROM runs WHERE id = 'r1'")).one() \
== ("{}", "exploratory")
schedules = _columns(conn, "schedules")
assert schedules["trigger_kind"][1].startswith("'cron'")
assert conn.execute(text("SELECT trigger_kind FROM schedules")).scalar_one() == "cron"
indexes = {r[0] for r in conn.execute(text("SELECT indexname FROM pg_indexes WHERE tablename = 'runs'"))}
assert {"ix_runs_schedule_id", "ix_runs_wf_ver_status"} <= indexes
# 同一个人在同一个应用里两段没归档的助手对话:created_by 是迁移补上的列(NULL 不受约束),唯一索引照样建上
convs = {r[0] for r in conn.execute(text("SELECT indexname FROM pg_indexes WHERE tablename = 'conversations'"))}
assert "ux_conversations_app_owner" in convs
def test_old_postgres_duplicates_are_settled_before_the_unique_index(scratch_pg):
with scratch_pg.begin() as conn:
conn.execute(text("CREATE TABLE conversations (id VARCHAR PRIMARY KEY, kind VARCHAR, archived BOOLEAN, "
"app_id VARCHAR, created_by VARCHAR, last_active_at TIMESTAMP)"))
for cid, at in (("c1", "2026-01-01"), ("c2", "2026-01-03"), ("c3", "2026-01-02")):
conn.execute(text("INSERT INTO conversations VALUES (:i, 'app', false, 'a1', '甲', :at)"),
{"i": cid, "at": at})
db_base._migrate(conn)
kept = conn.execute(text("SELECT id FROM conversations WHERE archived = false")).scalars().all()
assert kept == ["c2"], "留最近活跃的那一段,别的归档"
with pytest.raises(Exception, match="ux_conversations_app_owner"):
with conn.begin_nested():
conn.execute(text("INSERT INTO conversations VALUES ('c4', 'app', false, 'a1', '甲', '2026-02-01')"))
async def test_new_postgres_databases_have_the_partial_unique_indexes():
"""新库靠 create_all:部分唯一索引在 PostgreSQL 上也是部分索引(postgresql_where),条件里的布尔写成 false。"""
if not PG:
pytest.skip("要 AGENTLAB_TEST_DATABASE_URL(scripts/test-postgres.sh)")
async with db_base.engine.connect() as conn:
rows = await conn.execute(text("SELECT indexname, indexdef FROM pg_indexes WHERE indexname LIKE 'ux_%'"))
got = {r[0]: r[1] for r in rows}
assert "WHERE (((kind)::text = 'app'::text) AND (archived = false))" in got["ux_conversations_app_owner"]
assert "WHERE ((status)::text = ANY" in got["ux_catalog_draft_jobs_active"]
+2 -7
View File
@@ -446,7 +446,7 @@ async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_even
"""核对过状态、还没写入的那一下,接着跑的条件更新(runner._claim)先提交了:追加判定的条件更新落空,
照样不记——核对和写入是同一行上的条件更新,不是先读后写。"""
from app.core.bus import bus
from app.core.config import settings
from tests import appdb
Models(monkeypatch, verdicts={"新客": "contradicted"})
row = await finish(weekly(claims="judge"))
@@ -456,12 +456,7 @@ async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_even
def claimed_in_between(run_id, value):
# _append_judged 读完运行、分配序号之前正好调用它:在这里用另一条连接把运行占成 running
if run_id == row.id:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (row.id,))
db.commit()
finally:
db.close()
appdb.execute(("UPDATE runs SET status = 'running' WHERE id = ?", (row.id,)))
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", claimed_in_between)
+9 -34
View File
@@ -16,7 +16,6 @@ from __future__ import annotations
import asyncio
import json
import sqlite3
from collections import Counter
import pytest
@@ -25,7 +24,6 @@ from pydantic import BaseModel
from sqlalchemy import select
from app.core.bus import bus
from app.core.config import settings
from app.core.events import EventType
from app.db.base import SessionLocal
from app.db.models import Approval, Run, RunEvent
@@ -34,6 +32,7 @@ from app.engine.context import NodeError
from app.engine.runner import RunManager, run_manager, verify_manifest
from app.engine.schema import NodeType
from app.tools.registry import register
from tests import appdb
#: Agent 场景里工具真正执行的次数。注册表是进程级的,工具只注册一次,计数按测试清零
CALLS: list[str] = []
@@ -162,12 +161,7 @@ async def _pending(run_id: str) -> list[Approval]:
def _sql(statement: str, *params) -> None:
"""另开一条连接改库:模拟别处(恢复、继续运行、另一个写入者)抢在中间提交。"""
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute(statement, params)
db.commit()
finally:
db.close()
appdb.execute((statement, params))
# --------------------------------------------------------------------------
@@ -226,12 +220,7 @@ async def test_a_resume_that_wins_the_row_leaves_nothing_behind(monkeypatch):
def resumed_in_between(run_id, value):
if run_id == paused.id:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (paused.id,))
db.commit()
finally:
db.close()
appdb.execute(("UPDATE runs SET status = 'running' WHERE id = ?", (paused.id,)))
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", resumed_in_between)
@@ -252,14 +241,9 @@ async def test_events_written_elsewhere_in_between_win(monkeypatch):
def written_in_between(run_id, value):
if run_id == paused.id:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute("INSERT INTO run_events (id, run_id, seq, type, node_id, ts, data) "
"VALUES ('elsewhere-1', ?, ?, 'log', NULL, 0, '{}')", (paused.id, taken))
db.execute("UPDATE runs SET last_seq = ? WHERE id = ?", (taken, paused.id))
db.commit()
finally:
db.close()
appdb.execute(("INSERT INTO run_events (id, run_id, seq, type, node_id, ts, data) "
"VALUES ('elsewhere-1', ?, ?, 'log', NULL, 0, '{}')", (paused.id, taken)),
("UPDATE runs SET last_seq = ? WHERE id = ?", (taken, paused.id)))
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", written_in_between)
@@ -352,12 +336,8 @@ async def test_published_only_once_stored_and_never_when_not_written(monkeypatch
async def stored_when_published(event):
if event.type == EventType.APPROVAL_INQUIRY:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
(count,) = db.execute("SELECT count(*) FROM run_events WHERE run_id = ? AND seq = ?",
(event.run_id, event.seq)).fetchone()
finally:
db.close()
[(count,)] = appdb.query("SELECT count(*) FROM run_events WHERE run_id = ? AND seq = ?",
event.run_id, event.seq)
published.append((event.seq, count))
await publish(event)
@@ -430,12 +410,7 @@ async def test_a_sealed_run_that_changes_in_between_gets_nothing(monkeypatch, ch
def changed_in_between(run_id, value):
if run_id == final.id:
db = sqlite3.connect(settings.db_path, timeout=5)
try:
db.execute(change, (final.id,))
db.commit()
finally:
db.close()
appdb.execute((change, (final.id,)))
set_seq(run_id, value)
monkeypatch.setattr(bus, "set_seq", changed_in_between)
+6 -4
View File
@@ -322,15 +322,16 @@ async def test_launch_events_are_written_with_the_run_or_not_at_all():
from app.db.base import engine
from app.engine.launch import launch_run
from tests.appdb import refuse_inserts
lib = await upstream_with_two_versions()
wf, _ = await workflow(weekly(lib, upgrade_policy="dual"), published=1, versions=(1,))
before = await _run_count()
# 口径升版处置那条写不进去。走异步引擎建删触发器:同步连接等写锁时会卡住事件循环,而持锁的正是这个循环里的写入
create, drop = refuse_inserts("run_events", "NEW.type = 'caliber.upgrade'", name="refuse_upgrade_note")
async with engine.begin() as conn:
await conn.execute(text(
"CREATE TRIGGER refuse_upgrade_note BEFORE INSERT ON run_events WHEN NEW.type = 'caliber.upgrade' "
"BEGIN SELECT RAISE(ABORT, 'refused by test'); END"))
for sql in create:
await conn.execute(text(sql))
try:
async with SessionLocal() as session:
with pytest.raises(IntegrityError):
@@ -338,7 +339,8 @@ async def test_launch_events_are_written_with_the_run_or_not_at_all():
assert await _run_count() == before, "没写进治理事件,却留下了一条运行"
finally:
async with engine.begin() as conn:
await conn.execute(text("DROP TRIGGER IF EXISTS refuse_upgrade_note"))
for sql in drop:
await conn.execute(text(sql))
async def test_an_exploratory_launch_takes_the_saved_graph_and_the_run_defaults():
+4 -2
View File
@@ -112,9 +112,11 @@ def test_a_missing_table_is_left_alone():
async def test_new_databases_index_schedule_id():
from sqlalchemy import inspect
async with app_engine.connect() as conn:
names = {row[1] for row in (await conn.execute(text("PRAGMA index_list(runs)"))).all()}
assert "ix_runs_schedule_id" in names
indexes = await conn.run_sync(lambda sync: inspect(sync).get_indexes("runs"))
assert {ix["name"]: ix["column_names"] for ix in indexes}.get("ix_runs_schedule_id") == ["schedule_id"]
async def test_a_migrated_row_reads_back_empty_options():
+3 -7
View File
@@ -254,11 +254,10 @@ async def test_abandoning_while_the_pause_is_being_recorded(slow, monkeypatch):
async def test_an_approval_closed_just_before_it_is_marked_answered(slow, monkeypatch):
"""放弃恰好提交在「读到这条审批还是 pending」和「把它记成已回复」之间。"""
import sqlite3
from sqlalchemy import event
from app.db.base import engine
from tests import appdb
run = await run_manager.start(graph=_graph(gate=True), input_payload={"q": "x"})
await _until(lambda: _is(run.id, "interrupted"))
@@ -268,11 +267,8 @@ async def test_an_approval_closed_just_before_it_is_marked_answered(slow, monkey
def abandon_first(conn, cursor, statement, params, context, executemany):
if armed["on"] and statement.lstrip().upper().startswith("UPDATE APPROVALS"):
armed["on"] = False
other = sqlite3.connect(engine.url.database, timeout=5)
other.execute("UPDATE approvals SET status='cancelled' WHERE run_id=?", (run.id,))
other.execute("UPDATE runs SET status='cancelled' WHERE id=?", (run.id,))
other.commit()
other.close()
appdb.execute(("UPDATE approvals SET status='cancelled' WHERE run_id=?", (run.id,)),
("UPDATE runs SET status='cancelled' WHERE id=?", (run.id,)))
event.listen(engine.sync_engine, "before_cursor_execute", abandon_first)
try:
+5 -1
View File
@@ -23,6 +23,7 @@ from typing import Any
import pytest
from sqlalchemy import create_engine, select, text
from sqlalchemy.exc import IntegrityError
from app.core.config import settings
from app.data import raw_store, recipe_parsers, recipe_types, table_versions
@@ -32,6 +33,7 @@ from app.db.base import SessionLocal, utcnow
from app.db.models import (
DataSource, ImportStaging, SourceSnapshot, TableBuild, TableImport, TableRecipe,
)
from app.db.portable import unique_violation
@pytest.fixture(autouse=True)
@@ -268,8 +270,10 @@ async def test_recipe_seq_is_unique_per_source():
session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256=RECIPE_SHA))
await session.commit()
session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256=RECIPE_SHA))
with pytest.raises(Exception, match="UNIQUE"):
with pytest.raises(IntegrityError) as caught:
await session.commit()
# 撞的正是 (source_id, seq) 这一条唯一约束(两种库的报错写法不同,按表和列认)
assert unique_violation(caught.value, "table_recipes", "source_id", "seq")
# --------------------------------------------------------------------------