mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 13:22:01 +00:00
合入 W2-a 的复核修复(nr8/a:导入的 NOT NULL 补值与孤儿行、收尾 dispose、sslmode 换算、PostgreSQL 写入去 NUL、不分大小写搜索 contains_ci、json_eq 两库同语义、JSON 守卫、用户可见列表补决胜键、bubblewrap --clearenv、部署脚本与原子写的几处、README 连接数预算)
# Conflicts: # backend/app/api/schedules.py # backend/app/main.py
This commit is contained in:
@@ -71,6 +71,12 @@ os.environ.setdefault("AGENTLAB_SCHEDULER", "0")
|
||||
|
||||
import pytest # noqa: E402
|
||||
|
||||
# 整个 JSON 值做比较、去重、分组、排序时编译就报错(tests/jsonguard.py):PostgreSQL 的 json 不支持,SQLite 的测试也拦下
|
||||
from app.db.base import engine as _guarded_engine # noqa: E402
|
||||
from tests.jsonguard import install as _json_guard # noqa: E402
|
||||
|
||||
_json_guard(_guarded_engine.sync_engine.dialect)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _prepare_schema():
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""测试时挂在应用库编译器上的守卫:整个 JSON 值做比较、去重、分组、排序就报错(两种库都挂,跑全量时一直生效)。
|
||||
|
||||
PostgreSQL 的 json 没有相等和大小的运算符:`=`、`!=`、`IN`、`IS DISTINCT FROM`、`<` 之类,SELECT DISTINCT、GROUP BY、
|
||||
ORDER BY、UNION(去重的那几种)碰到 json 类型的值都报错;SQLite 照常执行。只在 PostgreSQL 上跑到才发现太晚:这里在编译
|
||||
那一刻就拦下,SQLite 的测试也拦得到。按表达式的类型认,不按写法认:跨行的 order_by、变量别名、
|
||||
`select(Run.origin).distinct()`、JSON 的下标(`Run.origin["k"] == 1`,取出来的还是 json)都算。按键筛选用 db/portable 的
|
||||
json_eq 等(类型是布尔、文本),不受影响。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON
|
||||
from sqlalchemy.sql import operators
|
||||
from sqlalchemy.sql.elements import UnaryExpression
|
||||
|
||||
|
||||
class JsonMisuse(AssertionError):
|
||||
"""整个 JSON 值被拿去比较、去重、分组或排序。"""
|
||||
|
||||
|
||||
_COMPARE = {operators.eq, operators.ne, operators.in_op, operators.not_in_op, operators.is_distinct_from,
|
||||
operators.is_not_distinct_from, operators.lt, operators.le, operators.gt, operators.ge}
|
||||
|
||||
|
||||
def _json(element: Any) -> bool:
|
||||
while isinstance(element, UnaryExpression) and element.modifier is not None:
|
||||
element = element.element
|
||||
return isinstance(getattr(element, "type", None), JSON)
|
||||
|
||||
|
||||
class JsonGuard:
|
||||
"""混进编译器类里用:class Guarded(JsonGuard, 原来的编译器)。"""
|
||||
|
||||
def visit_binary(self, binary: Any, **kw: Any) -> str:
|
||||
if binary.operator in _COMPARE and (_json(binary.left) or _json(binary.right)):
|
||||
raise JsonMisuse(f"整个 JSON 值做了比较({binary.operator.__name__}):PostgreSQL 的 json 没有这个运算符,"
|
||||
"按键筛选用 db/portable.json_eq 等")
|
||||
return super().visit_binary(binary, **kw) # type: ignore[misc]
|
||||
|
||||
def visit_select(self, select: Any, **kw: Any) -> str:
|
||||
if select._distinct and any(_json(c) for c in select.selected_columns):
|
||||
raise JsonMisuse("SELECT DISTINCT 里有 JSON 列:PostgreSQL 的 json 没有相等运算符,去不了重")
|
||||
if any(_json(c) for c in select._group_by_clauses):
|
||||
raise JsonMisuse("GROUP BY 了 JSON 值:PostgreSQL 的 json 没有相等运算符")
|
||||
if any(_json(c) for c in select._order_by_clauses):
|
||||
raise JsonMisuse("ORDER BY 了 JSON 值:PostgreSQL 的 json 没有大小")
|
||||
return super().visit_select(select, **kw) # type: ignore[misc]
|
||||
|
||||
def visit_compound_select(self, cs: Any, **kw: Any) -> str:
|
||||
keyword = str(getattr(cs.keyword, "value", cs.keyword)).upper()
|
||||
if "ALL" not in keyword and any(_json(c) for c in cs.selected_columns):
|
||||
raise JsonMisuse(f"{keyword or 'UNION'} 去重时碰到 JSON 列:PostgreSQL 的 json 没有相等运算符")
|
||||
return super().visit_compound_select(cs, **kw) # type: ignore[misc]
|
||||
|
||||
|
||||
def install(dialect: Any) -> None:
|
||||
"""挂到应用库引擎的方言实例上(PostgreSQL 时是 db/postgres 换过的编译器,SQLite 时是原来的)。"""
|
||||
base = dialect.statement_compiler
|
||||
if issubclass(base, JsonGuard):
|
||||
return
|
||||
dialect.statement_compiler = type(f"JsonGuarded{base.__name__}", (JsonGuard, base), {})
|
||||
@@ -77,16 +77,67 @@ def test_content_addressed_artifacts_are_written_atomically(tmp_path, monkeypatc
|
||||
assert _leftovers(folder) == []
|
||||
|
||||
|
||||
async def test_file_write_tool_appends_by_replacing_the_whole_file():
|
||||
async def test_file_write_tool_overwrites_atomically_and_appends_in_place():
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.tools.builtin.files import file_write
|
||||
|
||||
ctx = SimpleNamespace(sandbox_session=f"atomic-{os.getpid()}")
|
||||
assert (await file_write(ctx, path="notes/a.txt", content="第一行\n"))["ok"]
|
||||
out = await file_write(ctx, path="notes/a.txt", content="第二行\n", append=True)
|
||||
from app.core.config import settings
|
||||
|
||||
path = settings.session_dir(ctx.sandbox_session) / "notes" / "a.txt"
|
||||
inode = path.stat().st_ino
|
||||
out = await file_write(ctx, path="notes/a.txt", content="第二行\n", append=True)
|
||||
assert path.read_text(encoding="utf-8") == "第一行\n第二行\n" and out["bytes"] == len("第一行\n第二行\n".encode())
|
||||
assert path.stat().st_ino == inode, "追加不换文件"
|
||||
await file_write(ctx, path="notes/a.txt", content="重写\n")
|
||||
assert path.read_text(encoding="utf-8") == "重写\n"
|
||||
assert _leftovers(path.parent) == []
|
||||
|
||||
|
||||
def test_create_falls_back_when_the_filesystem_cannot_hard_link(tmp_path, monkeypatch):
|
||||
"""exFAT、一部分 SMB / FUSE 挂载不支持硬链接(EPERM、ENOTSUP、EXDEV):退回 O_EXCL 直接建,照样只建一次,不崩。"""
|
||||
import errno
|
||||
|
||||
def no_links(src, dst):
|
||||
raise OSError(errno.EPERM, "Operation not permitted")
|
||||
|
||||
monkeypatch.setattr(os, "link", no_links)
|
||||
target = tmp_path / ".secret"
|
||||
assert atomic.create_text(target, "第一把", mode=0o600) is True
|
||||
assert atomic.create_text(target, "第二把", mode=0o600) is False
|
||||
assert target.read_text() == "第一把" and stat.S_IMODE(target.stat().st_mode) == 0o600
|
||||
assert _leftovers(tmp_path) == []
|
||||
|
||||
|
||||
def test_replacing_keeps_the_permissions_of_the_old_file(tmp_path):
|
||||
target = tmp_path / "run.sh"
|
||||
target.write_text("echo 1")
|
||||
target.chmod(0o750)
|
||||
atomic.write_text(target, "echo 2")
|
||||
assert target.read_text() == "echo 2" and stat.S_IMODE(target.stat().st_mode) == 0o750
|
||||
|
||||
|
||||
def test_appending_keeps_the_same_file(tmp_path):
|
||||
"""追加写不换文件:inode、权限都不变,原有内容不动(以前整份读出来重写、再改名)。"""
|
||||
target = tmp_path / "log.txt"
|
||||
target.write_text("一\n")
|
||||
target.chmod(0o640)
|
||||
before = target.stat().st_ino
|
||||
atomic.append_bytes(target, "二\n".encode())
|
||||
assert target.read_text() == "一\n二\n"
|
||||
assert target.stat().st_ino == before and stat.S_IMODE(target.stat().st_mode) == 0o640
|
||||
assert _leftovers(tmp_path) == []
|
||||
|
||||
|
||||
def test_the_data_dir_carries_a_marker_deploy_can_recognise():
|
||||
"""数据目录里有 .agentlab-data:库在 PostgreSQL、密钥由环境变量给时,deploy.sh 也认得出这是 AgentLab 的数据目录。"""
|
||||
from app.core.config import DATA_DIR_MARKER, settings
|
||||
|
||||
settings.ensure_dirs()
|
||||
marker = settings.data_dir / DATA_DIR_MARKER
|
||||
assert marker.is_file()
|
||||
text = marker.read_text(encoding="utf-8")
|
||||
settings.ensure_dirs()
|
||||
assert marker.read_text(encoding="utf-8") == text, "已有的不覆盖"
|
||||
|
||||
@@ -103,3 +103,58 @@ def test_auto_falls_back_to_local_and_says_why(monkeypatch: pytest.MonkeyPatch)
|
||||
# 节点上选 fast 档也不能点到一个起不来的 bubblewrap
|
||||
monkeypatch.setattr(mgr.sys, "platform", "linux")
|
||||
assert manager._resolve_named("fast") is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 环境变量:沙箱里的代码看不到后端进程的(模型的 API Key、带密码的 AGENTLAB_DATABASE_URL)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_SECRETS = {"AGENTLAB_DATABASE_URL": "postgresql+asyncpg://u:not-a-real-password@h/db",
|
||||
"ANTHROPIC_API_KEY": "sk-ant-not-real", "OPENAI_API_KEY": "sk-not-real"}
|
||||
|
||||
|
||||
def _setenvs(argv: list[str]) -> dict[str, str]:
|
||||
return {argv[i + 1]: argv[i + 2] for i, a in enumerate(argv) if a == "--setenv"}
|
||||
|
||||
|
||||
def test_the_sandbox_starts_from_a_cleared_environment(monkeypatch, tmp_path):
|
||||
from app.sandbox.base import SandboxLimits
|
||||
|
||||
for key, value in _SECRETS.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
argv = bw.BubblewrapSandbox()._argv(tmp_path, SandboxLimits(), ["/usr/bin/python3", "/work/x.py"],
|
||||
{"RUN_FLAG": "1", "HOME": "/work/custom"})
|
||||
# --clearenv 在所有 --setenv 前面:bwrap 按先后处理,放后面就把设好的也清掉了
|
||||
assert argv[1] == "--clearenv" and argv.index("--clearenv") < argv.index("--setenv")
|
||||
env = _setenvs(argv)
|
||||
assert env["RUN_FLAG"] == "1" and env["HOME"] == "/work/custom", "调用方给的照旧生效,能盖过缺省的"
|
||||
assert env["PATH"] == "/usr/bin:/bin"
|
||||
assert not any(k.startswith("AGENTLAB_") or k.endswith("_API_KEY") for k in env)
|
||||
assert not any(v in " ".join(argv) for v in _SECRETS.values())
|
||||
|
||||
|
||||
async def test_bwrap_itself_gets_only_a_minimal_environment(monkeypatch, tmp_path):
|
||||
for key, value in _SECRETS.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
class Done:
|
||||
returncode = 0
|
||||
pid = 1
|
||||
|
||||
async def communicate(self):
|
||||
return b"", b""
|
||||
|
||||
async def fake_exec(*argv, **kwargs):
|
||||
seen["argv"], seen["env"] = list(argv), kwargs.get("env")
|
||||
return Done()
|
||||
|
||||
box = bw.BubblewrapSandbox()
|
||||
monkeypatch.setattr(bw.BubblewrapSandbox, "available", staticmethod(lambda: True))
|
||||
monkeypatch.setattr(box, "_session_dir", lambda _sid: tmp_path)
|
||||
monkeypatch.setattr(bw.asyncio, "create_subprocess_exec", fake_exec)
|
||||
result = await box.run("print(1)", env={"RUN_FLAG": "1"})
|
||||
assert result.ok, result.error
|
||||
assert seen["env"] is not None and set(seen["env"]) == {"PATH"}, "bwrap 进程本身不继承后端的环境变量"
|
||||
assert not any(v in " ".join(seen["argv"]) for v in _SECRETS.values())
|
||||
assert _setenvs(seen["argv"])["RUN_FLAG"] == "1"
|
||||
|
||||
@@ -14,6 +14,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.core.config import settings
|
||||
from app.db import to_postgres
|
||||
@@ -68,6 +69,11 @@ async def main():
|
||||
s.add(source)
|
||||
await s.flush()
|
||||
s.add(SourceChange(source_id=source.id, kind="column_added", fingerprint="f" * 64, table_name="订单"))
|
||||
dropped = SourceChange(source_id=source.id, kind="column_removed", fingerprint="d" * 64, table_name="订单")
|
||||
s.add(dropped)
|
||||
await s.flush()
|
||||
# 删掉最后一条:SQLite 的 AUTOINCREMENT 计数器停在 2,表里的 MAX 是 1。导过去之后新的变化要从 3 编起
|
||||
await s.delete(dropped)
|
||||
long = Run(workflow_name="长" * 300, last_seq=5_000_000_000, origin={"b": 2, "a": 1})
|
||||
sql_null = Run(workflow_name="SQL 的空", origin={"x": 1})
|
||||
json_null = Run(workflow_name="JSON 的空", origin=None)
|
||||
@@ -263,7 +269,7 @@ async def test_a_waiting_run_is_imported_and_finishes_on_postgresql(source_dir):
|
||||
seqs = done["seqs"]
|
||||
assert seqs == sorted(set(seqs)) and seqs[-1] > made["last_seq"], "接着导入前的序号往下编"
|
||||
assert done["artifact"] == {"区域": "虚构", "数": 1}
|
||||
assert done["new_change_seq"] > 1, "自增主键的序列接在导入的最大值后面"
|
||||
assert done["new_change_seq"] == 3, "序列接在 SQLite 的 AUTOINCREMENT 计数器后面,删过的号不复用(召回按它当游标)"
|
||||
|
||||
|
||||
@needs_pg
|
||||
@@ -294,3 +300,137 @@ async def test_a_target_with_the_service_still_connected_is_refused(source_dir):
|
||||
with pytest.raises(CopyRefused, match="服务的连接"):
|
||||
await copy_to_postgres(data, target, progress=lambda _: None)
|
||||
assert _sql(target, "SELECT count(*) FROM information_schema.tables WHERE table_name = 'runs'") == [(0,)]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 老库:补列时没带缺省值的 NOT NULL 列、SQLite 当时没强制外键留下的孤儿行
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _legacy_source(folder: Path) -> Path:
|
||||
"""照模型建一个 SQLite 应用库,再做旧:定时任务的 trigger 列删掉、按第一版迁移(不带缺省值)补回来,存量行是 NULL;
|
||||
关着外键写几行孤儿行(对话轮次指向已删的运行、事件指向已删的运行、轮次指向已删的对话)。"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import create_engine, insert
|
||||
|
||||
from app.db import models # noqa: F401
|
||||
from app.db.base import Base
|
||||
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
path = folder / "agentlab.db"
|
||||
engine = create_engine(f"sqlite:///{path}")
|
||||
tables = Base.metadata.tables
|
||||
try:
|
||||
Base.metadata.create_all(engine)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(insert(tables["schedules"]).values(id="s1", workflow_id="w1", cron="0 9 * * *",
|
||||
timezone="Asia/Shanghai"))
|
||||
conn.execute(insert(tables["schedule_fires"]).values(
|
||||
id="f1", schedule_id="s1", fire_at=datetime(2026, 1, 1, tzinfo=timezone.utc)))
|
||||
conn.execute(insert(tables["conversations"]).values(id="c1", title="虚构的对话"))
|
||||
conn.execute(insert(tables["conversation_turns"]).values(id="t1", conversation_id="c1", seq=0,
|
||||
run_id="run-gone"))
|
||||
conn.execute(insert(tables["conversation_turns"]).values(id="t2", conversation_id="conv-gone", seq=0))
|
||||
conn.execute(insert(tables["run_events"]).values(id="e1", run_id="run-gone", seq=1, type="log", ts=0.0,
|
||||
data={}))
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("ALTER TABLE schedules DROP COLUMN \"trigger\""))
|
||||
conn.execute(text("ALTER TABLE schedules ADD COLUMN \"trigger\" JSON"))
|
||||
finally:
|
||||
engine.dispose()
|
||||
return path
|
||||
|
||||
|
||||
def test_old_sqlite_databases_get_null_not_null_columns_filled(tmp_path):
|
||||
"""SQLite 这边:启动时的迁移把老版本补列留下的 NULL 填成模型的缺省值,两种库存的一样。"""
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from app.db.base import _migrate
|
||||
|
||||
path = _legacy_source(tmp_path / "old")
|
||||
engine = create_engine(f"sqlite:///{path}")
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
assert conn.execute(text("SELECT \"trigger\" FROM schedules")).scalar_one() is None
|
||||
_migrate(conn)
|
||||
assert conn.execute(text("SELECT \"trigger\" FROM schedules")).scalar_one() == "{}"
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_orphan_rows_are_nulled_or_left_out_on_the_snapshot(tmp_path):
|
||||
path = _legacy_source(tmp_path / "old")
|
||||
fixed: dict[str, int] = {}
|
||||
orphans = to_postgres.settle_orphans(path, fixed=fixed)
|
||||
assert fixed == {"外键指向已不存在的行,置空": 1}
|
||||
assert orphans == {"conversation_turns": 1, "run_events": 1}
|
||||
import sqlite3
|
||||
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
assert db.execute("SELECT id, run_id FROM conversation_turns").fetchall() == [("t1", None)]
|
||||
assert db.execute("SELECT count(*) FROM run_events").fetchone() == (0,)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@needs_pg
|
||||
async def test_an_old_database_with_nulls_and_orphans_imports_cleanly(tmp_path):
|
||||
"""真实开发库上核对不过的那两种:定时任务 trigger 为 NULL(NOT NULL 列),对话轮次指向已删的运行。导入照样核对通过,
|
||||
孤儿行照实列出、不算失败,重跑也一样。"""
|
||||
from tests.appdb import fresh_postgres_database
|
||||
|
||||
folder = tmp_path / "old"
|
||||
_legacy_source(folder)
|
||||
with fresh_postgres_database("legacy") as target:
|
||||
report = await copy_to_postgres(folder, target, progress=lambda _: None)
|
||||
lines = to_postgres.render(report)
|
||||
assert report.ok, "\n".join(lines)
|
||||
assert report.orphans == {"conversation_turns": 1, "run_events": 1}
|
||||
assert report.fixed.get("外键指向已不存在的行,置空") == 1
|
||||
assert any("孤儿行" in line and "run_events 1 行" in line for line in lines)
|
||||
assert _sql(target, "SELECT CAST(\"trigger\" AS TEXT) FROM schedules") == [("{}",)]
|
||||
assert _sql(target, "SELECT count(*) FROM schedule_fires") == [(1,)]
|
||||
assert _sql(target, "SELECT run_id FROM conversation_turns") == [(None,)]
|
||||
again = await copy_to_postgres(folder, target, progress=lambda _: None)
|
||||
assert again.ok and again.orphans == report.orphans
|
||||
|
||||
|
||||
def test_null_not_null_values_fall_back_to_the_model_default():
|
||||
from app.db.base import Base
|
||||
|
||||
schedules = Base.metadata.tables["schedules"]
|
||||
fixed: dict[str, int] = {}
|
||||
fix = to_postgres._Fixer(fixed)
|
||||
assert fix.value(schedules.c["trigger"], None) == "{}" # JSON 列按原文绑定
|
||||
assert fix.value(schedules.c["trigger_kind"], None) == "cron"
|
||||
assert fix.value(schedules.c["target_id"], None) is None # 可空的照旧是 NULL
|
||||
assert fix.value(schedules.c["cron"], None) is None # 没有缺省值的:照实交给库去拒
|
||||
assert fixed == {"NOT NULL 列的空值填成缺省值": 2}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 停服务之后数据目录里不留 -wal(导入按它认服务停没停)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
START_STOP = r'''
|
||||
from fastapi.testclient import TestClient
|
||||
from app.main import app
|
||||
|
||||
with TestClient(app) as client:
|
||||
assert client.get("/api/health").status_code == 200
|
||||
'''
|
||||
|
||||
|
||||
def test_stopping_the_service_leaves_no_wal_behind(tmp_path):
|
||||
data = tmp_path / "data"
|
||||
data.mkdir()
|
||||
env = {**os.environ, "AGENTLAB_DATA_DIR": str(data), "AGENTLAB_DATABASE_URL": "", "AGENTLAB_CHECKPOINT_URL": "",
|
||||
"AGENTLAB_SCHEDULER": "0", "PYTHONPATH": str(BACKEND)}
|
||||
done = subprocess.run([sys.executable, "-c", START_STOP], env=env, capture_output=True, text=True, timeout=180,
|
||||
cwd=str(data))
|
||||
assert done.returncode == 0, done.stderr[-4000:]
|
||||
assert (data / "agentlab.db").is_file() and (data / "checkpoints.db").is_file()
|
||||
assert sorted(p.name for p in data.iterdir() if p.name.endswith(("-wal", "-shm"))) == []
|
||||
assert to_postgres.source_in_use(data / "agentlab.db", data / "checkpoints.db") is None
|
||||
|
||||
@@ -45,6 +45,21 @@ def test_every_migrated_column_is_in_the_models():
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_every_not_null_migrated_column_gets_a_value_for_old_rows():
|
||||
"""迁移补的列模型里是 NOT NULL 的,存量行不能留 NULL(PostgreSQL 的表照模型带 NOT NULL,导入时整行插不进):要么补列时
|
||||
带 DEFAULT 回填,要么登记在 _NOT_NULL_BACKFILLS 里每次启动按模型的缺省值填。"""
|
||||
bad = []
|
||||
for table, column, ddl in db_base._COLUMN_MIGRATIONS:
|
||||
col = Base.metadata.tables[table].c[column]
|
||||
if col.nullable:
|
||||
continue
|
||||
if not re.search(r"\bDEFAULT\b", ddl, re.I) and (table, column) not in db_base._NOT_NULL_BACKFILLS:
|
||||
bad.append((table, column))
|
||||
assert bad == []
|
||||
for table, column in db_base._NOT_NULL_BACKFILLS:
|
||||
assert db_base.model_default(Base.metadata.tables[table].c[column])[0], (table, column)
|
||||
|
||||
|
||||
def test_checkpoint_tables_do_not_clash_with_app_tables():
|
||||
"""断点和应用表放在同一个库、同一个 schema:两边的表名不能撞。名字从两个 saver 自己的建表语句里取。"""
|
||||
import inspect as pyinspect
|
||||
@@ -58,22 +73,40 @@ def test_checkpoint_tables_do_not_clash_with_app_tables():
|
||||
assert created & set(Base.metadata.tables) == set()
|
||||
|
||||
|
||||
def test_no_query_compares_sorts_or_groups_a_whole_json_column():
|
||||
"""PostgreSQL 的 json 没有相等运算符:整列做 ==、!=、in_、DISTINCT、ORDER BY、GROUP BY 都会报错(SQLite 不报)。
|
||||
按里面某个键筛选走 db/portable 的 json_eq 等。扫一遍应用代码里「模型.JSON列」后面跟着这几样的写法。"""
|
||||
from pathlib import Path
|
||||
def test_the_compile_guard_catches_whole_json_values_however_they_are_written():
|
||||
"""整个 JSON 值做比较、去重、分组、排序:编译时就报错(tests/jsonguard.py,conftest 挂在应用库引擎上,跑全量时一直生效)。
|
||||
按表达式的类型认,逐行正则抓不到的写法也抓得到;按键筛选的 json_eq 等、IS NULL、UNION ALL 不受影响。"""
|
||||
from sqlalchemy import union, union_all
|
||||
|
||||
from sqlalchemy import JSON
|
||||
from tests.jsonguard import JsonMisuse
|
||||
|
||||
columns = sorted({f"{m.class_.__name__}.{a.key}" for m in Base.registry.mappers for a in m.column_attrs
|
||||
if isinstance(a.columns[0].type, JSON)})
|
||||
names = "|".join(re.escape(c) for c in columns)
|
||||
bad = re.compile(rf"(?<![\w.])({names})\b(\s*(==|!=)|\.(in_|not_in|distinct|is_distinct_from|like|contains)\()"
|
||||
rf"|(order_by|group_by|distinct)\([^)]*(?<![\w.])({names})\b(?!\[)")
|
||||
app = Path(__file__).resolve().parents[1] / "app"
|
||||
hits = [f"{path.relative_to(app)}:{i}" for path in app.rglob("*.py")
|
||||
for i, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1) if bad.search(line)]
|
||||
assert columns and hits == []
|
||||
dialect = db_base.engine.sync_engine.dialect
|
||||
alias = Run.origin
|
||||
caught = [
|
||||
select(Run.origin).distinct(),
|
||||
select(Run).distinct(),
|
||||
select(Run.id).order_by(
|
||||
Run.created_at,
|
||||
Run.origin,
|
||||
),
|
||||
select(Run.id).order_by(alias.desc()),
|
||||
select(Run.id).where(alias == {"a": 1}),
|
||||
select(Run.id).where(Run.input.in_([{"q": 1}])),
|
||||
select(Run.id).where(Run.origin["k"] == 1),
|
||||
select(func.count()).select_from(Run).group_by(Run.options),
|
||||
union(select(Run.origin), select(Run.input)),
|
||||
]
|
||||
for statement in caught:
|
||||
with pytest.raises(JsonMisuse):
|
||||
statement.compile(dialect=dialect)
|
||||
fine = [
|
||||
select(Run.origin).where(json_eq(Run.origin, "k", 1), Run.origin.is_(None),
|
||||
Run.origin != None), # noqa: E711 - 写成 IS NOT NULL,json 上也能用
|
||||
select(Run.id).where(json_text(Run.origin, "k") == "x").order_by(json_text(Run.origin, "k")).distinct(),
|
||||
union_all(select(Run.origin), select(Run.input)),
|
||||
]
|
||||
for statement in fine:
|
||||
statement.compile(dialect=dialect)
|
||||
|
||||
|
||||
def test_postgres_columns_are_unbounded_varchar_bigint_and_json():
|
||||
@@ -134,6 +167,16 @@ def test_database_urls_are_normalized_and_never_logged_with_the_password():
|
||||
# asyncpg 的 ssl= 换成 libpq 的 sslmode=,只给 asyncpg 看的参数不带给 psycopg
|
||||
assert libpq_conninfo("postgresql+asyncpg://u:pw@h/db?ssl=require&prepared_statement_cache_size=0") \
|
||||
== "postgresql://u:pw@h/db?sslmode=require"
|
||||
# 应用库地址写 libpq 的 sslmode=:换成 asyncpg 的 ssl=(以前原样交给 asyncpg,连的时候 TypeError);断点库换回 sslmode=
|
||||
app_url = normalize_database_url("postgresql://u:pw@h/db?sslmode=verify-full")
|
||||
assert app_url == "postgresql+asyncpg://u:pw@h/db?ssl=verify-full"
|
||||
assert libpq_conninfo(app_url) == "postgresql://u:pw@h/db?sslmode=verify-full"
|
||||
for bad, says in (("postgresql://u@h/db?sslrootcert=/etc/ca.pem", "PGSSLROOTCERT"),
|
||||
("postgresql://u@h/db?sslmode=strict", "sslmode=strict"),
|
||||
("postgresql://u@h/db?ssl=require&sslmode=require", "只写一个"),
|
||||
("postgresql://u@h/db?application_name=x", "application_name")):
|
||||
with pytest.raises(ValueError, match=says):
|
||||
normalize_database_url(bad)
|
||||
assert "pw" not in describe("postgresql+asyncpg://u:pw@h:5433/db")
|
||||
for bad in ("mysql://u@h/db", "sqlite:///x.db", "postgresql://h", "这不是地址"):
|
||||
with pytest.raises(ValueError):
|
||||
@@ -145,6 +188,15 @@ def test_database_urls_are_normalized_and_never_logged_with_the_password():
|
||||
assert Settings(database_url=" ").db_dialect == "sqlite"
|
||||
|
||||
|
||||
def test_a_postgres_url_without_the_driver_installed_says_what_to_install(monkeypatch):
|
||||
import importlib.util
|
||||
|
||||
real = importlib.util.find_spec
|
||||
monkeypatch.setattr(importlib.util, "find_spec", lambda name, *a, **k: None if name == "asyncpg" else real(name, *a, **k))
|
||||
with pytest.raises(RuntimeError, match=r"backend\[postgres\]"):
|
||||
db_base.make_engine("postgresql+asyncpg://u:pw@127.0.0.1:1/db")
|
||||
|
||||
|
||||
def test_the_startup_line_names_the_database_without_the_password():
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
@@ -188,24 +240,39 @@ async def test_json_helpers_match_the_same_rows_on_both_databases():
|
||||
{"flag": False}, # 3
|
||||
{}, # 4
|
||||
None, # 5(JSON 的 null)
|
||||
{"v": True, "n": 1.0, "b": False}, # 6
|
||||
{"v": 1, "n": 1, "b": 0}, # 7
|
||||
])
|
||||
# 带类型的相等:数字 3 不等于字符串 "3"
|
||||
# 按 JSON 的类型比:数字 3 不等于字符串 "3";true 不等于 1、false 不等于 0(SQLite 的 json_extract 单用时分不开);
|
||||
# 数 1 等于 1.0
|
||||
assert await _match(tag, json_eq(Run.origin, "v", 3)) == [0]
|
||||
assert await _match(tag, json_eq(Run.origin, "v", "3")) == [1]
|
||||
assert await _match(tag, json_eq(Run.origin, "s", "3")) == [0]
|
||||
assert await _match(tag, json_eq(Run.origin, "v", 1)) == [7]
|
||||
assert await _match(tag, json_eq(Run.origin, "v", True)) == [6]
|
||||
assert await _match(tag, json_eq(Run.origin, "b", False)) == [6]
|
||||
assert await _match(tag, json_eq(Run.origin, "b", 0)) == [7]
|
||||
assert await _match(tag, json_eq(Run.origin, "n", 1)) == [6, 7]
|
||||
assert await _match(tag, json_eq(Run.origin, "n", 1.0)) == [6, 7]
|
||||
with pytest.raises(TypeError):
|
||||
json_eq(Run.origin, "v", {"a": 1})
|
||||
# None:没有这个键、或者值是 null
|
||||
assert await _match(tag, json_eq(Run.origin, "k", None)) == [1, 3, 4, 5]
|
||||
assert await _match(tag, json_eq(Run.origin, "k", None)) == [1, 3, 4, 5, 6, 7]
|
||||
# 标记位:true 和非零的数算真
|
||||
assert await _match(tag, json_flag(Run.origin, "flag")) == [0, 1]
|
||||
from sqlalchemy import not_
|
||||
|
||||
assert await _match(tag, not_(json_flag(Run.origin, "flag"))) == [2, 3, 4, 5]
|
||||
# 文本
|
||||
assert await _match(tag, not_(json_flag(Run.origin, "flag"))) == [2, 3, 4, 5, 6, 7]
|
||||
# 文本:字符串是它本身,数、true / false 是 JSON 写法(两种库都是)
|
||||
assert await _match(tag, json_text(Run.origin, "k") == "乙") == [2]
|
||||
assert await _match(tag, func.coalesce(json_text(Run.origin, "k"), "").in_(["甲", ""])) == [0, 1, 3, 4, 5]
|
||||
assert await _match(tag, func.coalesce(json_text(Run.origin, "k"), "").in_(["甲", ""])) == [0, 1, 3, 4, 5, 6, 7]
|
||||
assert await _match(tag, json_text(Run.origin, "v") == "3") == [0, 1]
|
||||
assert await _match(tag, json_text(Run.origin, "v") == "1") == [7]
|
||||
assert await _match(tag, json_text(Run.origin, "v") == "true") == [6]
|
||||
assert await _match(tag, json_text(Run.origin, "n") == "1.0") == [6]
|
||||
assert await _match(tag, json_text(Run.origin, "uses").contains("r-22", autoescape=True)) == [2]
|
||||
# 是不是字符串:input.question 偶数行是数字、奇数行是字符串
|
||||
assert await _match(tag, json_is_string(Run.input, "question")) == [1, 3, 5]
|
||||
assert await _match(tag, json_is_string(Run.input, "question")) == [1, 3, 5, 7]
|
||||
|
||||
|
||||
async def test_json_columns_keep_the_order_of_keys():
|
||||
@@ -313,6 +380,46 @@ async def test_text_sorts_and_compares_byte_by_byte_like_sqlite():
|
||||
assert lower == sum(1 for n in names if n < "a")
|
||||
|
||||
|
||||
async def test_nul_characters_in_text_columns():
|
||||
"""文本列里的 NUL:SQLite 照存;PostgreSQL 的 text / varchar 不收,写入时去掉(同 JSON 列),不让整条写入失败。"""
|
||||
async with SessionLocal() as session:
|
||||
row = Run(workflow_name="带\x00NUL\x00的名字", error="错\x00误")
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
rid = row.id
|
||||
await session.execute(text("UPDATE runs SET end_reason = :r WHERE id = :i"), {"r": "x\x00y", "i": rid})
|
||||
await session.commit()
|
||||
async with SessionLocal() as session:
|
||||
got = (await session.execute(select(Run.workflow_name, Run.error, Run.end_reason).where(Run.id == rid))).one()
|
||||
if db_base.engine.dialect.name == "postgresql":
|
||||
assert tuple(got) == ("带NUL的名字", "错误", "xy")
|
||||
else:
|
||||
assert tuple(got) == ("带\x00NUL\x00的名字", "错\x00误", "x\x00y")
|
||||
|
||||
|
||||
async def test_case_insensitive_search_folds_like_the_database_does():
|
||||
"""contains_ci:ASCII 两种库都不分大小写;非 ASCII(É/é、Ж/ж、全角A/a)按库自己会不会折——SQLite 看构建,PostgreSQL 看
|
||||
建库的区域设置(列本身是 COLLATE "C",列上的 ILIKE 只认 ASCII,所以模式带 COLLATE "default")。% 和 _ 按字面。"""
|
||||
from app.db.portable import contains_ci
|
||||
|
||||
tag = uuid.uuid4().hex[:12]
|
||||
async with SessionLocal() as session:
|
||||
session.add(Run(workflow_name=f"大小写-{tag}-Éclair Жук Ab Report_100%"))
|
||||
await session.commit()
|
||||
pg = db_base.engine.dialect.name == "postgresql"
|
||||
folds = (await session.execute(text(
|
||||
"SELECT lower('ÉЖA' COLLATE \"default\") = 'éжa'" if pg else "SELECT lower('ÉЖA') = 'éжa'"))).scalar()
|
||||
|
||||
async def hit(needle: str) -> bool:
|
||||
return bool((await session.execute(select(func.count()).select_from(Run).where(
|
||||
Run.workflow_name.like(f"大小写-{tag}-%"), contains_ci(Run.workflow_name, needle)))).scalar())
|
||||
|
||||
assert await hit("report") and await hit("REPORT_100%")
|
||||
assert not await hit("report-100") and not await hit("100%%")
|
||||
assert await hit("éclair") == bool(folds) and await hit("жук") == bool(folds) and await hit("ab") == bool(folds)
|
||||
assert await hit("Éclair") and await hit("Жук")
|
||||
|
||||
|
||||
async def test_timestamps_read_back_as_aware_utc():
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
|
||||
@@ -178,7 +178,9 @@ async def test_a_second_reindex_while_one_is_running_is_refused(client, monkeypa
|
||||
|
||||
async def slow(session, collection=None, *, on_progress=None):
|
||||
await gate.wait()
|
||||
return await real(session, collection, on_progress=on_progress)
|
||||
# 放行之后只重建一个空集合:这里测的是「同时只跑一个」,不是重建本身。按全部集合重建要把同一个 xdist 进程里
|
||||
# 别的测试留下的几百段都重算一遍,满载时超过 _wait_done 的 10 秒,后面两条测试跟着撞上「正在重建」
|
||||
return await real(session, "reindex_gate_empty", on_progress=on_progress)
|
||||
|
||||
monkeypatch.setattr(kb, "reindex", slow)
|
||||
try:
|
||||
@@ -211,7 +213,7 @@ async def test_two_reindexes_started_at_the_same_moment_do_not_both_run(client,
|
||||
async def slow(session, collection=None, *, on_progress=None):
|
||||
started.append(collection)
|
||||
await gate.wait()
|
||||
return await real(session, collection, on_progress=on_progress)
|
||||
return await real(session, "reindex_gate_empty", on_progress=on_progress) # 理由同上一条
|
||||
|
||||
monkeypatch.setattr(kb, "reindex", slow)
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user