diff --git a/.env.example b/.env.example index cede4581..582ab6c4 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,56 @@ # AGENTLAB_PORT=8000 # AGENTLAB_WEB_PORT=5273 +# ---- 应用库 ---- +# 不设:数据目录里的 SQLite(data/agentlab.db,断点 data/checkpoints.db),单机单进程。 +# 设成 PostgreSQL 时整个应用库和断点都放进去(要 [postgres] 额外依赖,environment.yml 已带);数据目录照样要有。 +# 从现有的 SQLite 数据目录搬过去:python -m app.db copy-to-postgres data(README「用 PostgreSQL」) +# AGENTLAB_DATABASE_URL=postgresql+asyncpg://agentlab:密码@127.0.0.1:5432/agentlab +# 要 SSL 时加 ?sslmode=require(verify-ca / verify-full 同理);证书文件用环境变量 PGSSLROOTCERT / PGSSLCERT / PGSSLKEY,不写在地址里 +# AGENTLAB_DATABASE_URL=postgresql+asyncpg://agentlab:密码@db.example.com:5432/agentlab?sslmode=verify-full +# 断点放另一个库时才设(不设就和应用库同一个库) +# AGENTLAB_CHECKPOINT_URL=postgresql://agentlab:密码@127.0.0.1:5432/agentlab_checkpoints +# 启动时库还连不上(和库一起起、库还在初始化)最多等几秒,按退避重试;密码不对、库不存在这种不等。缺省 90 +# AGENTLAB_DB_STARTUP_TIMEOUT=90 + +# ---- 连接池(PostgreSQL;每个进程一套)---- +# 常驻几条,缺省 10;设成 0 不用连接池、每次新建连接 +# AGENTLAB_DB_POOL_SIZE=10 +# 高峰再多开几条,缺省 20 +# AGENTLAB_DB_MAX_OVERFLOW=20 +# 池子满了等一条空出来最多等几秒,缺省 30 +# AGENTLAB_DB_POOL_TIMEOUT=30 +# 一条连接用多久换新的(秒),缺省 1800 +# AGENTLAB_DB_POOL_RECYCLE=1800 + +# ---- PgBouncer(README「多进程」的「直连与 PgBouncer」)---- +# 直连地址(写法同 AGENTLAB_DATABASE_URL):进程间通知、跨进程锁、断点这几样依赖会话状态的连接走它,普通查询走上面的地址。 +# 不设就全用 AGENTLAB_DATABASE_URL(直连 PostgreSQL、或 PgBouncer 的会话模式时都不用设) +# AGENTLAB_DATABASE_DIRECT_URL=postgresql+asyncpg://agentlab:密码@db.example.com:5432/agentlab +# AGENTLAB_DATABASE_URL 指的是 PgBouncer 的事务模式时设成 1(缺省 0):不用客户端连接池、关掉预备语句缓存; +# 这时必须另给上面的直连地址,没给就拒绝启动 +# AGENTLAB_DB_PGBOUNCER=0 + +# ---- 多进程(README「多进程」;只有应用库是 PostgreSQL 时才能大于 1)---- +# 几个工作进程,缺省 1(scripts/prod.sh、Docker 的入口按它起;dev.sh 总是一个进程、热重载) +# AGENTLAB_WORKERS=1 +# 每个进程在库里登记、续租约:租约多长(秒,缺省 30)、隔多久续一次(秒,缺省 5)。租约到期没续上算进程没了,别的进程接手它的活 +# AGENTLAB_WORKER_LEASE_S=30 +# AGENTLAB_WORKER_HEARTBEAT_S=5 +# 后台调度循环(定时任务、事件派发、流程时限)开不开,缺省开;拷一份数据目录另起的服务要关成 0。一拍隔几秒,缺省 15 +# AGENTLAB_SCHEDULER=1 +# AGENTLAB_SCHEDULER_TICK_S=15 +# 构建号:进程登记的代码版本里带上它(新旧版本不许同时连一个库)。Docker 镜像构建时自动写进去,不设就用 git 提交 +# AGENTLAB_BUILD= + +# ---- 开发用 PostgreSQL(./scripts/dev.sh --postgres)---- +# 这几项只有 dev.sh 读:写成环境变量或写在这个 .env 里都行(环境变量优先)。数据在 docker 数据卷里, +# 密码第一次自动生成在 data/.postgres-dev/,dev.sh 照它设 AGENTLAB_DATABASE_URL(盖过上面写的) +# AGENTLAB_DEV_PG_CONTAINER=agentlab-dev-postgres +# AGENTLAB_DEV_PG_PORT=55432 +# AGENTLAB_DEV_PG_VOLUME=agentlab-dev-pgdata +# AGENTLAB_DEV_PG_IMAGE=postgres:16 + # ---- 沙箱 ---- # auto(挑第一个已就绪的:microvm → bubblewrap → seatbelt → local) # 可选值:microvm | seatbelt | bubblewrap | local | off diff --git a/Dockerfile b/Dockerfile index 0a2dfa22..54359a04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,8 @@ SH WORKDIR /app COPY LICENSE COMMERCIAL-LICENSE.md /app/ COPY backend/app /app/backend/app +# 锁文件留一份:进程登记的代码版本把它算进源码指纹(app/cluster/version.py:升了依赖的进程也算版本不同) +COPY backend/requirements.lock /app/backend/requirements.lock COPY --from=web /src/frontend/dist /app/web ENV AGENTLAB_WEB_DIST=/app/web \ @@ -93,10 +95,14 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --start-interval=2s - # 版本号放在最后:它每次打包都不同,放前面会让装依赖那几层的缓存全部失效 ARG VERSION=dev +# 进程登记时带上构建号(app/cluster/version.py:库里还有活着的、版本不同的进程时新进程不许加入) +ENV AGENTLAB_BUILD=${VERSION} LABEL org.opencontainers.image.title="AgentLab" \ org.opencontainers.image.description="可视化 Agent 编排实验台" \ org.opencontainers.image.version="${VERSION}" \ agentlab.microvm="${WITH_MICROVM}" -# 单进程:运行管理器在进程内、数据库是 SQLite,不能开多个 worker -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] +# 工作进程数照 AGENTLAB_WORKERS(缺省 1)。大于 1 要应用库是 PostgreSQL(AGENTLAB_DATABASE_URL):SQLite 只支持单进程, +# 服务启动时拒绝第二个进程(app/cluster/guard.py)。exec:uvicorn 直接收 SIGTERM,走完整的优雅退出 +# 进程数问后端(python -m app.cluster workers,环境变量和 .env 一起读;SQLite 而大于 1 时照实报错、不起) +CMD ["sh", "-c", "n=\"$(python -m app.cluster workers)\" || exit 2; exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers \"$n\""] diff --git a/README.md b/README.md index 2c2f2f2c..35959acd 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - [快速开始](#快速开始) · [生产部署](#生产部署) · [核心能力](#核心能力) · [受限动态编排](#受限动态编排模板定骨架agent-填关节) - [可点击证据](#可点击证据报告里的每个数都点得开出处) · [评测与口径账本](#回归评测与口径账本) · [应用](#应用已发布版本的使用面) · [身份与权限](#身份与权限) · [能力登记与端点](#能力登记与端点) - [数据目录](#数据目录让助手和-agent-看懂业务数据) · [工作流形态](#工作流形态) · [节点类型](#节点类型) · [架构](#架构) -- [关键设计决策](#关键设计决策) · [运行环境](#运行环境) · [常见问题](#常见问题) · [开发与验证](#开发与验证) +- [关键设计决策](#关键设计决策) · [运行环境](#运行环境) · [用 PostgreSQL](#用-postgresql) · [常见问题](#常见问题) · [开发与验证](#开发与验证) --- @@ -44,6 +44,7 @@ conda env create -f environment.yml # 首次:创建 agentlab 环境并安装 后端仍只听 127.0.0.1,页面里的 `/api` 由前端开发服务器在本机转发。**默认不开登录**:同一网络里打得开页面的人 都能用全部功能——跑工作流、花你配的模型额度、读数据源、执行沙箱代码,审批和发起记的是各自填的署名、未经核实。 只在信得过的网络里这样用;多人使用请开登录,见[身份与权限](#身份与权限)。 +应用库要用 PostgreSQL 时加 `--postgres`(本机 docker 起一个开发用的库,见[用 PostgreSQL](#用-postgresql))。 认不出的参数直接报错,`./scripts/dev.sh --help` 看用法。 **无需配置 API Key 即可完整体验。** 内置 Mock provider 产出流式回复、工具调用与 @@ -60,14 +61,13 @@ Mock 不会写引用标记:探索运行里报告撰写节点照样跑完,但 ## 生产部署 -**不用 Docker:** `./scripts/prod.sh` 构建前端并由后端一并托管页面,单进程、不热重载,启动于 +**不用 Docker:** `./scripts/prod.sh` 构建前端并由后端一并托管页面,缺省一个进程(`--workers` 起多个,要 PostgreSQL)、不热重载,启动于 http://127.0.0.1:8000 。前端产物不存在或比源码旧时才重新构建,`--build` / `--no-build` 强制构建或跳过。 `--port`(或 `AGENTLAB_PORT`)改端口;`--host` 监听所有网卡,默认同样**不开登录**,多人使用请开登录 ([身份与权限](#身份与权限))。 -数据目录默认为仓库下的 `data/`,`AGENTLAB_DATA_DIR` 可改。只能单进程运行:运行管理器在进程内、数据库是 SQLite;后台任务 -(探查、起草、剖析、评测、影子起草、口径比对与召回)的排队、限流和按需探查的锁也都在进程内,多进程部署要先换成库里的锁。 -上传表格的版本存储同样只支持单进程运行(不要给 uvicorn 加 `--workers`):启动时会锁住数据目录,同一个数据目录 -起了第二个进程时,后起的那个不做任何清理,上传、启用版本等写操作一律返回 503。 +数据目录默认为仓库下的 `data/`,`AGENTLAB_DATA_DIR` 可改。应用库默认是数据目录里的 SQLite,**只支持单进程**:同一个数据目录 +已经有一个服务在用时,第二个进程拒绝启动、照实报错(不要给 uvicorn 加 `--workers`,`--workers` 大于 1 时 `prod.sh` 先拦下)。 +换成 PostgreSQL([用 PostgreSQL](#用-postgresql))之后可以多进程:`./scripts/prod.sh --workers 4`,规矩见[多进程](#多进程)。 按期累积的数据源每个历史版本是一份完整副本,磁盘占用随期数和被引用的版本数增长。除当前版本和被运行引用的 版本外,每个数据源再保留最近启用过的 3 个版本,其余的会被回收、无法再启用。 @@ -75,7 +75,10 @@ http://127.0.0.1:8000 。前端产物不存在或比源码旧时才重新构建 `--platform linux/arm64` 或 `linux/amd64`),连同一键部署脚本、配置模板和部署说明打成 `release/agentlab-<版本>-linux-.tar`。拷到只装了 Docker Engine 的离线机器上,解包、按需修改 `agentlab.env`、运行 `./deploy.sh` 即可,不需要 docker compose。安装、升级、备份、沙箱权限与常见问题见 -[deploy/README.md](deploy/README.md)。 +[deploy/README.md](deploy/README.md)。应用库可以用包里带的 PostgreSQL:新部署 `./deploy.sh install --postgres`(旁边再起一个 +PostgreSQL 容器,端口不对外,密码自动生成在数据目录里),已经在用 SQLite 的 `./deploy.sh migrate-to-postgres`(停服务、备份、 +导入、核对,哪一步出错都退回 SQLite);`upgrade` 前自动备份库,表结构和默认数据启动时自动补齐。三种放法见 deploy/README.md +「用 PostgreSQL」。 镜像里默认带 microVM 的运行时和沙箱镜像(约 150 MB,`--no-microvm` 可不带)。目标机有 KVM 时,deploy.sh 先用临时容器试启动一台 microVM,起得来就把 `/dev/kvm` 交给容器,代码在 microVM 里执行,部署时不用联网。 @@ -461,8 +464,8 @@ flowchart LR 一次运行只看得到开始时已有的记录;使用者逐行只看得到自己提交的(「记录列表」组件),正式运行的汇总是全量的。 表单只加可选字段时同一版本加列,改类型、删字段、加必填字段出新版本,旧记录照原样保留:看板查新一版时, 旧版本只读并进同一张 `records`(旧版本没有的字段、类型改得对不上的字段为空),各版本都按运行 -开始时的记录固定;新一版发布之前已经发起、还在半路的实例照常写完旧版本,之后发起的只写新一版。记录表按单进程锁写入, -面向人工填报的频率,不适合每秒多次的高频写入。表单里的附件先上传(`data/files/`,按内容哈希存、只读)、提交时绑到 +开始时的记录固定;新一版发布之前已经发起、还在半路的实例照常写完旧版本,之后发起的只写新一版。记录表的写入全站一个接一个 +(多进程时是库里的锁),面向人工填报的频率,不适合每秒多次的高频写入。表单里的附件先上传(`data/files/`,按内容哈希存、只读)、提交时绑到 这次运行,读得到那次运行或那一行记录的人才能下载;上传后 24 小时没提交的自动清理。 **验收用的假外部系统**:`python scripts/fake-erp.py --port 8765`,可以切换「正常 / 写成功后超时 / 500 / 4xx」,接口见 @@ -739,8 +742,8 @@ flowchart TB B5["sandbox
microVM / Seatbelt / bubblewrap"] end subgraph ST["存储"] - S1[("agentlab.db
业务数据")] - S2[("checkpoints.db
执行断点")] + S1[("agentlab.db 或 PostgreSQL
业务数据")] + S2[("checkpoints.db 或同一个 PostgreSQL
执行断点")] S3[("artifacts/
内容寻址工件库")] end FE <-->|REST + WebSocket| BE @@ -991,6 +994,7 @@ sudo setfacl -m u:$USER:rw /dev/kvm # 当前会话立即可用 pip install -e 'backend[docs]' -c backend/requirements.lock # PDF / Word / PowerPoint / Excel 解析 pip install -e 'backend[db]' -c backend/requirements.lock # MySQL / PostgreSQL / Oracle 驱动 pip install -e 'backend[microvm]' -c backend/requirements.lock # microVM 沙箱 +pip install -e 'backend[postgres]' -c backend/requirements.lock # 应用库用 PostgreSQL(asyncpg、psycopg 3、官方断点库) pip install -e 'backend[dev]' -c backend/requirements.lock # pytest ``` @@ -1001,8 +1005,16 @@ pip install -e 'backend[dev]' -c backend/requirements.lock # pytest 装包时以它为约束,所以换一台机器装到的也是同一套。 LangGraph / LangChain 一族在 `pyproject.toml` 里只放行到下一个小版本:引擎直接依赖 -它们的断点格式、interrupt / resume 和流式事件,这些在小版本里也改过行为。aiosqlite -与 SQLAlchemy 同理——`db/base.py` 给 aiosqlite 打了补丁,连接池断开连接时会走到它。 +它们的断点格式、interrupt / resume 和流式事件,这些在小版本里也改过行为。断点格式本身在 +`langgraph-checkpoint` 里(langgraph 只要求它 <5,所以单独卡住);SQLite、PostgreSQL 两个断点库 +(`langgraph-checkpoint-sqlite`、`langgraph-checkpoint-postgres`)都按它读写,`copy-to-postgres` 搬断点也靠两边是同一版格式。 +`langgraph-checkpoint-postgres` 3.1.3 起要 `langgraph-checkpoint` 4.3,锁在 3.1.2,升级时三个一起动。aiosqlite +与 SQLAlchemy 同理——`db/base.py` 给 aiosqlite 打了补丁,连接池断开连接时会走到它;应用库换成 PostgreSQL 之后 +补丁照样打着:上传表格、手工登记的 SQLite 数据源还走 aiosqlite。 + +`[postgres]` 里是应用库用 PostgreSQL 时要的:SQLAlchemy 这边走 asyncpg(和 `[db]` 里连业务库的是同一个包), +断点库走官方的 `langgraph-checkpoint-postgres`,它只认 psycopg 3 和 `psycopg-pool`。psycopg 带 `[binary]`: +wheel 里自带 libpq,镜像和本机都不用另装 PostgreSQL 客户端。 升级依赖要主动做,改完跑一遍测试再提交: @@ -1010,7 +1022,7 @@ LangGraph / LangChain 一族在 `pyproject.toml` 里只放行到下一个小版 ./scripts/lock-deps.sh --upgrade-package langgraph # 升一个包(连带它必须跟着动的) ./scripts/lock-deps.sh --upgrade # 全部升到 pyproject 允许的最新 ./scripts/lock-deps.sh # 只加了新依赖:补上它,已锁的不动 -pip install -e 'backend[microvm,dev,db,docs]' -c backend/requirements.lock # 装到当前环境 +pip install -e 'backend[microvm,dev,db,docs,postgres]' -c backend/requirements.lock # 装到当前环境 ``` 重新生成锁文件需要 [uv](https://docs.astral.sh/uv/);装依赖本身只用 pip。 @@ -1020,7 +1032,196 @@ pip install -e 'backend[microvm,dev,db,docs]' -c backend/requirements.lock # 全部位于 `data/`:`agentlab.db` 业务数据、`checkpoints.db` 执行断点、 `artifacts/` 工件库、`workspace/` 沙箱文件、`uploads/` 上传文件、`records/` 填报登记的记录表、`files/` 表单附件、 `.secret_key` 加密密钥。 -删除 `data/` 即恢复出厂设置。 +删除 `data/` 即恢复出厂设置。应用库用 PostgreSQL 时(下一节)不再用 `agentlab.db`、`checkpoints.db`,业务数据和断点 +都在 PostgreSQL 里;别的照旧在 `data/`,数据目录照样要有、要备份。 + +### 用 PostgreSQL + +缺省的应用库是数据目录里的 SQLite:单机单进程,什么都不用装。换成 PostgreSQL 之后才能多进程([多进程](#多进程))。要把应用库(工作流、运行、事件、审批、待办、数据目录、 +评测……)和执行断点放进 PostgreSQL,设一项: + +```bash +AGENTLAB_DATABASE_URL=postgresql+asyncpg://agentlab:<密码>@127.0.0.1:5432/agentlab # postgresql:// 也认 +``` + +- 要 `[postgres]` 额外依赖(`environment.yml`、Docker 镜像都已带上);库要先建好(`CREATE DATABASE agentlab`), + 表由服务启动时建,老版本建的库启动时补列补索引,同 SQLite。启动日志第一句写用的是哪种库、哪台主机哪个库(不写密码)。 +- 没装 `[postgres]` 就设了这一项,启动时直接报要装什么(`pip install -e 'backend[postgres]' -c backend/requirements.lock`)。 +- SSL:地址后加 `?sslmode=require`(或 `verify-ca`、`verify-full`;asyncpg 的写法 `?ssl=require` 也认),启动时换成 asyncpg + 认的参数。证书文件不写在地址里(asyncpg 不收 `sslrootcert` 这类参数,写了启动就报错),设环境变量 `PGSSLROOTCERT`、 + `PGSSLCERT`、`PGSSLKEY`——应用库的 asyncpg 和断点库的 libpq 都认。地址里别的 libpq 参数(`application_name`、 + `connect_timeout` 之类)asyncpg 也不收,同样启动时报出来。 +- 断点(LangGraph 的 checkpoints、checkpoint_blobs、checkpoint_writes、checkpoint_migrations 四张表,不和应用的表重名) + 缺省放在同一个库;要放别处设 `AGENTLAB_CHECKPOINT_URL=postgresql://…`(这一项是 libpq 的写法,`sslrootcert` 等可以直接写)。 +- 连接池:每个进程常驻 `AGENTLAB_DB_POOL_SIZE`(缺省 10)条、高峰再多开 `AGENTLAB_DB_MAX_OVERFLOW`(缺省 20)条,取连接前探活, + 30 分钟换一次;断点库另有一个小池子。前面有 PgBouncer 时照下面「直连与 PgBouncer」的三种部署写(事务模式要开 + `AGENTLAB_DB_PGBOUNCER=1` 并另给直连地址)。 +- 连接数的账:每个进程最多 `AGENTLAB_DB_POOL_SIZE + AGENTLAB_DB_MAX_OVERFLOW`(缺省 10 + 20 = 30)条应用库连接,断点库的池子 + 最多 `min(AGENTLAB_MAX_CONCURRENT_RUNS, 20)` 条(至少 4,缺省 20),合起来一个进程最多 50 条;导入命令另占几条。PostgreSQL + 缺省 `max_connections=100`,还要留几条给 `psql`、备份和超级用户(`superuser_reserved_connections` 缺省 3):一个进程够用, + 起几个进程就是几倍——要么把 `max_connections` 调大(约「进程数 × 50 + 20」),要么调小两边的池子(比如 POOL_SIZE=5、 + MAX_OVERFLOW=10、MAX_CONCURRENT_RUNS=10,一个进程最多 25 条),要么前面放 PgBouncer。 +- 和 SQLite 上的行为一致:字符串列不限长(SQLite 本来就不查长度,模型里写的长度只是示意)、文本按字节比大小和排序 + (列上带 COLLATE "C",同 SQLite,不看建库时选的区域设置)、整数列是 BIGINT、JSON 列存原文(PostgreSQL 的 json,不是 JSONB: + JSONB 会重排对象里的键)、时间存不带时区的 UTC、排序时可空列的 NULL 照 SQLite 的位置(升序在前、降序在后;只管直接按列 + 排的,按聚合、coalesce 之类的表达式排的不管,列表都另带了 id 之类的决胜键,并列时两种库排出来一样)、按 JSON 里的键筛选 + 按 JSON 的类型比(true 不等于 1)。两处 SQLite 照存而 PostgreSQL 不收的值会被换掉:JSON 里的 NaN / Infinity 写成 null, + 文本(JSON 里的、普通文本列里的)中的 NUL 字符去掉。 +- 不分大小写的搜索(运行列表按名字、对象索引按注释):文本列是 COLLATE "C",列上的 ILIKE 只认 ASCII 的大小写,所以按建库时的 + 区域设置折大小写——建库用 `en_US.UTF-8`、`C.UTF-8` 这类区域,É/é、全角A/a 都认;用纯 `C` 区域建的库只认 ASCII。 +- `AGENTLAB_DATA_DIR` 照旧要设:工件、上传表格的版本库、原件、附件、记录表、沙箱工作区、`.secret_key` 都还是文件。 +- 启动时库还连不上(和库的容器一起起、库还在初始化):按退避重试(1、2、4、5、5… 秒),每次打一条「应用库还没就绪」,最多等 + `AGENTLAB_DB_STARTUP_TIMEOUT`(缺省 90)秒再报错退出;密码不对、库不存在这种等也没用的(SQLSTATE 28、3D、42 类)立刻报错。 + 建表、补列补索引在库里的咨询锁里做,默认数据在跨进程的锁里补,几个工作进程同时起各只做一次。 + +**和库一起起。** 部署:离线包的 `./deploy.sh install --postgres` 带一个内置 PostgreSQL,`migrate-to-postgres` 把 SQLite 的部署切过去 +([deploy/README.md](deploy/README.md)「用 PostgreSQL」)。开发:`./scripts/dev.sh --postgres` 起(已经有就复用)一个本地的 +`postgres:16` 容器 `agentlab-dev-postgres`,只开在 `127.0.0.1:55432`,数据在 docker 数据卷 `agentlab-dev-pgdata`,密码第一次 +自动生成在 `data/.postgres-dev/`(后端按 `PGPASSFILE` 读),后端照它设 `AGENTLAB_DATABASE_URL`;Ctrl-C 只停前后端,库的容器 +留着下次复用。容器名、端口、数据卷、镜像用 `AGENTLAB_DEV_PG_CONTAINER`、`AGENTLAB_DEV_PG_PORT`、`AGENTLAB_DEV_PG_VOLUME`、 +`AGENTLAB_DEV_PG_IMAGE` 改(环境变量,或写在 `.env` 里,环境变量优先;`.env.example` 列着应用库、连接池、PgBouncer、多进程和这几项)。不加 `--postgres` 和原来一样:SQLite,不用 docker。 + +**从 SQLite 搬过来。** `python -m app.db copy-to-postgres` 把数据目录里的 `agentlab.db` 和 `checkpoints.db` 导进 PostgreSQL, +停在审批上的、中断了的运行导过去之后照样能接着跑: + +```bash +# 1. 停服务(导入会检查:来源旁边还留着 agentlab.db-wal / checkpoints.db-wal,或者目标库上已经有服务连着,就拒绝) +# 2. 备份数据目录(停着服务拷一份) +# 3. 只核对:数来源有多少、目标上已有多少,什么都不写 +cd backend && python -m app.db copy-to-postgres ../data postgresql+asyncpg://agentlab:<密码>@127.0.0.1:5432/agentlab --dry-run +# 4. 导入:建表、按外键先后逐表分批插入,结束时逐表按主键核对、断点按条数核对,有没导进去的退出码非零 +cd backend && python -m app.db copy-to-postgres ../data postgresql+asyncpg://agentlab:<密码>@127.0.0.1:5432/agentlab +# 5. 设 AGENTLAB_DATABASE_URL(AGENTLAB_DATA_DIR 还指着原来的数据目录),起服务 +``` + +- 目标地址不给就用 `AGENTLAB_DATABASE_URL`;`--checkpoint-url` 把断点导到另一个库;`--work-dir` 放快照(要留出和两个库一样大的 + 空间,缺省系统临时目录)。读的是 SQLite 备份接口拷出的一份快照,在快照上先补到当前的表结构,来源的内容不改。 +- 重跑是幂等的:目标上已有同一主键(或撞了别的唯一约束)的行一律不动,只补上还没有的,不重复、也不覆盖——所以导入之后又在 + SQLite 上改过的行,重跑不会带过去;要整份重来就清空目标库再导。断点同理,已有的不重写。 +- 能查到的只有上面两样:服务连着别的数据目录、另一台机器上的服务连着目标库,查不出来,要自己保证。服务正常停下时会关掉 + 所有连接、-wal 随之消失;还在的是上次强杀留下的,确认服务已停后加 `--assume-stopped`。 +- 老库里 SQLite 存得下、PostgreSQL 不收的会先在快照上处理、在报告里照实列出:老版本补列时没带缺省值、留着 NULL 的 NOT NULL 列 + 填成模型的缺省值(启动时的迁移在 SQLite 上也这么填,两边存的一样);**孤儿行**——外键指向已经不在的行(SQLite 当时没强制 + 外键留下的)——外键是 SET NULL 的置空,别的(CASCADE、不可空)不导入,按表列出条数,不算核对失败(父行被删时它们本来就该 + 跟着删)。自增主键的序列接在 SQLite 用过的最大号后面(删过的号不复用)。 +- 导完 `agentlab.db` 留在数据目录里也没关系(可以当回退用):它照样算 AgentLab 自己的文件,不能登记成数据源。 + +**备份。** PostgreSQL 用 `pg_dump`(应用表和断点在同一个库里就一次带上),数据目录照旧整份拷。两样最好在停服务时一起做; +不停服务的话先 dump 库、再拷数据目录:工件、原件按内容寻址、只增不改,库里引用到的文件拷数据目录时都在。例外是填报的记录表 +`records/*.db`:SQLite 文件、原地追加写,开着服务直接拷可能拷到写了一半的——要么停服务拷,要么一个个用 SQLite 的备份接口拷 +(`sqlite3 data/records/x.v1.db ".backup /备份目录/x.v1.db"`)。 + +**多台机器**见下一节[多进程](#多进程)。 + +**测试。** `scripts/test-postgres.sh` 用本机 docker 已有的 `postgres:16` 镜像起一个临时容器(只绑 127.0.0.1 的随机端口,跑完删掉), +在上面跑后端全量;参数原样交给 pytest。只对 SQLite 成立的测试标了 `sqlite_only`,在 PostgreSQL 上跳过。 + +问数据每一轮的事件日志(`agentlab.db` 的 `chat_turn_events`:助手的操作、阶段、运行、复核、终局,供页面先回放再接实时)只为 +进行中的轮次服务:已结束(成功、失败、取消、中断)超过 14 天的轮次由调度循环每小时清一次、整轮删除;重开已结束的一轮读的是 +对话里那一行(答案、图、复核、用量、几次尝试),不受影响。助手等模型时的心跳只在实时流上发、不落库。 + +### 多进程 + +应用库是 PostgreSQL 时可以起几个工作进程,指着同一个库、同一个数据目录: + +```bash +AGENTLAB_DATABASE_URL=postgresql+asyncpg://… ./scripts/prod.sh --workers 4 # 一台机器、uvicorn 起 4 个进程共用一个端口 +AGENTLAB_WORKERS=4 … # Docker:agentlab.env 里加这一行(deploy/README.md) +``` + +几个容器、几台机器各起一份也行(前面挂一个负载均衡,WebSocket、SSE 不需要粘性会话)。**SQLite 只支持单进程**:数据目录上有一把 +文件锁,第二个进程拒绝启动、照实报错;`AGENTLAB_WORKERS` 大于 1 而应用库是 SQLite 时同样拒绝。 + +| 配置 | 缺省 | 说的是什么 | +| --- | --- | --- | +| `AGENTLAB_WORKERS` | 1 | 起几个工作进程(`prod.sh`、Docker 的入口按它给 uvicorn `--workers`) | +| `AGENTLAB_WORKER_HEARTBEAT_S` | 5 | 每个进程隔多久续一次租约、取一次发给自己的请求(停止之类) | +| `AGENTLAB_WORKER_LEASE_S` | 30 | 租约多长(至少是心跳的 3 倍):进程没了(强杀、断电)多久之后别的进程接手 | +| `AGENTLAB_SCHEDULER_TICK_S` | 15 | 调度循环一拍的间隔 | + +连接数照上一节「连接数的账」按进程数乘,另外每个进程多一条专门收发进程间通知(LISTEN / NOTIFY)的连接,不占连接池;跨进程的锁 +(会话级咨询锁)持锁期间占一条。 + +**直连与 PgBouncer。** 有几样连接依赖会话状态:收发进程间通知的那一条(LISTEN)、跨进程的锁(会话级咨询锁,取锁和放锁要在同一条 +会话上)、断点库(预备语句,建表时取会话锁)。普通查询走 `AGENTLAB_DATABASE_URL`;设了 `AGENTLAB_DATABASE_DIRECT_URL`(写法同它) +时,上面这几样走直连地址(断点库缺省也走它,`AGENTLAB_CHECKPOINT_URL` 另给的除外);不设就全用同一个地址。三种部署: + +| 部署 | 怎么设 | +| --- | --- | +| 直连 PostgreSQL | 只设 `AGENTLAB_DATABASE_URL`。最简单,多进程建议这样 | +| PgBouncer 会话模式(`pool_mode=session`) | `AGENTLAB_DATABASE_URL` 指 PgBouncer 就行:会话模式下一个客户端连接独占一条服务端连接,会话状态都在 | +| PgBouncer 事务模式(`pool_mode=transaction`) | `AGENTLAB_DATABASE_URL` 指 PgBouncer,另设 `AGENTLAB_DB_PGBOUNCER=1` 和 `AGENTLAB_DATABASE_DIRECT_URL`(直连 PostgreSQL,或指 PgBouncer 的会话模式那个库)。没给直连地址时拒绝启动 | + +`AGENTLAB_DB_PGBOUNCER=1` 做的事(只对普通地址):不用客户端的连接池(交给 PgBouncer);关掉 SQLAlchemy 和 asyncpg 的预备语句缓存 +(事务模式下下一条语句可能落在另一条服务端连接上,缓存着的预备语句在那边不存在);asyncpg 每条语句仍先预备再执行(SQLAlchemy 的 +asyncpg 方言一律这样做,同一个事务里、在同一条服务端连接上),名字用随机的,不和别的客户端留下的撞名——PgBouncer 那边要配 +`server_reset_query = DISCARD ALL` 加 `server_reset_query_always = 1` 收掉留下的预备语句,或用 1.21 以上开 `max_prepared_statements`。 +事务级的咨询锁(发名额、同源队、建表时用)在事务模式下照常成立:取锁、放锁在同一个事务里。这几条照 SQLAlchemy、asyncpg、psycopg +的文档和代码核过,没有在真的 PgBouncer 上跑过(本机没有它的镜像)。 + +**认领和租约。** 每个进程在库里登记一行(主机、pid、启动时的随机串),隔一拍心跳续租约。运行、问数据的一轮、后台任务(探查、起草、 +剖析、评测、影子起草)、处理中的文档、执行中的发布单都记着认领它的进程;租约按进程算,进程没了它认领的行一起到期。平稳关停的进程 +交班、删掉自己的登记,别的进程马上接手;强杀的等租约过期。启动时的整理只碰属主不在了的行,绝不碰别的活进程的。接手的规矩: + +| 什么 | 属主不在了之后 | +| --- | --- | +| 在跑的运行 | 标中断(可从断点继续运行),补一条「服务上次未正常关闭」——不自动重跑有副作用的步骤,人点「继续运行」 | +| 排队的运行 | 标失败(从没开始执行,没有断点,「恢复」只会伪造一份结果),补一条失败原因 | +| 停在审批上的运行 | 不属于任何进程:谁收到决定谁接着跑,同一个决定只会被一个进程执行(审批和运行都是带条件的更新) | +| 问数据的一轮 | 还在建图的标中断(可「重试本轮」);已经发起运行的,接手的进程接着等这次运行、照常复核收尾 | +| 起草、剖析、探查、评测、影子起草 | 标中断(没做完的表、题记停止),界面上点「重试」接着做 | +| 处理中的文档、执行中的发布单 | 文档标失败(请重新上传);发布单改成停下、停在没做完的那一步,可以接着做 | +| 待审批、结果未知的写入 | 缺了待办的补建;结果未知、还没安排核实的补上核实(所属运行还在别的活进程里执行的不碰) | + +**停止。** 运行、问数据的一轮、后台任务在别的进程上跑时,「停止」写一条给属主的请求(PostgreSQL 上 NOTIFY 叫醒它,最迟一拍心跳), +属主照本进程的规矩停、收尾(`run.cancelled` 带上是谁停的),界面上和单进程时一样;属主不在了的按上表接手。 + +**运行并发。** 每段执行(发起、审批之后接着跑、继续运行)开始前要一个名额,按来源分四类,几个进程合起来算:交互(问数据、画布 +试运行、手动发起的正式运行、蓝图试运行、个人应用刷新)、应用(应用入口、按需视图、钩子、接口)、定时(定时任务、事件触发)、系统 +(评测、口径召回、对账、补算、按口径正式出具)。全站上限取 `AGENTLAB_MAX_CONCURRENT_RUNS`(缺省 20),其中一部分是**给交互的留底** +(缺省全站上限的 1/4、至少 1、至多全站上限减 1):应用、定时、系统三类合起来永远用不到留底,有人问数据、试运行时总有名额;留底以外的 +它们可以**借用**——三类各自的上限缺省就是「全站上限 − 留底」,没人问数据时评测也能开满,设置里可以调小。交互自己的上限缺省等于全站 +上限。等名额时按优先级(缺省交互 40 > 应用 30 > 定时 20 > 系统 10,同级先来先得),已经在执行的不打断。在「设置 → 偏好设置 → 运行 +并发」里改(管理员),改了最迟几秒各进程按新的发。单进程时也照这套发名额。 + +**调度当班。** 只有一个进程当班(库里的一份租约):定时认领、事件派发、Webhook 投递、流程的时限和核实、每天的汇总、各类周期清理、 +口径比对的补比和自动召回的续发、接手不在了的进程留下的行,都只在当班的进程里做;其余进程待命,当班的没了待命的接班(平稳关停的马上 +交班)。切换的那一会儿重复不了:定时认领是条件更新、触发和投递记录有唯一约束。访问密钥的请求记录攒在各进程自己的内存里,各自隔一拍 +心跳写库。 + +**实时流。** 运行事件、问数据一轮的事件都先落库。页面的 WebSocket / SSE 连到的进程不在执行它时,从库里按 seq 追:执行的进程每提交 +一批事件发一个 NOTIFY(只带运行 / 轮次的 id 和 seq),别的进程收到就回库里读;NOTIFY 丢了也有兜底(运行隔 2 秒、问数据隔一拍心跳回库里 +看一次)。不落库的那两样——正文 token、思考增量,问数据助手的保活心跳——只转给有人在看的进程(攒一小段放进 NOTIFY 里)。回放、重连、 +心跳的说法不变;序号由执行它的进程发,别的进程往执行中的运行里记事件交给它记,停着的运行追加时按库里的最后一条接号(条件更新防撞号)。 + +**进程里的账和缓存。** 要全站一致的都进了库:每日花费账(裁判、核查、评测)先占住那一行再读写;访问密钥每分钟限流、登录失败的退避 +记在库里;按需探查、加入范围、上传表格的版本存储、记录表的写入、按需视图的单飞和每日上限、个人应用刷新、出具、召回、补算、定时发起 +用跨进程的锁(PostgreSQL 的咨询锁);同源队(起草、剖析、探查一个接一个)记在库里。登录设置、对外访问地址、向量模型、能力登记的治理 +快照、数据源的连接池、应用页面的缓存仍在各进程内存里,改的那个进程马上刷新,别的进程收到通知(或最迟一拍心跳)跟着重读。纯性能的缓存 +留在进程里、过期无害:问数据的向量、取值猜测、表结构快照、取数快照的遮罩集合、OIDC 的元数据、DoH 解析、xlsx 的列扫描、口径比对的 +SQL 形状。各进程各自的并发闸门(口径比对、自动对比、起草的模型调用、上传解析)按进程算,进程多了总量跟着多。 + +**多台机器。** 几台机器要用同一份数据目录:放在共享存储(NFS 之类)上,每台挂在**同一个绝对路径**(库里记着上传表格各版本库文件的 +绝对路径),各台机器的时钟仍要对时(NTP):租约按库的时钟算,机器之间差多少都不影响谁算活着,但「停了两个租约时长」这类判断看的是各机器按本机时钟写下的时间。库服务器自己的时钟往前跳超过一个租约时(少见),所有进程续租时都会发现自己的租约已经过期、一起隔离:执行到一半的运行标成中断,在运行详情里点「继续运行」从断点接着执行;后台任务按「属主不在了」的规矩处理;各进程自动换身份重新登记,之后发起的照常。往回跳时不隔离,只是真没了的进程要多等跳回去的那么久才被接手。本机墙钟跳了(NTP 一步校时、手动改时间)不会隔离:判续没续晚看的是计入休眠的单调钟,对不上时以库里的租约为准。文件都是先写临时文件再改名,读的一方看不到写了一半的;写上传版本库、记录表 +在库里的锁下一个接一个。但要说实话:数据目录里还有 SQLite 文件——上传表格的各版本库(发布后只读)、填报的记录表(`records/*.db`, +原地追加写)、沙箱里代码自己建的库——SQLite 文件放在网络文件系统上,读写一致性靠文件锁,而 NFS 上的文件锁不可靠:一台机器在追加时 +另一台读,可能读到不一致的页。所以多台机器时,记录表写得多的部署建议用一台机器多进程,或者把数据目录放在支持可靠文件锁的存储上。 +沙箱会话(microVM)在各进程自己手上,不跨进程共享。 + +**升级。** 不支持新旧版本的进程同时连着一个库,代码里拦着:每个进程登记时带上代码版本(包版本、构建号或 git 提交、源码指纹——源码加上依赖锁文件,只升了依赖也算版本不同)和 +库结构版本,库里还有活着的、版本不同的进程时,新版本的进程在补列之前就拒绝启动,报错里列出全部这样的进程、各是什么版本、心跳还在不在走;同版本的照常 +加入。升级步骤: + +1. 停掉全部旧版本的进程(平稳关停的马上删掉自己的登记;强杀的等它的租约过期,新进程启动时会等); +2. 备份(`pg_dump` 加数据目录,见上一节); +3. 起新版本:第一个进程补列补索引,之后的照常加入。 + +只停了一部分就起新版本,新版本起不来,旧版本照常跑,不会两边各按各的写同一批行。 + +**测试。** `scripts/test-multiprocess.sh` 用本机 docker 的 `postgres:16` 起临时库,在上面起两个工作进程,用演示模型跑端到端 +(`backend/tests/test_multiprocess_e2e.py`,缺省跳过,`AGENTLAB_MULTIPROCESS_E2E=1` 时跑):运行由任一进程认领、另一个进程上的 +流收得到实时事件、同一个决定只执行一次、跨进程停止、定时只触发一次、评测和问数据并发时优先级生效、每日花费上限不越过、强杀一个进程 +之后被接手或照实标中断。检查脚本的 `--stacks`(隔离的副本服务)只支持 SQLite。 --- @@ -1082,6 +1283,12 @@ CHECK_DATA_SRC=<数据目录> node scripts/check-all.mjs --stacks 2 --lanes 12 node scripts/check-all.mjs evidence publish ``` +应用库换成 PostgreSQL 跑同一套(起临时的 `postgres:16` 容器,跑完删掉,见[用 PostgreSQL](#用-postgresql)): + +```bash +PYTHON=$(conda run -n agentlab which python) ./scripts/test-postgres.sh +``` + pytest 前面的 `env -u` 不能省:没配 Key 的 provider 会退回环境变量里的密钥,启动时的 seed 也会拿它种一个 provider——环境里留着真密钥,测试就可能真的去调接口。 diff --git a/backend/app/api/access_keys.py b/backend/app/api/access_keys.py index ba50d365..f7caecba 100644 --- a/backend/app/api/access_keys.py +++ b/backend/app/api/access_keys.py @@ -131,6 +131,9 @@ async def put_settings(payload: SettingsIn, session: AsyncSession = Depends(get_ data={"before": bool(before.get(PERSONAL_KEYS)), "after": payload.personal_keys}) await session.commit() remember_settings(value) + from app.cluster import caches + + await caches.broadcast("auth_settings") return await get_settings() @@ -148,7 +151,7 @@ async def reference() -> dict[str, Any]: async def list_service_accounts(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: _require_admin() rows = list((await session.execute(select(User).where(User.kind == keys.SERVICE_KIND) - .order_by(User.created_at))).scalars()) + .order_by(User.created_at, User.id))).scalars()) counts: dict[str, int] = {} for owner_id in (await session.execute(select(AccessKey.owner_user_id).where( AccessKey.owner_user_id.in_([r.id for r in rows] or [""]), AccessKey.revoked_at.is_(None)))).scalars(): @@ -229,7 +232,7 @@ async def list_keys(session: AsyncSession = Depends(get_session)) -> dict[str, A principal = current_principal() if principal.mode == "off" or not principal.user_id: return {"items": []} - query = select(AccessKey).order_by(AccessKey.created_at.desc()) + query = select(AccessKey).order_by(AccessKey.created_at.desc(), AccessKey.id.desc()) if not _is_admin(): query = query.where(AccessKey.owner_user_id == principal.user_id) rows = list((await session.execute(query)).scalars()) diff --git a/backend/app/api/actions.py b/backend/app/api/actions.py index 1ece0340..47fe2f3e 100644 --- a/backend/app/api/actions.py +++ b/backend/app/api/actions.py @@ -25,7 +25,7 @@ async def list_actions( limit: int = Query(default=100, ge=1, le=500), ) -> dict[str, Any]: ensure_role("builder") - stmt = select(ActionRecord).order_by(ActionRecord.created_at.desc()).limit(limit) + stmt = select(ActionRecord).order_by(ActionRecord.created_at.desc(), ActionRecord.id.desc()).limit(limit) if run_id: stmt = stmt.where(ActionRecord.run_id == run_id) if app_id: diff --git a/backend/app/api/apps.py b/backend/app/api/apps.py index 38f99a3a..a4f2bbb3 100644 --- a/backend/app/api/apps.py +++ b/backend/app/api/apps.py @@ -249,7 +249,7 @@ async def list_apps(include_archived: bool = False, scope: str | None = None, from app.blueprints.personal import allowed principal = _principal(x_actor) - query = select(App).order_by(App.updated_at.desc()) + query = select(App).order_by(App.updated_at.desc(), App.id.desc()) if not include_archived: query = query.where(App.status != "archived") if scope == "personal": @@ -434,7 +434,7 @@ async def assistant_conversation(app_id: str, session: AsyncSession = Depends(ge mine = Conversation.created_by == who if who else Conversation.created_by.is_(None) lookup = (select(Conversation).where(Conversation.kind == "app", Conversation.app_id == app.id, Conversation.archived.is_(False), mine) - .order_by(Conversation.last_active_at.desc()).limit(1)) + .order_by(Conversation.last_active_at.desc(), Conversation.id.desc()).limit(1)) # 取或建在进程里按 (应用, 人) 串行:编辑器开发模式下 effect 执行两次、同一个人开两个标签页,都会同时来取。库里有 # (app_id, created_by) 的部分唯一索引(ux_conversations_app_owner):别处(另一个进程)抢先建了的,这次插入在保存点里 # 被拒,改用那一段——历史才不会被拆开。没署名(关登录又没填)的不受索引约束,靠这把进程内的锁 @@ -552,6 +552,9 @@ async def publish_draft(session: AsyncSession, app: App, *, note: str, dispositi await session.rollback() raise CodedHTTPException(409, "有人刚刚发布了这个应用,请刷新后再试", PUBLISH_CONFLICT) from e clear_cache() + from app.cluster import caches + + await caches.broadcast("app_pages") return {"ok": True, "version": number, "level": level, "spec_hash": row.spec_hash, "issues": issues} @@ -689,7 +692,7 @@ async def _hook_state(session: AsyncSession, app: App) -> tuple[dict[str, Any], version = await _previous(session, app) rows: dict[str, AppHook] = {} for row in (await session.execute(select(AppHook).where(AppHook.app_id == app.id) - .order_by(AppHook.created_at))).scalars(): + .order_by(AppHook.created_at, AppHook.id))).scalars(): rows.setdefault(row.name, row) return declarations(app.draft or {}), declarations(version.spec if version is not None else {}), rows @@ -1045,4 +1048,7 @@ async def backfill(app_id: str, view_id: str, session: AsyncSession = Depends(ge except LaunchRefused as e: raise _view_refused(e) from e clear_cache() + from app.cluster import caches + + await caches.broadcast("app_pages") return out diff --git a/backend/app/api/auth.py b/backend/app/api/auth.py index 45fd4d20..73b32785 100644 --- a/backend/app/api/auth.py +++ b/backend/app/api/auth.py @@ -94,13 +94,13 @@ async def login(payload: LoginIn, request: Request, response: Response, if mode == "off": raise HTTPException(409, "未开启登录,无需登录") keys = (f"u:{payload.username.strip().lower()}", f"ip:{_client_ip(request)}") - if wait := sessions.wait_seconds(*keys): + if wait := await sessions.wait_for(*keys): raise HTTPException(429, f"登录失败次数过多,请 {wait} 秒后再试") user = (await session.execute(select(User).where(User.username == payload.username.strip()))).scalar_one_or_none() if user is None or user.disabled or not verify_password(payload.password, user.password_hash): - sessions.failed(*keys) + await sessions.note_failed(*keys) raise HTTPException(401, "用户名或密码不正确") - sessions.succeeded(*keys) + await sessions.note_succeeded(*keys) token = await sessions.create_session(session, user, ip=_client_ip(request), user_agent=request.headers.get("user-agent")) await session.commit() @@ -317,6 +317,10 @@ async def put_settings(payload: SettingsIn, request: Request, response: Response user_agent=request.headers.get("user-agent")) await session.commit() remember_settings(value) + # 多进程:别的进程的登录设置缓存也要换(开没开登录!),见 app/cluster/caches.py + from app.cluster import caches + + await caches.broadcast("auth_settings") out = _settings_out(value) if token is not None and created is not None: _login_response(request, response, token) diff --git a/backend/app/api/blueprints.py b/backend/app/api/blueprints.py index 0979f507..3c51ad1c 100644 --- a/backend/app/api/blueprints.py +++ b/backend/app/api/blueprints.py @@ -264,13 +264,14 @@ async def list_blueprints(scope: Literal["shared", "personal"] | None = None, # 候选只有自己的、发布成了应用的(要不要列看应用的管理权,shared_listed 逐个应用核对) query = query.where(or_(Blueprint.owner == (principal.username or ""), Blueprint.app_id.is_not(None))) else: - query = query.order_by(Blueprint.updated_at.desc()).limit(_LIST_MAX) + query = query.order_by(Blueprint.updated_at.desc(), Blueprint.id.desc()).limit(_LIST_MAX) rows += await shared_listed(session, principal, list((await session.execute(query)).scalars())) if scope in (None, "personal") and await allowed(session, principal): mine = Blueprint.scope == "personal" if principal.mode != "off": mine = mine & (Blueprint.owner == (principal.username or "")) - rows += list((await session.execute(select(Blueprint).where(mine).order_by(Blueprint.updated_at.desc()) + rows += list((await session.execute(select(Blueprint).where(mine) + .order_by(Blueprint.updated_at.desc(), Blueprint.id.desc()) .limit(_LIST_MAX))).scalars()) rows.sort(key=lambda bp: bp.updated_at or bp.created_at, reverse=True) return {"items": [await blueprint_out(session, bp) for bp in rows[:_LIST_MAX]]} diff --git a/backend/app/api/compare.py b/backend/app/api/compare.py index 16fbf75f..eda5a4a2 100644 --- a/backend/app/api/compare.py +++ b/backend/app/api/compare.py @@ -120,7 +120,7 @@ async def pick_base(session: Any, target: Run, *, schedule_only: bool = False) - Run.created_at < target.created_at) base = (await session.execute( select(Run).where(*earlier, Run.status == "succeeded", Run.manifest_hash.is_not(None)) - .order_by(Run.created_at.desc()).limit(1))).scalars().first() + .order_by(Run.created_at.desc(), Run.id.desc()).limit(1))).scalars().first() if base is None: continue between = await session.scalar(select(func.count()).select_from(Run).where( @@ -184,9 +184,15 @@ async def _side(run: Run, sealed: _Sealed, masks: _Masks, kinds: dict[str, Any]) for artifact, info in sealed.queries.items()} returned = _returned(sealed) - def sources(started: int) -> recompute.Sources: - return recompute.Sources(load=loader, queries=tuple(sealed.queries), outputs=_outputs_before(sealed, started), - masked=masked, returned=returned, finished=_finished_before(sealed, started)) + def sources(started: int, card_node: str = "") -> recompute.Sources: + # 节点产出、跑完的时刻按口径卡所在那一层的 id 交进去(engine/lineage.in_scope,同推演 api/whatif._sources):子工作流 + # 里的卡(「子工作流节点 id/卡」)取值链写的是子图里的 id,只看同一张子图里的节点,父图里同名的不算 + from app.engine.lineage import in_scope + + return recompute.Sources(load=loader, queries=tuple(sealed.queries), + outputs=in_scope(_outputs_before(sealed, started), card_node), + masked=masked, returned=returned, + finished=in_scope(_finished_before(sealed, started), card_node)) return engine.Side(run=run_ref(run), graph=sealed.graph, events=sealed.events, cards=cards, load=loader, masked=masked, sources=sources, dialects=kinds, inputs=dict(run.input or {})) diff --git a/backend/app/api/conversations.py b/backend/app/api/conversations.py index 4d969ba3..94dc93a1 100644 --- a/backend/app/api/conversations.py +++ b/backend/app/api/conversations.py @@ -390,7 +390,7 @@ async def list_conversations( # 开登录后的 user 角色:只列自己建的;建立人为空的老对话不列 query = query.where(Conversation.created_by == mine) rows = list((await session.execute( - query.order_by(Conversation.last_active_at.desc()) + query.order_by(Conversation.last_active_at.desc(), Conversation.id.desc()) )).scalars()) # 应用的助手对话只列能管理的应用的(一个应用一次判断) managed: dict[str, bool] = {} diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 63ee57f6..540c082a 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -327,7 +327,19 @@ skip_if。这些字段按受限的 Python 表达式求值,变量直接写路 - edges 里每条边 {source, target, sourceHandle?} - branch 节点的出边必须带 sourceHandle,取值是 cases 里的 key,另外要有一条 sourceHandle="default" 兜底 - loop 节点:sourceHandle="body" 连循环体,循环体最后一个节点再连回 loop 节点;sourceHandle="done" 连循环结束后的去向 -- human 节点(approve 模式):sourceHandle 用 "approved" 和 "rejected" +- human 节点(approve 模式):批准走 sourceHandle "approved",驳回走下面的「驳回」出口 +- 「驳回」出口:会被人驳回的节点可以多接一条 sourceHandle="rejected" 的边,驳回(包括超时驳回)之后沿它往下走。 + 能接的只有:human(审批、补充输入、编辑稿件三种模式)、tool(工具调用审批、写外部系统前的动作确认)、 + code(代码执行审批)、dual_check(双盲复算的裁决);别的节点接了画布校验会报错。 + · 什么时候接:驳回之后还要做事——通知申请人(接一个通知或写说明的节点,再到成果节点)、退回修改(连回要改的上游 + 那一步,改完再到这个节点问一次)、改走另一条路(比如出一份「已退回」的成果)。驳回时这一步照常结束,下游用 + {{ nodes.关卡id.note }} 取驳回备注、{{ nodes.关卡id.actor }} 取谁驳回的;动作确认被驳回时写入不会发出 + · 不接时:补充输入、编辑稿件、动作确认、代码和工具审批、双盲裁决被驳回,运行就此结束、记为被驳回(是人的决定,不算失败); + 审批模式不接时这一路到审批为止、不再往下走(运行照常结束,没有成果)。审批写了 stop_on_reject(驳回即终止运行)就不要再接驳回出口: + 驳回时运行直接结束,那条边走不到 + · 驳回那条路必须走到成果节点,或者在一个不再往下连的节点结束(退回修改绕回上游、改完再问一次也算),否则驳回之后 + 只能一直绕下去,发布检查会报错 + · 驳回不是故障:「失败重试」「出错时继续」不管驳回,不要为了驳回之后接着走去配它们 - 一个节点连出多条普通边 = 并行执行,多条边汇入同一节点 = 等待汇聚 """ diff --git a/backend/app/api/copilot_blueprint.py b/backend/app/api/copilot_blueprint.py index 32cc2064..515f0ba2 100644 --- a/backend/app/api/copilot_blueprint.py +++ b/backend/app/api/copilot_blueprint.py @@ -207,9 +207,12 @@ async def _conversation(session: AsyncSession, blueprint: Any, actor: str | None async def _open_turn(session: AsyncSession, conversation: Any, question: str) -> str: from sqlalchemy import func, select + from app.chat.job import hold_conversation from app.db.base import utcnow from app.db.models import ConversationTurn + # 先占住这段对话再取最大值往下接(同 api/conversations 开一轮):多进程时同时两次开轮不撞号 + await hold_conversation(session, conversation.id) seq = int((await session.execute(select(func.coalesce(func.max(ConversationTurn.seq), -1)) .where(ConversationTurn.conversation_id == conversation.id))).scalar_one()) + 1 turn = ConversationTurn(conversation_id=conversation.id, seq=seq, question=question, status="running") diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index b1450b87..7a1c2f54 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -280,7 +280,7 @@ async def _recipe_infos(session: AsyncSession, rows: list[DataSource]) -> dict[s for st in (await session.execute( select(ImportStaging).where( ImportStaging.source_id.in_(ids), ImportStaging.status.in_(table_versions.STAGING_OPEN), - ).order_by(ImportStaging.created_at) + ).order_by(ImportStaging.created_at, ImportStaging.id) )).scalars(): # 按创建时间升序,后面的覆盖前面的:留下最近的那一个 out.setdefault(st.source_id, {})["open_staging"] = { diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index 37da3987..325f6ae5 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -285,7 +285,7 @@ async def _overview(session: AsyncSession, suite_ids: list[str]) -> dict[str, di for r in reversed(same)] schedules = (await session.execute(select(Schedule).where( Schedule.target_kind == "eval_suite", Schedule.target_id.in_(suite_ids)) - .order_by(Schedule.created_at))).scalars() + .order_by(Schedule.created_at, Schedule.id))).scalars() for row in schedules: try: text = cron.describe(row.cron, row.timezone) @@ -299,7 +299,7 @@ async def _overview(session: AsyncSession, suite_ids: list[str]) -> dict[str, di @router.get("/suites") async def list_suites(archived: bool = False, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """题集(归档的不列,archived=true 时一起列)。每个题集另带 latest、trend、schedules(_overview)。""" - query = select(EvalSuite).order_by(EvalSuite.updated_at.desc()) + query = select(EvalSuite).order_by(EvalSuite.updated_at.desc(), EvalSuite.id.desc()) if not archived: query = query.where(EvalSuite.archived.is_(False)) rows = list((await session.execute(query)).scalars()) @@ -457,7 +457,7 @@ async def preview_sql(suite_id: str, request: Request, session: AsyncSession = D async with SessionLocal() as read: refs = (await read.execute(select(Artifact.id).where( Artifact.run_id == outcome.run_ids[0], Artifact.kind == "query_snapshot", - Artifact.node_id == gold_rules.QUERY_NODE).order_by(Artifact.created_at.desc()))).scalars().all() + Artifact.node_id == gold_rules.QUERY_NODE).order_by(Artifact.created_at.desc(), Artifact.id.desc()))).scalars().all() snapshot = None for ref in refs: try: @@ -537,7 +537,7 @@ async def _run_queries(session: AsyncSession, run_id: str) -> list[dict[str, Any from app.db.models import Artifact rows = (await session.execute(select(Artifact).where(Artifact.run_id == run_id, Artifact.kind == QUERY_SNAPSHOT_KIND) - .order_by(Artifact.created_at))).scalars().all() + .order_by(Artifact.created_at, Artifact.id))).scalars().all() out = [] for ref in rows: try: @@ -780,7 +780,8 @@ def _snapshot_error(e: snapshots.SnapshotInvalid) -> CodedHTTPException: @router.get("/catalog-snapshots") async def list_snapshots(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - rows = list((await session.execute(select(EvalCatalogSnapshot).order_by(EvalCatalogSnapshot.created_at.desc()) + rows = list((await session.execute(select(EvalCatalogSnapshot) + .order_by(EvalCatalogSnapshot.created_at.desc(), EvalCatalogSnapshot.id.desc()) .limit(200))).scalars()) await snapshots.reconcile_stale(session, rows) return {"snapshots": [snapshots.snapshot_out(r) for r in rows]} @@ -976,7 +977,8 @@ async def list_evaluations(suite_id: str, limit: int = 50, session: AsyncSession await _suite(session, suite_id) rows = list((await session.execute(select(EvalRun).where(EvalRun.suite_id == suite_id) - .order_by(EvalRun.created_at.desc()).limit(max(1, min(limit, 200))))).scalars()) + .order_by(EvalRun.created_at.desc(), EvalRun.id.desc()) + .limit(max(1, min(limit, 200))))).scalars()) await runner.reconcile_stale(session, rows) return {"evaluations": [_evaluation_out(r) for r in rows]} @@ -1185,7 +1187,7 @@ async def get_pair(pair_id: str, session: AsyncSession = Depends(get_session)) - from app.eval import compare, runner rows = list((await session.execute(select(EvalRun).where(EvalRun.pair_id == pair_id) - .order_by(EvalRun.created_at))).scalars()) + .order_by(EvalRun.created_at, EvalRun.id))).scalars()) if not rows: raise CodedHTTPException(404, "这次 A/B 不存在", PAIR_NOT_FOUND) await runner.reconcile_stale(session, rows) diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 82f491a0..8acb580d 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -223,9 +223,13 @@ class _Sealed: 当「已封存」展示——只看 sealed 的消费方会把一条断掉的封存链画成绿的。""" return self.sealed and self.seal["ok"] is True - def nodes_of(self, node_type: str) -> list[str]: - return [str(n.get("id")) for n in self.graph.get("nodes") or [] - if isinstance(n, dict) and n.get("type") == node_type] + def nodes_of(self, node_type: str, *, nested: bool = False) -> list[str]: + """这次运行的图上这一类的节点;nested 为真时连子工作流里跑过的一起给(「子工作流节点 id/…」,类型照 node.started 记的)。""" + out = [str(n.get("id")) for n in self.graph.get("nodes") or [] + if isinstance(n, dict) and n.get("type") == node_type] + if nested: + out += [nid for nid, kind in self.types.items() if "/" in nid and kind == node_type] + return out def payload_of(self, node_ids: list[str]) -> dict[str, Any] | None: """这几个节点里最后跑完的那个的产出,从封存的事件指向的工件里取。""" @@ -1764,7 +1768,8 @@ def _reviews(sealed: _Sealed, report: _Report 审稿记录和计数,按它认(只认审稿记录存上了的),产出拼成 {challenge_artifact, counts, on_refuted: fail}。运行因此 失败了,被推翻的是哪几句、反方凭什么,证据面板里照样看得到。 """ - reviewers = set(sealed.nodes_of("challenge")) + # 子工作流里的反方审稿也认(「sub/chal」):它审的是子工作流里那份报告(审稿记录按全局 id 记,「sub/write」) + reviewers = set(sealed.nodes_of("challenge", nested=True)) if not reviewers: return spec: GraphSpec | None = None @@ -1843,7 +1848,8 @@ def _challenge_of(sealed: _Sealed, report: _Report, unit_id: str) -> dict[str, A unreviewed = output.get("unreviewed") found = {"verdict": verdict, "node_id": node_id, "artifact": artifact, "on_refuted": str(output.get("on_refuted") or "degrade"), - "report_label": writer.title if writer else report.node_id, + # 子工作流里的报告(「sub/write」)不在运行的图上:写「子工作流 › 报告」,不写内部 id + "report_label": writer.title if writer else _node_label(sealed, report.node_id), "counts": _challenge_counts(output.get("counts"), claims), **({"unreviewed": unreviewed} if isinstance(unreviewed, int) and not isinstance(unreviewed, bool) and unreviewed >= 0 else {})} @@ -4184,7 +4190,7 @@ async def _append_judged(sealed: _Sealed, report: _Report, units: list[str], 核对和写入是同一个事务里的条件更新,和接着跑占住运行(runner._claim)抢的是同一行:谁先提交算谁的。 """ from app.core.bus import bus - from app.engine.runner import _stage, run_manager + from app.engine.runner import _seq_floor, _stage, run_manager same_seal = (Run.manifest_seq.is_(None) if sealed.run_manifest_seq is None else Run.manifest_seq == sealed.run_manifest_seq, @@ -4194,8 +4200,9 @@ async def _append_judged(sealed: _Sealed, report: _Report, units: list[str], run = await session.get(Run, sealed.run_id) if run is None: return None - # 进程重启过的话内存里的序号从 0 起:接在库里最后一条后面,不和已有的撞号 - bus.set_seq(sealed.run_id, run.last_seq or 0) + # 接在库里最后一条后面(进程重启过、或者别的进程追加过的话,内存里的序号对不上) + last = run.last_seq + bus.set_seq(sealed.run_id, await _seq_floor(session, sealed.run_id, last)) event = run_manager._event( sealed.run_id, EventType.EVIDENCE_JUDGED, node_id=report.node_id, data={ "report": report.node_id, "doc_artifact": report.doc_artifact, "units": units, @@ -4206,8 +4213,11 @@ async def _append_judged(sealed: _Sealed, report: _Report, units: list[str], "calls": outcome["calls"], "duration_ms": outcome["duration_ms"], "budget": outcome["budget"], "limits_hit": outcome["limits_hit"], "unjudged": outcome["unjudged"], "gaps": outcome["gaps"], "notes": outcome["notes"], "skipped": outcome["skipped"], "post_seal": True}) + # last_seq 也要还是读到的那个:多进程时别的进程可能同时往这次运行里追加(另一次裁判、批注),各拿着同一个号就撞了; + # 同 engine/run_append.append_events,谁先提交算谁的,后到的这一次不记(判定照样交回给点的人) claimed = await session.execute( - update(Run).where(Run.id == sealed.run_id, Run.status.in_(_TERMINAL), *same_seal) + update(Run).where(Run.id == sealed.run_id, Run.status.in_(_TERMINAL), *same_seal, + Run.last_seq.is_(None) if last is None else Run.last_seq == last) .values(last_seq=event.seq)) if claimed.rowcount != 1: # 运行已经不是那份封存下的终态了。烧掉一个序号不要紧:不落库的事件(令牌流)本来就占序号, diff --git a/backend/app/api/governance.py b/backend/app/api/governance.py index a5a73074..9fd5351b 100644 --- a/backend/app/api/governance.py +++ b/backend/app/api/governance.py @@ -52,7 +52,7 @@ async def tool_usage( for (rid,) in await session.execute( select(Run.id) .where(Run.workflow_id == workflow_id, counted_runs()) - .order_by(Run.created_at.desc()) + .order_by(Run.created_at.desc(), Run.id.desc()) .limit(limit_runs) ) ] diff --git a/backend/app/api/identity.py b/backend/app/api/identity.py index 6a7c2530..adb972a8 100644 --- a/backend/app/api/identity.py +++ b/backend/app/api/identity.py @@ -49,7 +49,7 @@ def user_out(row: User) -> dict[str, Any]: @router.get("/users") async def list_users(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - rows = (await session.execute(select(User).order_by(User.created_at))).scalars() + rows = (await session.execute(select(User).order_by(User.created_at, User.id))).scalars() return {"items": [user_out(r) for r in rows]} @@ -294,7 +294,7 @@ async def candidates(purpose: str, app_id: str | None = None, entry: str | None @router.get("/audit") async def list_audit(subject_kind: str | None = None, subject_id: str | None = None, limit: int = 200, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - query = select(AuditEvent).order_by(AuditEvent.at.desc()).limit(max(1, min(limit, 1000))) + query = select(AuditEvent).order_by(AuditEvent.at.desc(), AuditEvent.id.desc()).limit(max(1, min(limit, 1000))) if subject_kind: query = query.where(AuditEvent.subject_kind == subject_kind) if subject_id: diff --git a/backend/app/api/knowledge.py b/backend/app/api/knowledge.py index a9dac9b2..0f2bb700 100644 --- a/backend/app/api/knowledge.py +++ b/backend/app/api/knowledge.py @@ -221,7 +221,7 @@ async def list_documents( collection: str | None = None, session: AsyncSession = Depends(get_session), ) -> list[Document]: - stmt = select(Document).order_by(Document.created_at.desc()).limit(300) + stmt = select(Document).order_by(Document.created_at.desc(), Document.id.desc()).limit(300) if collection: stmt = stmt.where(Document.collection == collection) return list((await session.execute(stmt)).scalars()) @@ -404,6 +404,10 @@ async def set_embedding( else: session.add(Setting(key=emb.EMBEDDING_SETTING_KEY, value=value)) await session.commit() + # 多进程:别的进程照新的设置换向量模型(不换的话它们按旧模型检索、写旧模型的向量) + from app.cluster import caches + + await caches.broadcast("embeddings") return { "embedder": emb.embedder_id(), "dim": emb.embedder_dim(), @@ -473,8 +477,10 @@ class _ReindexJob: } -#: 最近一次重建。只在进程里记:重建是幂等的,服务重启丢了进度,再点一次就是 +#: 最近一次重建(本进程里的)。重建是幂等的,服务重启丢了进度,再点一次就是。多进程时进度另记在设置表里(REINDEX_KEY, +#: 带上做它的进程),别的进程上的「有没有在跑」「到哪了」读那一份;同一时间只跑一个靠跨进程的锁(kb-reindex) _REINDEX: _ReindexJob | None = None +REINDEX_KEY = "cluster:kb_reindex" #: 后台任务要有人拿着引用,否则跑到一半可能被回收 _REINDEX_TASK: Any = None @@ -483,9 +489,59 @@ def reindex_job() -> _ReindexJob | None: return _REINDEX -async def _run_reindex(job: _ReindexJob) -> None: +async def _save_state(job: _ReindexJob) -> None: + """把进度记进设置表(多进程时别的进程读它;SQLite 只有一个进程,不记)。写不进去只记日志。""" + from app.cluster import counters, identity + + if not counters.shared(): + return + from app.db.base import SessionLocal + from app.db.models import Setting + + try: + async with SessionLocal() as session: + row = await session.get(Setting, REINDEX_KEY) + value = {**job.view(), "owner": identity.owner()} + if row is None: + session.add(Setting(key=REINDEX_KEY, value=value)) + else: + row.value = value + await session.commit() + except Exception: # noqa: BLE001 + logger.exception("重建索引的进度没记进库") + + +async def _stored_state() -> dict[str, Any] | None: + """别的进程记的那一份:还写着在跑、做它的进程却不在了的,照实说没做完。SQLite 只有一个进程,没有这一份。""" + from app.cluster import counters, state + + if not counters.shared(): + return None + from app.db.base import SessionLocal + from app.db.models import Setting + + async with SessionLocal() as session: + row = await session.get(Setting, REINDEX_KEY) + value = dict(row.value) if row is not None and isinstance(row.value, dict) else None + if value is None: + return None + owner = value.pop("owner", None) + if value.get("state") == "running" and not (await state.live_for([owner])).alive(owner, local=False): + value.update(state="failed", error="重建索引未完成:做这件事的服务进程停止了", hint="再点一次重建就是") + return value + + +async def _run_reindex(job: _ReindexJob, lock: Any = None) -> None: + import asyncio + from app.db.base import SessionLocal, utcnow + async def report() -> None: + while True: + await _save_state(job) + await asyncio.sleep(1.0) + + reporter = asyncio.ensure_future(report()) try: async with SessionLocal() as session: out = await kb.reindex(session, job.collection, on_progress=job.chunk_step) @@ -499,6 +555,10 @@ async def _run_reindex(job: _ReindexJob) -> None: job.state, job.error, job.hint = "failed", f"重建索引未完成:{reason}", hint or None finally: job.finished_at = utcnow() + reporter.cancel() + await _save_state(job) + if lock is not None: + await lock.give_back() @kb_router.post("/reindex") @@ -538,12 +598,22 @@ async def reindex( 409, f"正在重建索引({_REINDEX.collection or '全部集合'}," f"已完成 {running['done']}/{running['total']}),请等待完成后再操作", ) + # 多进程:别的进程上正在重建的,跨进程的锁占不到(app/cluster/mutex),同样 409,说法照它记在库里的进度 + from app.cluster import mutex + + lock = mutex.lock("kb-reindex") + if not await lock.try_take(): + other = await _stored_state() or {} + raise HTTPException( + 409, f"正在重建索引({other.get('collection') or '全部集合'}," + f"已完成 {other.get('done', 0)}/{other.get('total', 0)}),请等待完成后再操作", + ) job = _REINDEX = _ReindexJob(collection, chunks, memories) if background: - _REINDEX_TASK = asyncio.create_task(_run_reindex(job)) + _REINDEX_TASK = asyncio.create_task(_run_reindex(job, lock)) return JSONResponse(job.view(), status_code=202) - await _run_reindex(job) + await _run_reindex(job, lock) if job.state == "failed": raise HTTPException(502, job.error + (f"。{job.hint}" if job.hint else "")) return job.result @@ -553,8 +623,13 @@ async def reindex( async def reindex_status() -> dict[str, Any]: """最近一次重建的进度:{state, phase, total, done, chunks, memories, result, error…}。 - 还没重建过返回 {state: "idle"}。 + 还没重建过返回 {state: "idle"}。多进程时本进程没在做的,读做它的进程记在库里的那一份。 """ + if _REINDEX is not None and _REINDEX.state == "running": + return _REINDEX.view() + stored = await _stored_state() + if stored is not None and (_REINDEX is None or str(stored.get("started_at") or "") > _REINDEX.view()["started_at"]): + return stored return _REINDEX.view() if _REINDEX is not None else {"state": "idle"} @@ -635,7 +710,7 @@ class SkillOut(SkillIn): @skills_router.get("", response_model=list[SkillOut]) async def list_skills(session: AsyncSession = Depends(get_session)) -> list[Skill]: - return list((await session.execute(select(Skill).order_by(Skill.created_at.desc()))).scalars()) + return list((await session.execute(select(Skill).order_by(Skill.created_at.desc(), Skill.id.desc()))).scalars()) @skills_router.post("", response_model=SkillOut, status_code=201) diff --git a/backend/app/api/ledger.py b/backend/app/api/ledger.py index 2fe472ec..c938d611 100644 --- a/backend/app/api/ledger.py +++ b/backend/app/api/ledger.py @@ -86,7 +86,7 @@ async def effects(workflow_id: str | None = None, version: int | None = None, ap query = query.where(ActionRecord.state == state) if capability: query = query.where(ActionRecord.capability == capability) - rows = (await session.execute(query.order_by(ActionRecord.created_at.desc()).limit(limit))).scalars() + rows = (await session.execute(query.order_by(ActionRecord.created_at.desc(), ActionRecord.id.desc()).limit(limit))).scalars() return {"items": [effect_row(r) for r in rows]} diff --git a/backend/app/api/notify.py b/backend/app/api/notify.py index 62592fe3..a120585e 100644 --- a/backend/app/api/notify.py +++ b/backend/app/api/notify.py @@ -127,7 +127,7 @@ async def list_channels() -> dict[str, Any]: from app.db.models import User principal = current_principal() - stmt = select(NotifyChannel).order_by(NotifyChannel.created_at) + stmt = select(NotifyChannel).order_by(NotifyChannel.created_at, NotifyChannel.id) if principal.mode != "off": # 个人渠道只给本人看 stmt = stmt.where(or_(NotifyChannel.scope != "user", NotifyChannel.owner_id == principal.user_id)) @@ -295,7 +295,7 @@ async def list_deliveries( format 是渠道的格式:邮件渠道成功的那条界面上写「已交给邮件服务器」,不写「已送达」(退信不追踪)。""" stmt = (select(NotifyDelivery, NotifyChannel.format) .join(NotifyChannel, NotifyChannel.id == NotifyDelivery.channel_id) - .order_by(NotifyDelivery.created_at.desc()).limit(limit)) + .order_by(NotifyDelivery.created_at.desc(), NotifyDelivery.id.desc()).limit(limit)) principal = current_principal() if principal.mode != "off" and not principal.at_least("builder"): stmt = stmt.where(NotifyChannel.scope == "user", NotifyChannel.owner_id == principal.user_id) diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index bd87fc14..fe570b3c 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -213,7 +213,7 @@ async def list_runs( """ stmt = _filtered(select(Run), workflow_id=workflow_id, status=status, run_class=run_class, q=q, schedule_id=schedule_id, origin_kind=origin_kind) - stmt = stmt.order_by(Run.created_at.desc()).limit(limit) + stmt = stmt.order_by(Run.created_at.desc(), Run.id.desc()).limit(limit) if before is not None: stmt = stmt.where(Run.created_at < before) return list((await session.execute(stmt)).scalars()) @@ -265,7 +265,10 @@ def _filtered(stmt: Any, *, workflow_id: str | None, status: str | None, run_cla if classes := _csv(run_class): stmt = stmt.where(Run.run_class.in_(classes)) if q and q.strip(): - stmt = stmt.where(Run.workflow_name.ilike(f"%{_like(q.strip())}%", escape="\\")) + # 不分大小写(db/portable.contains_ci):PostgreSQL 的文本列按字节排序(COLLATE "C"),列上的 ILIKE 只认 ASCII 的大小写 + from app.db.portable import contains_ci + + stmt = stmt.where(contains_ci(Run.workflow_name, q.strip())) if schedule_id: stmt = stmt.where(Run.schedule_id == schedule_id) if origins := _csv(origin_kind): @@ -281,11 +284,6 @@ def _csv(value: str | None) -> list[str]: return [v.strip() for v in (value or "").split(",") if v.strip()] -def _like(text: str) -> str: - """用户输入里的 % 和 _ 按字面匹配,不当通配符。""" - return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - - @router.get("/{run_id}", response_model=RunOut) async def get_run(run_id: str, session: AsyncSession = Depends(get_session)) -> RunOut: run = await session.get(Run, run_id) @@ -343,7 +341,7 @@ async def list_artifacts( ) -> list[dict[str, Any]]: """这次运行产出的全部工件引用。审阅出具物时从这里下钻到完整证据。""" rows = await session.execute( - select(Artifact).where(Artifact.run_id == run_id).order_by(Artifact.created_at) + select(Artifact).where(Artifact.run_id == run_id).order_by(Artifact.created_at, Artifact.id) ) return [ { @@ -637,18 +635,22 @@ async def stream_run(websocket: WebSocket, run_id: str) -> None: await _end(websocket, run.status if run else None) return - # 占位期间攒下的那几条历史里也有,按 seq 滤掉,否则前端同一步收两遍 - pump = asyncio.create_task(_pump(websocket, subscription.stream(after=last_seq), run_id)) - try: - # 客户端断开时读操作会抛异常,用它来结束这条连接 - while True: - await websocket.receive_text() - except WebSocketDisconnect: - pass - finally: - pump.cancel() - with contextlib.suppress(asyncio.CancelledError): - await pump + # 占位期间攒下的那几条历史里也有,按 seq 滤掉,否则前端同一步收两遍。多进程时这次运行可能在别的进程里执行: + # 跟随它(app/cluster/relay.py),本进程从库里按 seq 追、在本进程的总线上发布,这里收到的和在执行的进程上一样 + from app.cluster.relay import relay + + with relay.follow_run(run_id, last_seq): + pump = asyncio.create_task(_pump(websocket, subscription.stream(after=last_seq), run_id)) + try: + # 客户端断开时读操作会抛异常,用它来结束这条连接 + while True: + await websocket.receive_text() + except WebSocketDisconnect: + pass + finally: + pump.cancel() + with contextlib.suppress(asyncio.CancelledError): + await pump except WebSocketDisconnect: return except Exception: # noqa: BLE001 @@ -753,7 +755,7 @@ async def list_approvals( .join(Task, Task.approval_id == Approval.id, isouter=True) # 别人的个人应用刷新、个人蓝图试运行上的审批不列(只给拥有者,同运行列表) .where(or_(Run.id.is_(None), not_(hidden_runs(current_principal())))) - .order_by(Approval.created_at.desc()) + .order_by(Approval.created_at.desc(), Approval.id.desc()) .limit(limit) ) if status != "all" and (statuses := _csv(status)): diff --git a/backend/app/api/schedules.py b/backend/app/api/schedules.py index fe66843e..d9548802 100644 --- a/backend/app/api/schedules.py +++ b/backend/app/api/schedules.py @@ -467,12 +467,12 @@ async def _fire_outs(session: AsyncSession, fires: list[ScheduleFire]) -> list[d if fires: rows = (await session.execute( select(WebhookDelivery).where(WebhookDelivery.fire_id.in_([f.id for f in fires])) - .order_by(WebhookDelivery.created_at) + .order_by(WebhookDelivery.created_at, WebhookDelivery.id) )).scalars() for d in rows: notified.setdefault(d.fire_id, []).append({"event": d.event, "status": d.status, "at": _iso(d.created_at)}) # 正在发起的触发:库里是占位的「未能发起」(为了进程被强杀时照实说),进程还在、只是还没发起完,按已发起给 - launching = {f.id for f in fires if service.launching(f.id)} + launching = {f.id for f in fires if service.launching(f.id, f)} return [{ "id": f.id, "fire_at": _iso(f.fire_at), @@ -516,9 +516,9 @@ async def _schedule_outs(session: AsyncSession, rows: list[Schedule]) -> list[di from app import scheduler on = scheduler.scheduler_enabled() - for r in rows: - if _event_kind(r): - states[r.id] = {**await outbox.pending_for(session, r), "scheduler_on": on} + # 一次查齐(outbox.pending_for_many),不再每行各查一次 + waiting = await outbox.pending_for_many(session, [r for r in rows if _event_kind(r)]) + states = {sid: {**state, "scheduler_on": on} for sid, state in waiting.items()} def target(r: Schedule) -> dict[str, Any]: kind, target_id = service.target_of(r) @@ -592,36 +592,73 @@ async def list_variables(tz: str | None = Query(default=None, alias="timezone"), return service.variables(tz=zone, event=event) +#: 选对象时一页最多给几个(按名字排;更多的按关键字查) +TRIGGER_OPTIONS_LIMIT = 100 + + @router.get("/trigger-options") -async def trigger_options(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: +async def trigger_options(event: str | None = None, q: str | None = None, selected: str | None = None, + limit: int = Query(default=TRIGGER_OPTIONS_LIMIT, ge=1, le=500), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """新建、修改事件触发任务时能选的:每种事件的对象(能不能选、为什么不能,由这里判,界面不自己判)、「只认某种来源」的 来源、「只认某个应用」的应用、去抖的缺省值、触发链的上限。 - {events: [{event, label, subject_label, objects: [{id, name, ok, reason}]}], origins: [{kind, label}], + 对象按名字排、一页最多 limit 个,more 说后面还有没有:数据源、工作流成百上千时不一次全给,界面按关键字(q,名字里含它) + 再查。event 给了只查这一种事件的对象;selected 是当前选中的那一个(修改时),总在里面,不被分页挤掉、不显示成「已不存在」。 + + {events: [{event, label, subject_label, objects: [{id, name, ok, reason}], more}], origins: [{kind, label}], apps: [{id, name}], default_debounce_s, max_debounce_s, max_depth}""" from app.db.models import App, DataSource - sources = list((await session.execute(select(DataSource).order_by(DataSource.name))).scalars()) - flows = list((await session.execute(select(Workflow).where(Workflow.is_template.is_(False)) - .order_by(Workflow.name))).scalars()) - tables = [] - for src in sources: - if (src.origin or "manual") != "upload": - continue - ok = await triggers.accumulating(session, src) - tables.append({"id": src.id, "name": src.name, "ok": ok, - "reason": None if ok else "每次上传替换整张表,没有「新的一期」:要用这个事件请在导入设置里改为按期累积"}) - objects = { - "table.version": tables, - "workflow.finished": [{"id": w.id, "name": w.name, "ok": bool(w.published_version), - "reason": None if w.published_version else "还没有发布版本:只有正式运行成功结束才算事件"} - for w in flows], - "source.changed": [{"id": src.id, "name": src.name, "ok": True, "reason": None} for src in sources], - } - apps = (await session.execute(select(App.id, App.name).where(App.status != "archived").order_by(App.name))).all() + if event is not None and event not in triggers.EVENTS: + raise _invalid(f"不认识的事件种类:{event}", "event") + wanted = [event] if event else list(triggers.EVENTS) + words = (q or "").strip() + + async def page(model: Any, *conds: Any) -> tuple[list[Any], bool]: + query = select(model).where(*conds) + if words: + # 名字里含这几个字,不分大小写:两种库同一个说法(db/portable.contains_ci,PostgreSQL 上按库的区域设置折大小写) + from app.db.portable import contains_ci + + query = query.where(contains_ci(model.name, words)) + rows = list((await session.execute(query.order_by(model.name, model.id).limit(limit + 1))).scalars()) + more = len(rows) > limit + rows = rows[:limit] + if selected and all(r.id != selected for r in rows): + if (picked := (await session.execute(select(model).where(model.id == selected, *conds))).scalar_one_or_none()): + rows.append(picked) + return rows, more + + async def tables() -> tuple[list[dict[str, Any]], bool]: + # 只有上传表格有「新的一期」:按来源筛在库里做(老行没写来源的是手工接的源) + rows, more = await page(DataSource, DataSource.origin == "upload") + out = [] + for src in rows: + ok = await triggers.accumulating(session, src) + out.append({"id": src.id, "name": src.name, "ok": ok, + "reason": None if ok else "每次上传替换整张表,没有「新的一期」:要用这个事件请在导入设置里改为按期累积"}) + return out, more + + async def flows() -> tuple[list[dict[str, Any]], bool]: + rows, more = await page(Workflow, Workflow.is_template.is_(False)) + return [{"id": w.id, "name": w.name, "ok": bool(w.published_version), + "reason": None if w.published_version else "还没有发布版本:只有正式运行成功结束才算事件"} + for w in rows], more + + async def sources() -> tuple[list[dict[str, Any]], bool]: + rows, more = await page(DataSource) + return [{"id": src.id, "name": src.name, "ok": True, "reason": None} for src in rows], more + + makers = {"table.version": tables, "workflow.finished": flows, "source.changed": sources} + entries = [] + for e in wanted: + objects, more = await makers[e]() + entries.append({"event": e, "label": triggers.EVENT_LABEL[e], "subject_label": triggers.SUBJECT_LABEL[e], + "objects": objects, "more": more}) + apps = (await session.execute(select(App.id, App.name).where(App.status != "archived").order_by(App.name, App.id))).all() return { - "events": [{"event": e, "label": triggers.EVENT_LABEL[e], "subject_label": triggers.SUBJECT_LABEL[e], - "objects": objects[e]} for e in triggers.EVENTS], + "events": entries, "origins": [{"kind": k, "label": triggers.ORIGIN_LABEL[k]} for k in triggers.FORMAL_ORIGINS], "apps": [{"id": i, "name": n} for i, n in apps], "default_debounce_s": triggers.DEFAULT_DEBOUNCE_S, "max_debounce_s": triggers.MAX_DEBOUNCE_S, @@ -700,7 +737,7 @@ async def _preview_event(session: AsyncSession, body: dict[str, Any]) -> dict[st @router.get("") async def list_schedules(session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - rows = list((await session.execute(select(Schedule).order_by(Schedule.created_at.desc()))).scalars()) + rows = list((await session.execute(select(Schedule).order_by(Schedule.created_at.desc(), Schedule.id.desc()))).scalars()) return {"items": await _schedule_outs(session, rows)} diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index c9b32957..a1ba2b51 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -134,7 +134,7 @@ def _to_out(row: Provider) -> ProviderOut: @router.get("/providers", response_model=list[ProviderOut]) async def list_providers(session: AsyncSession = Depends(get_session)) -> list[ProviderOut]: - rows = (await session.execute(select(Provider).order_by(Provider.created_at))).scalars() + rows = (await session.execute(select(Provider).order_by(Provider.created_at, Provider.id))).scalars() return [_to_out(r) for r in rows] @@ -900,11 +900,22 @@ def _mail_public(stored: Any) -> dict[str, Any]: return mail.public_value(stored) +#: 运行并发一组(app/cluster/slots.py) +CONCURRENCY_GROUP = "concurrency" + + +def concurrency_values(stored: Any) -> dict[str, Any]: + from app.cluster import slots + + return slots.values(stored) + + def _internal(key: str) -> bool: """通知自己记的账(每天汇总的认领行、自动建过邮件渠道的人,notify/digest.STATE_PREFIX):不是用户设置,不列、不收。""" from app.notify.digest import STATE_PREFIX - return key.startswith(STATE_PREFIX) + # cluster: 开头的是多进程协调自己记的(重建索引的进度之类,app/cluster),同样不是用户设置 + return key.startswith(STATE_PREFIX) or key.startswith("cluster:") DEFAULT_SETTINGS: dict[str, Any] = { @@ -945,6 +956,8 @@ DEFAULT_SETTINGS: dict[str, Any] = { EVAL_GROUP: eval_values(None), # 邮件服务(SMTP):通知渠道的邮件格式用它发信。缺省关 MAIL_GROUP: _mail_public(None), + # 运行并发(app/cluster/slots.py):全站同时运行的上限,交互、应用、定时、系统四类各自的上限和优先级。多进程时几个进程合起来算 + CONCURRENCY_GROUP: concurrency_values(None), } @@ -1011,6 +1024,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str merged[APPS_GROUP] = apps_values(stored.get(APPS_GROUP)) merged[LEDGER_GROUP] = ledger_values(stored.get(LEDGER_GROUP)) merged[EVAL_GROUP] = eval_values(stored.get(EVAL_GROUP)) + merged[CONCURRENCY_GROUP] = concurrency_values(stored.get(CONCURRENCY_GROUP)) # 口令不回(password 恒为 null),只说配没配(password_set) merged[MAIL_GROUP] = _mail_public(stored.get(MAIL_GROUP)) for key, value in stored.items(): @@ -1122,6 +1136,14 @@ async def put_settings( if problem := eval_problem(payload.values[EVAL_GROUP]): raise HTTPException(422, problem) payload.values[EVAL_GROUP] = eval_values(payload.values[EVAL_GROUP]) + if CONCURRENCY_GROUP in payload.values: + from app.cluster import slots + + if problem := slots.problem(payload.values[CONCURRENCY_GROUP]): + raise HTTPException(422, problem) + saved_row = await session.get(Setting, CONCURRENCY_GROUP) + payload.values[CONCURRENCY_GROUP] = slots.merged(saved_row.value if saved_row is not None else None, + payload.values[CONCURRENCY_GROUP]) if MAIL_GROUP in payload.values: from app.notify import mail @@ -1144,9 +1166,17 @@ async def put_settings( else: session.add(Setting(key=key, value=value)) await session.commit() + if CONCURRENCY_GROUP in payload.values: + # 本进程马上按新的发名额(别的进程最迟几秒,slots.POLICY_TTL_S) + from app.cluster import slots + + slots.forget_policy() if NOTIFY_GROUP in payload.values: # 中间件、单点登录的回调按进程缓存取对外访问地址:写完库就刷新,不用重启 public_url.remember(payload.values[NOTIFY_GROUP][public_url.SETTING_KEY]) + from app.cluster import caches + + await caches.broadcast("public_url") return await get_settings(session) @@ -1226,4 +1256,14 @@ async def system_info() -> dict[str, Any]: "max_run_seconds": app_settings.max_run_seconds, "model_timeout_seconds": app_settings.model_timeout_seconds, }, + # 多进程协调(app/cluster):登记着的工作进程、谁在调度当班、各类运行在跑 / 在等几个 + "cluster": await _cluster_info(), } + + +async def _cluster_info() -> dict[str, Any]: + from app.cluster import identity, leader, slots, workers + + return {"me": identity.owner(), "database": app_settings.db_dialect, "workers": await workers.listing(), + "leader": await leader.current(), "runs": await slots.snapshot(), + "lease_s": workers.lease_s(), "heartbeat_s": workers.heartbeat_s()} diff --git a/backend/app/api/shares.py b/backend/app/api/shares.py index 27006a5a..5f0bd679 100644 --- a/backend/app/api/shares.py +++ b/backend/app/api/shares.py @@ -206,7 +206,7 @@ async def list_shares(app_id: str, session: AsyncSession = Depends(get_session), app, _ = await _managed(session, app_id, x_actor) rows = list((await session.execute(select(ViewShare).where(ViewShare.app_id == app.id) - .order_by(ViewShare.created_at.desc()))).scalars()) + .order_by(ViewShare.created_at.desc(), ViewShare.id.desc()))).scalars()) version = await _previous(session, app) titles = _titles(version.spec if version is not None else None) return {"enabled": await _enabled(session), "embed_origins": bool(settings.embed_origins), diff --git a/backend/app/api/source_changes.py b/backend/app/api/source_changes.py index 904fc00b..f3f5c44f 100644 --- a/backend/app/api/source_changes.py +++ b/backend/app/api/source_changes.py @@ -79,7 +79,7 @@ async def upkeep(session: AsyncSession = Depends(get_session)) -> dict[str, Any] counts = await sc.counts_by_source(session, ids) schedules: dict[str, list[dict[str, Any]]] = {} for r in (await session.execute(select(Schedule).where(Schedule.target_kind == "source_maintenance") - .order_by(Schedule.created_at))).scalars(): + .order_by(Schedule.created_at, Schedule.id))).scalars(): schedules.setdefault(r.target_id or "", []).append({ "id": r.id, "enabled": r.enabled, "cron": r.cron, "timezone": r.timezone, "description_text": _describe(r.cron, r.timezone), @@ -249,7 +249,7 @@ async def get_maintenance(source_id: str, session: AsyncSession = Depends(get_se row = await _source(session, source_id) rows = list((await session.execute(select(Schedule).where( Schedule.target_kind == "source_maintenance", Schedule.target_id == row.id) - .order_by(Schedule.created_at))).scalars()) + .order_by(Schedule.created_at, Schedule.id))).scalars()) outs = await _schedule_outs(session, rows) fire_ids = [o["last_fire"]["id"] for o in outs if o.get("last_fire")] inputs = dict((await session.execute(select(ScheduleFire.id, ScheduleFire.inputs).where( @@ -310,7 +310,7 @@ async def background_jobs(source_id: str, session: AsyncSession = Depends(get_se brief = await module.job_brief(session, job) if job is not None else None if job is None or brief is None: continue - ahead = runtime.lanes.ahead(row.id, job.id) + ahead = await runtime.lanes.ahead(row.id, job.id) position = len(ahead) if ahead is not None else None found.append((position, job.created_at, { "kind": kind, "id": job.id, "status": brief["status"], "total": brief.get("total", 0), diff --git a/backend/app/api/workflows.py b/backend/app/api/workflows.py index 6355d69f..a536b491 100644 --- a/backend/app/api/workflows.py +++ b/backend/app/api/workflows.py @@ -131,7 +131,7 @@ async def _workflow_out(session: AsyncSession, workflow: Workflow) -> WorkflowOu @router.get("", response_model=list[WorkflowOut]) async def list_workflows(session: AsyncSession = Depends(get_session)) -> list[WorkflowOut]: rows = list( - (await session.execute(select(Workflow).order_by(Workflow.updated_at.desc()))).scalars() + (await session.execute(select(Workflow).order_by(Workflow.updated_at.desc(), Workflow.id.desc()))).scalars() ) counts = await _run_counts(session) levels = await _published_levels(session) diff --git a/backend/app/apps/access.py b/backend/app/apps/access.py index c3121f13..27a9b4f5 100644 --- a/backend/app/apps/access.py +++ b/backend/app/apps/access.py @@ -72,7 +72,7 @@ def role_of(principal: Principal, app: Any, grants: list[Any]) -> str | None: async def grants_of(session: AsyncSession, app_id: str) -> list[AppGrant]: return list((await session.execute(select(AppGrant).where(AppGrant.app_id == app_id) - .order_by(AppGrant.created_at))).scalars()) + .order_by(AppGrant.created_at, AppGrant.id))).scalars()) async def app_role(session: AsyncSession, principal: Principal, app: Any) -> str | None: diff --git a/backend/app/apps/backfill.py b/backend/app/apps/backfill.py index 97630d6e..eab353a8 100644 --- a/backend/app/apps/backfill.py +++ b/backend/app/apps/backfill.py @@ -219,8 +219,11 @@ async def run_backfill(session: AsyncSession, app: Any, version: AppVersion, vie _LOCKS: dict[tuple[str, str], asyncio.Lock] = {} -def _lock(app_id: str, view_id: str) -> asyncio.Lock: - return _LOCKS.setdefault((app_id, view_id), asyncio.Lock()) +def _lock(app_id: str, view_id: str) -> Any: + """跨进程的锁(app/cluster/mutex):同一个视图的补算多进程时也只跑一份。""" + from app.cluster import mutex + + return mutex.lock(f"app-backfill:{app_id}:{view_id}") async def _run_backfill(session: AsyncSession, app: Any, version: AppVersion, view_id: str, *, principal: Any, diff --git a/backend/app/apps/derive.py b/backend/app/apps/derive.py index 831bffe6..307b96df 100644 --- a/backend/app/apps/derive.py +++ b/backend/app/apps/derive.py @@ -65,7 +65,7 @@ async def derive_app(session: AsyncSession, workflow_id: str, version: int, *, n state = PinState(alias=PIN, pin={"workflow_id": workflow_id, "version": version}, workflow=workflow, row=row) schedule = (await session.execute(select(Schedule).where( Schedule.workflow_id == workflow_id, Schedule.run_class == "formal", Schedule.enabled.is_(True)) - .order_by(Schedule.created_at))).scalars().first() + .order_by(Schedule.created_at, Schedule.id))).scalars().first() return await _derive(session, row.graph or {}, title=workflow.name, name=name, description=workflow.description or "", pin={"alias": PIN, "workflow_id": workflow_id, "version": version, "graph_hash": state.hash, diff --git a/backend/app/apps/resolve.py b/backend/app/apps/resolve.py index 1f45c938..f033259f 100644 --- a/backend/app/apps/resolve.py +++ b/backend/app/apps/resolve.py @@ -163,7 +163,7 @@ async def sealed_runs(session: AsyncSession, workflow_id: Any, version: Any, gra or_(Run.origin_kind.is_(None), Run.origin_kind.in_(SEALED_ORIGINS))) if isinstance(origin, str) and origin.startswith("schedule:"): query = query.where(Run.schedule_id == origin[len("schedule:"):]) - rows = list((await session.execute(query.order_by(Run.finished_at.desc(), Run.created_at.desc()) + rows = list((await session.execute(query.order_by(Run.finished_at.desc(), Run.created_at.desc(), Run.id.desc()) .limit(max(n, 1) * 3 + 5))).scalars()) out: list[Run] = [] for run in rows: @@ -213,7 +213,7 @@ async def trend_runs(session: AsyncSession, app_id: str | None, workflow_id: Any rows = (await session.execute(select(Run).where( Run.app_id == app_id, Run.origin_kind == BACKFILL_ORIGIN, Run.workflow_id == workflow_id, Run.version == version, Run.version_hash == graph_hash, Run.run_class == "formal", - Run.status == "succeeded").order_by(Run.created_at.desc()).limit(max(n, 1) * 3 + 5))).scalars() + Run.status == "succeeded").order_by(Run.created_at.desc(), Run.id.desc()).limit(max(n, 1) * 3 + 5))).scalars() seen = {input_key(r.input) for r in own} for run in rows: key = input_key(run.input) @@ -387,10 +387,11 @@ def _records_gap(entry: Any) -> str | None: _LOCKS: dict[str, asyncio.Lock] = {} -def _lock(key: str) -> asyncio.Lock: - if key not in _LOCKS: - _LOCKS[key] = asyncio.Lock() - return _LOCKS[key] +def _lock(key: str) -> Any: + """按需视图的单飞锁(app/cluster/mutex,多进程之间也是一把):按缓存键(视图 + 参数)取,同一个键正在运行的那次接上、不另发。""" + from app.cluster import mutex + + return mutex.lock(f"app-view:{key}") def normalize_params(form: dict[str, Any] | None, params: Any) -> dict[str, Any]: @@ -470,24 +471,36 @@ async def run_view(session: AsyncSession, *, app: Any, version: AppVersion, view raise FormRejected(errors) inputs = normalize_params(form, params) key = await cache_key(session, state, inputs) + # 锁按缓存键(这个视图、这组参数):管的是去重——同一个键正在运行的那次接上,不另算一遍。每日上限不靠它:发起之前在库里先占 + # 一份这个应用今天的额度(app/cluster/quota,带条件的加一),几个键、几个进程同时刷新也正好到上限为止,发起不成退回 + from app.cluster import quota + async with _lock(key): run = await _cached(session, key) if run is not None: return run, run.status == "succeeded" - # 「每天」按服务端所在时区的自然日算 - start = datetime.now().astimezone().replace(hour=0, minute=0, second=0, microsecond=0).astimezone(timezone.utc) - used = await session.scalar(select(func.count(Run.id)).where( - Run.app_id == app.id, Run.origin_kind == "app_view", Run.created_at >= start)) limit = await daily_limit(session) - if int(used or 0) >= limit: + + async def used_today(s: AsyncSession) -> int: + # 「每天」按服务端所在时区的自然日算 + start = datetime.now().astimezone().replace(hour=0, minute=0, second=0, microsecond=0).astimezone(timezone.utc) + return int(await s.scalar(select(func.count(Run.id)).where( + Run.app_id == app.id, Run.origin_kind == "app_view", Run.created_at >= start)) or 0) + + day = await quota.take(f"app_view:{app.id}", limit, seed=used_today) + if day is None: raise ViewRefused(429, f"这个应用今天的按需运行已达到上限({limit} 次),请明天再试,或请管理员调整上限", DAILY_LIMIT) - run = await launch_run( - session, workflow_id=state.pin["workflow_id"], run_class="formal", version=state.pin["version"], - expect_hash=state.pin.get("graph_hash"), input_payload=inputs, actor=actor, origin_kind="app_view", - app_id=app.id, app_version=version.version, - origin={"view": view_id, "spec_hash": version.spec_hash, "actor_verified": principal.verified, - "auth_mode": principal.mode, **_key_trace(principal)}) + try: + run = await launch_run( + session, workflow_id=state.pin["workflow_id"], run_class="formal", version=state.pin["version"], + expect_hash=state.pin.get("graph_hash"), input_payload=inputs, actor=actor, origin_kind="app_view", + app_id=app.id, app_version=version.version, + origin={"view": view_id, "spec_hash": version.spec_hash, "actor_verified": principal.verified, + "auth_mode": principal.mode, **_key_trace(principal)}) + except BaseException: + await quota.give_back(f"app_view:{app.id}", day=day) + raise ttl = source.get("ttl_s") if ttl is None and await reads_live(session, state.graph): ttl = TTL_DEFAULT @@ -1062,7 +1075,7 @@ async def _task_states(session: AsyncSession, run: Run) -> dict[str, Any]: from app.db.models_process import Task rows = (await session.execute(select(Task.node_id, Task.status).where(Task.run_id == run.id) - .order_by(Task.created_at))).all() + .order_by(Task.created_at, Task.id))).all() return {str(node): {"status": status} for node, status in rows if node} @@ -1523,7 +1536,7 @@ async def _trails(session: AsyncSession, runs: list[Run]) -> dict[str, _Trail]: if not ids: return out for row in (await session.execute(select(Task.run_id, Task.node_id, Task.kind, Task.status, Task.outcome) - .where(Task.run_id.in_(ids)).order_by(Task.created_at))).all(): + .where(Task.run_id.in_(ids)).order_by(Task.created_at, Task.id))).all(): out[row.run_id].tasks.append(row) for row in (await session.execute(select(ActionRecord).where(ActionRecord.run_id.in_(ids)) .order_by(ActionRecord.created_at, ActionRecord.attempt))).scalars(): @@ -1582,6 +1595,10 @@ async def _rejected_exits(session: AsyncSession, runs: list[Run], out: dict[str, names = {str(v.get("node_id")) for v in via or [] if isinstance(v, dict) and v.get("node_id")} if names: gates[run_id] = names + for v in via or []: + # 子工作流里的关卡(「sub/x」)不在运行的图上:名字照终态事件上记的(「子工作流 › x」),不写内部 id + if isinstance(v, dict) and v.get("node_id") and isinstance(v.get("label"), str) and v["label"]: + out[run_id].end_labels.setdefault(str(v["node_id"]), v["label"]) if not gates: return wanted = sorted({n for names in gates.values() for n in names}) @@ -1755,7 +1772,7 @@ async def list_instances(session: AsyncSession, app_id: str, *, mine: str | None query = query.where(Run.started_by == mine) if status: query = query.where(Run.status == status) - rows = list((await session.execute(query.order_by(Run.created_at.desc()).limit(max(1, min(limit, 200)) * 2))) + rows = list((await session.execute(query.order_by(Run.created_at.desc(), Run.id.desc()).limit(max(1, min(limit, 200)) * 2))) .scalars()) if entry: rows = [r for r in rows if (r.origin or {}).get("entry") == entry] @@ -1906,7 +1923,7 @@ async def _task_list(session: AsyncSession, ctx: _Page, props: dict[str, Any]) - | ((TaskAssignment.principal_kind == "user") & (TaskAssignment.principal == (p.username or ""))) | ((TaskAssignment.principal_kind == "group") & TaskAssignment.principal.in_(list(p.groups) or [""]))) query = query.where(Task.id.in_(mine)) - rows = list((await session.execute(query.order_by(Task.created_at.desc()).limit(100))).scalars()) + rows = list((await session.execute(query.order_by(Task.created_at.desc(), Task.id.desc()).limit(100))).scalars()) runs = {run_id: str((origin or {}).get("title") or name or "") for run_id, origin, name in (await session.execute( select(Run.id, Run.origin, Run.workflow_name).where(Run.id.in_({t.run_id for t in rows})))).all()} return {"scope": scope, "items": [_task_row(t, runs.get(t.run_id)) for t in rows]} @@ -1964,11 +1981,11 @@ async def _process_data(session: AsyncSession, ctx: _Page, ctype: Any, kind: str return {"state": "empty"} if kind == "task": task = (await session.execute(select(Task).where(Task.run_id == run.id, Task.node_id == node_id) - .order_by(Task.created_at.desc()))).scalars().first() + .order_by(Task.created_at.desc(), Task.id.desc()))).scalars().first() if task is None: return {"state": "empty", "node": node_id} rows = list((await session.execute(select(TaskAssignment).where(TaskAssignment.task_id == task.id) - .order_by(TaskAssignment.created_at))).scalars()) + .order_by(TaskAssignment.created_at, TaskAssignment.id))).scalars()) reader = principal_with_actor(ctx.principal, ctx.actor) resolved_at = _aware(task.resolved_at).isoformat() if task.resolved_at else None signoffs = ctype == "task_form" and any( diff --git a/backend/app/auth/access.py b/backend/app/auth/access.py index 3c04db08..07eda877 100644 --- a/backend/app/auth/access.py +++ b/backend/app/auth/access.py @@ -62,6 +62,7 @@ def hidden_runs(principal: Any) -> Any: from app.db.models import Run from app.db.models_apps import App from app.db.models_blueprints import Blueprint + from app.db.portable import json_text if principal.mode == "off": return false() @@ -72,7 +73,7 @@ def hidden_runs(principal: Any) -> Any: kind = func.coalesce(Run.origin_kind, "") return or_( and_(kind == "personal_app", or_(Run.app_id.is_(None), Run.app_id.not_in(mine))), - and_(kind == "trial", func.coalesce(func.json_extract(Run.origin, "$.blueprint_id"), "").in_(others)), + and_(kind == "trial", func.coalesce(json_text(Run.origin, "blueprint_id"), "").in_(others)), ) diff --git a/backend/app/auth/keys.py b/backend/app/auth/keys.py index 1d4c5dc3..b4d82771 100644 --- a/backend/app/auth/keys.py +++ b/backend/app/auth/keys.py @@ -39,7 +39,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Any, Awaitable, Callable -from sqlalchemy import delete, func, select +from sqlalchemy import delete, select from starlette.requests import HTTPConnection from app.api.coded import CodedHTTPException @@ -255,6 +255,16 @@ async def introspect(session: Any, principal: Any) -> dict[str, Any]: _BUCKETS: dict[str, tuple[float, float]] = {} +async def take_shared(key_id: str) -> int: + """同 take,多进程(PostgreSQL)时几个进程共用一个桶(记在库里,app/cluster/counters.py):不然每个进程各有一个满桶, + 实际的上限是进程数乘每分钟 RATE_PER_MINUTE 次。SQLite 只有一个进程,照旧用进程里的桶。""" + from app.cluster import counters + + if counters.shared(): + return await counters.take_token(f"key:{key_id}", per_minute=RATE_PER_MINUTE) + return take(key_id) + + def take(key_id: str, *, now: float | None = None) -> int: """这把密钥再发一个请求:放行交回 0,超了交回还要等几秒(Retry-After)。令牌桶:满桶 RATE_PER_MINUTE 个,每秒补 RATE_PER_MINUTE / 60 个。""" @@ -535,12 +545,13 @@ async def started_before(principal: Any, idem_key: str) -> str | None: """拥有者按这个幂等键用密钥发起过的实例(来源里记着 idempotency_key):处理到一半服务停了、同一个键重新认领时交回它。""" from app.db.base import SessionLocal from app.db.models import Run + from app.db.portable import json_eq async with SessionLocal() as session: return await session.scalar(select(Run.id).where( Run.origin_kind == "app_entry", Run.started_by == principal.username, - func.json_extract(Run.origin, "$.idempotency_key") == idem_key, - func.json_extract(Run.origin, "$.via") == "key").order_by(Run.created_at.desc()).limit(1)) + json_eq(Run.origin, "idempotency_key", idem_key), + json_eq(Run.origin, "via", "key")).order_by(Run.created_at.desc(), Run.id.desc()).limit(1)) # ---------------------------------------------------------------- 请求记录(A6) @@ -649,7 +660,8 @@ async def recent_requests(session: Any, key_id: str, limit: int = 100) -> list[d except Exception: # noqa: BLE001 - 写不进去只是少几条最新的,照常给库里有的 logger.exception("访问密钥的请求记录写库失败") rows = (await session.execute(select(AccessKeyRequest).where(AccessKeyRequest.key_id == key_id) - .order_by(AccessKeyRequest.at.desc()).limit(max(1, min(limit, 200))))).scalars() + .order_by(AccessKeyRequest.at.desc(), AccessKeyRequest.id.desc()) + .limit(max(1, min(limit, 200))))).scalars() return [{"at": r.at.isoformat() if r.at else None, "method": r.method, "route": r.route, "status": r.status, "run_id": r.run_id, "ip": r.ip} for r in rows] diff --git a/backend/app/auth/middleware.py b/backend/app/auth/middleware.py index 40152483..9e90f469 100644 --- a/backend/app/auth/middleware.py +++ b/backend/app/auth/middleware.py @@ -197,7 +197,7 @@ class AuthMiddleware: keys.note(principal.key_id, method=method, route=path, status=principal.status, run_id=None, ip=ip) await _reply(send, principal.status, principal.detail, principal.code) return - if wait := keys.take(principal.key_id): + if wait := await keys.take_shared(principal.key_id): keys.note(principal.key_id, method=method, route=path, status=429, run_id=None, ip=ip) await _reply(send, 429, f"这把访问密钥的请求太频繁(每分钟最多 {keys.RATE_PER_MINUTE} 次),请 {wait} 秒后再试", keys.KEY_RATE_LIMITED, [(b"retry-after", str(wait).encode())]) diff --git a/backend/app/auth/sessions.py b/backend/app/auth/sessions.py index 2951366e..fe637d04 100644 --- a/backend/app/auth/sessions.py +++ b/backend/app/auth/sessions.py @@ -129,3 +129,30 @@ def failed(*keys: str) -> None: def succeeded(*keys: str) -> None: for key in keys: _failures.pop(key, None) + + +# 多进程(PostgreSQL)时登录失败记在库里(app/cluster/counters.py),几个进程看到的是同一份:不然把尝试分散到几个进程上, +# 退避就弱了几倍。SQLite 只有一个进程,照旧用上面进程里的那份 + +async def wait_for(*keys: str) -> int: + from app.cluster import counters + + return await counters.wait_seconds(*keys) if counters.shared() else wait_seconds(*keys) + + +async def note_failed(*keys: str) -> None: + from app.cluster import counters + + if counters.shared(): + await counters.failed(*keys, free_tries=_FREE_TRIES, max_wait_s=_MAX_WAIT_S) + else: + failed(*keys) + + +async def note_succeeded(*keys: str) -> None: + from app.cluster import counters + + if counters.shared(): + await counters.succeeded(*keys) + else: + succeeded(*keys) diff --git a/backend/app/auth/signing.py b/backend/app/auth/signing.py index 4c2060a3..18279e22 100644 --- a/backend/app/auth/signing.py +++ b/backend/app/auth/signing.py @@ -23,6 +23,7 @@ from typing import Any from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from app.core import atomic from app.core.config import settings #: 断言放在这个请求头里 @@ -35,14 +36,13 @@ _KEY_FILE = ".assertion_ed25519.pem" @lru_cache(maxsize=1) def _private_key() -> Ed25519PrivateKey: path = settings.data_dir / _KEY_FILE - if path.exists(): - key = serialization.load_pem_private_key(path.read_bytes(), password=None) - assert isinstance(key, Ed25519PrivateKey) - return key - key = Ed25519PrivateKey.generate() - path.write_bytes(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, - serialization.NoEncryption())) - path.chmod(0o600) + if not path.exists(): + # 只建一次(core/atomic.create_bytes):几个进程同时第一次签名,后到的读先到的那一把,验签的一方只认得一把 + fresh = Ed25519PrivateKey.generate() + atomic.create_bytes(path, fresh.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption()), mode=0o600) + key = serialization.load_pem_private_key(path.read_bytes(), password=None) + assert isinstance(key, Ed25519PrivateKey) return key diff --git a/backend/app/blueprints/personal.py b/backend/app/blueprints/personal.py index 4c73a020..99c1b5e2 100644 --- a/backend/app/blueprints/personal.py +++ b/backend/app/blueprints/personal.py @@ -342,8 +342,11 @@ async def event(session: AsyncSession, blueprint: Any, op: str, principal: Any, _LOCKS: dict[str, asyncio.Lock] = {} -def _lock(who: str | None) -> asyncio.Lock: - return _LOCKS.setdefault(who or "", asyncio.Lock()) +def _lock(who: str | None) -> Any: + """跨进程的锁(app/cluster/mutex):查上限和发起在一起,多进程时同一个人连点两下也不会多跑一次。""" + from app.cluster import mutex + + return mutex.lock(f"personal-refresh:{who or ''}") def _today_start() -> datetime: @@ -416,9 +419,10 @@ async def refresh(session: AsyncSession, app: Any, *, principal: Any) -> tuple[A def _runs_of(app_id: str, graph_hash: str) -> Any: from app.db.models import Run + from app.db.portable import json_eq return select(Run).where(Run.app_id == app_id, Run.origin_kind == ORIGIN, - func.json_extract(Run.origin, "$.graph_hash") == graph_hash) + json_eq(Run.origin, "graph_hash", graph_hash)) async def latest_refresh(session: AsyncSession, app_id: str, graph_hash: str, *, active_only: bool = False) -> Any: @@ -428,7 +432,7 @@ async def latest_refresh(session: AsyncSession, app_id: str, graph_hash: str, *, query = _runs_of(app_id, graph_hash) if active_only: query = query.where(Run.status.in_(("queued", "running", "interrupted"))) - return (await session.execute(query.order_by(Run.created_at.desc()).limit(1))).scalars().first() + return (await session.execute(query.order_by(Run.created_at.desc(), Run.id.desc()).limit(1))).scalars().first() async def exploratory_runs(session: AsyncSession, app_id: str, graph_hash: str | None, n: int, @@ -444,7 +448,7 @@ async def exploratory_runs(session: AsyncSession, app_id: str, graph_hash: str | query = _runs_of(app_id, graph_hash).where(Run.status == "succeeded") if owner: query = query.where(Run.started_by == owner) - rows = list((await session.execute(query.order_by(Run.finished_at.desc(), Run.created_at.desc()) + rows = list((await session.execute(query.order_by(Run.finished_at.desc(), Run.created_at.desc(), Run.id.desc()) .limit(max(n, 1) * 3 + 5))).scalars()) out = [] for run in rows: @@ -467,7 +471,7 @@ async def review_task(session: AsyncSession, blueprint_id: str, *, pending: bool query = select(Task).where(Task.kind == REVIEW, Task.correlation_key == f"{REVIEW_KEY}{blueprint_id}") if pending: query = query.where(Task.status == "pending") - return (await session.execute(query.order_by(Task.created_at.desc()).limit(1))).scalars().first() + return (await session.execute(query.order_by(Task.created_at.desc(), Task.id.desc()).limit(1))).scalars().first() async def _review_targets(session: AsyncSession, to: Any) -> list[tuple[str, str]]: @@ -567,7 +571,7 @@ async def _copy_by(session: AsyncSession, blueprint_id: str, who: str | None) -> query = select(Blueprint).where(Blueprint.copied_from == blueprint_id) query = query.where(Blueprint.owner == who) if who else query.where(Blueprint.owner.is_(None)) - return (await session.execute(query.order_by(Blueprint.created_at.desc()).limit(1))).scalars().first() + return (await session.execute(query.order_by(Blueprint.created_at.desc(), Blueprint.id.desc()).limit(1))).scalars().first() async def review_published(session: AsyncSession, blueprint: Any, *, app_id: str, principal: Any) -> str | None: @@ -607,7 +611,7 @@ async def settle_review(session: AsyncSession, original: Any) -> str | None: return None published = (await session.execute(select(Blueprint).where( Blueprint.copied_from == original.id, Blueprint.status == "published", Blueprint.app_id.is_not(None)) - .order_by(Blueprint.updated_at.desc()).limit(1))).scalars().first() + .order_by(Blueprint.updated_at.desc(), Blueprint.id.desc()).limit(1))).scalars().first() if published is None or await review_task(session, original.id) is None: return None publisher = SimpleNamespace(username=published.owner, verified=False, groups=(), mode="local", @@ -666,7 +670,7 @@ async def review_payload(session: AsyncSession, task: Any) -> dict[str, Any]: .order_by(BlueprintEvent.seq.desc()).limit(1))).scalars().first() copies = (await session.execute(select(Blueprint.id, Blueprint.owner, Blueprint.status, Blueprint.app_id) .where(Blueprint.copied_from == blueprint.id) - .order_by(Blueprint.created_at))).all() + .order_by(Blueprint.created_at, Blueprint.id))).all() return {"kind": REVIEW, "blueprint_id": blueprint.id, "app_id": task.app_id, "name": blueprint.name, "purpose": _purpose(blueprint), "template_id": blueprint.template_id, "submitted_by": submitted.actor if submitted is not None else blueprint.owner, diff --git a/backend/app/blueprints/publish_plan.py b/backend/app/blueprints/publish_plan.py index 80438a16..bc5a90db 100644 --- a/backend/app/blueprints/publish_plan.py +++ b/backend/app/blueprints/publish_plan.py @@ -629,8 +629,12 @@ async def _start(session: AsyncSession, plan_id: str, *, principal: Principal, w # 前面建好的东西不见了:接着做会把后面几步接到一个对不上的状态上,只能重新出单(新单按现在的样子重新算) raise PlanRefused(409, "这份发布单前面建好的东西不见了,不能接着做:请重新出一份发布单", OBJECT_MISSING) # 条件更新抢到执行权:两个人同时点「确认发布」,只有先到的那个做 + # 属主记成本进程(多进程:别的进程启动时只收拾属主不在了的那些,见 startup) + from app.db.base import current_owner + claimed = await session.execute(update(PublishPlan).where(PublishPlan.id == row.id, PublishPlan.status == want) - .values(status="running").execution_options(synchronize_session=False)) + .values(status="running", owner=current_owner()) + .execution_options(synchronize_session=False)) if claimed.rowcount != 1: await session.rollback() raise PlanRefused(409, "这份发布单刚刚被别人开始执行了,请刷新后查看进度", PLAN_STATE) @@ -1010,15 +1014,24 @@ _STEPS = { # -------------------------------------------------------------------------- -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """上次进程没了时还在执行的发布单(status=running,执行是在请求里做的,进程一没就再没有人接着推):改成停下,停在第一个 还没做完的步骤上、写明执行中断,这样就能「接着做」。那一步可能已经做了一半:接着做时按图哈希复用版本、按应用 id 更新 - 同一个应用,不会另建一份工作流版本或应用。交回整理了几份。""" + 同一个应用,不会另建一份工作流版本或应用。交回整理了几份。 + + 只收拾属主不在了的(多进程:别的活着的进程正在执行的不碰;live 见 app/cluster/workers.Liveness,不给就当场查库)。""" from app.apps.audit import record + from app.cluster import claims, state from app.db.base import SessionLocal + live = live if live is not None else await state.current(fresh=True) + claimed = [plan_id for plan_id, _ in await claims.orphans(PublishPlan, live, running=lambda _id: False, + statuses=("running",))] + if not claimed: + return 0 async with SessionLocal() as session: - rows = list((await session.execute(select(PublishPlan).where(PublishPlan.status == "running"))).scalars()) + rows = list((await session.execute(select(PublishPlan).where(PublishPlan.id.in_(claimed), + PublishPlan.status == "running"))).scalars()) for row in rows: kind = next((st.get("kind") for st in row.steps or [] if st.get("status") not in ("done", "skipped")), None) if kind: diff --git a/backend/app/blueprints/store.py b/backend/app/blueprints/store.py index 377a0b3f..cdfba70f 100644 --- a/backend/app/blueprints/store.py +++ b/backend/app/blueprints/store.py @@ -152,8 +152,14 @@ async def _verified_reuse(session: AsyncSession, op: dict[str, Any]) -> dict[str async def _next_seq(session: AsyncSession, blueprint_id: str) -> int: - from app.db.models_blueprints import BlueprintEvent + """这个蓝图下一条留痕的序号。先占住蓝图那一行(PostgreSQL 上是行锁,SQLite 上拿到写锁)再取最大值往下接:多进程时同一个 + 蓝图同时两处记留痕(卡片上改、助手提的),一个接一个,不撞 (blueprint_id, seq) 的唯一约束。updated_at 不动。""" + from sqlalchemy import update + from app.db.models_blueprints import Blueprint, BlueprintEvent + + await session.execute(update(Blueprint).where(Blueprint.id == blueprint_id) + .values(updated_at=Blueprint.updated_at).execution_options(synchronize_session=False)) return int((await session.execute(select(func.coalesce(func.max(BlueprintEvent.seq), 0)) .where(BlueprintEvent.blueprint_id == blueprint_id))).scalar_one()) + 1 diff --git a/backend/app/capabilities/registry.py b/backend/app/capabilities/registry.py index ea63d849..7b7068f1 100644 --- a/backend/app/capabilities/registry.py +++ b/backend/app/capabilities/registry.py @@ -127,10 +127,9 @@ async def _discover(session: AsyncSession) -> dict[str, dict[str, Any]]: async def sync_all(session: AsyncSession) -> int: """把内置工具、数据源、自定义工具、MCP 同步进 capabilities。返回变动的行数。启动时和各对象增删改时调。""" - from sqlalchemy.dialects.sqlite import insert - from app.db.base import utcnow from app.db.models import Capability + from app.db.portable import insert_ignore found = await _discover(session) existing = {row.name: row for row in (await session.execute(select(Capability))).scalars()} @@ -169,8 +168,10 @@ async def sync_all(session: AsyncSession) -> int: row.synced_at = now changed += 1 if fresh: - # 两个请求同时同步(列表页和发起运行撞在一起):后到的那条插入什么都不做,不报主键冲突。不加锁(单进程 + SQLite) - done = await session.execute(insert(Capability).values(fresh).on_conflict_do_nothing(index_elements=["name"])) + # 两个请求同时同步(列表页和发起运行撞在一起):后到的那条插入什么都不做,不报主键冲突。不加锁:两种库都是同一条 + # INSERT … ON CONFLICT DO NOTHING,多个进程同时同步也一样 + done = await session.execute( + insert_ignore(session, Capability).values(fresh).on_conflict_do_nothing(index_elements=["name"])) changed += done.rowcount or 0 if changed or fresh: await session.commit() @@ -183,7 +184,7 @@ def _platform_defaults_apply(row: Any, name: str) -> bool: return name in GOVERNANCE_DEFAULTS and row.version == 1 and row.updated_by is None -async def refresh_cache(session: AsyncSession) -> None: +async def refresh_cache(session: AsyncSession, *, broadcast: bool = True) -> None: """重算登记的进程内快照。能作证的端点读取:绑定登记端点的 http 自定义工具,治理后效果为 read,声明了 tabular。 Python 自定义工具(用户代码在算)和 MCP(没有结构化回执)不算,不管登记里写了什么。""" from app.capabilities.endpoints import bound_endpoint_ref @@ -199,6 +200,11 @@ async def refresh_cache(session: AsyncSession) -> None: _endpoint_prod.clear() _endpoint_prod.update({e.id: bool((e.environments or {}).get("prod")) for e in (await session.execute(select(Endpoint))).scalars()}) + if broadcast: + # 多进程:别的进程的这份快照也要重算(app/cluster/caches.py) + from app.cluster import caches + + await caches.broadcast("capabilities") def cached_view(name: str) -> dict[str, Any] | None: diff --git a/backend/app/chat/drive.py b/backend/app/chat/drive.py index f9f1f899..95b4fe82 100644 --- a/backend/app/chat/drive.py +++ b/backend/app/chat/drive.py @@ -100,6 +100,20 @@ def add_usage(into: dict[str, Any] | None, more: Mapping[str, Any] | None) -> di return out +async def _sealed(run_id: str, timeout: float) -> None: + """等终态的运行封存好(manifest_hash 写上),最多 timeout 秒。""" + from app.db.base import SessionLocal + from app.db.models import Run + + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + async with SessionLocal() as session: + if await session.scalar(select(Run.manifest_hash).where(Run.id == run_id)): + return + await asyncio.sleep(0.1) + + class TurnDriver: """一轮问数据。drive() 跑完一轮,结果在 turn 上;on_phase(阶段) 在每一步开始时调。子类给差异(见模块说明)。 @@ -244,6 +258,9 @@ class TurnDriver: if row is None or row.status in ("succeeded", "failed", "cancelled", "interrupted"): # 封存在终态之后:等执行任务收完尾(含封存),评分只读封存范围 await run_manager.wait_idle(run_id, timeout=CANCEL_GRACE_S) + if row is not None and row.status != "interrupted" and not run_manager.is_active(run_id): + # 多进程:运行在别的进程里执行(审批之后被别的进程接着跑了),本进程等不到它的执行任务,回库里等封存 + await _sealed(run_id, CANCEL_GRACE_S) async with SessionLocal() as session: return await session.get(Run, run_id) await asyncio.sleep(0.2) diff --git a/backend/app/chat/job.py b/backend/app/chat/job.py index 2d04bf4d..438f6fba 100644 --- a/backend/app/chat/job.py +++ b/backend/app/chat/job.py @@ -16,7 +16,8 @@ **事件日志**(chat/log.py;每条 {seq, type, data, ts}): turn.attempt {mode, attempts, scope, max_steps?} 这一次尝试开始了(发起、重试本轮、重新运行、跑一下、继续运行、接着收尾) - op {...} 助手的一条操作原样(含心跳、查看、final、usage) + op {...} 助手的一条操作原样(含查看、final、usage;心跳只落换了阶段的那一拍,同一阶段接着的只在实时流上发, + 见 chat/log.beat) stage {stage} 到了哪一步:build 建图、run 运行、waiting 停在审批上等人、review 复核 run {run_id, status, run_class} 发起了一次运行(页面据此接上运行自己的实时流看过程) review {result} 复核结论原样(含 usage、retry_instruction) @@ -54,7 +55,7 @@ from app.chat import log as turn_log from app.chat import record from app.chat.drive import TurnDriver from app.core import job_runtime as runtime -from app.db.base import SessionLocal, utcnow +from app.db.base import SessionLocal, current_owner, utcnow logger = logging.getLogger(__name__) @@ -100,6 +101,34 @@ def principal_of(snapshot: Mapping[str, Any] | None) -> Any: mode=str(s.get("mode") or "off"), via=str(s.get("via") or "session")) +async def current_principal(snapshot: Mapping[str, Any] | None) -> tuple[Any, str | None]: + """发起时的身份快照 → 接手这一刻的身份:(身份, 不能接着做的原因)。 + + 关着登录时发起的(快照里 mode 是 off、没有用户名)照快照:没有账号可核对。开着登录的按用户名读现在的账号:不在了、 + 停用了交回原因;角色、组、显示名换成现在的(降了角色就按降了之后的做);按现在的身份已经用不了问数据(使用者、管理员 + 没开问数据,同 api/deps._chat_scope)也交回原因。认证方式、是否核实过照快照。""" + from app.api.deps import Principal + from app.auth import user_can_chat + from app.auth.sessions import groups_of + from app.db.models_identity import User + + s = dict(snapshot or {}) + name = s.get("username") + if not s or str(s.get("mode") or "off") == "off" or not name: + return principal_of(snapshot), None + async with SessionLocal() as session: + user = (await session.execute(select(User).where(User.username == name))).scalar_one_or_none() + if user is None or user.disabled: + return None, f"发起这一轮的账号「{name}」已{'停用' if user is not None else '不存在'},本轮没有接着收尾" + groups = await groups_of(session, user.id) + principal = Principal(username=user.username, role=user.role, display_name=user.display_name or user.username, + user_id=user.id, groups=groups, verified=bool(s.get("verified")), + mode=str(s.get("mode") or "off"), via=str(s.get("via") or "session")) + if not principal.at_least("builder") and not (principal.at_least("user") and user_can_chat()): + return None, f"发起这一轮的账号「{name}」已经没有问数据的权限,本轮没有接着收尾" + return principal, None + + # -------------------------------------------------------------------------- # 驱动:问数据和评测不一样的那几项 # -------------------------------------------------------------------------- @@ -113,6 +142,8 @@ class ChatDriver(TurnDriver): self.execution = execution self._thinking = "" self._thinking_at = time.monotonic() + #: 最近一次落库的心跳的阶段:同一阶段接着的心跳不落库(on_op) + self._beat_phase: Any = None #: 这一轮的查看记录(存进 meta.looks) self.looks: list[dict[str, Any]] = [] #: 等不及的那次复核(还在跑):收尾之后由它把用量记进这一轮 @@ -132,6 +163,13 @@ class ChatDriver(TurnDriver): await self._flush_thinking() return await self._flush_thinking() + if kind == "heartbeat": + if op.get("phase") == self._beat_phase: + # 同一阶段接着的心跳是纯保活:只在实时流上发、不落库(chat/log.beat;以前每 3 秒落一条) + turn_log.beat(self.execution.turn_id, op) + return + # 换了阶段的那一拍(规划 → 查看 → 搭建……)说的是到了哪一步,照落:刷新、重开时过程里那一行阶段要回放得出来 + self._beat_phase = op.get("phase") await self.execution.emit("op", op) if kind == "look" and (look := record.look_event(op)) is not None: self.looks.append(look) @@ -147,6 +185,7 @@ class ChatDriver(TurnDriver): self._thinking_at = time.monotonic() async def _generate(self, instruction: str, base_graph: dict[str, Any] | None) -> tuple[list[dict[str, Any]], str | None]: + self._beat_phase = None # 每一次生成从头算:第一拍心跳照落(页面按它画「规划」那一行) try: return await super()._generate(instruction, base_graph) finally: @@ -336,6 +375,12 @@ class ChatTurnExecution(runtime.JobRun): # ---------------- 执行 ---------------- async def _run(self) -> None: + try: + await self._execute() + finally: + turn_log.forget(self.turn_id) # 进程里记的最近一拍心跳(chat/log.beat)跟着这一次执行收掉 + + async def _execute(self) -> None: from app.api.deps import set_principal from app.chat import turn as policy from app.db.models import ConversationTurn @@ -352,8 +397,13 @@ class ChatTurnExecution(runtime.JobRun): self.request = dict(job.request or {}) self.turn_seq = row.seq self.run_id = job.run_id - # 这一轮按发起的人的身份跑:answer_ops 里的 restricted()、对话归属,发起运行时的遮罩、只读都看它 - set_principal(principal_of(job.principal)) + # 这一轮按发起的人的身份跑:answer_ops 里的 restricted()、对话归属,发起运行时的遮罩、只读都看它。按这一刻的账号重新核对 + # (current_principal):服务重启后接着收尾时,快照是发起那一刻的,之后停用了、降了角色的不能照快照接着做 + principal, refused = await current_principal(job.principal) + if refused is not None: + await self._identity_refused(refused) + return + set_principal(principal) scope = [s.get("id") for s in self.request.get("scope") or [] if isinstance(s, Mapping) and s.get("id")] driver = self.driver = ChatDriver(self, question=row.question, datasource_ids=scope) driver.turn.attempts = int(job.attempt or 0) @@ -381,9 +431,10 @@ class ChatTurnExecution(runtime.JobRun): fields, meta, outcome, output, usage = await _settled(driver, self) await self.save(meta=meta, usage=usage, **fields) await self._end(outcome, output) - # 晚到的复核先登记、再改任务状态:实时流看到任务结束时,要知道还得等这一次复核回来 + # 晚到的复核先登记、再改任务状态:实时流看到任务结束时,要知道还得等这一次复核回来(库里也记一笔:别的进程上的流) if driver.late_review is not None: _track_late(self.turn_id, driver.late_review) + await _mark_late(self.turn_id, True) await self._job(status="succeeded", stage=None, finished_at=utcnow()) async def _end(self, outcome: str, output: Any = None) -> None: @@ -417,6 +468,16 @@ class ChatTurnExecution(runtime.JobRun): await self._end("cancelled") await self._job(status="cancelled", stage=None, finished_at=utcnow()) + async def _identity_refused(self, reason: str) -> None: + """发起这一轮的账号接手时已经停用、或者已经没有问数据的权限:照实标中断、写清原因,不再往下做(不建图、不等运行、 + 不复核)。已经发起的那次运行不动——停在审批上的照旧等人,在记录页照常看得到;这一轮不替一个已经无权的人收它的尾。""" + await self._settle() + if not self.ended: + await self.save(status="error", error=reason, + meta={"cause": {"kind": "identity", "message": reason}, **_timing(self.request)}) + await self._end("error") + await self._job(status="interrupted", stage=None, error=reason, finished_at=utcnow()) + async def _failed(self, reason: str) -> None: await self._crash_out(reason) @@ -439,6 +500,11 @@ class ChatTurnManager(runtime.JobManager): def __init__(self) -> None: super().__init__(ChatTurnExecution, label="问数据的一轮", name="chat-turn") + def model(self) -> Any: + from app.db.models_chat import ChatTurnJob + + return ChatTurnJob + def start(self, job_id: str) -> bool: # type: ignore[override] """同一轮在这个进程里还有一个后台任务活着就不起第二个、也不把登记换成新的(换了的话,停止只停得到后起的那个, 先起的那个跑完把「已取消」盖成「已完成」)。起了交回 True。""" @@ -483,13 +549,46 @@ def _track_late(turn_id: str, task: asyncio.Task[Any]) -> None: watcher = asyncio.ensure_future(finish()) _late[turn_id] = watcher + watcher.add_done_callback(lambda _t: _later(_mark_late(turn_id, False))) + + +async def _mark_late(turn_id: str, pending: bool) -> None: + """晚到的复核记在任务行上(late_review_at):多进程时别的进程上的流也知道还要等它(stream_status)。""" + from app.db.models_chat import ChatTurnJob + + try: + async with SessionLocal() as session: + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id == turn_id).values( + late_review_at=utcnow() if pending else None).execution_options(synchronize_session=False)) + await session.commit() + except Exception: # noqa: BLE001 - 记不上只是别的进程上的流可能早一点结束 + logger.exception("问数据:记晚到的复核状态出错") + if not pending: + turn_log.notify(turn_id) + + +def _later(coro: Any) -> None: + with contextlib.suppress(RuntimeError): + asyncio.ensure_future(coro) def late_pending(turn_id: str) -> bool: + """本进程里晚到的复核还在不在跑。""" task = _late.get(turn_id) return task is not None and not task.done() +async def late_waiting(job: Any) -> bool: + """晚到的复核还要不要等:本进程里在跑,或者库里记着、记它的进程(这一轮的属主)还活着。""" + if late_pending(job.id): + return True + if getattr(job, "late_review_at", None) is None: + return False + from app.cluster import state + + return (await state.live_for([job.owner])).alive(job.owner) + + # -------------------------------------------------------------------------- # 写那一行 # -------------------------------------------------------------------------- @@ -935,7 +1034,7 @@ async def _claim(session: AsyncSession, *, conversation: Any, question: str, dat row.meta = {k: v for k, v in new_meta.items() if v is not None} row.status, row.error, row.answer, row.review = "running", "", "", {} row.run_id = None - values = {"status": "queued", "stage": None, "mode": mode, "request": request, + values = {"status": "queued", "stage": None, "mode": mode, "request": request, "owner": current_owner(), "run_id": run_id if mode in ("continue", "resume") else None, "attempt": 0, "principal": principal_snapshot(), "actor": actor, "error": None, "started_at": None, "finished_at": None} @@ -1023,13 +1122,14 @@ async def state(session: AsyncSession, turn_id: str) -> dict[str, Any] | None: from app.db.models_chat import ChatTurnJob job = await session.get(ChatTurnJob, turn_id) - return None if job is None else job_view(job) + return None if job is None else job_view(job, late=await late_waiting(job)) -def job_view(job: Any) -> dict[str, Any]: +def job_view(job: Any, *, late: bool | None = None) -> dict[str, Any]: return {"status": job.status, "stage": job.stage, "mode": job.mode, "run_id": job.run_id, "attempt": job.attempt, "last_seq": job.last_seq, "attempt_seq": job.attempt_seq, "error": job.error, - "late_review": late_pending(job.id)} + "late_review": (late_pending(job.id) or getattr(job, "late_review_at", None) is not None) if late is None + else late} async def stream_status(turn_id: str) -> str | None: @@ -1040,7 +1140,7 @@ async def stream_status(turn_id: str) -> str | None: job = await session.get(ChatTurnJob, turn_id) if job is None: return "gone" - if job.status in runtime.ACTIVE or late_pending(turn_id): + if job.status in runtime.ACTIVE or await late_waiting(job): return None return job.status @@ -1071,8 +1171,10 @@ async def forget(session: AsyncSession, conversation_id: str) -> None: # -------------------------------------------------------------------------- -async def startup(*, resume: bool | None = None) -> dict[str, int]: - """服务启动时(run_manager.setup 之后:服务重启时在跑的运行这时已经标成挂起): +async def startup(*, resume: bool | None = None, live: Any = None) -> dict[str, int]: + """服务启动时(run_manager.setup 之后:服务重启时在跑的运行这时已经标成挂起);多进程时当班进程隔一会儿再来一次 + (app/cluster/reaper.py),只碰属主不在了的轮次(进程没了;或者就是本进程、却不在本进程里跑),别的活着的进程在跑的一概 + 不碰。认领那一下是带条件的 UPDATE,几个进程同时接手同一轮只有一个改得动;接手的进程照下面的规矩办: - 还在建图(没有运行)的这一轮标中断:那一行记 error 和原因(build_interrupted),可以重试本轮; - 已经发起了运行的:resume 为真(缺省看 AGENTLAB_SCHEDULER)就接着等那次运行、照常复核收尾;为假就停着(任务记 @@ -1085,11 +1187,17 @@ async def startup(*, resume: bool | None = None) -> dict[str, int]: from app.db.models import Conversation, ConversationTurn from app.db.models_chat import ChatTurnJob + from app.cluster import claims, state + resume = bool(settings.scheduler) if resume is None else resume counts = {"interrupted": 0, "resumed": 0, "parked": 0, "legacy": 0} + live = live if live is not None else await state.current(fresh=True) + claimed = [job_id for job_id, _ in await claims.orphans(ChatTurnJob, live, running=manager.running, + statuses=runtime.ACTIVE)] async with SessionLocal() as session: - jobs = [j for j in (await session.execute(select(ChatTurnJob).where( - ChatTurnJob.status.in_(runtime.ACTIVE)))).scalars() if not manager.running(j.id)] + jobs = list((await session.execute(select(ChatTurnJob).where(ChatTurnJob.id.in_(claimed), + ChatTurnJob.status.in_(runtime.ACTIVE)))).scalars()) \ + if claimed else [] legacy = list((await session.execute( select(ConversationTurn).join(Conversation, Conversation.id == ConversationTurn.conversation_id) .outerjoin(ChatTurnJob, ChatTurnJob.id == ConversationTurn.id) @@ -1114,7 +1222,21 @@ async def startup(*, resume: bool | None = None) -> dict[str, int]: await session.commit() turn_log.notify(job.id) counts["parked"] += 1 - for row in legacy: + # 晚到的复核记在库里、记它的进程却不在了:不会再回来,别让别的进程上的流一直等 + async with SessionLocal() as session: + late = (await session.execute(select(ChatTurnJob.id, ChatTurnJob.owner).where( + ChatTurnJob.late_review_at.is_not(None)))).all() + gone = [turn_id for turn_id, owner in late if not live.alive(owner, local=late_pending(turn_id))] + if gone: + async with SessionLocal() as session: + await session.execute(update(ChatTurnJob).where(ChatTurnJob.id.in_(gone)).values(late_review_at=None) + .execution_options(synchronize_session=False)) + await session.commit() + for turn_id in gone: + turn_log.notify(turn_id) + # 老轮次(没有任务行)只在启动时整理:当班进程隔一会儿来的那一次(live.trust_mine)不碰——老页面开的轮次(不占锁的写入) + # 正在建图时也是这个样子 + for row in [] if live.trust_mine else legacy: if turn_run_id(row): continue # 有运行的老轮次:页面打开时请服务端接着收尾 async with SessionLocal() as session: @@ -1156,5 +1278,5 @@ async def shutdown() -> None: __all__ = ["APPROVAL_POLL_S", "AUTO_RETRY_MODES", "BUSY", "CONCURRENCY_CLASS", "ChatDriver", "ChatTurnExecution", "MODES", "REVIEW_WAIT_S", "TURN_BUSY", "TURN_FINISHED", "TURN_INVALID", "TurnRefused", "cancel", "forget", - "job_view", "late_pending", "manager", "principal_of", "principal_snapshot", "shutdown", "start", "startup", - "state", "stop_conversation", "stream_status"] + "current_principal", "job_view", "late_pending", "manager", "principal_of", "principal_snapshot", "shutdown", + "start", "startup", "state", "stop_conversation", "stream_status"] diff --git a/backend/app/chat/log.py b/backend/app/chat/log.py index 9e73c8b1..1f151097 100644 --- a/backend/app/chat/log.py +++ b/backend/app/chat/log.py @@ -5,34 +5,57 @@ 也不用像运行的事件总线那样处理慢消费者的积压。订阅方在读库**之前**先登记等待(watch),读完没东西才睡:读和写之间新写进来的 那一条会把它叫醒,不会漏掉。 -叫醒只在本进程里(进程内的事件);隔一拍心跳(HEARTBEAT_S)订阅方也会回库里看一次,所以换成多进程(第 2 波)时,别的进程写的 -事件最迟一拍心跳就送到——那时再换成 PostgreSQL 的 LISTEN/NOTIFY,这里的接口不用变。 +叫醒在本进程里是进程内的事件;多进程(PostgreSQL)时另发 NOTIFY(app/cluster/relay.py),别的进程写的事件、改的状态马上叫醒这里 +的流;NOTIFY 丢了的话,隔一拍心跳(HEARTBEAT_S)订阅方也会回库里看一次。SQLite 只有一个进程,不额外轮询。 线上格式(SSE,每条一行 data: ):{seq, type, data, ts};回放完、这一轮也不在进行中了,补一条不带 seq 的 {type: stream.end, status}(任务的状态)再关;隔一拍没有新事件发一行注释心跳「: ping」(页面据此知道连接还活着)。 事件种类见 app/chat/job.py 的模块说明。 + +**助手的心跳只落换了阶段的那一拍。** 模型几秒没出声时助手发一拍 {op: heartbeat, phase, elapsed_ms}(页面据此写阶段、已等多久): +以前每 3 秒落一条,一轮几百条。现在换了阶段的那一拍(规划 → 查看 → 搭建……,说的是到了哪一步)照落,刷新、重开时过程里那一行 +阶段回放得出来;同一阶段接着的那些是纯保活,只记在本进程里(beat:每一轮最近的一拍),流上发一条不带 seq 的 {type: op, data}; +重连时流先补最近的那一拍。之后落了任何一条事件,那一拍就作废(心跳只说「上一条事件之后还在等」,重连时补旧的会让页面的进度 +退回去)。其余事件照落:跨进程的实时流从库里按 seq 追;保活心跳不落库,执行这一轮的进程只在别的进程有流在看时把它转过去 +(app/cluster/relay.py 的 w / b 两种通知),丢了无妨——心跳本来就不进回放。 + +**清理。** 已结束(成功、失败、取消、中断)超过 RETAIN_DAYS 天的轮次整轮删事件日志(purge,挂在调度循环的那一拍里,一小时最多一次): +页面重开已结束的一轮读的是 conversation_turns 那一行(答案、图、复核、用量、几次尝试都在那里),只有还在进行中的轮次才回放 +事件日志。任务行留着:last_seq 接着发号,之后在这一轮上重试、继续运行不撞号。 """ from __future__ import annotations import asyncio import contextlib +import itertools import json +import logging import time from collections import defaultdict from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from datetime import datetime, timedelta from typing import Any -from sqlalchemy import func, select, update +from sqlalchemy import delete, func, select, update from sqlalchemy.ext.asyncio import AsyncSession from app.db.base import SessionLocal +logger = logging.getLogger(__name__) + #: 隔多久没有新事件就发一拍心跳(秒),也是订阅方回库里看一次的间隔 HEARTBEAT_S = 15.0 #: 一次回放最多读多少条 READ_BATCH = 500 #: 回放、实时推送完、这一轮不在进行中时的结束标记(不落库、不带 seq) STREAM_END = "stream.end" +#: 已结束的轮次的事件日志留几天 +RETAIN_DAYS = 14 +#: 清理多久最多跑一次(秒)、一次最多清几轮 +PURGE_EVERY_S = 3600.0 +PURGE_BATCH = 500 +#: 结束了的任务状态(core/job_runtime 的状态词里不再往下走的那几种) +FINISHED = ("succeeded", "failed", "cancelled", "interrupted") class _Waiters: @@ -63,8 +86,43 @@ waiters = _Waiters() def notify(turn_id: str) -> None: - """这一轮有了变化(新事件、任务状态变了):叫醒本进程里等着的订阅方。""" + """这一轮有了变化(新事件、任务状态变了):叫醒本进程里等着的订阅方;多进程时另发 NOTIFY t|turn_id,别的进程上的流 + 也叫醒(app/cluster/relay.py;SQLite 只有一个进程,什么都不发)。""" waiters.notify(turn_id) + from app.cluster.signal import signals + + signals.publish("t", turn_id) + + +#: 每一轮最近的一拍心跳:(第几拍, 操作)。只在本进程里,不落库(见模块说明) +_beats: dict[str, tuple[int, dict[str, Any]]] = {} +_beat_no = itertools.count(1) + + +def beat(turn_id: str, op: dict[str, Any]) -> None: + """助手的一拍心跳:记下最近的这一拍、叫醒订阅方,由流发出去(不落库、不占 seq)。别的进程上有流在看这一轮时转过去 + (app/cluster/relay.py:不落库的东西只能这样跨进程)。""" + _beats[turn_id] = (next(_beat_no), dict(op)) + waiters.notify(turn_id) + from app.cluster.relay import relay + + relay.beat(turn_id, op) + + +def remote_beat(turn_id: str, op: dict[str, Any]) -> None: + """别的进程转过来的一拍心跳(这一轮在那边执行、这边有流在看):同本进程的一拍。""" + _beats[turn_id] = (next(_beat_no), dict(op)) + waiters.notify(turn_id) + + +def latest_beat(turn_id: str) -> dict[str, Any] | None: + got = _beats.get(turn_id) + return dict(got[1]) if got else None + + +def forget(turn_id: str) -> None: + """这一轮结束了:进程里记的那一拍清掉。""" + _beats.pop(turn_id, None) async def add(session: AsyncSession, turn_id: str, type_: str, data: dict[str, Any] | None = None) -> int: @@ -91,6 +149,8 @@ async def append(session: AsyncSession, turn_id: str, type_: str, data: dict[str """追加一条,提交,叫醒订阅方。交回它的 seq(见 add)。""" seq = await add(session, turn_id, type_, data) await session.commit() + # 落了新事件,上一拍心跳作废:重连时不再补它(心跳只说「上一条事件之后还在等」) + _beats.pop(turn_id, None) notify(turn_id) return seq @@ -124,8 +184,19 @@ async def stream(turn_id: str, after: int, *, active: Callable[[], Awaitable[str """SSE 的每一帧:先回放 seq > after 的,再推实时的;这一轮不在进行中、也没有新的了,补一条 stream.end 再结束。 active() 交回这一轮还在进行中时为 None,不在进行中时交回任务的状态(stream.end 里带的就是它)。""" - beat = HEARTBEAT_S if heartbeat is None else heartbeat + from app.cluster.relay import relay + + with relay.watch_turn(turn_id): + async for frame_ in _stream(turn_id, after, active=active, heartbeat=heartbeat): + yield frame_ + + +async def _stream(turn_id: str, after: int, *, active: Callable[[], Awaitable[str | None]], + heartbeat: float | None) -> AsyncIterator[str]: + interval = HEARTBEAT_S if heartbeat is None else heartbeat last = after + #: 这条连接发过的最近一拍助手心跳(_beats 的第几拍):新连上的从 0 起,先补最近的那一拍 + sent = 0 while True: with waiters.watch(turn_id) as woke: rows = await read(turn_id, last) @@ -138,11 +209,48 @@ async def stream(turn_id: str, after: int, *, active: Callable[[], Awaitable[str if status is not None: yield frame({"type": STREAM_END, "status": status, "data": {"status": status}}) return + got = _beats.get(turn_id) + if got is not None and got[0] > sent: + sent = got[0] + # 不带 seq:页面按 seq 补齐时不算它(frontend/src/api/chatTurns.ts) + yield frame({"type": "op", "data": got[1], "ts": time.time()}) + continue try: - await asyncio.wait_for(woke.wait(), timeout=beat) + await asyncio.wait_for(woke.wait(), timeout=interval) except asyncio.TimeoutError: yield PING -__all__ = ["HEARTBEAT_S", "PING", "READ_BATCH", "STREAM_END", "add", "append", "frame", "notify", "read", "stream", "waiters", - "wire"] +async def purge(now: datetime | None = None, *, retain_days: float = RETAIN_DAYS, limit: int = PURGE_BATCH) -> list[str]: + """结束超过 retain_days 天的轮次整轮删事件日志(见模块说明),交回清了哪几轮。任务行、conversation_turns 那一行不动。 + 结束的时刻按任务的 finished_at,没记的按 updated_at。""" + from app.db.base import utcnow + from app.db.models_chat import ChatTurnEvent, ChatTurnJob + + cutoff = (now or utcnow()) - timedelta(days=retain_days) + async with SessionLocal() as session: + ended = select(ChatTurnJob.id).where(ChatTurnJob.status.in_(FINISHED), + func.coalesce(ChatTurnJob.finished_at, ChatTurnJob.updated_at) < cutoff) + turns = list((await session.execute(select(ChatTurnEvent.turn_id).where(ChatTurnEvent.turn_id.in_(ended)) + .distinct().limit(limit))).scalars()) + if turns: + await session.execute(delete(ChatTurnEvent).where(ChatTurnEvent.turn_id.in_(turns)) + .execution_options(synchronize_session=False)) + await session.commit() + return turns + + +_last_purge = 0.0 + + +async def purge_tick(now: datetime | None = None) -> list[str]: + """调度循环每一拍调(scheduler/loop.process_due,那一拍不一定给 now):一小时最多清一次,不给 now 按当前时间算。""" + global _last_purge + if time.monotonic() - _last_purge < PURGE_EVERY_S and _last_purge: + return [] + _last_purge = time.monotonic() + return await purge(now) + + +__all__ = ["FINISHED", "HEARTBEAT_S", "PING", "READ_BATCH", "RETAIN_DAYS", "STREAM_END", "add", "append", "beat", "forget", + "frame", "latest_beat", "notify", "purge", "purge_tick", "read", "stream", "waiters", "wire"] diff --git a/backend/app/chat/questions.py b/backend/app/chat/questions.py index 99d61240..aaa7efd5 100644 --- a/backend/app/chat/questions.py +++ b/backend/app/chat/questions.py @@ -98,6 +98,7 @@ async def recent_questions(session: AsyncSession, principal: Any, *, limit: int) """最近问过的问题,从新到旧,最多 limit 条(取法见模块开头)。principal 用来去掉这个人读不了的只给拥有者的运行。""" from app.auth.access import hidden_runs from app.db.models import Conversation, ConversationTurn, Run + from app.db.portable import json_is_string from app.engine.origin import counted_runs hidden = hidden_runs(principal) @@ -107,13 +108,13 @@ async def recent_questions(session: AsyncSession, principal: Any, *, limit: int) .join(Conversation, Conversation.id == ConversationTurn.conversation_id) .outerjoin(Run, Run.id == ConversationTurn.run_id) .where(Conversation.kind == "chat", func.trim(ConversationTurn.question) != "", counted_runs(), not_(hidden)) - .order_by(ConversationTurn.created_at.desc()).limit(limit) + .order_by(ConversationTurn.created_at.desc(), ConversationTurn.id.desc()).limit(limit) )).all() runs = (await session.execute( select(Run.id, Run.input, Run.workflow_name, Run.created_at, Run.origin_kind) .where(Run.run_class != "formal", func.coalesce(Run.origin_kind, "") != "chat", counted_runs(), not_(hidden), - func.json_type(Run.input, "$.question") == "text") - .order_by(Run.created_at.desc()).limit(limit) + json_is_string(Run.input, "question")) + .order_by(Run.created_at.desc(), Run.id.desc()).limit(limit) )).all() out = [Question(text=question.strip(), at=at, run_id=run_id, turn_id=turn_id, workflow=workflow, origin="chat") for turn_id, question, at, run_id, workflow in turns] diff --git a/backend/app/cluster/__init__.py b/backend/app/cluster/__init__.py new file mode 100644 index 00000000..78e4742c --- /dev/null +++ b/backend/app/cluster/__init__.py @@ -0,0 +1,14 @@ +"""多进程协调(docs/design/orchestration-round/00-总方案.md §5 W2-b):应用库是 PostgreSQL 时可以起几个工作进程(uvicorn +--workers N,或者几个容器指着同一个库、同一个数据目录),SQLite 只支持单进程。 + +- identity:这个进程是谁(OWNER);workers:登记、租约、谁还活着;state:手上的那一份「谁还活着」 +- leader:调度当班(全站只许一个进程做的周期活);service:心跳、起停 +- requests:发给别的进程的请求(停止、往运行里记事件);signal:PostgreSQL 的 LISTEN / NOTIFY 叫醒 +- slots:运行的并发名额(四类:交互、应用、定时、系统,各有全站上限和优先级) +- reaper:接手属主不在了的行;mutex:跨进程的互斥锁(PostgreSQL 的咨询锁);guard:SQLite 的单进程守卫 +- counters:按时间窗的计数(访问密钥限流、登录失败),PostgreSQL 记在库里、SQLite 在进程里 +""" +from app.cluster.identity import owner +from app.cluster.service import leading, on_beat, on_leadership, start, started, stop + +__all__ = ["leading", "on_beat", "on_leadership", "owner", "start", "started", "stop"] diff --git a/backend/app/cluster/__main__.py b/backend/app/cluster/__main__.py new file mode 100644 index 00000000..04824c79 --- /dev/null +++ b/backend/app/cluster/__main__.py @@ -0,0 +1,24 @@ +"""python -m app.cluster workers:照后端自己的配置(环境变量和 .env 一起读)说起几个工作进程,scripts/prod.sh 和 Docker 的入口按它给 +uvicorn --workers(复核 R-b4:写在 .env 里的 AGENTLAB_DATABASE_URL、AGENTLAB_WORKERS,shell 脚本自己读不到)。应用库是 SQLite 而进程数 +大于 1 时照实报错、退出码 2(SQLite 只支持单进程)。""" +from __future__ import annotations + +import sys + + +def main(argv: list[str]) -> int: + if argv[:1] != ["workers"]: + print("用法:python -m app.cluster workers", file=sys.stderr) + return 64 + from app.cluster.guard import SINGLE_PROCESS + from app.core.config import settings + + if settings.db_dialect == "sqlite" and settings.workers > 1: + print(SINGLE_PROCESS.format(dir=settings.data_dir, hint=f"AGENTLAB_WORKERS={settings.workers}"), file=sys.stderr) + return 2 + print(settings.workers) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/backend/app/cluster/caches.py b/backend/app/cluster/caches.py new file mode 100644 index 00000000..d9232492 --- /dev/null +++ b/backend/app/cluster/caches.py @@ -0,0 +1,125 @@ +"""进程里缓存着的东西跨进程失效(docs/design/orchestration-round/00-总方案.md §5 W2-b「进程内的账和缓存」)。 + +有几样东西每个进程读进内存、改的时候只刷新改它的那个进程:登录设置(开没开登录!)、对外访问地址、向量模型、能力登记的 +治理快照、数据源的连接池、应用页面的渲染缓存。多进程时别的进程会一直用旧的——登录方式那一样是安全问题。 + +做法:改的进程照旧刷新自己,再 broadcast(名字, 键) 写一条 cache_events、发 NOTIFY cache;别的进程收到通知(或者隔一拍心跳, +NOTIFY 丢了的兜底)按 id 往后读,调登记过的处理(on)清掉、重读自己那一份。处理按名字登记在各自的模块里。SQLite 只有 +一个进程,broadcast 什么都不做(改的进程已经刷新过自己)。 +""" +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Callable +from datetime import timedelta +from typing import Any + +from sqlalchemy import delete, func, select + +from app.cluster import identity +from app.cluster.signal import signals +from app.core.config import settings +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +_handlers: dict[str, Callable[[str | None], Any]] = {} +_last_seen: int | None = None +#: 往回多读几条(复核 R-b5):id 是先取后提交的,id 大的先提交时游标会越过 id 小的那一条——每次从游标往回留这么多条重读,按处理过 +#: 的 id 去重 +LOOKBACK = 200 +_done: set[int] = set() +_lock: asyncio.Lock | None = None + + +def on(name: str, handler: Callable[[str | None], Any]) -> None: + """登记名为 name 的缓存怎么失效:handler(键),键为 None 是整个清掉。可以是协程函数。""" + _handlers[name] = handler + + +def _enabled() -> bool: + from app.cluster.service import started + + return settings.db_dialect == "postgresql" and started() + + +async def broadcast(name: str, key: str | None = None) -> None: + """name(某个键)变了:告诉别的进程。改的进程自己要先刷新好自己那一份(这里不调本进程的处理)。""" + from app.db.models_cluster import CacheEvent + + if not _enabled(): + return + try: + async with SessionLocal() as session: + session.add(CacheEvent(name=name, key=key, owner=identity.owner())) + await session.commit() + except Exception: # noqa: BLE001 - 写不进去只是别的进程晚一点(它们的缓存各有自己的兜底) + logger.exception("通知别的进程缓存「%s」变了,没写进库", name) + return + signals.publish("cache", name) + + +async def start() -> None: + """起服务时:从现在的最后一条往后读(之前的变化,自己启动时读进来的已经是新的)。""" + global _last_seen, _floor + from app.db.models_cluster import CacheEvent + + signals.on("cache", lambda _parts: asyncio.ensure_future(poll())) + async with SessionLocal() as session: + _last_seen = int(await session.scalar(select(func.coalesce(func.max(CacheEvent.id), 0))) or 0) + _floor = _last_seen + _done.clear() + + +#: 起服务时已有的最后一条:往回重读不越过它(那之前的变化,自己启动时读进来的已经是新的) +_floor = 0 + + +async def poll() -> int: + """读别的进程写的、还没处理过的几条,照着失效。交回处理了几条。""" + global _last_seen, _lock + from app.db.models_cluster import CacheEvent + + if not _enabled() or _last_seen is None: + return 0 + if _lock is None: + _lock = asyncio.Lock() + async with _lock: + async with SessionLocal() as session: + rows = (await session.execute(select(CacheEvent).where(CacheEvent.id > max(_floor, _last_seen - LOOKBACK)) + .order_by(CacheEvent.id).limit(500))).scalars().all() + done = 0 + for row in rows: + if int(row.id) in _done: + continue + _done.add(int(row.id)) + _last_seen = max(_last_seen, int(row.id)) + if row.owner == identity.owner(): + continue + handler = _handlers.get(row.name) + if handler is None: + continue + try: + out = handler(row.key) + if asyncio.iscoroutine(out): + await out + done += 1 + except Exception: # noqa: BLE001 + logger.exception("按别的进程的通知重读缓存「%s」出错", row.name) + if len(_done) > 4 * LOOKBACK: + low = _last_seen - 2 * LOOKBACK + _done.difference_update({i for i in _done if i <= low}) + return done + + +async def prune() -> int: + from app.db.models_cluster import CacheEvent + + async with SessionLocal() as session: + done = await session.execute(delete(CacheEvent).where(CacheEvent.created_at < utcnow() - timedelta(hours=1))) + await session.commit() + return int(done.rowcount or 0) + + +__all__ = ["broadcast", "on", "poll", "prune", "start"] diff --git a/backend/app/cluster/claims.py b/backend/app/cluster/claims.py new file mode 100644 index 00000000..6c75e746 --- /dev/null +++ b/backend/app/cluster/claims.py @@ -0,0 +1,45 @@ +"""认领属主不在了的行(cluster/reaper 的各种规矩共用):先看库里还没结束的那些行和它们的属主,属主不在了的逐行用带条件的 +UPDATE 改成自己(状态还是刚才看到的、属主还是刚才看到的那个)——几个进程同时接手,同一行只有一个改得动。""" +from __future__ import annotations + +from collections.abc import Callable, Iterable +from typing import Any + +from sqlalchemy import select, update + +from app.cluster import identity +from app.cluster.workers import Liveness +from app.db.base import SessionLocal + + +def same(column: Any, value: Any) -> Any: + """column 还是 value(None 用 IS NULL 比)。""" + return column.is_(None) if value is None else column == value + + +async def orphans(model: Any, live: Liveness, *, running: Callable[[str], bool], statuses: Iterable[str], + extra: Any = None) -> list[tuple[str, str]]: + """model 里状态在 statuses、属主不在了的行认领过来(owner 改成本进程),交回 [(id, 原来的状态)]。running:这一行在本进程里 + 是不是真的在跑。extra:另加的筛选条件。""" + wanted = tuple(statuses) + stmt = select(model.id, model.owner, model.status).where(model.status.in_(wanted)) + if extra is not None: + stmt = stmt.where(extra) + async with SessionLocal() as session: + rows = (await session.execute(stmt)).all() + me = identity.owner() + out: list[tuple[str, str]] = [] + for row_id, owner, status in rows: + if live.alive(owner, local=running(row_id)): + continue + async with SessionLocal() as session: + done = await session.execute(update(model).where(model.id == row_id, model.status == status, + same(model.owner, owner)) + .values(owner=me).execution_options(synchronize_session=False)) + await session.commit() + if done.rowcount == 1: + out.append((row_id, status)) + return out + + +__all__ = ["orphans", "same"] diff --git a/backend/app/cluster/counters.py b/backend/app/cluster/counters.py new file mode 100644 index 00000000..63025527 --- /dev/null +++ b/backend/app/cluster/counters.py @@ -0,0 +1,117 @@ +"""几个进程共用的计数:访问密钥每分钟限流的令牌桶(auth/keys)、登录失败的退避(auth/sessions)。 + +应用库是 PostgreSQL(可以多进程)时记在库里(rate_counters),一次请求一句带条件的 upsert:令牌桶「补多少、扣一个」在一句 +INSERT … ON CONFLICT DO UPDATE … WHERE 里算完,几个进程同时来也扣不过头;登录失败先把次数加一、占住那一行,再按次数算要等 +多久。SQLite 只有一个进程,计数照旧在进程里(调用方自己的那份),不为每个请求多写一次库——单进程时进程就是全站。 + +时间用墙钟(time.time):几台机器各自的单调时钟不能比,墙钟要对时(NTP),同租约。 +""" +from __future__ import annotations + +import math +import time +from datetime import datetime, timedelta, timezone + +from sqlalchemy import delete, select + +from app.cluster.service import started +from app.core.config import settings +from app.db.base import SessionLocal, utcnow + + +def shared() -> bool: + """计数要不要记在库里:PostgreSQL、而且起了协调(服务在跑)。测试、脚本里直接调用时照旧在进程里。""" + return settings.db_dialect == "postgresql" and started() + + +def _expiry(seconds: float) -> datetime: + return utcnow() + timedelta(seconds=seconds) + + +async def take_token(key: str, *, per_minute: int, now: float | None = None) -> int: + """令牌桶:满桶 per_minute 个,每秒补 per_minute / 60 个。放行交回 0,超了交回还要等几秒。""" + from sqlalchemy.dialects.postgresql import insert + + from app.db.models_cluster import RateCounter as Row + + rate = per_minute / 60.0 + cap = float(per_minute) + moment = time.time() if now is None else now + refill = func_least(cap, Row.value + (moment - Row.at) * rate) + stmt = insert(Row).values(key=key, count=1, value=cap - 1.0, at=moment, expires_at=_expiry(120)) \ + .on_conflict_do_update(index_elements=[Row.key], + set_={"value": refill - 1.0, "at": moment, "count": Row.count + 1, + "expires_at": _expiry(120)}, + where=refill >= 1.0) \ + .returning(Row.value) + async with SessionLocal() as session: + got = (await session.execute(stmt)).first() + if got is not None: + await session.commit() + return 0 + row = await session.get(Row, key) + await session.rollback() + tokens = min(cap, float(row.value) + (moment - float(row.at)) * rate) if row is not None else 0.0 + return max(1, math.ceil((1.0 - tokens) / rate)) + + +def func_least(a: float, b: object) -> object: + from sqlalchemy import func + + return func.least(a, b) + + +async def wait_seconds(*keys: str) -> int: + """登录失败退避:这几个键里还要等多久(秒),不用等为 0。""" + from app.db.models_cluster import RateCounter as Row + + if not keys: + return 0 + now = time.time() + async with SessionLocal() as session: + rows = (await session.execute(select(Row.value).where(Row.key.in_([f"login:{k}" for k in keys])))).scalars() + until = max((float(v) for v in rows), default=0.0) + return int(until - now) + 1 if until > now else 0 + + +async def failed(*keys: str, free_tries: int, max_wait_s: int) -> None: + """记一次登录失败:次数加一,满 free_tries 次之后要等 2^(次数-free_tries) 秒(最多 max_wait_s)。""" + from sqlalchemy import update + from sqlalchemy.dialects.postgresql import insert + + from app.db.models_cluster import RateCounter as Row + + now = time.time() + async with SessionLocal() as session: + for key in keys: + name = f"login:{key}" + count = (await session.execute( + insert(Row).values(key=name, count=1, value=0.0, at=now, expires_at=_expiry(86400)) + .on_conflict_do_update(index_elements=[Row.key], set_={"count": Row.count + 1, "at": now, + "expires_at": _expiry(86400)}) + .returning(Row.count))).scalar_one() + wait = 0 if count < free_tries else min(2 ** (count - free_tries), max_wait_s) + await session.execute(update(Row).where(Row.key == name).values(value=now + wait) + .execution_options(synchronize_session=False)) + await session.commit() + + +async def succeeded(*keys: str) -> None: + from app.db.models_cluster import RateCounter as Row + + async with SessionLocal() as session: + await session.execute(delete(Row).where(Row.key.in_([f"login:{k}" for k in keys]))) + await session.commit() + + +async def prune(now: datetime | None = None) -> int: + """到期的计数删掉(当班进程做)。""" + from app.db.models_cluster import RateCounter as Row + + async with SessionLocal() as session: + done = await session.execute(delete(Row).where(Row.expires_at < (now or datetime.now(timezone.utc)))) + await session.commit() + return int(done.rowcount or 0) + + +__all__ = ["failed", "prune", "shared", "succeeded", "take_token", "wait_seconds"] diff --git a/backend/app/cluster/fence.py b/backend/app/cluster/fence.py new file mode 100644 index 00000000..ec1eecc7 --- /dev/null +++ b/backend/app/cluster/fence.py @@ -0,0 +1,191 @@ +"""自我隔离(fencing):本进程续不上租约的时候,当作自己认领的行可能已经被别的进程接手,停下手上的执行、不再写。 + +**为什么。** 进程卡住了但没死(SIGSTOP、虚拟机暂停、事件循环被长同步代码卡住、库一时连不上):别的进程看它的租约过期了,把它在跑 +的运行标中断、交给别人继续运行。它一缓过来,手上的执行接着往下走:照样写事件(和接手的那一边撞号)、照样写终态(把接手的结果 +盖掉)。所以: + +- 续租约在单独的任务、单独的连接上做(cluster/service.py),不被同一拍别的活拖住;最近一次续上的时刻记在这里(renewed)。 +- 本进程的执行在动手写之前问一句 check():离上次续上已经超过一个租约时长(别的进程按两个租约时长才接手,中间留着一段),或者这段 + 执行是在上一次隔离之前开始的(世代不对),就抛 Fenced——写事件、写断点、写终态、写待办和动作、发外部写入之前都问。只对绑定了 + 世代的执行(运行的一段执行、后台任务)抛;别的调用者问到时只隔离、不抛。 +- 一旦发现续晚了(trip):世代加一,手上的执行全部停掉(停的时候不写终态,那些行交给接手的进程按「属主不在了」处理),换一个新的 + 身份重新登记;新身份登记上之前新开始的执行先等着(ready),登记上之后照常。 + +**用哪只钟(复核 R-b7、R-b14)。** 别的进程判「租约过期」用的是库的时钟(cluster/workers:续租写 lease_until = 库的 now() + +租约),跟各台机器的墙钟差多少无关。这边判「续晚了」用计入休眠的钟:time.monotonic() 在 macOS(mach_absolute_time)、Linux +(CLOCK_MONOTONIC)上都不计休眠——笔记本合盖、虚拟机挂起超过两个租约,对面早已接手,这边还以为刚续过。所以 Linux 用 +CLOCK_BOOTTIME,macOS 用 clock_gettime(CLOCK_MONOTONIC)(这个计入休眠),只看它;墙钟会被 NTP 一步校时、手动改时间、虚拟机 +对时拨来拨去,取得到计入休眠的钟时不拿它判(否则单进程墙钟一跳就把自己隔离了)。两只都取不到时退回 monotonic,墙钟当后备: +两只里走得多的那只说过了一个租约就算续晚了。两只钟对不上(jumped,差出一个租约以上)时不直接隔离:先续一次租,续租那一句按库的 +时钟核对「自己的租约还没过期」,以它为准——续上了记一条日志照常,续不上(库里自己的租约已经过期、或者登记不见了)才隔离。 +虚拟机被宿主挂起时客户机的 CLOCK_BOOTTIME 也可能停着:那时靠续租那一句(最迟一拍心跳之后)发现过期。 + +Fenced 是 BaseException:节点、工具里常见的 except Exception 拦不住它,一路退到执行的最外层。没起协调(测试、脚本)时什么都不查。 +""" +from __future__ import annotations + +import asyncio +import contextvars +import logging +import sys +import time +from collections.abc import Callable +from typing import Any + +logger = logging.getLogger(__name__) + + +class Fenced(BaseException): + """本进程续不上租约了(或者这段执行属于上一次隔离之前):不许再写。""" + + +def _sleep_aware_clock() -> tuple[Callable[[], float], bool]: + """计入休眠的单调钟,和它是不是真的计入休眠。""" + if sys.platform.startswith("linux") and hasattr(time, "CLOCK_BOOTTIME"): + return (lambda: time.clock_gettime(time.CLOCK_BOOTTIME)), True + if sys.platform == "darwin" and hasattr(time, "CLOCK_MONOTONIC"): + return (lambda: time.clock_gettime(time.CLOCK_MONOTONIC)), True + return time.monotonic, False + + +_clock, SLEEP_AWARE = _sleep_aware_clock() +#: 墙钟(单独一个名字,测试里拨它) +_wall: Callable[[], float] = time.time + +_generation = 0 +_renewed: float | None = None +_renewed_wall: float | None = None +_active = False +_handlers: list[Callable[[], Any]] = [] +_bound: contextvars.ContextVar[int | None] = contextvars.ContextVar("agentlab_fence_generation", default=None) + + +def now() -> float: + """计入休眠的单调钟的读数(本进程里判时长都用它:cluster/leader 判还当不当班也是)。""" + return _clock() + + +def activate() -> None: + """起协调、续上了第一次租约时(PostgreSQL;SQLite 只有一个进程,不用隔离)。""" + global _active + _active = True + renewed() + + +def deactivate() -> None: + global _active, _renewed, _renewed_wall + _active, _renewed, _renewed_wall = False, None, None + + +def renewed(*, ago: float = 0.0) -> None: + """续上了一次租约(ago:其实是多少秒以前续上的,测试里装「卡了一阵」用)。""" + global _renewed, _renewed_wall + _renewed, _renewed_wall = _clock() - ago, _wall() - ago + + +def _since() -> tuple[float, float] | None: + """离上次续上:(计入休眠的钟走了多少, 墙钟走了多少)。""" + if _renewed is None or _renewed_wall is None: + return None + return _clock() - _renewed, _wall() - _renewed_wall + + +def generation() -> int: + return _generation + + +def bind() -> None: + """这段执行属于现在这一世代(运行的一段执行、后台任务、问数据的一轮开始时调)。""" + _bound.set(_generation) + + +def stale() -> bool: + """离上次续上租约是不是已经超过一个租约时长:取得到计入休眠的钟时只看它;取不到时它(monotonic)和墙钟里走得多的那只说了算。 + 隔离之后、新身份还没登记上的那一段也算(pending)。""" + if not _active: + return False + since = _since() + if since is None: + return True + from app.cluster.workers import lease_s + + return (since[0] if SLEEP_AWARE else max(since)) > lease_s() + + +def pending() -> bool: + """隔离过、换的新身份还没登记上。""" + return _active and _renewed is None + + +async def ready(timeout: float | None = None) -> None: + """新开始的一段执行(运行、后台任务)在 bind() 之前等一等:隔离之后、新身份还没登记上时,它认领的行在别的进程眼里属主 + 不在,会被当成没人管接手(复核 R-b9)。最多等一个租约时长(timeout),等不到抛 Fenced——不开始,那几行由别的进程照「属主 + 不在了」处理(cluster/service 那边同时会放弃这个一直没登记上的身份)。""" + if not pending(): + return + from app.cluster.workers import lease_s + + deadline = _clock() + (lease_s() if timeout is None else timeout) + while pending(): + if _clock() >= deadline: + raise Fenced() + await asyncio.sleep(0.05) + + +def jumped() -> bool: + """两只钟差出一个租约以上(墙钟往前或往后跳了,或者单调钟没计入休眠)。续租时问:是的话不直接隔离,以续租那一句按库的时钟 + 核对的结果为准(cluster/service.renew)。""" + if not _active: + return False + since = _since() + if since is None: + return False + from app.cluster.workers import lease_s + + return abs(since[0] - since[1]) > lease_s() + + +def fenced_here() -> bool: + """当前这段执行还能不能写:世代不对(隔离之前开始的)或者租约续晚了。只说绑定了世代的执行(bind 过的)。""" + if not _active: + return False + bound = _bound.get() + return bound is not None and (bound != _generation or stale()) + + +def check() -> None: + """写之前问一句。续晚了就隔离(谁先问到都一样);抛 Fenced 只对绑定了世代的执行(运行的一段执行、后台任务:bind 过的)—— + 别的调用者(HTTP 处理、调度那一拍、进程间请求)不是认领了行在跑的执行,抛给它们只会把调度循环之类的打停(复核 R-b9)。""" + if not _active: + return + if _renewed is not None and stale(): + trip("执行要写的时候发现租约已经续晚了") + bound = _bound.get() + if bound is not None and (bound != _generation or stale()): + raise Fenced() + + +def on_trip(handler: Callable[[], Any]) -> None: + if handler not in _handlers: + _handlers.append(handler) + + +def trip(why: str) -> bool: + """隔离:世代加一,叫各处停掉手上的执行(cluster/service 再换身份重新登记)。已经在这一世代隔离过的不重复。交回这一次有没有动手。""" + global _generation, _renewed, _renewed_wall + if not _active or _renewed is None: + return False + _generation += 1 + _renewed = _renewed_wall = None + logger.error("本进程的租约续晚了(%s):当作认领的运行、问数据、后台任务可能已被别的进程接手,停掉手上的执行、不写终态," + "换一个身份重新登记", why) + for handler in list(_handlers): + try: + handler() + except Exception: # noqa: BLE001 + logger.exception("隔离时停执行出错") + return True + + +__all__ = ["Fenced", "SLEEP_AWARE", "activate", "bind", "check", "deactivate", "fenced_here", "generation", "jumped", "now", + "on_trip", "pending", "ready", "renewed", "stale", "trip"] diff --git a/backend/app/cluster/guard.py b/backend/app/cluster/guard.py new file mode 100644 index 00000000..355b8a94 --- /dev/null +++ b/backend/app/cluster/guard.py @@ -0,0 +1,86 @@ +"""SQLite 只支持单进程:服务启动时拿数据目录上的文件锁(data/.agentlab.lock,flock 独占、不等),拿不到就拒绝启动。 + +**为什么拒绝而不是凑合。** SQLite 应用库下,运行的序号、并发名额、同源队、调度当班……这些在库里协调的东西照样能用,但 +上传表格的版本库、填报记录表、断点库都是数据目录里的 SQLite 文件,两个进程一起写同一个文件靠的是文件锁,换不成库里的锁; +uvicorn --workers 2 起来的第二个进程以前也只是悄悄地不写上传版本(503),别的照跑、各算各的,最难查。现在照实说:第二个进程 +起不来,报错写明「SQLite 只支持单进程;要多进程把应用库换成 PostgreSQL」。 + +进程退出(含强杀)时操作系统放掉文件锁,重启(dev.sh 的 --reload 先停旧的再起新的)不受影响。同一个进程里重复取(测试里 +一次次起停服务)是幂等的;数据目录换了就放掉旧的、对新目录重取。没有 fcntl 的平台(Windows)只记一条警告。 +""" +from __future__ import annotations + +import logging +import os +from pathlib import Path + +from app.core.config import settings + +logger = logging.getLogger(__name__) + +LOCK_FILE = ".agentlab.lock" + +SINGLE_PROCESS = ("应用库是 SQLite 时只支持单进程:数据目录 {dir} 已经被另一个 AgentLab 进程使用({hint})。" + "SQLite 不能多进程部署——不要给 uvicorn 加 --workers、不要让两个服务指着同一个数据目录;" + "要多进程,把应用库换成 PostgreSQL(AGENTLAB_DATABASE_URL,见 README「用 PostgreSQL」「多进程」)") + + +class SingleProcessOnly(RuntimeError): + """SQLite 下第二个进程启动:拒绝。""" + + +_held: tuple[Path, int] | None = None + + +def _path() -> Path: + return settings.data_dir / LOCK_FILE + + +def acquire() -> None: + """SQLite 时取数据目录上的独占锁,拿不到抛 SingleProcessOnly。PostgreSQL 时什么都不做(可以多进程)。""" + global _held + if settings.db_dialect != "sqlite": + return + if settings.workers > 1: + raise SingleProcessOnly(SINGLE_PROCESS.format(dir=settings.data_dir, hint=f"AGENTLAB_WORKERS={settings.workers}")) + path = _path() + if _held is not None and _held[0] == path: + return + release() + try: + import fcntl + except ImportError: + logger.warning("这个平台没有 fcntl,无法确认数据目录只被一个进程使用:SQLite 只支持单进程部署") + return + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + holder = "" + try: + holder = path.read_text(encoding="utf-8").strip() + except OSError: + pass + raise SingleProcessOnly(SINGLE_PROCESS.format( + dir=settings.data_dir, hint=f"占着它的是 {holder}" if holder else "另一个进程占着它")) from None + try: + os.ftruncate(fd, 0) + os.write(fd, f"pid {os.getpid()}".encode()) + except OSError: + pass + _held = (path, fd) + + +def release() -> None: + global _held + if _held is not None: + try: + os.close(_held[1]) + except OSError: + pass + _held = None + + +__all__ = ["LOCK_FILE", "SINGLE_PROCESS", "SingleProcessOnly", "acquire", "release"] diff --git a/backend/app/cluster/identity.py b/backend/app/cluster/identity.py new file mode 100644 index 00000000..7a554012 --- /dev/null +++ b/backend/app/cluster/identity.py @@ -0,0 +1,56 @@ +"""这个工作进程是谁:主机 + pid + 启动随机串(OWNER)。 + +库里认领了的行(运行、问数据的一轮、后台任务、并发名额、同源队)都记着 owner;进程在 cluster_workers 里登记、续租约, +别的进程据此判断「这一行的属主还在不在」。随机串让同一个 pid 被复用(容器里 pid 常是 1)、或者同一个进程里测试一次次起停 +服务时,都分得清是哪一次启动认领的。 + +在导入时定下:建表的缺省值(db 模型的 owner 列)直接取它,不必等服务启动。fork 出来的子进程(uvicorn --workers)各自重新导入 +应用、各有各的;万一是 fork 之后没重新导入(multiprocessing 的 fork 方式),pid 对不上时 owner() 重新生成。 +""" +from __future__ import annotations + +import os +import secrets +import socket + +_HOST = (socket.gethostname() or "localhost")[:80] + + +def _make(pid: int) -> str: + return f"{_HOST}:{pid}:{secrets.token_hex(4)}" + + +_PID = os.getpid() +OWNER = _make(_PID) + + +def owner() -> str: + """这个进程的 owner id。""" + global OWNER, _PID + pid = os.getpid() + if pid != _PID: + _PID, OWNER = pid, _make(pid) + return OWNER + + +def regenerate() -> str: + """换一个身份(隔离之后重新登记用,cluster/fence.py):同一个 pid、新的随机串。之前认领的行算「属主不在了」。""" + global OWNER, _PID + _PID = os.getpid() + OWNER = _make(_PID) + return OWNER + + +def host() -> str: + return _HOST + + +def describe(value: str | None) -> str: + """写进日志、界面的样子:主机和 pid(随机串不写)。""" + if not value: + return "(没有记录)" + parts = value.split(":") + return f"{parts[0]} 上的进程 {parts[1]}" if len(parts) >= 2 else value + + +__all__ = ["OWNER", "describe", "host", "owner", "regenerate"] diff --git a/backend/app/cluster/leader.py b/backend/app/cluster/leader.py new file mode 100644 index 00000000..b532b11e --- /dev/null +++ b/backend/app/cluster/leader.py @@ -0,0 +1,132 @@ +"""调度当班:全站只许一个进程做的事(定时认领、事件派发、Webhook 投递、流程的时限和核实、各类周期清理、接手不在了的 +进程留下的活)由库里的一份租约定下谁做(cluster_leases 的 name=scheduler 那一行)。 + +一句条件 UPDATE 同时管「接班」和「续租」:这一行的 owner 是自己,或者租约已经过期(当班的进程没了)、没人当班,就改成自己、 +租约往后推。改到了一行就是当班。当班的进程平稳关停时交班(owner 清空),待命的下一拍心跳就接上;强杀的等租约过期。 + +两个进程短暂地都以为自己当班(续租那一拍库慢了、本地还没过期)时,重复不了:定时认领本来就是条件 UPDATE、触发记录 +(任务, 时刻, 种类) 唯一,事件派发按投递记录的唯一约束,待办的升级、核实各自也是条件更新。当班只是让它们平常不撞。 +本地另留一截余量(MARGIN):租约快到期还没续上时,自己先当作不当班了。 + +租约按库的时钟写、按库的时钟判(复核 R-b7,同 cluster/workers):机器之间墙钟差多少都不会让两边都以为租约归自己。本地判「还当 +不当班」不看墙钟,看计入休眠的单调钟(cluster/fence.now):从发出续租那一刻起算一个租约时长,库里的租约只会更晚到期。 +""" +from __future__ import annotations + +import logging +from datetime import datetime, timedelta + +from sqlalchemy import case, or_, update + +from app.cluster import fence, identity +from app.cluster.workers import db_now, lease_s +from app.db.base import SessionLocal + +logger = logging.getLogger(__name__) + +SCHEDULER = "scheduler" +#: 本地判断「还在当班」时留的余量(秒) +MARGIN_S = 2.0 + + +class Leadership: + def __init__(self, name: str) -> None: + self.name = name + #: 库里记的到期时刻(库的时钟;给人看、交班用) + self.until: datetime | None = None + #: 本地判还当不当班用的到期时刻(cluster/fence.now 的读数) + self.local_until: float | None = None + self.epoch = 0 + + def single(self) -> None: + """只有本进程(SQLite):直接算当班,不写库。""" + self.until = datetime.max.replace(tzinfo=None) + self.local_until = float("inf") + + def drop(self) -> None: + """本地当作不当班了(隔离之后换身份时:库里那份租约属于旧身份)。""" + self.until = self.local_until = None + + def held(self) -> bool: + """本地看:还在当班(租约离到期还有余量)。""" + if self.until is None or self.local_until is None: + return False + if self.until.tzinfo is None: + return True # single():只有本进程 + # 余量不超过租约的四分之一:租约调得很短(验收脚本)时也留得出当班的时间 + margin = min(MARGIN_S, lease_s() / 4) + return fence.now() + margin < self.local_until + + async def claim(self) -> bool: + """接班或续租。交回这一拍之后是不是当班。""" + from app.db.models_cluster import ClusterLease + from app.db.portable import insert_ignore + + started = fence.now() + me = identity.owner() + mine = ClusterLease.owner == me + async with SessionLocal() as session: + now = await db_now(session) + until = now + timedelta(seconds=lease_s()) + done = await session.execute( + update(ClusterLease).where( + ClusterLease.name == self.name, + or_(mine, ClusterLease.owner.is_(None), ClusterLease.lease_until.is_(None), + ClusterLease.lease_until < now)) + .values(owner=me, lease_until=until, + acquired_at=case((mine, ClusterLease.acquired_at), else_=now), + epoch=case((mine, ClusterLease.epoch), else_=ClusterLease.epoch + 1)) + .returning(ClusterLease.epoch) + .execution_options(synchronize_session=False)) + epoch = done.scalar_one_or_none() + if epoch is None: + # 还没有这一行(新库):插一行,撞了说明别人刚插上,这一拍不当班 + inserted = await session.execute(insert_ignore(session, ClusterLease.__table__).values( + name=self.name, owner=me, lease_until=until, acquired_at=now, epoch=1) + .on_conflict_do_nothing(index_elements=["name"])) + epoch = 1 if inserted.rowcount == 1 else None + await session.commit() + was = self.held() + if epoch is None: + if was or self.until is not None: + logger.warning("调度当班交给了别的进程(本进程 %s 不再当班)", identity.describe(me)) + self.drop() + return False + if not was and self.epoch != epoch: + logger.info("本进程(%s)接手调度当班(第 %d 班)", identity.describe(me), epoch) + self.until, self.local_until, self.epoch = until, started + lease_s(), int(epoch) + return True + + async def release(self) -> None: + """交班(平稳关停时)。""" + from app.db.models_cluster import ClusterLease + + if self.until is None: + return + single = self.until.tzinfo is None + self.drop() + if single: + return + async with SessionLocal() as session: + await session.execute(update(ClusterLease).where( + ClusterLease.name == self.name, ClusterLease.owner == identity.owner()) + .values(owner=None, lease_until=None).execution_options(synchronize_session=False)) + await session.commit() + + +scheduler = Leadership(SCHEDULER) + + +async def current(name: str = SCHEDULER) -> dict[str, object] | None: + """库里记着谁当班(系统信息、验收脚本看)。""" + from app.db.models_cluster import ClusterLease + + async with SessionLocal() as session: + row = await session.get(ClusterLease, name) + if row is None: + return None + return {"owner": row.owner, "until": row.lease_until.isoformat() if row.lease_until else None, "epoch": row.epoch, + "me": row.owner == identity.owner()} + + +__all__ = ["Leadership", "MARGIN_S", "SCHEDULER", "current", "scheduler"] diff --git a/backend/app/cluster/mutex.py b/backend/app/cluster/mutex.py new file mode 100644 index 00000000..96214ba3 --- /dev/null +++ b/backend/app/cluster/mutex.py @@ -0,0 +1,147 @@ +"""跨进程的互斥锁:本进程里一把 asyncio 锁(按事件循环各一把),应用库是 PostgreSQL 时另取一把会话级咨询锁 +(pg_advisory_lock,按锁名的哈希取号),持锁期间占着一条专门的连接,放锁时在同一条连接上放。SQLite 只有一个进程, +本进程的锁就是全站的锁。 + +用在「先看库里有什么、再改文件」这种一个库事务包不住的地方:上传表格的版本存储(发布、回收、启动清理、固定版本, +data/table_versions.store_lock)、填报记录表的追加(records/store)。 + +用法同 asyncio.Lock:`async with lock:`。另留 acquire() / release() / _waiters / locked(),只碰本进程那一把(测试用来在 +本进程里卡住锁)。不可重入。 +""" +from __future__ import annotations + +import asyncio +import hashlib +import weakref +from typing import Any + +from app.core.config import settings + + +def advisory_key(name: str) -> int: + """锁名 → 咨询锁的号(有符号 64 位)。加前缀,不和别处直接写的号(db/base 的建表锁、断点库的建表锁)撞。""" + digest = hashlib.blake2b(f"agentlab:{name}".encode(), digest_size=8).digest() + return int.from_bytes(digest, "big", signed=True) + + +class ClusterLock: + def __init__(self, name: str) -> None: + self.name = name + self.key = advisory_key(name) + self._local = asyncio.Lock() + self._conn: Any = None + + # 只碰本进程那一把(同 asyncio.Lock) + async def acquire(self) -> bool: + return await self._local.acquire() + + def release(self) -> None: + self._local.release() + + def locked(self) -> bool: + return self._local.locked() + + @property + def _waiters(self) -> Any: + return self._local._waiters + + async def try_take(self) -> bool: + """不等:本进程、别的进程都没人占着就占上,交回占没占到。占到了用 give_back 放(可以在另一个任务里放:后台任务做完才放)。""" + if self._local.locked(): + return False + await self._local.acquire() + if settings.db_dialect != "postgresql": + return True + try: + from sqlalchemy import text + + from app.db.base import session_engine + + conn = await session_engine().connect() + try: + conn = await conn.execution_options(isolation_level="AUTOCOMMIT") + got = bool(await conn.scalar(text("SELECT pg_try_advisory_lock(:k)"), {"k": self.key})) + except BaseException: + await conn.close() + raise + if not got: + await conn.close() + self._local.release() + return False + self._conn = conn + except BaseException: + if self._local.locked() and self._conn is None: + self._local.release() + raise + return True + + async def give_back(self) -> None: + """放掉 try_take 占到的。""" + await self.__aexit__(None, None, None) + + async def __aenter__(self) -> ClusterLock: + await self._local.acquire() + if settings.db_dialect != "postgresql": + return self + try: + from sqlalchemy import text + + from app.db.base import session_engine + + # 会话级的锁:走直连(AGENTLAB_DATABASE_DIRECT_URL,没设就是应用库)——PgBouncer 事务模式下取锁、放锁可能落在 + # 两条不同的服务端连接上 + conn = await session_engine().connect() + try: + conn = await conn.execution_options(isolation_level="AUTOCOMMIT") + await conn.execute(text("SELECT pg_advisory_lock(:k)"), {"k": self.key}) + except BaseException: + await conn.close() + raise + self._conn = conn + except BaseException: + self._local.release() + raise + return self + + async def __aexit__(self, *exc: object) -> None: + conn, self._conn = self._conn, None + try: + if conn is not None: + from sqlalchemy import text + + try: + await asyncio.shield(conn.execute(text("SELECT pg_advisory_unlock(:k)"), {"k": self.key})) + finally: + # 放不掉(连接断了)也要关:会话一结束,库那边的咨询锁随之释放 + await asyncio.shield(conn.close()) + finally: + self._local.release() + + +class NamedLocks: + """按事件循环各一组的命名锁(测试每个用例一个循环:asyncio.Lock 不能换循环用)。""" + + def __init__(self) -> None: + self._by_loop: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, ClusterLock]] = \ + weakref.WeakKeyDictionary() + + def get(self, name: str) -> ClusterLock: + loop = asyncio.get_running_loop() + group = self._by_loop.get(loop) + if group is None: + group = self._by_loop[loop] = {} + lock = group.get(name) + if lock is None: + lock = group[name] = ClusterLock(name) + return lock + + +locks = NamedLocks() + + +def lock(name: str) -> ClusterLock: + """名为 name 的跨进程锁(本事件循环里同一个名字是同一把)。""" + return locks.get(name) + + +__all__ = ["ClusterLock", "NamedLocks", "advisory_key", "lock", "locks"] diff --git a/backend/app/cluster/quota.py b/backend/app/cluster/quota.py new file mode 100644 index 00000000..bb811ab8 --- /dev/null +++ b/backend/app/cluster/quota.py @@ -0,0 +1,72 @@ +"""每日额度:先占一份额度再做事,做不成退回(ledger/issue 的出具上限、apps/resolve 的按需视图上限)。 + +**为什么不靠锁。** 以前是「数一数今天做了几次,没到上限就发起」,数和发起之间靠一把锁串行;多进程时那把锁要么只管本进程(几个 +进程一起越过上限),要么全站一把(几份出具只能一个接一个发起)。现在额度记在库里(rate_counters 里按「名字 + 本地日期」一行), +占额度是一句带条件的 UPDATE:还没到上限才加一,否则什么都不改——两种库上这一句都是原子的,几个进程、几个协程同时来也正好 +占到上限为止,发起本身可以同时做。发起失败(被拒、出错)把那一份退回。 + +这一天第一次占的时候按库里已有的记录起算(seed:比如今天已经出具过几次),之后只看这一行。进程在占了额度、发起之前没了的, +那一份不退:宁可这一天少出一次,不会多出。 +""" +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from datetime import datetime, timedelta +from typing import Any + +from sqlalchemy import select, update + +from app.db.base import SessionLocal, utcnow + + +def today() -> str: + """本地日期(每日上限按服务端所在时区的自然日算,同以前按本地零点数)。""" + return datetime.now().astimezone().date().isoformat() + + +def _key(name: str, day: str) -> str: + return f"quota:{name}:{day}" + + +async def take(name: str, limit: int, *, seed: Callable[[Any], Awaitable[int]] | None = None, + day: str | None = None) -> str | None: + """占一份额度:这一天占过的还不到 limit 就加一、交回占在了哪一天(本地日期),到了交回 None(什么都不改)。seed(session): + 这一天第一次占时已经用掉了几份(按库里的记录数)。退回时把交回的日期给 give_back(复核 R-b11:23:59:59 占、00:00:01 发起 + 失败,要退回前一天那一份,不是第二天的)。""" + from app.db.models_cluster import RateCounter as Row + from app.db.portable import insert_ignore + + day = day or today() + key = _key(name, day) + async with SessionLocal() as session: + if await session.scalar(select(Row.key).where(Row.key == key)) is None: + used = int(await seed(session)) if seed is not None else 0 + await session.execute(insert_ignore(session, Row.__table__).values( + key=key, count=used, value=0.0, at=0.0, expires_at=utcnow() + timedelta(days=3)) + .on_conflict_do_nothing(index_elements=["key"])) + done = await session.execute(update(Row).where(Row.key == key, Row.count < int(limit)) + .values(count=Row.count + 1).execution_options(synchronize_session=False)) + await session.commit() + return day if done.rowcount == 1 else None + + +async def give_back(name: str, *, day: str | None = None) -> None: + """退回一份(占了额度、却没发起成)。day 给 take 交回的那个日期(不给按今天,只有测试这么用)。""" + from app.db.models_cluster import RateCounter as Row + + key = _key(name, day or today()) + async with SessionLocal() as session: + await session.execute(update(Row).where(Row.key == key, Row.count > 0).values(count=Row.count - 1) + .execution_options(synchronize_session=False)) + await session.commit() + + +async def used(name: str, *, day: str | None = None) -> int | None: + """这一天占了几份(没占过为 None)。""" + from app.db.models_cluster import RateCounter as Row + + async with SessionLocal() as session: + return await session.scalar(select(Row.count).where(Row.key == _key(name, day or today()))) + + +__all__ = ["give_back", "take", "today", "used"] diff --git a/backend/app/cluster/reaper.py b/backend/app/cluster/reaper.py new file mode 100644 index 00000000..a48e6013 --- /dev/null +++ b/backend/app/cluster/reaper.py @@ -0,0 +1,56 @@ +"""接手属主不在了的行(进程没了、租约过期):各种东西各按各的规矩,在各自的模块里登记(register),这里按登记的先后 +一个个调。 + +谁调:每个进程启动时调一次(那时自己手上什么都没在跑,这一次也收拾自己上一次留下的——测试里同一个进程一次次起停服务), +之后当班的进程隔一会儿(或者收到别的进程走了的通知)再调。各规矩里认领那一下都是带条件的 UPDATE(状态没变、属主还是 +刚才看到的那个),几个进程同时接手同一行只有一个改得动;活着的进程认领的行一概不碰(Liveness.alive)。 + +登记的规矩(启动时的整理就是它们):运行(标中断 / 排队的标失败,不自动重跑)、问数据的一轮(建图中的标中断;发起了运行的 +接着等、照常复核收尾)、数据目录起草 / 剖析 / 探查、评测、影子起草(标中断)、文档处理(标失败)、发布单(改成停下)…… +见各模块的 reap / startup。 +""" +from __future__ import annotations + +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +from app.cluster.workers import Liveness + +logger = logging.getLogger(__name__) + +Reaper = Callable[[Liveness], Awaitable[Any]] +_REAPERS: list[tuple[str, Reaper]] = [] + + +def register(name: str, fn: Reaper) -> Reaper: + """登记一种接手规矩(同名的换成新的:模块重新导入时不重复登记)。""" + for i, (known, _) in enumerate(_REAPERS): + if known == name: + _REAPERS[i] = (name, fn) + return fn + _REAPERS.append((name, fn)) + return fn + + +def names() -> list[str]: + return [name for name, _ in _REAPERS] + + +async def reap(live: Liveness) -> dict[str, Any]: + """按登记的先后调一遍,交回各自整理了多少。一种出错不影响后面的。""" + out: dict[str, Any] = {} + for name, fn in list(_REAPERS): + try: + got = await fn(live) + except Exception: # noqa: BLE001 + logger.exception("接手不在了的进程留下的「%s」出错,下一次再试", name) + continue + if got: + out[name] = got + if out: + logger.info("接手了不在了的进程留下的:%s", out) + return out + + +__all__ = ["Reaper", "names", "reap", "register"] diff --git a/backend/app/cluster/relay.py b/backend/app/cluster/relay.py new file mode 100644 index 00000000..6d0ed68f --- /dev/null +++ b/backend/app/cluster/relay.py @@ -0,0 +1,378 @@ +"""实时流跨进程(docs/design/orchestration-round/00-总方案.md §5 W2-b「实时流跨进程」)。 + +**为什么。** 运行的事件先落库、再在本进程的事件总线上推给订阅方(core/bus);问数据一轮的事件同样先落库,再叫醒本进程里 +等着的流(chat/log)。多进程时,页面的 WebSocket / SSE 连到的进程不一定是执行这次运行、这一轮的那个进程。 + +**怎么做。** 订阅方在哪个进程,就由那个进程的转发器(Relay)从库里按 seq 追: +- 运行:WebSocket 接上之后登记跟随(follow_run)。执行它的进程每提交一条事件,发 NOTIFY r|run_id|seq(cluster/signal, + 攒一下批量发);跟随它的进程收到后,只要这次运行不在本进程里执行,就从库里读 seq 大于手上游标的事件、在本进程的总线上 + 发布一遍(relayed,不再往外转),订阅方收到的和执行它的进程上的一模一样,序号由执行的进程发、不重复。到了终态(或者 + 挂起、属主不在了被标中断)再关掉本进程的订阅方,同执行进程上的规矩。NOTIFY 丢了也不怕:跟随的运行隔 POLL_S 秒回库里看一次。 +- 流式增量(正文 token、思考 delta)不落库:跟随的进程隔几秒发一次「我在看」(w|r:run_id),执行的进程只给有人在看的运行把 + 增量攒一小段、放进载荷转过去(e|run_id|[事件…])。 +- 问数据一轮:流本来就是从库里读的,这里只管叫醒——追加一条事件、任务状态变了都发 t|turn_id,收到的进程叫醒本进程里等着 + 这一轮的流;助手的保活心跳(不落库,chat/log.beat)同样只给有人在看的那几轮转过去(b|turn_id|心跳)。 +- SQLite 只有一个进程:signal 不收不发,这里什么都不做(单进程照旧走进程内的叫醒,不额外轮询)。 + +WebSocket / SSE 的回放、重连、心跳的说法都不变:回放照旧读库,转发器只补「实时」那一段。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import time +from collections.abc import Iterator +from typing import Any + +from sqlalchemy import select + +from app.cluster.signal import signals +from app.core.events import EventType, RunEventModel +from app.db.base import SessionLocal + +logger = logging.getLogger(__name__) + +#: 在看的一方隔多久重发一次「我在看」;执行的一方收到后记多久 +WATCH_S = 5.0 +WATCH_TTL_S = 15.0 +#: 流式增量攒多久转一次 +EPHEMERAL_FLUSH_S = 0.05 +#: 跟随的运行隔多久回库里看一次(NOTIFY 丢了的兜底) +POLL_S = 2.0 +#: 一条载荷最多多少字节(PostgreSQL 的 NOTIFY 上限是 8000) +PAYLOAD_MAX = 7000 +#: 往回多读几条(复核 R-b5):号是先取后提交的,号大的先提交时游标会越过号小的那一条;每次从游标往回留这么多条重读,按已经 +#: 发布过的 seq 去重 +LOOKBACK = 64 +#: 不落库的事件(同 engine/runner._EPHEMERAL) +EPHEMERAL = {EventType.LLM_TOKEN, EventType.LLM_THINKING_DELTA} +_TERMINAL = ("succeeded", "failed", "cancelled") + + +class Relay: + def __init__(self) -> None: + #: 本进程里在跟随的运行:{run_id: 跟随的连接数}、游标(已经在本进程发布过的最大 seq) + self._follow: dict[str, int] = {} + self._cursor: dict[str, int] = {} + #: 本进程已经发布过的 seq(游标往回 LOOKBACK 条以内的):往回重读时去重 + self._seen: dict[str, set[int]] = {} + #: 接上时回放到了哪一条:往回重读不越过它(那之前的回放已经发过) + self._floor: dict[str, int] = {} + self._pumping: dict[str, asyncio.Lock] = {} + #: 本进程里在看的问数据一轮 + self._turns: dict[str, int] = {} + #: 别的进程在看的(执行的一方记):{run_id / turn_id: 到期的时刻} + self._watched_runs: dict[str, float] = {} + self._watched_turns: dict[str, float] = {} + #: 攒着要转的流式增量:{run_id: [事件]} + self._buffer: dict[str, list[dict[str, Any]]] = {} + self._flusher: asyncio.Task[None] | None = None + self._loop_task: asyncio.Task[None] | None = None + self._installed = False + + # ---------------- 起停 ---------------- + + def install(self) -> None: + """登记收通知的处理、接上事件总线(起服务时,PostgreSQL 才有意义;重复调用无害)。""" + from app.core.bus import bus + + if not self._installed: + signals.on("r", self._on_run) + signals.on("e", self._on_ephemeral, raw=True) + signals.on("w", self._on_watch) + signals.on("t", self._on_turn) + signals.on("b", self._on_beat, raw=True) + signals.on_resync(self._resync) + self._installed = True + bus.tap = self._tap + + async def start(self) -> None: + self.install() + if signals.active and (self._loop_task is None or self._loop_task.done()): + self._loop_task = asyncio.create_task(self._loop(), name="agentlab-cluster-relay") + + async def stop(self) -> None: + from app.core.bus import bus + + if bus.tap == self._tap: + bus.tap = None + for task in (self._loop_task, self._flusher): + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self._loop_task = self._flusher = None + self._buffer.clear() + + # ---------------- 跟随(订阅方所在的进程) ---------------- + + @contextlib.contextmanager + def follow_run(self, run_id: str, after: int = 0) -> Iterator[None]: + """这条连接在看这次运行(WebSocket 接上、回放完之后):在本进程里没在执行时从库里追。""" + self._follow[run_id] = self._follow.get(run_id, 0) + 1 + self._cursor[run_id] = max(self._cursor.get(run_id, 0), int(after or 0)) + # 回放已经发到 after 为止(订阅位那边按 after 去重):往回重读不越过它 + self._floor[run_id] = max(self._floor.get(run_id, 0), int(after or 0)) if run_id not in self._floor else \ + min(self._floor[run_id], int(after or 0)) + if signals.active: + signals.publish("w", f"r:{run_id}") + self._kick(run_id) + try: + yield + finally: + left = self._follow.get(run_id, 1) - 1 + if left <= 0: + self._follow.pop(run_id, None) + self._cursor.pop(run_id, None) + self._seen.pop(run_id, None) + self._floor.pop(run_id, None) + self._pumping.pop(run_id, None) + else: + self._follow[run_id] = left + + @contextlib.contextmanager + def watch_turn(self, turn_id: str) -> Iterator[None]: + """这条 SSE 在看这一轮:执行它的进程把保活心跳转过来。""" + self._turns[turn_id] = self._turns.get(turn_id, 0) + 1 + if signals.active: + signals.publish("w", f"t:{turn_id}") + try: + yield + finally: + left = self._turns.get(turn_id, 1) - 1 + if left <= 0: + self._turns.pop(turn_id, None) + else: + self._turns[turn_id] = left + + def _kick(self, run_id: str) -> None: + if run_id in self._follow: + task = asyncio.ensure_future(self.pump(run_id)) + task.add_done_callback(_quiet) + + async def pump(self, run_id: str) -> int: + """把库里 seq 大于游标的事件在本进程发布一遍(这次运行在本进程里执行时什么都不做)。交回发布了几条。""" + from app.core.bus import bus + from app.db.models import Run, RunEvent + from app.engine.runner import run_manager + + if run_id not in self._follow or run_manager.is_active(run_id): + return 0 + lock = self._pumping.setdefault(run_id, asyncio.Lock()) + async with lock: + after = self._cursor.get(run_id, 0) + floor = self._floor.get(run_id, 0) + seen = self._seen.setdefault(run_id, set()) + async with SessionLocal() as session: + # 先读状态、再读事件:终态和终态事件是同一个事务提交的,读到终态时终态事件一定也读得到 + status = await session.scalar(select(Run.status).where(Run.id == run_id)) + fetched = list((await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.seq > max(floor, after - LOOKBACK)) + .order_by(RunEvent.seq))).scalars()) + rows = [r for r in fetched if int(r.seq or 0) not in seen and int(r.seq or 0) > floor] + last_log = None + for row in rows: + self._cursor[run_id] = max(self._cursor.get(run_id, 0), int(row.seq or 0)) + seen.add(int(row.seq or 0)) + last_log = row + try: + event = RunEventModel(id=row.id, run_id=run_id, seq=row.seq, type=EventType(row.type), + node_id=row.node_id, ts=row.ts, data=row.data or {}) + except ValueError: + continue # 不认识的事件种类(更新的版本写的):回放时照样读得到,这里不转 + await bus.publish(event, relayed=True) + low = self._cursor.get(run_id, 0) - 2 * LOOKBACK + if len(seen) > 4 * LOOKBACK: + self._seen[run_id] = {q for q in seen if q > low} + ended = status is None or status in _TERMINAL or ( + status == "interrupted" and last_log is not None and last_log.type == EventType.LOG + and (last_log.data or {}).get("code") == "server_shutdown") + if ended and (rows or status is None or status in _TERMINAL): + # 同执行进程上的规矩(runner._finalize):到了终态、关停挂起的,让本进程的订阅方收到终态再结束 + await bus.close(run_id) + return len(rows) + + def _on_run(self, parts: list[str]) -> None: + if parts: + self._kick(parts[0]) + + def _on_ephemeral(self, parts: list[str]) -> None: + from app.core.bus import bus + from app.engine.runner import run_manager + + if len(parts) < 2 or parts[0] not in self._follow or run_manager.is_active(parts[0]): + return + try: + items = json.loads(parts[1]) + except ValueError: + return + for item in items if isinstance(items, list) else []: + try: + event = RunEventModel(**item) + except Exception: # noqa: BLE001 + continue + asyncio.ensure_future(bus.publish(event, relayed=True)).add_done_callback(_quiet) + + def _on_turn(self, parts: list[str]) -> None: + if parts: + from app.chat import log as turn_log + + turn_log.waiters.notify(parts[0]) + + def _on_beat(self, parts: list[str]) -> None: + if len(parts) < 2 or parts[0] not in self._turns: + return + try: + op = json.loads(parts[1]) + except ValueError: + return + if isinstance(op, dict): + from app.chat import log as turn_log + + turn_log.remote_beat(parts[0], op) + + async def _resync(self) -> None: + for run_id in list(self._follow): + await self.pump(run_id) + from app.chat import log as turn_log + + for turn_id in list(self._turns): + turn_log.waiters.notify(turn_id) + + async def _loop(self) -> None: + """跟随方:隔 WATCH_S 秒重发「我在看」,隔 POLL_S 秒回库里看一次跟随的运行。""" + last_watch = 0.0 + while True: + await asyncio.sleep(POLL_S) + now = time.monotonic() + if now - last_watch >= WATCH_S: + last_watch = now + for run_id in list(self._follow): + signals.publish("w", f"r:{run_id}") + for turn_id in list(self._turns): + signals.publish("w", f"t:{turn_id}") + for run_id in list(self._follow): + try: + await self.pump(run_id) + except Exception: # noqa: BLE001 + logger.debug("转发运行 %s 的事件出错", run_id, exc_info=True) + + # ---------------- 执行的一方 ---------------- + + def _on_watch(self, parts: list[str]) -> None: + if not parts or ":" not in parts[0]: + return + kind, target = parts[0].split(":", 1) + until = time.monotonic() + WATCH_TTL_S + if kind == "r": + self._watched_runs[target] = until + elif kind == "t": + self._watched_turns[target] = until + + def _watched(self, book: dict[str, float], key: str) -> bool: + until = book.get(key) + if until is None: + return False + if until < time.monotonic(): + book.pop(key, None) + return False + return True + + def turn_watched(self, turn_id: str) -> bool: + return signals.active and self._watched(self._watched_turns, turn_id) + + def _tap(self, event: RunEventModel) -> None: + if not signals.active: + return + if event.type in EPHEMERAL: + if self._watched(self._watched_runs, event.run_id): + self._buffer.setdefault(event.run_id, []).append(event.model_dump(mode="json")) + if self._flusher is None or self._flusher.done(): + self._flusher = asyncio.ensure_future(self._flush()) + return + if event.run_id in self._follow: + self._cursor[event.run_id] = max(self._cursor.get(event.run_id, 0), int(event.seq or 0)) + self._seen.setdefault(event.run_id, set()).add(int(event.seq or 0)) + signals.publish("r", event.run_id, event.seq) + + async def _flush(self) -> None: + await asyncio.sleep(EPHEMERAL_FLUSH_S) + buffer, self._buffer = self._buffer, {} + for run_id, items in buffer.items(): + for chunk in _chunks(items): + signals.publish("e", run_id, json.dumps(chunk, ensure_ascii=False, default=str), keep=True) + + def beat(self, turn_id: str, op: dict[str, Any]) -> None: + """问数据助手的一拍保活心跳:有别的进程在看这一轮就转过去。""" + if self.turn_watched(turn_id): + text = json.dumps(op, ensure_ascii=False, default=str) + if len(text.encode()) <= PAYLOAD_MAX: + signals.publish("b", turn_id, text) + + +def _size(item: dict[str, Any]) -> int: + return len(json.dumps(item, ensure_ascii=False, default=str).encode()) + + +def _split(item: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """太长的一条正文增量按字切成几条(复核 R-b6:以前截掉一截):按字切,不会切坏 UTF-8;每条都在 limit 字节以内。切不开的 + (不是正文增量)交回空。""" + data = dict(item.get("data") or {}) + text = data.get("delta") + if not isinstance(text, str) or set(data) != {"delta"}: + return [] + out: list[dict[str, Any]] = [] + while text: + low, high = 1, len(text) + while low < high: # 二分:最长的一段前缀,装得进 limit + mid = (low + high + 1) // 2 + if _size({**item, "data": {"delta": text[:mid]}}) <= limit: + low = mid + else: + high = mid - 1 + out.append({**item, "data": {"delta": text[:low]}}) + text = text[low:] + return out + + +def _chunks(items: list[dict[str, Any]]) -> Iterator[list[dict[str, Any]]]: + """按载荷上限切开(同一个节点接着的正文增量先合成一条;合成之后太长的再按字切成几条,不截)。""" + merged: list[dict[str, Any]] = [] + for item in items: + prev = merged[-1] if merged else None + if prev is not None and prev.get("type") == item.get("type") and prev.get("node_id") == item.get("node_id") \ + and set((prev.get("data") or {})) == {"delta"} == set((item.get("data") or {})): + prev["data"] = {"delta": str(prev["data"]["delta"]) + str(item["data"]["delta"])} + prev["seq"] = item.get("seq", prev.get("seq")) + continue + merged.append(dict(item)) + limit = PAYLOAD_MAX - 2 + pieces: list[dict[str, Any]] = [] + for item in merged: + if _size(item) <= limit: + pieces.append(item) + else: + pieces.extend(_split(item, limit)) # 不是正文增量又太长的(不该有)丢掉,丢了无妨:不落库的本来就不进回放 + chunk: list[dict[str, Any]] = [] + size = 2 + for item in pieces: + n = _size(item) + if chunk and size + n + 1 > PAYLOAD_MAX: + yield chunk + chunk, size = [], 2 + chunk.append(item) + size += n + 1 + if chunk: + yield chunk + + +def _quiet(task: asyncio.Future[Any]) -> None: + if not task.cancelled() and task.exception() is not None: + logger.debug("转发出错", exc_info=task.exception()) + + +relay = Relay() + +__all__ = ["EPHEMERAL", "POLL_S", "Relay", "WATCH_S", "relay"] diff --git a/backend/app/cluster/requests.py b/backend/app/cluster/requests.py new file mode 100644 index 00000000..2711d4d6 --- /dev/null +++ b/backend/app/cluster/requests.py @@ -0,0 +1,134 @@ +"""发给别的进程的请求(cluster_requests):停止它在跑的东西、往它在执行的运行里记一条事件。 + +**为什么不直接改库。** 一次运行、一轮问数据、一个后台任务的执行在认领它的那个进程里(一个 asyncio 任务),手上拿着模型调用、 +查询、写到一半的库操作。别的进程收到「停止」时能做的只是告诉它:写一条请求(收件人是这一行的 owner),PostgreSQL 上 NOTIFY +叫醒它;它隔一拍心跳(最迟 worker_heartbeat_s 秒)也会来取,照本进程里的停止办(停掉执行、按停止的原因收尾、发 run.cancelled +带上是谁停的),界面上的「停止」和单进程时一样。收件人不在了(租约过期)的请求不办:那一行按「属主不在了」的规矩接手 +(cluster/reaper.py),请求跟着删掉。 + +往运行里记事件同理:执行中的运行,序号由执行它的进程发(core/bus 的进程内计数),别的进程直接写会撞号;所以交给它写。 +""" +from __future__ import annotations + +import asyncio +import logging +import weakref +from collections.abc import Awaitable, Callable +from datetime import datetime, timedelta +from typing import Any + +from sqlalchemy import delete, select + +from app.cluster import identity +from app.cluster.signal import signals +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +#: (kind, target) → 办这件事的函数 (target_id, actor, payload) +_HANDLERS: dict[tuple[str, str], Callable[[str, str | None, dict[str, Any]], Awaitable[Any]]] = {} +#: 取请求时本进程里一次只取一份(心跳和 NOTIFY 同时来时不重复办) +_draining: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() + + +def handle(kind: str, target: str) -> Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]: + """登记一种请求怎么办(在各自的模块里,导入时登记)。""" + def wrap(fn: Callable[..., Awaitable[Any]]) -> Callable[..., Awaitable[Any]]: + _HANDLERS[(kind, target)] = fn + return fn + return wrap + + +async def send(owner: str, kind: str, target: str, target_id: str, *, actor: str | None = None, + payload: dict[str, Any] | None = None) -> None: + """发一条请求给 owner 那个进程(提交之后 NOTIFY 叫醒它)。""" + from app.db.models_cluster import ClusterRequest + + async with SessionLocal() as session: + session.add(ClusterRequest(owner=owner, kind=kind, target=target, target_id=target_id, actor=actor, + payload=dict(payload or {}))) + await session.commit() + signals.publish("q", owner) + + +async def drain() -> int: + """取走发给本进程的请求、照办(心跳每一拍、收到 NOTIFY 时)。交回办了几件。一件出错不影响别的。""" + from app.db.models_cluster import ClusterRequest + + loop = asyncio.get_running_loop() + lock = _draining.get(loop) + if lock is None: + lock = _draining[loop] = asyncio.Lock() + async with lock: + return await _drain() + + +async def _drain() -> int: + from app.db.models_cluster import ClusterRequest + + me = identity.owner() + async with SessionLocal() as session: + rows = list((await session.execute(select(ClusterRequest).where(ClusterRequest.owner == me) + .order_by(ClusterRequest.created_at, ClusterRequest.id))).scalars()) + if not rows: + return 0 + await session.execute(delete(ClusterRequest).where(ClusterRequest.id.in_([r.id for r in rows]))) + await session.commit() + done = 0 + for row in rows: + fn = _HANDLERS.get((row.kind, row.target)) + if fn is None: + logger.warning("收到不认识的进程间请求 %s/%s,不办", row.kind, row.target) + continue + # 各办各的、另起任务:停止要等执行收尾(最多十来秒),不能拖住心跳和后面几件 + task = asyncio.ensure_future(_handle(fn, row.kind, row.target, row.target_id, row.actor, dict(row.payload or {}))) + _running.add(task) + task.add_done_callback(_running.discard) + done += 1 + return done + + +#: 正在办的请求(另起的任务要有人拿着引用) +_running: set[asyncio.Future[Any]] = set() + + +async def _handle(fn: Callable[..., Awaitable[Any]], kind: str, target: str, target_id: str, actor: str | None, + payload: dict[str, Any]) -> None: + from app.cluster.fence import Fenced + + try: + await fn(target_id, actor, payload) + except (Exception, Fenced): # noqa: BLE001 - Fenced 是 BaseException,不接住就成了没人取的任务异常 + logger.exception("办进程间请求 %s/%s %s 出错", kind, target, target_id) + + +async def settle(timeout: float = 5.0) -> None: + """等手上正在办的请求办完(测试、关停用)。""" + if _running: + await asyncio.wait(list(_running), timeout=timeout) + + +async def discard_for(owners: set[str]) -> int: + """收件人已经不在了的请求删掉(接手的进程做:那些行按属主不在了的规矩处理)。""" + from app.db.models_cluster import ClusterRequest + + if not owners: + return 0 + async with SessionLocal() as session: + done = await session.execute(delete(ClusterRequest).where(ClusterRequest.owner.in_(sorted(owners)))) + await session.commit() + return int(done.rowcount or 0) + + +async def prune(now: datetime | None = None, *, older_than_s: float = 3600.0) -> int: + """很久没人取的请求删掉(当班进程做)。""" + from app.db.models_cluster import ClusterRequest + + floor = (now or utcnow()) - timedelta(seconds=older_than_s) + async with SessionLocal() as session: + done = await session.execute(delete(ClusterRequest).where(ClusterRequest.created_at < floor)) + await session.commit() + return int(done.rowcount or 0) + + +__all__ = ["discard_for", "drain", "handle", "prune", "send"] diff --git a/backend/app/cluster/service.py b/backend/app/cluster/service.py new file mode 100644 index 00000000..2d40ed08 --- /dev/null +++ b/backend/app/cluster/service.py @@ -0,0 +1,425 @@ +"""本进程在协调里的那一份:启动时登记、起心跳,关停时交班、删登记。 + +续租约在单独的任务、单独的连接上(不走连接池,不被同一拍别的活拖住);续晚了(离上次续上超过一个租约时长)就隔离:停掉手上的 +执行、不写终态,换一个身份重新登记(cluster/fence.py)。SQLite 只有一个进程:登记只写一次、不续租、本进程直接当班、接手只在 +启动时做,心跳只做本进程自己攒着的活。 + +心跳每一拍(AGENTLAB_WORKER_HEARTBEAT_S,缺省 5 秒;PostgreSQL): +1. (续租约另有任务); +2. 刷新「别的进程谁还活着」; +3. 取走发给本进程的请求照办(停止、往运行里记事件,cluster/requests.py); +4. 本进程自己的周期活(on_beat 登记的:访问密钥的请求记录写库……——这些是每个进程攒在自己内存里的,只能自己写); +5. 争 / 续调度当班(cluster/leader.py);当班的进程在有进程走了(少了一个、收到 gone 通知)或者隔一个租约时长时,接手 + 不在了的进程留下的行(cluster/reaper.py),顺手删掉早就过期的登记和没人取的请求。 + +调度循环(scheduler/loop.py)在每个进程里都起,每一拍先看本进程当不当班,不当班就什么都不做。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +import time +from collections.abc import Awaitable, Callable +from typing import Any + +from app.cluster import guard, identity, leader, reaper, requests, state, workers +from app.cluster.signal import signals +from app.core.config import settings + +logger = logging.getLogger(__name__) + +_beats: list[tuple[str, Callable[[], Awaitable[Any]]]] = [] +_leader_listeners: list[Callable[[bool], Any]] = [] + + +def on_beat(name: str, fn: Callable[[], Awaitable[Any]]) -> None: + """登记一件本进程每拍心跳都要做的事(同名换成新的)。""" + for i, (known, _) in enumerate(_beats): + if known == name: + _beats[i] = (name, fn) + return + _beats.append((name, fn)) + + +def on_leadership(fn: Callable[[bool], Any]) -> None: + """当班换了(接手 True、交出 False)时调。""" + if fn not in _leader_listeners: + _leader_listeners.append(fn) + + +def leading() -> bool: + """本进程现在当不当调度班。没起协调(测试、脚本里直接调调度循环、直接 run_manager.setup)时算当班:那时只有这一个进程。""" + return not service.started or leader.scheduler.held() + + +class _Service: + def __init__(self) -> None: + self.task: asyncio.Task[None] | None = None + self.lease_task: asyncio.Task[None] | None = None + self.wake: asyncio.Event | None = None + self.reap_now = False + self.last_reap = 0.0 + self.others: frozenset[str] = frozenset() + self.started = False + #: SQLite:守卫已经确认只有本进程——登记只写一次、不续租、本进程直接当班、接手只在启动时做(不为协调多写库) + self.single = False + self._lease_conn: Any = None + self.lease_wake: asyncio.Event | None = None + #: 隔离之后、新身份还没登记上:从什么时候起(cluster/fence.now 的读数) + self.pending_since: float | None = None + #: 隔离掉(或者一直没登记上、放弃了)的旧身份:新身份登记上之后收拾它们留下的登记、名额、请求、当班租约 + self.retired: list[str] = [] + + async def start(self) -> None: + """登记、开始收发通知、起心跳。SQLite 时先取单进程的守卫(拿不到抛 guard.SingleProcessOnly)。""" + if self.started: + return + guard.acquire() + self.single = settings.db_dialect == "sqlite" + await workers.register(exclusive=self.single) + live = await workers.liveness() + state.remember(live) + self.others = live.others + await signals.start() + signals.on("q", self._on_request) + signals.on("gone", self._on_gone) + if not self.single: + await signals.ready(timeout=5.0) + self.wake = asyncio.Event() + self.started = True + from app.cluster import caches, fence + + await caches.start() + if self.single: + # 只有本进程:当班不用争(不写库),心跳只做本进程自己攒着的活(访问密钥的请求记录) + leader.scheduler.single() + self.task = asyncio.create_task(self._light_loop(), name="agentlab-cluster-beats") + else: + await self._lead() + fence.activate() + fence.on_trip(self._on_fenced) + self.lease_wake = asyncio.Event() + self.lease_task = asyncio.create_task(self._lease_loop(), name="agentlab-cluster-lease") + self.task = asyncio.create_task(self._loop(), name="agentlab-cluster-heartbeat") + logger.info("工作进程 %s 已登记(%s,租约 %g 秒、心跳 %g 秒;现在另有 %d 个进程)", identity.describe(identity.owner()), + "SQLite,单进程" if self.single else "PostgreSQL,可以多进程", + workers.lease_s(), workers.heartbeat_s(), len(live.others)) + + def _on_request(self, parts: list[str]) -> None: + if parts and parts[0] == identity.owner() and self.wake is not None: + asyncio.ensure_future(requests.drain()) + + def _on_gone(self, _parts: list[str]) -> None: + self.reap_now = True + if self.wake is not None: + self.wake.set() + + # ---------------- 续租约:单独的任务、单独的连接 ---------------- + + async def _lease_conn_open(self) -> Any: + import asyncpg + + from app.cluster.signal import listen_dsn + from app.db.base import APPLICATION_NAME + + if self._lease_conn is None or self._lease_conn.is_closed(): + self._lease_conn = await asyncpg.connect(listen_dsn(), server_settings={"application_name": APPLICATION_NAME}, + timeout=max(1.0, workers.heartbeat_s())) + return self._lease_conn + + async def renew(self) -> bool: + """续一拍租约(在专用连接上,不走连接池:池子被占满时取连接要等,等得比租约还长)。按库的时钟续、按库的时钟核对(复核 + R-b7):库里自己的租约已经过期(或者登记不见了),别的进程可能已经接手,不再续这个身份——隔离(cluster/fence.py),换身份 + 重新登记。计入休眠的钟说续晚了,同样隔离;两只钟差出一个租约(墙钟跳了)只记一条日志,以库的核对为准(复核 R-b14)。 + 交回续没续上。""" + from app.cluster import fence + + if fence.stale(): + fence.trip("续租约的时候发现已经续晚了") + return False + # 两只钟对不上(墙钟跳了,或者单调钟没计入休眠):不直接隔离,以下面这一句按库的时钟核对的结果为准(复核 R-b14) + jumped = fence.jumped() + try: + conn = await self._lease_conn_open() + renewed = await asyncio.wait_for(conn.fetchval( + "UPDATE cluster_workers SET heartbeat_at = timezone('UTC', clock_timestamp()), " + "lease_until = timezone('UTC', clock_timestamp()) + make_interval(secs => $2) " + "WHERE id = $1 AND lease_until >= timezone('UTC', clock_timestamp()) RETURNING id", + identity.owner(), float(workers.lease_s())), timeout=workers.heartbeat_s()) + if renewed is None: + expired = await asyncio.wait_for(conn.fetchval( + "SELECT lease_until < timezone('UTC', clock_timestamp()) FROM cluster_workers WHERE id = $1", + identity.owner()), timeout=workers.heartbeat_s()) + except Exception: # noqa: BLE001 - 库一时连不上:这一拍没续上,下一拍再试(续晚了的话下一拍隔离) + logger.warning("续租约没续上,下一拍再试", exc_info=True) + with contextlib.suppress(Exception): + if self._lease_conn is not None: + await self._lease_conn.close() + self._lease_conn = None + return False + if renewed is None: + why = "按库的时钟本进程的租约已经过期" if expired else "本进程的登记不见了(被当成不在了)" + fence.trip(why + (";本机的墙钟和单调钟也差出了一个租约以上" if jumped else "")) + return False + if jumped: + logger.warning("本机的墙钟和单调钟差出一个租约以上(墙钟跳了:NTP 校时、手动改时间、虚拟机对时),按库的时钟本进程的租约" + "还没过期:照常续上,不隔离") + fence.renewed() + return True + + async def _lease_loop(self) -> None: + """续租约;隔离之后改成重新登记新身份。哪一下出了错(库一抖、版本守卫拦下)都不让这个任务退出——它一退,之后就再也没有 + 隔离保护了(复核 R-b9):照实记日志,退避着重试(最长一个租约时长)。""" + from app.cluster import fence + + delay = workers.heartbeat_s() + while True: + try: + if self.pending_since is not None: + await self.register_again() + else: + await self.renew() + delay = workers.heartbeat_s() + except asyncio.CancelledError: + raise + except (Exception, fence.Fenced): # noqa: BLE001 + delay = min(max(delay * 2, workers.heartbeat_s()), workers.lease_s()) + logger.exception("%s出错,%g 秒后再试", "隔离之后重新登记" if self.pending_since is not None else "续租约", delay) + if self.lease_wake is None: + await asyncio.sleep(delay) + continue + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self.lease_wake.wait(), timeout=delay) + self.lease_wake.clear() + + def _on_fenced(self) -> None: + """隔离了:手上的执行由各处自己停(runner、job_runtime 登记的处理);这里马上换身份(之后新建的行记新身份),叫续租的任务 + 去登记它。""" + from app.cluster import fence + + old = identity.owner() + self.retired.append(old) + new = identity.regenerate() + self.pending_since = fence.now() + leader.scheduler.drop() + logger.warning("隔离:换成新身份 %s(旧身份 %s),登记上之前新开始的执行先等着", new, old) + if self.lease_wake is not None: + self.lease_wake.set() + + async def register_again(self) -> None: + """隔离之后登记新身份。一个租约时长都没登记上(库连不上、版本守卫拦着)就放弃这个身份、再换一个:这期间等不到的执行已经 + 放弃开始(fence.ready),它们的行记着这个身份——它永远不登记,别的进程就照「属主不在了」处理那几行。""" + from app.cluster import fence + + assert self.pending_since is not None + if fence.now() - self.pending_since > workers.lease_s(): + gave_up = identity.owner() + self.retired.append(gave_up) + identity.regenerate() + self.pending_since = fence.now() + logger.error("新身份 %s 一个租约时长都没登记上,放弃它、换成 %s 再试", gave_up, identity.owner()) + await workers.register() + retired, self.retired = self.retired, [] + self.pending_since = None + fence.renewed() + logger.warning("隔离之后换了身份重新登记:%s → %s(之前认领的行由当班进程按「属主不在了」接手)", + "、".join(retired), identity.owner()) + try: + await self.tidy_retired(retired) + except Exception: # noqa: BLE001 - 收拾不了的等旧身份的租约过期、按「属主不在了」处理 + logger.exception("收拾旧身份留下的登记、名额、请求时出错(等它们的租约过期再按属主不在了处理)") + + async def tidy_retired(self, retired: list[str]) -> dict[str, int]: + """新身份登记上之后,收拾旧身份留下的(复核 R-b10):它的执行都已停下、写不了了,不必等它的租约过期——删掉它的登记(别的 + 进程马上按「属主不在了」接手它认领的运行、任务)、它占着的运行名额和同源队的位置,交出它拿着的当班租约;发给它的请求转到 + 新身份名下照办(停止请求对已经不在本进程执行的运行照停止的规矩办:属主不在了就接手、放弃)。""" + from sqlalchemy import delete, update + + from app.db.base import SessionLocal + from app.db.models_cluster import ClusterLease, ClusterRequest, ClusterWorker, LaneTicket, RunSlot + + if not retired: + return {} + me = identity.owner() + async with SessionLocal() as session: + out = { + "workers": (await session.execute(delete(ClusterWorker).where(ClusterWorker.id.in_(retired)))).rowcount, + "slots": (await session.execute(delete(RunSlot).where(RunSlot.owner.in_(retired)))).rowcount, + "lanes": (await session.execute(delete(LaneTicket).where(LaneTicket.owner.in_(retired)))).rowcount, + "requests": (await session.execute(update(ClusterRequest).where(ClusterRequest.owner.in_(retired)) + .values(owner=me).execution_options(synchronize_session=False))).rowcount, + "leases": (await session.execute(update(ClusterLease).where(ClusterLease.owner.in_(retired)) + .values(owner=None, lease_until=None) + .execution_options(synchronize_session=False))).rowcount, + } + await session.commit() + out = {k: int(v or 0) for k, v in out.items()} + for old in retired: + signals.publish("gone", old) + if out["slots"]: + signals.publish("slot") + if out["lanes"]: + signals.publish("lane") + if out["requests"]: + signals.publish("q", me) + self.reap_now = True + logger.warning("收拾旧身份 %s:删了登记 %d 条、运行名额 %d 个、同源队的位置 %d 个,%d 条发给它的请求转给 %s,交出当班租约 %d 份", + "、".join(retired), out["workers"], out["slots"], out["lanes"], out["requests"], me, out["leases"]) + return out + + # ---------------- 当班、心跳的其余活 ---------------- + + async def _lead(self) -> bool: + was = leader.scheduler.held() + try: + now_leading = await leader.scheduler.claim() + except Exception: # noqa: BLE001 - 库一时不通:本地按没当班算 + logger.exception("争调度当班时出错") + now_leading = False + if now_leading != was: + for fn in list(_leader_listeners): + try: + out = fn(now_leading) + if asyncio.iscoroutine(out): + await out + except Exception: # noqa: BLE001 + logger.exception("当班换了之后的处理出错") + if now_leading: + self.reap_now = True + return now_leading + + async def beat(self) -> None: + """心跳的一拍(续租约另在 _lease_loop;测试里直接调)。SQLite 只做本进程自己攒着的活;隔离之后新身份还没登记上时也只做 + 这些(不争当班、不接手)。""" + if self.single or self.pending_since is not None: + await self._own_work() + return + live = await workers.liveness() + state.remember(live) + if self.others - live.others: + self.reap_now = True + self.others = live.others + try: + await requests.drain() + except Exception: # noqa: BLE001 + logger.exception("取发给本进程的请求时出错") + try: + from app.cluster import caches + + await caches.poll() + except Exception: # noqa: BLE001 + logger.exception("读别的进程的缓存变化时出错") + await self._own_work() + if await self._lead() and (self.reap_now or time.monotonic() - self.last_reap >= workers.lease_s()): + # 隔一会儿的这一次信得过自己:属主是本进程的行(刚建、还没交给后台任务的)不当成没人管 + await self.reap(live.trusting()) + + async def _own_work(self) -> None: + for name, fn in list(_beats): + try: + await fn() + except Exception: # noqa: BLE001 + logger.exception("心跳里的「%s」出错,下一拍再试", name) + + async def reap(self, live: workers.Liveness | None = None) -> dict[str, Any]: + self.reap_now = False + self.last_reap = time.monotonic() + live = live or await state.current(fresh=True) + dead = await _dead_owners(live) + out = await reaper.reap(live) + if dead: + await requests.discard_for(dead) + with contextlib.suppress(Exception): + await workers.prune() + await requests.prune() + from app.cluster import caches, counters + + # 到期的计数、额度(每日额度两种库都记在库里) + await counters.prune() + if settings.db_dialect == "postgresql": + await caches.prune() + return out + + async def _loop(self) -> None: + from app.cluster import fence + + assert self.wake is not None + while True: + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self.wake.wait(), timeout=workers.heartbeat_s()) + self.wake.clear() + try: + await self.beat() + except asyncio.CancelledError: + raise + except (Exception, fence.Fenced): # noqa: BLE001 - 一拍出错(库一时不通)不该让心跳停下 + logger.exception("心跳这一拍出错,下一拍照常进行") + + async def _light_loop(self) -> None: + while True: + await asyncio.sleep(workers.heartbeat_s()) + await self._own_work() + + async def stop(self) -> None: + """关停:停心跳、交班、删登记(别的进程马上接手本进程留下的行),停收发通知。""" + if not self.started: + return + self.started = False + from app.cluster import fence + + for task in (self.lease_task, self.task): + if task is not None: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self.task = self.lease_task = None + self.pending_since, self.retired = None, [] + fence.deactivate() + with contextlib.suppress(Exception): + await leader.scheduler.release() + with contextlib.suppress(Exception): + await workers.unregister() + if self._lease_conn is not None: + with contextlib.suppress(Exception): + await self._lease_conn.close() + self._lease_conn = None + signals.publish("gone", identity.owner()) + await signals.stop() + state.forget() + guard.release() + + +async def startup_reap() -> dict[str, Any]: + """启动时的那一次接手(每个进程都做;那时本进程什么都还没在跑,属主是自己的只会是前一次起停留下的)。""" + return await service.reap(await state.current(fresh=True)) + + +async def _dead_owners(live: workers.Liveness) -> set[str]: + """库里登记着、租约已过期的进程(它们的请求删掉)。""" + from sqlalchemy import select + + from app.db.base import SessionLocal + from app.db.models_cluster import ClusterRequest + + async with SessionLocal() as session: + owners = set((await session.execute(select(ClusterRequest.owner).distinct())).scalars()) + return {o for o in owners if o != identity.owner() and o not in live.others} + + +service = _Service() + + +async def start() -> None: + await service.start() + + +async def stop() -> None: + await service.stop() + + +def started() -> bool: + return service.started + + +__all__ = ["leading", "on_beat", "on_leadership", "service", "start", "started", "startup_reap", "stop"] diff --git a/backend/app/cluster/signal.py b/backend/app/cluster/signal.py new file mode 100644 index 00000000..46785494 --- /dev/null +++ b/backend/app/cluster/signal.py @@ -0,0 +1,222 @@ +"""进程之间的叫醒:PostgreSQL 的 LISTEN / NOTIFY(频道 agentlab)。SQLite 只有一个进程,什么都不发、也不听——本进程里的 +叫醒(事件总线、问数据的等待者、名额和同源队的等待者)各自照旧。 + +**只叫醒,不带数据。** 载荷是「类别|发出方|哪一个|到第几条」:运行事件 r|…||,问数据一轮的事件 +t|…||,发给某个进程的请求 q|…|,名额 / 同源队有了空位 slot、lane|…|,进程走了 gone|…|, +缓存要重读 cache|…。发出方是 OWNER 的随机串:自己发的自己不处理(本进程里已经叫醒过了)。收到的一方按 id 回库里读——NOTIFY +丢了、连接断了重连期间漏了,靠各处的兜底轮询补上(运行的转发、问数据的流、名额等待都隔一会儿回库里看一次),叫醒只让它们不必 +等到那一拍。 + +例外是不落库的那两样:运行里的流式增量(正文 token、思考 delta)和问数据助手的保活心跳,只在有别的进程在看的时候(它们发 +w 登记「我在看」,cluster/relay.py)由执行的进程把内容放进载荷转过去(攒一小段合成一条、截在 7000 字节以内)。丢了无妨: +它们本来就不进回放,下一条落库的事件会把画面带到正确的样子。 + +**发在提交之后。** 调用方提交了事务再 publish:收到的一方去读时一定读得到。发是攒一下再批量发的(FLUSH_S),同一个 id 只发 +最后一个 seq:一次运行几十条事件,跨进程的订阅方只要知道「到第几条了」。用一条专门的连接(不占连接池),收和发都走它; +断了隔一会儿重连,重连之后叫一遍各处的「补一次」(on_resync),漏掉的那一段由它们回库里读。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +from app.cluster import identity +from app.core.config import settings + +logger = logging.getLogger(__name__) + +CHANNEL = "agentlab" +#: 攒多久发一批(秒) +FLUSH_S = 0.02 +#: 重连的间隔(秒) +RETRY_S = 2.0 + +Handler = Callable[[list[str]], Any] + + +def listen_dsn() -> str: + """收发通知那条专用连接的地址:LISTEN 要一条一直开着的会话,设了直连地址(AGENTLAB_DATABASE_DIRECT_URL)就走它——普通地址 + 可能是 PgBouncer 的事务模式,那里收不到通知。""" + from app.db.urls import libpq_conninfo + + return libpq_conninfo(settings.session_db_url) + + +def _tag() -> str: + return identity.owner().rsplit(":", 1)[-1] + + +class Signals: + def __init__(self) -> None: + self._handlers: dict[str, list[Handler]] = {} + self._raw: set[str] = set() + self._seq = 0 + self._resync: list[Callable[[], Any]] = [] + self._pending: dict[tuple[str, ...], str] = {} + self._conn: Any = None + self._task: asyncio.Task[None] | None = None + self._flusher: asyncio.Task[None] | None = None + self._wake: asyncio.Event | None = None + self._stopping = False + + # ---------------- 登记 ---------------- + + def on(self, kind: str, handler: Handler, *, raw: bool = False) -> None: + """收到 kind 这一类时调 handler(其余几段)。可以是协程函数。同一个 handler 重复登记无害。raw:载荷里带着内容(JSON), + 只按第一段 id 切开,交回 [id, 余下的原文]。""" + handlers = self._handlers.setdefault(kind, []) + if handler not in handlers: + handlers.append(handler) + if raw: + self._raw.add(kind) + + def on_resync(self, handler: Callable[[], Any]) -> None: + """重连之后(可能漏了通知)调一次。""" + if handler not in self._resync: + self._resync.append(handler) + + # ---------------- 发 ---------------- + + @property + def active(self) -> bool: + return self._task is not None and not self._task.done() + + def publish(self, kind: str, *parts: object, keep: bool = False) -> None: + """攒一条要发的(没在听,也就是 SQLite、没起服务的测试里,什么都不做)。同一类同一个 id 只留最后一条;keep 时每条都发 + (带内容的转发:前后几段都要)。""" + if not self.active: + return + text = [str(p) for p in parts] + if keep: + self._seq += 1 + key: tuple[str, ...] = (kind, "#", str(self._seq)) + else: + key = (kind, text[0]) if text else (kind,) + self._pending[key] = "|".join([kind, _tag(), *text]) + if self._wake is not None: + self._wake.set() + + async def _flush_loop(self) -> None: + assert self._wake is not None + while not self._stopping: + await self._wake.wait() + self._wake.clear() + await asyncio.sleep(FLUSH_S) + batch, self._pending = list(self._pending.values()), {} + if not batch or self._conn is None: + continue + try: + await self._conn.execute(f"SELECT pg_notify('{CHANNEL}', p) FROM unnest($1::text[]) AS p", batch) + except Exception: # noqa: BLE001 - 发不出去只是别的进程晚一拍知道(它们会回库里看) + logger.debug("NOTIFY 没发出去", exc_info=True) + + # ---------------- 收 ---------------- + + def _received(self, _conn: Any, _pid: int, _channel: str, payload: str) -> None: + head = payload.split("|", 2) + if len(head) < 2 or head[1] == _tag(): + return + kind, body = head[0], head[2] if len(head) > 2 else "" + rest = (body.split("|", 1) if body else []) if kind in self._raw else (body.split("|") if body else []) + for handler in list(self._handlers.get(kind, ())): + try: + out = handler(rest) + if asyncio.iscoroutine(out): + task = asyncio.ensure_future(out) + task.add_done_callback(_quiet) + except Exception: # noqa: BLE001 + logger.exception("处理进程间通知 %s 出错", kind) + + async def _listen(self) -> None: + import asyncpg + + from app.db.base import APPLICATION_NAME + + dsn = listen_dsn() + first = True + while not self._stopping: + lost = asyncio.Event() + try: + conn = await asyncpg.connect(dsn, server_settings={"application_name": APPLICATION_NAME}) + except Exception as e: # noqa: BLE001 + logger.warning("进程间通知的连接没连上(%s),%.0f 秒后重试", e, RETRY_S) + await asyncio.sleep(RETRY_S) + continue + try: + conn.add_termination_listener(lambda _c: lost.set()) + await conn.add_listener(CHANNEL, self._received) + self._conn = conn + if not first: + logger.info("进程间通知的连接已重连") + self._fire_resync() + first = False + await lost.wait() + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + logger.warning("进程间通知的连接出错,%.0f 秒后重连", RETRY_S, exc_info=True) + finally: + self._conn = None + with contextlib.suppress(Exception): + await asyncio.wait_for(conn.close(), timeout=2.0) + if not self._stopping: + await asyncio.sleep(RETRY_S) + + def _fire_resync(self) -> None: + for handler in list(self._resync): + try: + out = handler() + if asyncio.iscoroutine(out): + asyncio.ensure_future(out).add_done_callback(_quiet) + except Exception: # noqa: BLE001 + logger.exception("重连之后补一次出错") + + # ---------------- 起停 ---------------- + + async def start(self) -> None: + """应用库是 PostgreSQL 时开始收发;SQLite 什么都不做。""" + if settings.db_dialect != "postgresql" or self.active: + return + self._stopping = False + self._wake = asyncio.Event() + self._task = asyncio.create_task(self._listen(), name="agentlab-cluster-listen") + self._flusher = asyncio.create_task(self._flush_loop(), name="agentlab-cluster-notify") + + async def ready(self, timeout: float = 5.0) -> bool: + """等连接连上(测试、启动时用)。""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while self.active and self._conn is None and loop.time() < deadline: + await asyncio.sleep(0.02) + return self._conn is not None + + async def stop(self) -> None: + self._stopping = True + tasks = [t for t in (self._flusher, self._task) if t is not None] + # 停之前把攒着的发出去:关停时交班、删登记这几条别的进程要尽快知道 + if self._pending and self._conn is not None: + with contextlib.suppress(Exception): + await asyncio.wait_for(self._conn.execute( + f"SELECT pg_notify('{CHANNEL}', p) FROM unnest($1::text[]) AS p", list(self._pending.values())), + timeout=1.0) + self._pending = {} + for task in tasks: + task.cancel() + for task in tasks: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self._task = self._flusher = None + self._conn = None + + +def _quiet(task: asyncio.Future[Any]) -> None: + if not task.cancelled() and task.exception() is not None: + logger.error("处理进程间通知出错", exc_info=task.exception()) + + +signals = Signals() + +__all__ = ["CHANNEL", "Signals", "signals"] diff --git a/backend/app/cluster/slots.py b/backend/app/cluster/slots.py new file mode 100644 index 00000000..48878c6d --- /dev/null +++ b/backend/app/cluster/slots.py @@ -0,0 +1,436 @@ +"""运行的并发名额(run_slots,docs/design/orchestration-round/00-总方案.md §5 W2-b「优先级与并发」)。 + +**为什么。** 以前是 runner 里一把进程内的信号量(AGENTLAB_MAX_CONCURRENT_RUNS,缺省 20),谁先到谁先跑:一次评测几十道题、 +一批口径召回,就能把 20 个名额占满,问数据、画布试跑在后面排队。多进程之后信号量也只管得了本进程。 + +**现在。** 每段执行(发起、审批之后接着跑、继续运行;停在审批上的不占)开始前要一个名额,按运行的来源归四类: + + 交互 interactive:问数据、画布试跑、正式运行(手动点的)、蓝图试运行、个人应用的刷新——有人在屏幕前等 + 应用 app:应用入口、按需视图、外部系统的钩子、接口 + 定时 schedule:定时任务(含事件触发) + 系统 system:评测、口径召回、对账、补算、按口径正式出具——服务端替人跑的 + +每类有全站的并发上限和优先级(设置里「运行并发」一组,管理员可调),另有全站总上限,和一个**交互留底**: + +- 留底:全站名额里永远留给交互类的那几个(缺省全站上限的 1/4、至少 1;至多全站上限减 1)。应用、定时、系统三类合起来 + 永远用不到这一部分——有人问数据、试跑时总有名额,不用等评测、召回跑完。 +- 借用:没人问数据时,别的类可以借满到「全站上限 − 留底」(三类各自的上限缺省就是这个数,设置里可以调小)。评测照旧跑得快。 +- 交互自己的上限缺省 = 全站上限:留底之外空着的名额它也能用。 + +等着的按优先级高的先、同一优先级先来先得(缺省交互 40 > 应用 30 > 定时 20 > 系统 10);已经在跑的不抢。单进程时总上限和以前的 +信号量一样大。 + +**怎么发。** 要名额的一方先在本进程登记、再插一行「等着」,然后自己去发一次(grant:把所有等着的按规矩过一遍,谁够格就 +改成「已发」);没轮到就等叫醒(本进程放名额时叫醒,PostgreSQL 上别的进程放名额时 NOTIFY slot 叫醒),另隔一会儿自己再 +发一次兜底。发的那一下全站串行:PostgreSQL 上在事务里取咨询锁,SQLite 只有一个进程,本进程的锁就够。名额用完删掉那一行。 +属主不在了的行(进程没了;本进程里登记已经没了的——测试里事件循环关了留下的)发的时候顺手删掉,名额不会漏。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +import time +import weakref +from collections import Counter +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy import delete, select, text, update + +from app.cluster import identity +from app.cluster.signal import signals +from app.core.config import settings +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +CLASSES = ("interactive", "app", "schedule", "system") +CLASS_LABEL = {"interactive": "交互", "app": "应用", "schedule": "定时", "system": "系统"} +#: 运行来源 → 类别(engine/origin.ORIGIN_KINDS)。没记来源的(直接调 run_manager.start、升级前的)算交互:同以前不设限 +ORIGIN_CLASS = { + "chat": "interactive", "canvas": "interactive", "launcher": "interactive", "trial": "interactive", + "personal_app": "interactive", + "app_entry": "app", "app_view": "app", "hook": "app", "api": "app", + "schedule": "schedule", + "eval": "system", "recall": "system", "reconcile": "system", "app_backfill": "system", "ledger_issue": "system", +} +SETTING_KEY = "concurrency" +DEFAULT_PRIORITY = {"interactive": 40, "app": 30, "schedule": 20, "system": 10} +#: 上限、优先级的取值范围 +LIMIT_MAX = 1000 +PRIORITY_MAX = 100 +#: 等名额时隔多久自己再发一次(秒):PostgreSQL 上别的进程放的名额靠 NOTIFY 叫醒,这是兜底;SQLite 只有本进程放名额 +POLL_S = {"postgresql": 1.0, "sqlite": 5.0} +#: 设置读一次管多久(秒):改了设置,最迟这么久之后各进程按新的发 +POLICY_TTL_S = 3.0 +#: 咨询锁的号(发名额那一下全站串行) +_GRANT_KEY = 0x4167536C6F747321 # "AgSlots!" + + +def class_of(origin_kind: str | None) -> str: + return ORIGIN_CLASS.get(origin_kind or "", "interactive") + + +def _reserve_of(total: int, reserve: int | None) -> int: + """生效的交互留底:缺省全站上限的 1/4、至少 1;至多全站上限减 1(全站只有 1 个名额时没有留底,别的类才跑得起来)。""" + wanted = max(1, total // 4) if reserve is None else int(reserve) + return max(0, min(wanted, total - 1)) + + +def defaults(total: int | None = None, reserve: int | None = None) -> dict[str, Any]: + """缺省的设置(总上限取 AGENTLAB_MAX_CONCURRENT_RUNS):交互上限 = 全站上限,别的三类 = 全站上限 − 交互留底。""" + total = max(1, int(total if total is not None else settings.max_concurrent_runs)) + kept = _reserve_of(total, reserve) + others = max(1, total - kept) + return {"max_total": total, "interactive_reserve": kept, + "classes": {k: {"limit": total if k == "interactive" else others, "priority": DEFAULT_PRIORITY[k]} + for k in CLASSES}} + + +def _int_in(value: Any, low: int, high: int) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and low <= value <= high + + +def values(stored: Any) -> dict[str, Any]: + """存着的(可能缺项、存坏了)→ 生效的:缺的、坏的补缺省(缺省按存着的全站上限、留底算)。""" + stored = stored if isinstance(stored, dict) else {} + total = stored.get("max_total") if _int_in(stored.get("max_total"), 1, LIMIT_MAX) else None + reserve = stored.get("interactive_reserve") if _int_in(stored.get("interactive_reserve"), 0, LIMIT_MAX) else None + out = defaults(total, reserve) + classes = stored.get("classes") if isinstance(stored.get("classes"), dict) else {} + for k in CLASSES: + item = classes.get(k) if isinstance(classes.get(k), dict) else {} + if _int_in(item.get("limit"), 1, LIMIT_MAX): + out["classes"][k]["limit"] = item["limit"] + if _int_in(item.get("priority"), 0, PRIORITY_MAX): + out["classes"][k]["priority"] = item["priority"] + return out + + +def borrowable(pol: dict[str, Any]) -> int: + """应用、定时、系统三类合起来最多用几个名额:全站上限 − 交互留底。""" + return max(0, int(pol["max_total"]) - int(pol.get("interactive_reserve") or 0)) + + +def problem(sent: Any) -> str | None: + """设置接口写进来的这一组有没有问题(说明给人看)。只核对给了的项。""" + if not isinstance(sent, dict): + return "运行并发的设置需要是对象" + total = sent.get("max_total") + if total is not None and (not isinstance(total, int) or isinstance(total, bool) or not 1 <= total <= LIMIT_MAX): + return f"全站同时运行的上限需要是 1 到 {LIMIT_MAX} 之间的整数" + reserve = sent.get("interactive_reserve") + if reserve is not None and not _int_in(reserve, 0, LIMIT_MAX): + return f"给交互留底的名额需要是 0 到 {LIMIT_MAX} 之间的整数" + if reserve is not None and isinstance(total, int) and total > 1 and reserve >= total: + return "给交互留底的名额要小于全站同时运行的上限:留底占满了,应用、定时、系统就一个都运行不了" + classes = sent.get("classes") + if classes is None: + return None + if not isinstance(classes, dict): + return "各类运行的设置需要是对象" + for k, item in classes.items(): + if k not in CLASSES: + return f"没有「{k}」这一类运行(只有交互、应用、定时、系统)" + if not isinstance(item, dict): + return f"「{CLASS_LABEL[k]}」类的设置需要是对象" + limit, prio = item.get("limit"), item.get("priority") + if limit is not None and (not isinstance(limit, int) or isinstance(limit, bool) or not 1 <= limit <= LIMIT_MAX): + return f"「{CLASS_LABEL[k]}」类的并发上限需要是 1 到 {LIMIT_MAX} 之间的整数" + if prio is not None and (not isinstance(prio, int) or isinstance(prio, bool) or not 0 <= prio <= PRIORITY_MAX): + return f"「{CLASS_LABEL[k]}」类的优先级需要是 0 到 {PRIORITY_MAX} 之间的整数" + return None + + +def merged(saved: Any, sent: dict[str, Any]) -> dict[str, Any]: + """只给了一部分的(只改一类的上限)补上存着的另几项。存的是原样(没给的类别上限不写,按全站上限、留底现算缺省)。""" + base = dict(saved) if isinstance(saved, dict) else {} + out: dict[str, Any] = {k: base[k] for k in ("max_total", "interactive_reserve") if k in base} + for key in ("max_total", "interactive_reserve"): + if sent.get(key) is not None: + out[key] = sent[key] + classes = {k: dict(v) for k, v in (base.get("classes") or {}).items() if k in CLASSES and isinstance(v, dict)} + for k, item in (sent.get("classes") or {}).items(): + for field_ in ("limit", "priority"): + if item.get(field_) is not None: + classes.setdefault(k, {})[field_] = item[field_] + out["classes"] = classes + return out + + +_policy: tuple[float, dict[str, Any]] | None = None + + +async def policy(*, fresh: bool = False) -> dict[str, Any]: + """现在生效的设置(读一次管 POLICY_TTL_S 秒)。""" + global _policy + now = time.monotonic() + if not fresh and _policy is not None and now - _policy[0] < POLICY_TTL_S: + return _policy[1] + from app.db.models import Setting + + async with SessionLocal() as session: + row = await session.get(Setting, SETTING_KEY) + value = values(row.value if row is not None else None) + _policy = (now, value) + return value + + +def forget_policy() -> None: + """设置刚改了:本进程下一次重读(别的进程最迟 POLICY_TTL_S 秒)。""" + global _policy + _policy = None + + +# -------------------------------------------------------------------------- +# 本进程里要名额的登记 +# -------------------------------------------------------------------------- + + +@dataclass +class _Holder: + run_id: str + loop: asyncio.AbstractEventLoop + woke: asyncio.Event = field(default_factory=asyncio.Event) + #: 还在排队等名额(发下来之前) + queued: bool = True + #: 要名额时本进程的身份(名额行记的属主):放的时候按它删——隔离之后换了身份也删得到(复核 R-b10) + owner: str = "" + + +_holders: dict[str, _Holder] = {} +_grant_locks: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock] = weakref.WeakKeyDictionary() + + +def _local(run_id: str) -> bool: + """这一行在本进程里还有人在要 / 在用(别的事件循环留下的不算:那个循环已经没人会放它了)。""" + holder = _holders.get(run_id) + if holder is None: + return False + try: + return holder.loop is asyncio.get_running_loop() and not holder.loop.is_closed() + except RuntimeError: + return False + + +def wake_local() -> None: + for holder in list(_holders.values()): + if not holder.loop.is_closed(): + holder.loop.call_soon_threadsafe(holder.woke.set) + + +def _grant_lock() -> asyncio.Lock: + loop = asyncio.get_running_loop() + lock = _grant_locks.get(loop) + if lock is None: + lock = _grant_locks[loop] = asyncio.Lock() + return lock + + +@dataclass +class _MemRow: + run_id: str + klass: str + priority: int + requested_at: Any + owner: str = "" + granted: bool = False + + +#: SQLite 时的名额(复核 R-b3):只有一个进程,规矩照同一套、只是不落库——一段运行不为要名额多写三次库 +_mem: dict[str, _MemRow] = {} + + +def _in_memory() -> bool: + return settings.db_dialect == "sqlite" + + +def _pick(rows: list[Any], pol: dict[str, Any]) -> list[str]: + """按规矩挑这一次发给谁(库里的行、进程里的行同一套):优先级高的先、同级先来先得,类别上限、全站上限、交互留底。""" + used = Counter(r.klass for r in rows if r.granted) + total = sum(used.values()) + # 交互以外的三类合起来用了几个:到了「全站上限 − 交互留底」就不再发给它们(留底永远给交互) + others = total - used["interactive"] + others_cap = borrowable(pol) + limits = {k: int(v["limit"]) for k, v in pol["classes"].items()} + granted: list[str] = [] + waiting = sorted((r for r in rows if not r.granted), key=lambda r: (-int(r.priority or 0), r.requested_at, r.run_id)) + for row in waiting: + if total >= int(pol["max_total"]): + break + if used[row.klass] >= limits.get(row.klass, int(pol["max_total"])): + continue + if row.klass != "interactive" and others >= others_cap: + continue + used[row.klass] += 1 + total += 1 + if row.klass != "interactive": + others += 1 + granted.append(row.run_id) + return granted + + +async def grant() -> list[str]: + """把等着的按规矩过一遍,够格的改成已发;属主不在了的行删掉。交回这一次发出去的。全站串行(见模块说明)。""" + if _in_memory(): + pol = await policy() + for run_id in [r for r in _mem if not _local(r)]: + _mem.pop(run_id, None) + granted = _pick(list(_mem.values()), pol) + for run_id in granted: + _mem[run_id].granted = True + if granted: + wake_local() + return granted + from app.cluster import state + from app.db.models_cluster import RunSlot + + pol = await policy() + # 拿锁之前先备好「谁还活着」(必要时这时候查库),持锁期间只用持锁的那条连接 + live = await state.current() + known_dead = await _known_dead(live) + async with _grant_lock(): + async with SessionLocal() as session: + if session.get_bind().dialect.name == "postgresql": + await session.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": _GRANT_KEY}) + rows = list((await session.execute(select(RunSlot))).scalars()) + # 持着全站的锁时不另取连接查库(复核:连接池用尽时会卡到取连接超时):用锁之前拿好的那一份「谁还活着」,不认识的属主 + # 这一次当活着(不删它的行),下一次手上那份更新了再说 + me = identity.owner() + dead = [r.run_id for r in rows + if (r.owner == me and not _local(r.run_id)) or (r.owner != me and r.owner in known_dead)] + if dead: + await session.execute(delete(RunSlot).where(RunSlot.run_id.in_(dead))) + rows = [r for r in rows if r.run_id not in set(dead)] + granted = _pick(rows, pol) + now = utcnow() + if granted: + await session.execute(update(RunSlot).where(RunSlot.run_id.in_(granted)) + .values(granted=True, granted_at=now).execution_options(synchronize_session=False)) + await session.commit() + if dead: + logger.info("运行名额:收回了 %d 个属主已经不在的名额", len(dead)) + if granted: + wake_local() + signals.publish("slot") + return granted + + +async def _known_dead(live: Any) -> set[str]: + """名额表里属主不在手上那份「活着」里的那几个,拿锁之前当场再查一次登记:仍不在的算没了。""" + from app.cluster import state + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + owners = set((await session.execute(select(RunSlot.owner).distinct())).scalars()) + unknown = {o for o in owners if o and o != identity.owner() and o not in live.others} + if not unknown: + return set() + fresh = await state.current(fresh=True) + return {o for o in unknown if o not in fresh.others} + + +async def _granted(run_id: str) -> bool | None: + """这一行发了没有:True 发了,False 还在等,None 这一行不见了(被当成属主不在了删掉)。""" + if _in_memory(): + row = _mem.get(run_id) + return None if row is None else row.granted + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + return await session.scalar(select(RunSlot.granted).where(RunSlot.run_id == run_id)) + + +async def _request(run_id: str, klass: str, priority: int) -> str: + """排进名额的队。交回这一行记的属主(本进程此刻的身份)。""" + me = identity.owner() + if _in_memory(): + _mem[run_id] = _MemRow(run_id, klass, priority, utcnow(), me) + return me + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id)) + session.add(RunSlot(run_id=run_id, klass=klass, priority=priority, owner=me, granted=False, + requested_at=utcnow())) + await session.commit() + return me + + +async def _release(run_id: str, owner: str) -> None: + if _in_memory(): + _mem.pop(run_id, None) + return + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + await session.execute(delete(RunSlot).where(RunSlot.run_id == run_id, RunSlot.owner == owner)) + await session.commit() + + +@contextlib.asynccontextmanager +async def hold(run_id: str, origin_kind: str | None) -> AsyncIterator[str]: + """要一个名额(等到发给自己),出来时放掉。交回类别。取消(停止运行)在等的时候也照常生效。""" + klass = class_of(origin_kind) + pol = await policy() + priority = int(pol["classes"][klass]["priority"]) + holder = _Holder(run_id, asyncio.get_running_loop()) + _holders[run_id] = holder + poll = POLL_S.get(settings.db_dialect, 5.0) + try: + holder.owner = await _request(run_id, klass, priority) + while True: + holder.woke.clear() + await grant() + got = await _granted(run_id) + if got: + holder.queued = False + break + if got is None: + holder.owner = await _request(run_id, klass, priority) + continue + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(holder.woke.wait(), timeout=poll) + yield klass + finally: + if _holders.get(run_id) is holder: + _holders.pop(run_id, None) + try: + await asyncio.shield(_release(run_id, holder.owner or identity.owner())) + except Exception: # noqa: BLE001 - 放不掉的名额下次发的时候按「本进程里已经没人用」收回 + logger.exception("运行 %s 的名额没放掉,下次发名额时收回", run_id) + wake_local() + signals.publish("slot") + + +async def waiting(run_id: str, *, here: bool = False) -> bool: + """这次运行是不是正排着队等名额(评测每题的时限、算期望的时限不算这一段,eval/runner、eval/gold)。 + here:这次运行的执行任务在本进程里(还没开始要、或者已经放掉了名额的就不是在等,不用查库)。""" + holder = _holders.get(run_id) + if holder is not None: + return holder.queued + if here: + return False + return (await _granted(run_id)) is False + + +async def snapshot() -> dict[str, Any]: + """现在各类在跑、在等几个(系统信息、验收脚本看)。""" + if _in_memory(): + rows = [(r.klass, r.granted) for r in _mem.values() if _local(r.run_id)] + else: + from app.db.models_cluster import RunSlot + + async with SessionLocal() as session: + rows = list((await session.execute(select(RunSlot.klass, RunSlot.granted))).all()) + out = {k: {"running": 0, "waiting": 0} for k in CLASSES} # noqa: C420 + for klass, granted in rows: + out.setdefault(klass, {"running": 0, "waiting": 0})["running" if granted else "waiting"] += 1 + return {"policy": await policy(), "classes": out} + + +__all__ = ["CLASSES", "CLASS_LABEL", "ORIGIN_CLASS", "SETTING_KEY", "borrowable", "class_of", "defaults", "forget_policy", "grant", + "waiting", + "hold", "merged", "policy", "problem", "snapshot", "values", "wake_local"] diff --git a/backend/app/cluster/state.py b/backend/app/cluster/state.py new file mode 100644 index 00000000..5b4f58e2 --- /dev/null +++ b/backend/app/cluster/state.py @@ -0,0 +1,41 @@ +"""本进程手上的「别的进程谁还活着」:心跳每一拍刷新一次;碰到不认识的属主(刚起来的进程)当场再查一次库。""" +from __future__ import annotations + +import time +from collections.abc import Iterable + +from app.cluster import identity +from app.cluster.workers import Liveness, heartbeat_s, liveness + +_snapshot: tuple[float, Liveness] | None = None + + +def remember(live: Liveness) -> None: + global _snapshot + _snapshot = (time.monotonic(), live) + + +def forget() -> None: + global _snapshot + _snapshot = None + + +async def current(*, fresh: bool = False) -> Liveness: + """最近的一份(不超过一拍心跳),fresh 时当场查库。""" + if not fresh and _snapshot is not None and time.monotonic() - _snapshot[0] < heartbeat_s(): + return _snapshot[1] + live = await liveness() + remember(live) + return live + + +async def live_for(owners: Iterable[str | None]) -> Liveness: + """判断这几个属主用的一份:里面有手上这份不认识的别的进程,就当场查一次库(它可能刚起来)。""" + live = await current() + me = identity.owner() + if any(o and o != me and o not in live.others for o in owners): + live = await current(fresh=True) + return live + + +__all__ = ["current", "forget", "live_for", "remember"] diff --git a/backend/app/cluster/version.py b/backend/app/cluster/version.py new file mode 100644 index 00000000..be3df40b --- /dev/null +++ b/backend/app/cluster/version.py @@ -0,0 +1,225 @@ +"""这个进程是哪一版代码、要的是哪一版库结构;库里还有活着的、版本不同的进程时不许加入(不支持新旧版本混跑)。 + +**为什么要拦。** 升级时只停了一部分进程、或者新旧两批容器同时连着一个库:新版的进程启动时补列、补索引,旧版的进程不认识新的列和 +规矩(认领、租约、接手的说法),两边各按各的写同一批行。以前只在 README 里写「升级要先停掉全部进程」,现在代码里拦住。 + +- 代码版本 code_version:包版本 + 构建号(AGENTLAB_BUILD,Docker 镜像构建时写进去)或 git 提交 + 源码指纹(app/ 下全部 .py + 的内容,加上依赖锁文件 requirements.lock 的内容;Docker 镜像里也留着这个文件)。比的是源码指纹:同一份代码、同一套依赖一定相同, + 改了一行、升了一个依赖就不同,取不到 git、没有构建号时也区分得开。锁文件取不到(不是从仓库或镜像装的)时换成几个关键包的版本。 +- 库结构版本 schema_version:模型里各表各列(名字、类型、可空)加迁移清单的哈希。 + +怎么拦:PostgreSQL 时,补列之前先看一眼(preflight)——还有活着的、版本不同的进程就不启动;它的心跳停着(多半是强杀之后留下的、 +租约还没到期)就等它过期再起,心跳还在走就照实报错:哪几个进程、什么版本、要先停掉再升级。登记那一下再在事务级咨询锁里查一遍 +(cluster/workers.register),两个不同版本同时启动也只有先登记上的那一个起得来。SQLite 只有一个进程,不用看。 +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import subprocess +from functools import lru_cache +from pathlib import Path +from typing import Any + +from app.cluster import identity + +logger = logging.getLogger(__name__) + +_APP_DIR = Path(__file__).resolve().parents[1] + + +class VersionMismatch(RuntimeError): + """库里还有活着的、版本不同的进程。""" + + +#: 依赖锁文件(仓库里 backend/requirements.lock;Docker 镜像里 /app/backend/requirements.lock) +LOCK_FILE = _APP_DIR.parent / "requirements.lock" +#: 锁文件取不到时拿这几个包的版本代替(升级时最容易带出行为差别的那几个) +KEY_PACKAGES = ("fastapi", "pydantic", "sqlalchemy", "asyncpg", "psycopg", "langgraph", "langgraph-checkpoint", + "langgraph-checkpoint-postgres", "langchain-core", "starlette", "uvicorn") + + +def dependency_marker() -> bytes: + """依赖这一份进指纹的样子:锁文件的内容;取不到就是几个关键包的版本。""" + try: + return b"lock:" + LOCK_FILE.read_bytes() + except OSError: + pass + from importlib.metadata import PackageNotFoundError, version + + parts = [] + for name in KEY_PACKAGES: + try: + parts.append(f"{name}=={version(name)}") + except PackageNotFoundError: + parts.append(f"{name}==") + return ("packages:" + ";".join(parts)).encode() + + +@lru_cache(maxsize=1) +def source_fingerprint() -> str: + digest = hashlib.sha256() + for path in sorted(_APP_DIR.rglob("*.py")): + if "__pycache__" in path.parts: + continue + digest.update(str(path.relative_to(_APP_DIR)).encode()) + try: + digest.update(path.read_bytes()) + except OSError: + continue + # 依赖升级(锁文件变了)也算版本不同:代码一行没改、换了一版 SQLAlchemy / LangGraph 的进程同样不许混进来(复核 R-b13) + digest.update(b"\0deps\0" + dependency_marker()) + return digest.hexdigest()[:12] + + +def _package() -> str: + try: + from importlib.metadata import version + + return version("agentlab-backend") + except Exception: # noqa: BLE001 + return "0" + + +def _build() -> str: + if build := (os.environ.get("AGENTLAB_BUILD") or "").strip(): + return build[:40] + try: + out = subprocess.run(["git", "rev-parse", "--short=10", "HEAD"], cwd=str(_APP_DIR), capture_output=True, + text=True, timeout=3) + return out.stdout.strip() if out.returncode == 0 else "" + except (OSError, subprocess.SubprocessError): + return "" + + +@lru_cache(maxsize=1) +def code_version() -> str: + """包版本+构建号或 git 提交+源码指纹,例如 0.1.0+e6a3ee8f12+3f9a…(最后一段是比较用的)。""" + return "+".join(p for p in (_package(), _build(), source_fingerprint()) if p) + + +def fingerprint_of(version: str | None) -> str: + return (version or "").rsplit("+", 1)[-1] + + +@lru_cache(maxsize=1) +def schema_version() -> str: + from app.db import models # noqa: F401 模型都登记进 metadata + from app.db.base import _COLUMN_MIGRATIONS, _INDEX_MIGRATIONS, Base + + digest = hashlib.sha256() + for name in sorted(Base.metadata.tables): + table = Base.metadata.tables[name] + digest.update(name.encode()) + for col in table.columns: + digest.update(f"{col.name}:{col.type!r}:{col.nullable}".encode()) + digest.update(repr(_COLUMN_MIGRATIONS).encode() + repr(_INDEX_MIGRATIONS).encode()) + return digest.hexdigest()[:12] + + +def same(code: str | None, schema: str | None) -> bool: + return fingerprint_of(code) == fingerprint_of(code_version()) and (schema or "") == schema_version() + + +async def _peers(conn: Any) -> list[dict[str, Any]] | None: + """库里登记着的别的进程(id、版本、租约、心跳);表还不在(新库)交回 None。在一条连接上查(preflight 在建表之前)。""" + from sqlalchemy import inspect, text + + def has(sync_conn: Any) -> tuple[bool, set[str]]: + insp = inspect(sync_conn) + if not insp.has_table("cluster_workers"): + return False, set() + return True, {c["name"] for c in insp.get_columns("cluster_workers")} + + exists, cols = await conn.run_sync(has) + if not exists: + return None + extra = ", ".join(c for c in ("code_version", "schema_version") if c in cols) + rows = (await conn.execute(text(f"SELECT id, lease_until, heartbeat_at{', ' + extra if extra else ''} " + "FROM cluster_workers WHERE id <> :me"), {"me": identity.owner()})).mappings().all() + return [dict(r) for r in rows] + + +def _describe(peer: dict[str, Any], beating: set[str] | None = None) -> str: + code = peer.get("code_version") or "(升级前的版本,没有记版本)" + schema = peer.get("schema_version") or "未知" + state = "" + if beating is not None: + state = ",心跳在走" if peer["id"] in beating else ",心跳停着、等租约过期" + return f"{identity.describe(peer['id'])}(代码 {code},库结构 {schema}{state})" + + +async def preflight(*, observe_s: float | None = None) -> None: + """PostgreSQL 时、建表补列之前:库里还有活着的、版本不同的进程就不启动(见模块说明)。observe_s:看它心跳还在不在走的时长 + (缺省两拍心跳)。""" + from app.core.config import settings + + if settings.db_dialect != "postgresql": + return + from app.cluster.workers import heartbeat_s, lease_s + + observe = 2 * heartbeat_s() + 1.0 if observe_s is None else observe_s + loop = asyncio.get_running_loop() + started = loop.time() + first_beats: dict[str, Any] = {} + warned = False + while True: + mismatched = await _mismatched_alive() + if not mismatched: + return + for peer in mismatched: + first_beats.setdefault(peer["id"], peer.get("heartbeat_at")) + # 心跳还在走:真活着,照实报错。心跳停着的(强杀、断电留下的登记)等它们的租约过期。报错时列出全部活着的、版本不同的 + # 进程(复核 R-b13:只点名心跳在走的那个,人停掉它再起又会被另一个拦下) + beating = {p["id"] for p in mismatched if p.get("heartbeat_at") != first_beats.get(p["id"])} + waited = loop.time() - started + if beating or waited >= lease_s() * 2 + observe: + raise VersionMismatch(_message(mismatched, beating=beating)) + if waited >= observe and not warned: + warned = True + logger.warning("库里登记着 %d 个版本不同的进程,心跳停着(多半是强杀之后留下的):等它们的租约过期再启动", + len(mismatched)) + await asyncio.sleep(min(0.5, max(0.1, heartbeat_s() / 2))) + + +async def _mismatched_alive(now: Any = None) -> list[dict[str, Any]]: + """租约没过期、版本不同的别的进程。now 不给就按库的时钟(租约是按库的时钟写的,cluster/workers)。""" + from sqlalchemy import text + + from app.db.base import engine + + async with engine.connect() as conn: + peers = await _peers(conn) + if peers and now is None: + now = (await conn.execute(text("SELECT timezone('UTC', clock_timestamp())"))).scalar() + if not peers: + return [] + naive = _naive(now) + alive = [p for p in peers if p.get("lease_until") is not None and _naive(p["lease_until"]) > naive] + return [p for p in alive if not same(p.get("code_version"), p.get("schema_version"))] + + +def _naive(moment: Any) -> Any: + return moment.replace(tzinfo=None) if getattr(moment, "tzinfo", None) else moment + + +def _message(peers: list[dict[str, Any]], *, beating: set[str] | None = None) -> str: + listed = ";".join(_describe(p, beating) for p in peers) + return (f"库里还有 {len(peers)} 个版本不同的 AgentLab 进程活着:{listed}。本进程是代码 {code_version()}、库结构 " + f"{schema_version()}。不支持新旧版本同时运行:先停掉这些进程再启动新版本(升级步骤见 README「多进程」的「升级」)") + + +def check_rows(rows: list[Any], now: Any) -> None: + """登记那一下(cluster/workers.register,持着事务级咨询锁)再查一遍:别的活着的进程里有版本不同的就抛 VersionMismatch。 + now 是库的时钟。""" + naive = _naive(now) + bad = [{"id": r.id, "code_version": r.code_version, "schema_version": r.schema_version} for r in rows + if r.lease_until is not None and _naive(r.lease_until) > naive and r.id != identity.owner() + and not same(r.code_version, r.schema_version)] + if bad: + raise VersionMismatch(_message(bad)) + + +__all__ = ["VersionMismatch", "check_rows", "code_version", "preflight", "same", "schema_version", "source_fingerprint"] diff --git a/backend/app/cluster/workers.py b/backend/app/cluster/workers.py new file mode 100644 index 00000000..b71b0a2b --- /dev/null +++ b/backend/app/cluster/workers.py @@ -0,0 +1,187 @@ +"""工作进程的登记和租约(cluster_workers),以及「这一行的属主还在不在」。 + +- register / renew / unregister:启动时登记、隔一拍心跳续租约、平稳关停时删掉自己那一行(别的进程马上就知道它不在了, + 不必等租约到期)。 +- liveness():现在哪些别的进程还活着(租约过期不到一个租约时长:再晚才算不在了,见 cluster/fence.py)。各种「启动整理」「接手」都只碰属主不在了的行: + Liveness.alive(owner, local=…) —— 属主是别的进程:看它的租约;属主是自己:看这一行在本进程里是不是真的在跑(local), + 同一个进程里前一次起停留下的(测试里一个进程起停多次服务)照样算不在了;没有属主(升级前的行)算不在了。 + +租约用库的时钟(复核 R-b7):PostgreSQL 上续租、登记写的是 lease_until = 库的 now() + 租约,判活比的也是库的 now()——租约由 +各台机器续、由别的机器判,用同一只钟,机器之间墙钟差多少都不起作用(cluster/leader 的当班租约同理)。SQLite 只有一个进程,用本机 +的钟。别处按「停了两个租约时长」判的时间(待办、动作的 updated_at)仍是各进程按本机时钟写的:各台机器还是要对时(NTP)。 +""" +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any + +from sqlalchemy import DateTime, delete, func, select, update + +from app.cluster import identity +from app.core.config import settings +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + + +#: 登记时全站串行用的事务级咨询锁的号 +_REGISTER_KEY = 0x4167576F726B6572 # "AgWorker" + + +def lease_s() -> float: + """租约多长(秒):至少是心跳间隔的三倍——库一时慢了、一拍没续上,不算进程没了。""" + return max(float(settings.worker_lease_s), 3.0 * float(settings.worker_heartbeat_s)) + + +def heartbeat_s() -> float: + return float(settings.worker_heartbeat_s) + + +def _postgres(session: Any) -> bool: + return session.get_bind().dialect.name == "postgresql" + + +def clock(session: Any) -> Any: + """判租约用的「现在」(写进 UTCDateTime 列、跟它比的那种:UTC、不带时区)。PostgreSQL 是库的时钟(SQL 表达式),SQLite + 是本机的(只有一个进程)。""" + if _postgres(session): + return func.timezone("UTC", func.clock_timestamp(), type_=DateTime()) + return utcnow() + + +async def db_now(session: Any) -> datetime: + """库的时钟此刻的读数(带时区的 UTC):要在 Python 里拿它算、比的时候用(登记、当班)。""" + if _postgres(session): + value = await session.scalar(select(func.timezone("UTC", func.clock_timestamp()))) + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) + return utcnow() + + +@dataclass(frozen=True) +class Liveness: + """某一刻还活着的别的进程。""" + + others: frozenset[str] = frozenset() + #: 属主是自己的行一律算活着(当班进程隔一会儿接手时用:自己刚建、还没交给后台任务的行不能当成没人管)。启动时的那一次 + #: 不信:那时本进程什么都还没在跑,属主是自己的只会是同一个进程里前一次起停留下的 + trust_mine: bool = False + + def alive(self, owner: str | None, *, local: bool = False) -> bool: + """这一行的属主还在不在。local:这一行在本进程里是不是真的在跑(属主是自己时只看它)。""" + if not owner: + return False + if owner == identity.owner(): + return local or self.trust_mine + return owner in self.others + + def trusting(self) -> Liveness: + return Liveness(self.others, trust_mine=True) + + +async def liveness(now: datetime | None = None) -> Liveness: + """现在还活着(租约没过期)的别的进程。now 只给测试用(不给就是库的时钟)。""" + from app.db.models_cluster import ClusterWorker + + me = identity.owner() + # 租约过期之后再多等一个租约时长才算不在了:卡住了但没死的进程(SIGSTOP、虚拟机暂停、库一时连不上)自己在一个租约时长时就 + # 停手(cluster/fence.py),两边中间留着一段,不会它还在写、这边已经接手 + async with SessionLocal() as session: + floor = (now if now is not None else clock(session)) - timedelta(seconds=lease_s()) + ids = (await session.execute(select(ClusterWorker.id).where( + ClusterWorker.lease_until > floor, ClusterWorker.id != me))).scalars().all() + return Liveness(frozenset(ids)) + + +async def alive(owner: str | None, *, local: bool = False) -> bool: + """单独问一个属主(停止时找收件人用)。""" + if not owner or owner == identity.owner(): + return bool(owner) and local + return (await liveness()).alive(owner) + + +def _info() -> dict[str, Any]: + return {"data_dir": str(settings.data_dir), "workers": settings.workers} + + +async def register(*, exclusive: bool = False) -> None: + """登记自己(已有就续上)。exclusive:库里别的进程一律当成已经不在了,删掉它们的登记(SQLite:只有一个进程, + 守卫已经确认没有别的进程在用这个数据目录,见 cluster/guard.py)。""" + from app.db.models_cluster import ClusterWorker + + from app.cluster import version + + me = identity.owner() + async with SessionLocal() as session: + now = await db_now(session) + if exclusive: + await session.execute(delete(ClusterWorker).where(ClusterWorker.id != me)) + elif session.get_bind().dialect.name == "postgresql": + # 查「还有没有活着的、版本不同的进程」和登记在同一个事务里、全站串行:两个不同版本同时启动只有先登记上的起得来 + from sqlalchemy import text + + await session.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": _REGISTER_KEY}) + now = await db_now(session) # 等锁可能等了一会儿:按拿到锁之后的库时钟 + others = (await session.execute(select(ClusterWorker).where(ClusterWorker.id != me))).scalars().all() + version.check_rows(list(others), now) + row = await session.get(ClusterWorker, me) + if row is None: + session.add(ClusterWorker(id=me, host=identity.host(), pid=os.getpid(), started_at=now, heartbeat_at=now, + lease_until=now + timedelta(seconds=lease_s()), info=_info(), + code_version=version.code_version(), schema_version=version.schema_version())) + else: + row.heartbeat_at, row.lease_until = now, now + timedelta(seconds=lease_s()) + await session.commit() + + +async def renew() -> bool: + """续一拍租约(走连接池的那一种;服务里续租在专用连接上,见 cluster/service.renew)。自己那一行不见了、或者按库的时钟已经 + 过期了返回 False(不续)。""" + from app.db.models_cluster import ClusterWorker + + async with SessionLocal() as session: + now = clock(session) + done = await session.execute(update(ClusterWorker).where( + ClusterWorker.id == identity.owner(), ClusterWorker.lease_until >= now).values( + heartbeat_at=now, lease_until=now + timedelta(seconds=lease_s())) + .execution_options(synchronize_session=False)) + await session.commit() + return done.rowcount == 1 + + +async def unregister() -> None: + from app.db.models_cluster import ClusterWorker + + async with SessionLocal() as session: + await session.execute(delete(ClusterWorker).where(ClusterWorker.id == identity.owner())) + await session.commit() + + +async def prune(now: datetime | None = None) -> int: + """删掉早就过期的登记(当班进程做):租约过期超过十倍租约时长的。它们认领的行早已被接手。""" + from app.db.models_cluster import ClusterWorker + + async with SessionLocal() as session: + floor = (now if now is not None else clock(session)) - timedelta(seconds=10 * lease_s()) + done = await session.execute(delete(ClusterWorker).where(ClusterWorker.lease_until < floor)) + await session.commit() + return int(done.rowcount or 0) + + +async def listing() -> list[dict[str, Any]]: + """现在登记着的进程(系统信息页、验收脚本看)。""" + from app.db.models_cluster import ClusterWorker + + async with SessionLocal() as session: + now = await db_now(session) + rows = (await session.execute(select(ClusterWorker).order_by(ClusterWorker.started_at, ClusterWorker.id))).scalars().all() + return [{"id": r.id, "host": r.host, "pid": r.pid, "alive": r.lease_until > now, "me": r.id == identity.owner(), + "code_version": r.code_version, "schema_version": r.schema_version, + "started_at": r.started_at.isoformat() if r.started_at else None, + "heartbeat_at": r.heartbeat_at.isoformat() if r.heartbeat_at else None} for r in rows] + + +__all__ = ["Liveness", "alive", "clock", "db_now", "heartbeat_s", "lease_s", "listing", "liveness", "prune", "register", + "renew", "unregister"] diff --git a/backend/app/core/artifact_store.py b/backend/app/core/artifact_store.py index 0bbc8d7f..3837f934 100644 --- a/backend/app/core/artifact_store.py +++ b/backend/app/core/artifact_store.py @@ -5,6 +5,7 @@ import json from pathlib import Path from typing import Any +from app.core import atomic from app.core.config import settings # -------------------------------------------------------------------------- @@ -62,10 +63,9 @@ async def put_json( artifact_id = content_hash(text) path = _path_of(artifact_id) if not path.exists(): - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(".tmp") - tmp.write_text(text, encoding="utf-8") - tmp.rename(path) # 原子落盘,写一半崩了不会留下半个工件 + # 原子落盘(core/atomic):写一半崩了不会留下半个工件;临时文件名带随机后缀,两个进程同时写同一份内容 + # 各写各的,不会一个把另一个写到一半的截断(以前共用 .tmp,后到的会把先到的半成品改名成正式文件) + atomic.write_text(path, text) async with SessionLocal() as session: existing = await session.get(Artifact, (artifact_id, run_id or "")) diff --git a/backend/app/core/atomic.py b/backend/app/core/atomic.py new file mode 100644 index 00000000..8a3eb8e4 --- /dev/null +++ b/backend/app/core/atomic.py @@ -0,0 +1,139 @@ +"""数据目录里的文件一律先写临时文件(同目录、名字带随机后缀)再改名。 + +为什么:读的一方(同一台机器上的别的进程、共享存储另一头的机器)要么看到旧的完整文件,要么看到新的完整文件, +看不到写了一半的;进程写到一半没了,留下的是一个没人认的临时文件,不是半个正式文件。临时文件名带随机后缀: +两个进程同时写同一个文件(内容寻址的工件撞了同一份内容)各写各的,不会一个把另一个写到一半的截断。 + +- write_bytes / write_text:写完 os.replace 到目标(目标已有就替换,沿用它的权限); +- append_bytes:追加写,不走改名(理由见函数说明); +- create_bytes / create_text:只在目标还没有时建(os.link,已有就不动、返回 False)——给只该生成一次的东西用 + (加密密钥、签名私钥):两个进程同时第一次启动,后到的读先到的那一份,不会各拿一把。 + +上传表格的版本库、原件存档、记录表另有自己的写法(data/raw_store、data/table_versions,同样先写临时文件),不走这里。 +""" + +from __future__ import annotations + +import contextlib +import os +import secrets +from pathlib import Path + +#: 临时文件名里的记号:.<原名>.tmp-<随机>(同 data/raw_store.TMP_MARK 的写法) +TMP_MARK = ".tmp-" + + +def _tmp_for(path: Path) -> Path: + return path.parent / f".{path.name}{TMP_MARK}{secrets.token_hex(6)}" + + +def _write_tmp(path: Path, data: bytes, *, mode: int | None, fsync: bool) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = _tmp_for(path) + # 没给 mode 时照普通新建文件(0666 再受 umask 约束);给了的(密钥 0600)从建出来那一刻起就是那个权限,再 chmod + # 一次压过 umask + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666 if mode is None else 0o600) + try: + with os.fdopen(fd, "wb") as fh: + fh.write(data) + fh.flush() + if fsync: + os.fsync(fh.fileno()) + if mode is not None: + os.chmod(tmp, mode) + except BaseException: + with contextlib.suppress(OSError): + tmp.unlink() + raise + return tmp + + +def fsync_dir(folder: Path) -> None: + """把目录项(新建、改名)落盘。有的平台、文件系统打不开目录或不支持,就算了(同 data/raw_store.fsync_dir)。""" + try: + fd = os.open(folder, os.O_RDONLY) + except OSError: + return + try: + os.fsync(fd) + except OSError: + pass + finally: + os.close(fd) + + +def write_bytes(path: Path, data: bytes, *, mode: int | None = None, fsync: bool = False) -> None: + """原子地写(替换)path。没给 mode、原来已有这个文件时沿用它的权限(改名换掉的是文件本身,权限不跟着走)。""" + if mode is None: + with contextlib.suppress(OSError): + mode = path.stat().st_mode & 0o7777 + tmp = _write_tmp(path, data, mode=mode, fsync=fsync) + try: + os.replace(tmp, path) + except BaseException: + with contextlib.suppress(OSError): + tmp.unlink() + raise + + +def write_text(path: Path, text: str, *, encoding: str = "utf-8", mode: int | None = None, + fsync: bool = False) -> None: + write_bytes(path, text.encode(encoding), mode=mode, fsync=fsync) + + +def create_bytes(path: Path, data: bytes, *, mode: int | None = None, fsync: bool = True) -> bool: + """path 还没有时原子地建出来,返回 True;已经有了(别人先建了)什么都不动,返回 False。建成后把目录项也落盘。 + + 硬链接不了的文件系统(exFAT、一部分 SMB / FUSE 挂载报 EPERM、ENOTSUP、EXDEV)退回 O_CREAT | O_EXCL 直接建目标文件: + 仍然只有一个进程建得成,但写的那一小会儿别的进程可能读到不完整的内容——只在头一次生成时有这个窗口。""" + tmp = _write_tmp(path, data, mode=mode, fsync=fsync) + try: + os.link(tmp, path) + except FileExistsError: + return False + except OSError: + return _create_directly(path, data, mode=mode, fsync=fsync) + finally: + with contextlib.suppress(OSError): + tmp.unlink() + fsync_dir(path.parent) + return True + + +def _create_directly(path: Path, data: bytes, *, mode: int | None, fsync: bool) -> bool: + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666 if mode is None else 0o600) + except FileExistsError: + return False + try: + with os.fdopen(fd, "wb") as fh: + fh.write(data) + fh.flush() + if fsync: + os.fsync(fh.fileno()) + if mode is not None: + os.chmod(path, mode) + except BaseException: + with contextlib.suppress(OSError): + path.unlink() + raise + fsync_dir(path.parent) + return True + + +def append_bytes(path: Path, data: bytes, *, fsync: bool = False) -> None: + """追加写(O_APPEND):不换文件(inode、权限、别人手里开着的句柄都不变),原有内容一个字节都不动。 + + 不走「临时文件再改名」:那得把整份读出来重写,文件越大越慢,还会换掉 inode 和权限。代价是追加到一半时读的一方可能 + 看到半截新内容——它看到的永远是「原有的整份 + 新内容的一个前缀」,原有的内容不会丢。""" + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o666) + with os.fdopen(fd, "ab") as fh: + fh.write(data) + fh.flush() + if fsync: + os.fsync(fh.fileno()) + + +def create_text(path: Path, text: str, *, encoding: str = "utf-8", mode: int | None = None) -> bool: + return create_bytes(path, text.encode(encoding), mode=mode) diff --git a/backend/app/core/background.py b/backend/app/core/background.py new file mode 100644 index 00000000..2f47ea7c --- /dev/null +++ b/backend/app/core/background.py @@ -0,0 +1,28 @@ +"""进程里「发出去就不管」的后台任务(推送通知、评测收尾的清理):留着引用免得被回收,测试和关停时等它们做完。""" + +from __future__ import annotations + +import asyncio +from typing import Any + + +async def drain(tasks: set[asyncio.Future[Any]]) -> None: + """等 tasks 里本事件循环上的任务都做完。 + + 已经做完的直接从集合里摘掉,不等它们自己的 discard 回调:done 回调排在任务所在的事件循环上,那个循环要是已经不转了(测试里 + 每个测试一个事件循环,前一个测试末尾刚做完的推送)回调就永远不跑、任务一直留在集合里。以前的写法 `while tasks: await + gather(*tasks)` 碰上这种集合,gather 拿到的全是做完了的、立刻返回,await 不让出事件循环,就在这里空转——CPU 打满、整个 + 进程卡死(PostgreSQL 全量里 test_personal_apps 收尾时出现过)。别的事件循环上还没做完的等不到(不在这个循环上),不等; + 那个循环已经关了的一并摘掉。""" + loop = asyncio.get_running_loop() + while True: + for task in list(tasks): + if task.done() or (task.get_loop() is not loop and task.get_loop().is_closed()): + tasks.discard(task) + mine = [task for task in tasks if task.get_loop() is loop] + if not mine: + return + await asyncio.gather(*mine, return_exceptions=True) + + +__all__ = ["drain"] diff --git a/backend/app/core/bus.py b/backend/app/core/bus.py index a94713a1..732e21d1 100644 --- a/backend/app/core/bus.py +++ b/backend/app/core/bus.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import contextlib from collections import defaultdict +from collections.abc import Callable from typing import AsyncIterator from app.core.events import RunEventModel @@ -23,12 +24,16 @@ class EventBus: 只负责实时分发;历史事件由 DB 承担,订阅者接入时先读库补齐再接实时流, 这样刷新页面或中途接入都能看到完整时间线。 + + 多进程(app/cluster/relay.py):本进程发布的每一条也交给 tap(转发器据此给别的进程发通知、给在看的进程转流式增量); + 别的进程执行的运行,转发器从库里按 seq 读出来再在本进程发布一遍(relayed=True,不再交给 tap,免得来回转)。 """ def __init__(self) -> None: self._subs: dict[str, set[_Subscriber]] = defaultdict(set) self._seq: dict[str, int] = defaultdict(int) self._lock = asyncio.Lock() + self.tap: Callable[[RunEventModel], None] | None = None def next_seq(self, run_id: str) -> int: self._seq[run_id] += 1 @@ -38,7 +43,12 @@ class EventBus: """恢复一个已存在的 run 时,把序号接上,保证前端排序不乱。""" self._seq[run_id] = max(self._seq[run_id], value) - async def publish(self, event: RunEventModel) -> None: + async def publish(self, event: RunEventModel, *, relayed: bool = False) -> None: + if not relayed and self.tap is not None: + try: + self.tap(event) + except Exception: # noqa: BLE001 - 转发出错不能拦住本进程的推送 + pass for sub in list(self._subs.get(event.run_id, ())): try: sub.queue.put_nowait(event) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 955e7242..3fce235d 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -31,6 +31,10 @@ def sanitize_session(session_id: str | None) -> str: return safe[:_SESSION_MAX] or "default" +#: 数据目录里的标记文件(Settings.ensure_dirs 建,deploy/deploy.sh 认) +DATA_DIR_MARKER = ".agentlab-data" + + class Settings(BaseSettings): """全局配置。所有项都可以用 AGENTLAB_ 前缀的环境变量覆盖。""" @@ -47,6 +51,36 @@ class Settings(BaseSettings): # --- 存储 --- data_dir: Path = PROJECT_DIR / "data" + # 应用自己的库(工作流、运行、事件、审批……)。不设:数据目录里的 SQLite(agentlab.db,断点在 checkpoints.db), + # 单机单进程。设成 postgresql+asyncpg://用户:密码@主机:端口/库名(postgresql:// 也认)时整个应用库用 PostgreSQL, + # LangGraph 的断点也放进同一个库;checkpoint_url 另给就放那里。数据目录照旧要有:工件、上传表格、沙箱等仍是文件 + database_url: str = "" + checkpoint_url: str = "" + # 直连地址(可选,写法同 database_url):普通查询走 database_url(可以是 PgBouncer),依赖会话状态的连接走这里——进程间通知的 + # LISTEN 那一条、跨进程锁(会话级咨询锁)、断点库(预备语句、建表时的会话锁)。不设就全用 database_url + database_direct_url: str = "" + # database_url 是 PgBouncer 的事务模式(AGENTLAB_DB_PGBOUNCER=1):应用库不用客户端连接池(交给 PgBouncer)、关掉 asyncpg 和 + # SQLAlchemy 的预备语句缓存、预备语句用不会撞名的名字。这时必须另给 database_direct_url(事务模式下 LISTEN、会话级咨询锁不成立), + # 启动时检查 + db_pgbouncer: bool = False + # PostgreSQL 的连接池(SQLite 不用这几项)。pool_size=0 表示不用连接池、每次新建连接(前面有 PgBouncer 时用, + # 测试里也这么用:每个测试一个事件循环,asyncpg 的连接不能跨循环) + db_pool_size: int = 10 + db_max_overflow: int = 20 + db_pool_timeout: int = 30 + db_pool_recycle: int = 1800 + # 启动时库还连不上(和 PostgreSQL 容器一起起、库还在初始化)最多等多少秒,按退避重试、照实打日志;过了再报错退出。 + # 密码不对、库不存在这种等也没用的不等 + db_startup_timeout: int = 90 + + # --- 多进程(app/cluster,docs/design/orchestration-round §5 W2-b)--- + # 几个工作进程(scripts/prod.sh、Docker 的入口按它给 uvicorn --workers)。只有应用库是 PostgreSQL 时才能大于 1: + # SQLite 只支持单进程,启动时拒绝(app/cluster/guard.py) + workers: int = 1 + # 每个工作进程在库里登记一行:隔 worker_heartbeat_s 秒续一次租约,租约长 worker_lease_s 秒。进程没了(强杀、断电) + # 租约到期后,别的进程接手它认领的运行、问数据、后台任务(各按各的规矩)。租约要比心跳长几倍:库一时慢了不算进程没了 + worker_lease_s: float = 30.0 + worker_heartbeat_s: float = 5.0 # --- 前端构建产物(生产部署用)--- # 设了就由服务端一并托管页面(见 app/web.py),页面和接口同源、只起一个进程。 @@ -115,6 +149,8 @@ class Settings(BaseSettings): # (检查脚本的 --stacks、沙箱)必须关:库里的定时任务会照常到点发起运行、发出 webhook, # 拷贝里的任务和原库的是同一批,等于同一件事做两遍 scheduler: bool = True + # 调度循环一拍的间隔(秒):定时认领、事件派发、流程的时限和核实都按它查一次。缺省 15;验收脚本(多进程端到端)调小 + scheduler_tick_s: float = 15.0 # --- 工具 --- http_tool_timeout: int = 20 @@ -195,6 +231,37 @@ class Settings(BaseSettings): return True return value + @field_validator("database_url", "checkpoint_url", "database_direct_url", mode="before") + @classmethod + def _valid_postgres_url(cls, value: object, info: Any) -> object: + # 写错了启动时就报出来,别等到第一次查库。只认 PostgreSQL:SQLite 就是不设 + if value is None or (isinstance(value, str) and not value.strip()): + return "" + if not isinstance(value, str): + return value + from app.db.urls import normalize_database_url, normalize_checkpoint_url + + name = f"AGENTLAB_{(info.field_name or '').upper()}" + normalize = normalize_checkpoint_url if info.field_name == "checkpoint_url" else normalize_database_url + try: + return normalize(value.strip()) + except ValueError as e: + raise ValueError(f"{name}:{e}") from None + + @field_validator("workers") + @classmethod + def _valid_workers(cls, value: int) -> int: + if value < 1: + raise ValueError("AGENTLAB_WORKERS 至少是 1") + return value + + @field_validator("worker_heartbeat_s", "worker_lease_s") + @classmethod + def _positive_seconds(cls, value: float, info: Any) -> float: + if value <= 0: + raise ValueError(f"AGENTLAB_{(info.field_name or '').upper()} 要大于 0") + return value + @field_validator("web_dist", mode="before") @classmethod def _blank_is_unset(cls, value: object) -> object: @@ -223,13 +290,36 @@ class Settings(BaseSettings): @property def db_url(self) -> str: - return f"sqlite+aiosqlite:///{self.db_path}" + """应用库的 SQLAlchemy 地址:设了 database_url 用它(已规范成 postgresql+asyncpg://),否则数据目录里的 SQLite。""" + return self.database_url or f"sqlite+aiosqlite:///{self.db_path}" + + @property + def db_dialect(self) -> str: + """"sqlite" 或 "postgresql"。""" + return "postgresql" if self.database_url else "sqlite" @property def checkpoint_path(self) -> Path: - """LangGraph checkpointer 用独立的库文件,避免和业务表抢写锁。""" + """LangGraph checkpointer 用独立的库文件,避免和业务表抢写锁。只在 SQLite 时用。""" return self.data_dir / "checkpoints.db" + @property + def session_db_url(self) -> str: + """依赖会话状态的连接(LISTEN、会话级咨询锁、断点库)用的应用库地址:设了直连地址用它,否则同 database_url。""" + return self.database_direct_url or self.database_url + + @property + def checkpoint_conninfo(self) -> str | None: + """PostgreSQL 断点库的 libpq 连接串(psycopg 用):checkpoint_url 优先,否则由直连地址(没设就是 database_url)换算。 + SQLite 时 None。断点库用预备语句、建表时取会话级的锁,所以缺省走直连。""" + if self.checkpoint_url: + return self.checkpoint_url + if self.database_url: + from app.db.urls import libpq_conninfo + + return libpq_conninfo(self.session_db_url) + return None + @property def workspace_dir(self) -> Path: """沙箱与文件工具的可写根目录,一切文件访问都被限制在这里面。""" @@ -257,6 +347,16 @@ class Settings(BaseSettings): def ensure_dirs(self) -> None: for d in (self.data_dir, self.workspace_dir, self.uploads_dir): d.mkdir(parents=True, exist_ok=True) + # 认得出「这是 AgentLab 的数据目录」的标记:应用库放在 PostgreSQL 时这里没有 agentlab.db,设了 AGENTLAB_SECRET_KEY + # 时也没有 .secret_key。deploy.sh 改属主前按它认,免得 DATA_DIR 写错把别人的目录 chown 了。只建一次,不覆盖 + marker = self.data_dir / DATA_DIR_MARKER + if not marker.exists(): + from app.core import atomic + + try: + atomic.create_text(marker, "AgentLab 的数据目录(deploy.sh 按这个文件认,别删)\n") + except OSError: + pass @lru_cache diff --git a/backend/app/core/crypto.py b/backend/app/core/crypto.py index abca0b07..1aaffabc 100644 --- a/backend/app/core/crypto.py +++ b/backend/app/core/crypto.py @@ -6,6 +6,7 @@ import os from cryptography.fernet import Fernet, InvalidToken +from app.core import atomic from app.core.config import settings _PREFIX = "enc::" @@ -16,12 +17,11 @@ def _load_key() -> bytes: raw = settings.secret_key if not raw: key_file = settings.data_dir / ".secret_key" - if key_file.exists(): - raw = key_file.read_text().strip() - else: - raw = base64.urlsafe_b64encode(os.urandom(32)).decode() - key_file.write_text(raw) - key_file.chmod(0o600) + if not key_file.exists(): + # 只建一次(core/atomic.create_text):两个进程同时第一次启动,后到的读先到的那一把,不会各拿一把、 + # 各自加密的东西对方解不开 + atomic.create_text(key_file, base64.urlsafe_b64encode(os.urandom(32)).decode(), mode=0o600) + raw = key_file.read_text().strip() # 任意字符串都归一化成 32 字节 Fernet key digest = hashlib.sha256(raw.encode()).digest() return base64.urlsafe_b64encode(digest) diff --git a/backend/app/core/errors.py b/backend/app/core/errors.py index 94cc3966..01355678 100644 --- a/backend/app/core/errors.py +++ b/backend/app/core/errors.py @@ -35,8 +35,9 @@ _BUG = {"TypeError", "AttributeError", "KeyError", "NameError", "IndexError", _UNREACHABLE = ("connection refused", "nodename nor servname", "name or service not known", "all connection attempts failed", "network is unreachable") -#: SQLAlchemy 包过的驱动异常开头是「(sqlite3.OperationalError) 」 -_WRAPPED = re.compile(r"^\([\w.]+\)\s*") +#: SQLAlchemy 包过的驱动异常开头是「(sqlite3.OperationalError) 」;asyncpg(PostgreSQL)的后面还跟着 +#: 「: 」,一起去掉(直接拿 e.orig 的也只剩后一段,同样去掉) +_WRAPPED = re.compile(r"^(?:\([\w.]+\)\s*)?(?::\s*)?") # 内部符号:`a.b.._run()`、``、模块路径。对使用者没有意义 _LOCALS = re.compile(r"[\w.]*\.?[\w.]*(\(\))?") _REPR = re.compile(r"<(?:function|bound method|coroutine|object) [^>]*>") diff --git a/backend/app/core/job_runtime.py b/backend/app/core/job_runtime.py index f1f55b10..15775ee7 100644 --- a/backend/app/core/job_runtime.py +++ b/backend/app/core/job_runtime.py @@ -27,13 +27,16 @@ 表记 cancelled。**不自动续跑**:复制出来的数据目录(检查脚本、验收副本)启动时不能 自己去调模型、查业务库,界面让人点「重试」接着做。 -进程内登记、进程内的队:服务端是单进程部署,多进程部署时要换成库里的锁。 +**多进程**(docs/design/orchestration-round §5 W2-b,app/cluster):任务表的 owner 是认领它的进程(建任务时缺省就是建它的 +进程),执行在那个进程里;同源队记在库里(lane_tickets),几个进程共用一条;启动整理只接手属主不在了的任务 +(interrupt_on_startup,多进程时当班进程隔一会儿再来一次);停止落在别的进程上时给属主发停止请求(JobManager.stop)。 """ from __future__ import annotations import asyncio import contextlib import logging +import weakref from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from typing import Any @@ -75,6 +78,11 @@ class SerialWriter: self._lock = asyncio.Lock() async def __call__(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + from app.cluster import fence + + # 本进程续不上租约了(cluster/fence.py):这个任务可能已经被别的进程接手,不再写 + fence.check() + async def go() -> Any: async with self._lock: async with SessionLocal() as session: @@ -123,117 +131,293 @@ def lane_label(kind: str) -> str: class LaneTicket: - """在一个源的队里的一个位置。kind 是占着它的是什么(登记过的一种,见 register_lane_kind);job_id 是任务的(同步写 - 操作没有)。轮到它时 granted 置位。""" + """在一个源的队里的一个位置(库里 lane_tickets 的一行)。kind 是占着它的是什么(登记过的一种,见 register_lane_kind); + job_id 是任务的(同步写操作没有)。轮到它时 granted 置位。""" - def __init__(self, lanes: SourceLanes, source_id: str, kind: str, job_id: str | None, + def __init__(self, lanes: SourceLanes, row_id: int, source_id: str, kind: str, job_id: str | None, loop: asyncio.AbstractEventLoop) -> None: self._lanes = lanes + self.id = row_id self.source_id = source_id self.kind = kind self.job_id = job_id self.loop = loop self.granted = asyncio.Event() + self.woke = asyncio.Event() self.released = False async def wait(self) -> None: - """等轮到自己。""" - await self.granted.wait() + """等轮到自己:本进程里有人出队时叫醒,PostgreSQL 上别的进程出队时 NOTIFY 叫醒,另隔一会儿自己回库里看一次。""" + from app.core.config import settings - def release(self) -> None: + poll = LANE_POLL_S.get(settings.db_dialect, 5.0) + while not self.granted.is_set(): + self.woke.clear() + got = await self._lanes._advance(self.source_id, me=self) + if got or self.granted.is_set(): + self.granted.set() + return + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self.woke.wait(), timeout=poll) + + async def release(self) -> None: """出队(轮到了、没轮到都行,重复调用无害):排在后面的第一个接着开始。""" - self._lanes._release(self) + await self._lanes._release(self) + + +class LaneHolder: + """队里的一位(holders / LaneBusy 里给出的样子):kind、job_id、属主。""" + + def __init__(self, kind: str, job_id: str | None, owner: str | None) -> None: + self.kind = kind + self.job_id = job_id + self.owner = owner + + def __repr__(self) -> str: + return f"LaneHolder({self.kind!r}, {self.job_id!r})" class LaneBusy(Exception): - """不阻塞地试占时这个源的队里已经有人:holder 是排在最前面(正在做)的那一个。""" + """不阻塞地试占时这个源的队里已经有人:holder 是排在最前面(正在做)的那一位。""" - def __init__(self, holder: LaneTicket) -> None: + def __init__(self, holder: LaneHolder) -> None: super().__init__(holder.kind) self.holder = holder +#: 等轮到自己时隔多久回库里看一次(秒):PostgreSQL 上别的进程出队靠 NOTIFY 叫醒,这是兜底;SQLite 只有本进程 +LANE_POLL_S = {"postgresql": 1.0, "sqlite": 5.0} + + class SourceLanes: - """每个数据源一条先来先到的队。""" + """每个数据源一条先来先到的队,记在库里(lane_tickets),几个进程共用一条(docs/design/orchestration-round §5 W2-b)。 + + 排队 = 插一行(id 自增,先来的号小);轮到谁 = 这个源上没有已经轮到的、活着的那一行时,号最小的那一行改成「轮到了」。改的那 + 一下按源串行:PostgreSQL 上在事务里取这个源的咨询锁,SQLite 只有一个进程、本进程的锁就够。属主不在了的行(进程没了;本进程 + 里登记已经没了——测试里事件循环关了留下的)改的时候顺手删掉,后面的接着轮到。""" def __init__(self) -> None: - self._queues: dict[str, list[LaneTicket]] = {} + self._tickets: dict[int, LaneTicket] = {} + #: 本进程里正在排进某个源的队(插了行、还没登记好 LaneTicket)的个数:这期间属主是本进程、登记里还没有的那一行不能 + #: 当成没人占着删掉 + self._entering: dict[str, int] = {} + self._locks: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock] = weakref.WeakKeyDictionary() + self._listening = False - def _queue(self, source_id: str) -> list[LaneTicket]: - """这个源的队。别的事件循环留下的(测试每个用例一个循环)去掉:那个循环里没人会再出队了。""" - queue = self._queues.get(source_id) or [] + def _local_alive(self, row_id: int) -> bool: + ticket = self._tickets.get(row_id) + if ticket is None or ticket.released: + return False + try: + return ticket.loop is asyncio.get_running_loop() and not ticket.loop.is_closed() + except RuntimeError: + return False + + def _lock(self) -> asyncio.Lock: loop = asyncio.get_running_loop() - alive = [t for t in queue if t.loop is loop and not t.released] - if alive: - self._queues[source_id] = alive - alive[0].granted.set() - else: - self._queues.pop(source_id, None) - return alive + lock = self._locks.get(loop) + if lock is None: + lock = self._locks[loop] = asyncio.Lock() + return lock - def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: + def _listen(self) -> None: + if self._listening: + return + from app.cluster.signal import signals + + signals.on("lane", lambda parts: self._wake(parts[0] if parts else None)) + self._listening = True + + def _wake(self, source_id: str | None) -> None: + for ticket in list(self._tickets.values()): + if (source_id is None or ticket.source_id == source_id) and not ticket.loop.is_closed(): + ticket.loop.call_soon_threadsafe(ticket.woke.set) + + async def _locked(self, session: AsyncSession, source_id: str) -> None: + if session.get_bind().dialect.name == "postgresql": + from sqlalchemy import text + + from app.cluster.mutex import advisory_key + + await session.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": advisory_key(f"lane:{source_id}")}) + + async def _dead_owners(self, source_id: str) -> frozenset[str]: + """拿这个源的锁之前备好:队里别的进程的属主哪些确实不在了(手上那份不认识的当场再查一次登记)。持着咨询锁时不另取连接 + 查库(复核:连接池用尽时会卡到取连接超时);拿锁之后才出现的属主不在这里面,这一次当活着,下一次再说。""" + from app.cluster import identity, state + from app.db.models_cluster import LaneTicket as Row + + async with SessionLocal() as session: + owners = set((await session.execute(select(Row.owner).where(Row.source_id == source_id).distinct())) + .scalars()) + me = identity.owner() + others = {o for o in owners if o and o != me} + if not others: + return frozenset() + live = await state.live_for(others) + return frozenset(o for o in others if o not in live.others) + + async def _rows(self, session: AsyncSession, source_id: str, dead_owners: frozenset[str]) -> list[Any]: + """这个源队里活着的各行(号小的在前);属主不在了的删掉(不提交)。dead_owners 是拿锁之前备好的那一份:属主是别的进程的 + 只认它;属主是自己的看本进程里登记着没有(正有人排进来时一律算活着)。""" + from app.cluster import identity + from app.db.models_cluster import LaneTicket as Row + + rows = list((await session.execute(select(Row).where(Row.source_id == source_id).order_by(Row.id))).scalars()) + me = identity.owner() + entering = bool(self._entering.get(source_id)) + + def alive(row: Any) -> bool: + if not row.owner: + return False + if row.owner == me: + return entering or self._local_alive(row.id) + return row.owner not in dead_owners + + dead = [r.id for r in rows if not alive(r)] + if dead: + from sqlalchemy import delete + + await session.execute(delete(Row).where(Row.id.in_(dead))) + return [r for r in rows if r.id not in dead] + + async def _advance(self, source_id: str, *, me: LaneTicket | None = None) -> bool: + """这个源上没有轮到的就让号最小的轮到。交回 me 是不是轮到了。""" + from app.db.models_cluster import LaneTicket as Row + + granted_now: int | None = None + dead_owners = await self._dead_owners(source_id) + async with self._lock(), SessionLocal() as session: + await self._locked(session, source_id) + rows = await self._rows(session, source_id, dead_owners) + head = next((r for r in rows if r.granted), None) + if head is None and rows: + head = rows[0] + await session.execute(update(Row).where(Row.id == head.id).values(granted=True) + .execution_options(synchronize_session=False)) + granted_now = head.id + await session.commit() + if granted_now is not None: + ticket = self._tickets.get(granted_now) + if ticket is not None: + ticket.granted.set() + ticket.woke.set() + from app.cluster.signal import signals + + signals.publish("lane", source_id) + return me is not None and head is not None and head.id == me.id + + async def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: """排进队尾,马上返回(不等)。排在最前面的立刻 granted;调用方 await ticket.wait() 等轮到自己。kind 要登记过。""" + from app.cluster import identity + from app.db.models_cluster import LaneTicket as Row + if kind not in _LANE_KINDS: raise ValueError(f"同源队里没有登记「{kind}」这种事,先 register_lane_kind") - queue = self._queue(source_id) - ticket = LaneTicket(self, source_id, kind, job_id, asyncio.get_running_loop()) - queue.append(ticket) - self._queues[source_id] = queue - if len(queue) == 1: + self._listen() + self._entering[source_id] = self._entering.get(source_id, 0) + 1 + try: + async with SessionLocal() as session: + row = Row(source_id=source_id, kind=kind, job_id=job_id, owner=identity.owner(), granted=False, + created_at=utcnow()) + session.add(row) + await session.commit() + row_id = int(row.id) + ticket = LaneTicket(self, row_id, source_id, kind, job_id, asyncio.get_running_loop()) + self._tickets[row_id] = ticket + finally: + left = self._entering.get(source_id, 1) - 1 + if left > 0: + self._entering[source_id] = left + else: + self._entering.pop(source_id, None) + if await self._advance(source_id, me=ticket): ticket.granted.set() return ticket - def try_enter(self, source_id: str, kind: str) -> LaneTicket: - """队空着才占(马上就轮到);有人在队里抛 LaneBusy。""" - queue = self._queue(source_id) - if queue: - raise LaneBusy(queue[0]) - return self.enter(source_id, kind) + async def try_enter(self, source_id: str, kind: str) -> LaneTicket: + """队空着才占(马上就轮到);有人在队里抛 LaneBusy。查和占在同一个事务里、按源串行。""" + from app.cluster import identity + from app.db.models_cluster import LaneTicket as Row - def holders(self, source_id: str) -> list[LaneTicket]: + if kind not in _LANE_KINDS: + raise ValueError(f"同源队里没有登记「{kind}」这种事,先 register_lane_kind") + self._listen() + dead_owners = await self._dead_owners(source_id) + async with self._lock(), SessionLocal() as session: + await self._locked(session, source_id) + rows = await self._rows(session, source_id, dead_owners) + if rows: + head = next((r for r in rows if r.granted), rows[0]) + await session.commit() + raise LaneBusy(LaneHolder(head.kind, head.job_id, head.owner)) + row = Row(source_id=source_id, kind=kind, job_id=None, owner=identity.owner(), granted=True, + created_at=utcnow()) + session.add(row) + await session.commit() + row_id = int(row.id) + ticket = LaneTicket(self, row_id, source_id, kind, None, asyncio.get_running_loop()) + ticket.granted.set() + self._tickets[row_id] = ticket + return ticket + + async def holders(self, source_id: str) -> list[LaneHolder]: """队里的各位,排在最前面(正在做)的在前。""" - return list(self._queue(source_id)) + dead_owners = await self._dead_owners(source_id) + async with SessionLocal() as session: + rows = await self._rows(session, source_id, dead_owners) + await session.commit() + rows.sort(key=lambda r: (not r.granted, r.id)) + return [LaneHolder(r.kind, r.job_id, r.owner) for r in rows] - def ahead(self, source_id: str, job_id: str) -> list[str] | None: + async def ahead(self, source_id: str, job_id: str) -> list[str] | None: """排在这个任务前面的各位是什么(kind,正在做的在前);这个任务不在队里(刚建、后台还没排进来)返回 None。""" - queue = self._queue(source_id) - for i, ticket in enumerate(queue): - if ticket.job_id == job_id: - return [t.kind for t in queue[:i]] + holders = await self.holders(source_id) + for i, holder in enumerate(holders): + if holder.job_id == job_id: + return [h.kind for h in holders[:i]] return None - def _release(self, ticket: LaneTicket) -> None: + async def _release(self, ticket: LaneTicket) -> None: + from sqlalchemy import delete + + from app.db.models_cluster import LaneTicket as Row + if ticket.released: return ticket.released = True - queue = self._queues.get(ticket.source_id) - if not queue or ticket not in queue: - return - head = queue[0] is ticket - queue.remove(ticket) - if not queue: - self._queues.pop(ticket.source_id, None) - elif head: - queue[0].granted.set() + self._tickets.pop(ticket.id, None) + + async def go() -> None: + async with SessionLocal() as session: + await session.execute(delete(Row).where(Row.id == ticket.id)) + await session.commit() + await self._advance(ticket.source_id) + + try: + await asyncio.shield(go()) + except Exception: # noqa: BLE001 - 删不掉的那一行下次排队时按「本进程里已经没人占着」收回 + logger.exception("同源队里的一位没能出队(源 %s),下次排队时收回", ticket.source_id) + self._wake(ticket.source_id) @contextlib.asynccontextmanager async def hold(self, source_id: str, kind: str, job_id: str | None = None) -> AsyncIterator[LaneTicket]: """阻塞地占:排队,轮到了再进去,出来时出队。""" - ticket = self.enter(source_id, kind, job_id) + ticket = await self.enter(source_id, kind, job_id) try: await ticket.wait() yield ticket finally: - ticket.release() + await ticket.release() - @contextlib.contextmanager - def try_hold(self, source_id: str, kind: str) -> Iterator[LaneTicket]: + @contextlib.asynccontextmanager + async def try_hold(self, source_id: str, kind: str) -> AsyncIterator[LaneTicket]: """不阻塞地试占(同步起草、导入目录):占不到抛 LaneBusy;占到了,执行期间新建的任务排在后面。""" - ticket = self.try_enter(source_id, kind) + ticket = await self.try_enter(source_id, kind) try: yield ticket finally: - ticket.release() + await ticket.release() lanes = SourceLanes() @@ -287,26 +471,48 @@ async def cancel_queued(session: AsyncSession, job_model: Any, task_model: Any, async def claim(session: AsyncSession, job_model: Any, job_id: str) -> bool: - """认领:排队中 → 进行中。开始之前就被停止了返回 False。""" + """认领:排队中 → 进行中,属主记成本进程(多进程:执行就在这里)。开始之前就被停止了返回 False。""" + from app.db.base import current_owner + + values: dict[str, Any] = {"status": "running", "started_at": utcnow()} + if hasattr(job_model, "owner"): + values["owner"] = current_owner() result = await session.execute( update(job_model).where(job_model.id == job_id, job_model.status == "queued") - .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) + .values(**values).execution_options(synchronize_session=False)) await session.commit() return result.rowcount == 1 async def interrupt_on_startup(job_model: Any, close: Callable[[AsyncSession, str], Awaitable[None]], *, - running: Callable[[str], bool]) -> int: - """服务启动时:排队、进行中、而且不在这个进程里跑的任务逐个 close(各自记成 interrupted、没结束的表记 cancelled)。 - 不自动续跑。返回整理了几个。""" + running: Callable[[str], bool], live: Any = None) -> int: + """接手属主不在了的任务(服务启动时;多进程时当班进程隔一会儿再来一次,见 app/cluster/reaper.py):排队、进行中、属主 + 不在了(进程没了,或者就是本进程、却不在本进程里跑)的任务认领过来、逐个 close(各自记成 interrupted、没结束的表记 + cancelled)。别的活着的进程在跑的一概不碰。不自动续跑。返回整理了几个。 + + live:判断属主在不在的那一份(cluster.workers.Liveness),不给就当场查库(启动时的口径:属主是自己的只看本进程)。""" + from app.cluster import claims, state + + live = live if live is not None else await state.current(fresh=True) + ids = [job_id for job_id, _ in await claims.orphans(job_model, live, running=running, statuses=ACTIVE)] async with SessionLocal() as session: - ids = [job_id for job_id in (await session.execute( - select(job_model.id).where(job_model.status.in_(ACTIVE)))).scalars() if not running(job_id)] for job_id in ids: await close(session, job_id) return len(ids) +async def stale(rows: Any, *, running: Callable[[str], bool]) -> list[Any]: + """读接口当场整理用:rows 里记着排队、进行中,属主却不在了的(属主是本进程的看 running——同以前「这个进程里却没在跑」, + 别的进程看租约)。""" + from app.cluster import state + + rows = [r for r in rows if r.status in ACTIVE] + if not rows: + return [] + live = await state.live_for([getattr(r, "owner", None) for r in rows]) + return [r for r in rows if not live.alive(getattr(r, "owner", None), local=running(r.id))] + + # ========================================================================== # 在后台执行 # ========================================================================== @@ -335,14 +541,27 @@ class JobRun: await self.writer.settle() async def run(self) -> None: + from app.cluster import fence + + try: + await fence.ready() # 隔离之后新身份还没登记上时先等一等(cluster/fence.ready) + except fence.Fenced: + logger.error("%s %s:本进程隔离之后一直没重新登记上,不开始(由别的进程按「属主不在了」处理)", self.kind, self.job_id) + return + fence.bind() try: try: await self._queue() await self._run() + except fence.Fenced: + # 本进程续不上租约了:停手、不写收尾(这个任务由接手的进程按「属主不在了」处理) + logger.error("%s %s:本进程续不上租约,停手、不写收尾", self.kind, self.job_id) except asyncio.CancelledError: current = asyncio.current_task() if current is not None: current.uncancel() + if fence.fenced_here(): + return await self._stopped() except JobFailed as e: await self._failed(str(e)) @@ -351,7 +570,7 @@ class JobRun: finally: # 收尾写完才出队:排在后面的任务开始时,这个任务已经记成结束了 if self.ticket is not None: - self.ticket.release() + await self.ticket.release() def lane_of(self, job: Any) -> str | None: """这个任务排在哪条队里:缺省是任务的数据源(job.source_id)。不占源的任务(以后的评测:开始时拍目录快照,不挡 @@ -365,7 +584,7 @@ class JobRun: source_id = self.lane_of(job) if job is not None and job.status == "queued" else None if source_id is None: return - self.ticket = lanes.enter(source_id, self.kind, self.job_id) + self.ticket = await lanes.enter(source_id, self.kind, self.job_id) await self.ticket.wait() async def _run(self) -> None: @@ -382,14 +601,38 @@ class JobRun: class JobManager: - """进程内的一种后台任务:一个 asyncio 任务一件事,停的时候给宽限。factory 按任务编号造一次执行(JobRun);label 是 - 日志里的叫法(「数据目录起草任务」),name 是 asyncio 任务名的前缀。""" + """一种后台任务:一个 asyncio 任务一件事,停的时候给宽限。factory 按任务编号造一次执行(JobRun);label 是日志里的叫法 + (「数据目录起草任务」),name 是 asyncio 任务名的前缀,也是跨进程停止请求里的 target。 + + 执行在认领它的那个进程里(任务表的 owner 列,建任务时缺省就是建它的进程)。停止落在别的进程上时(多进程,见 + app/cluster):stop 给属主发一条停止请求,属主照本进程的规矩停、收尾,这里等它收完(最多 grace 秒);属主不在了的交给 + 调用方按「不在这个进程里跑」处理(同以前「记录对不上:直接记停止」)。""" def __init__(self, factory: Callable[[str], JobRun], *, label: str, name: str) -> None: self._runs: dict[str, JobRun] = {} self._factory = factory self.label = label self.name = name + from app.cluster import fence, requests + + requests.handle("stop", name)(self._stop_requested) + fence.on_trip(self._stop_fenced) + + def _stop_fenced(self) -> None: + """本进程隔离了(cluster/fence.py):停掉手上的任务,停的时候不写收尾。""" + for job_id in list(self._runs): + run = self._runs.get(job_id) + if run is not None and run.task is not None and not run.task.done(): + run.stop_reason = run.stop_reason or "fenced" + run.task.cancel() + + def model(self) -> Any: + """任务表(停止时读属主用):缺省取 factory 上的 model。""" + return getattr(self._factory, "model", None) + + async def _stop_requested(self, job_id: str, actor: str | None, payload: dict[str, Any]) -> None: + await self.stop(job_id, str(payload.get("reason") or "user"), grace=float(payload.get("grace") or STOP_GRACE_S), + remote=False) def _alive(self, job_id: str) -> JobRun | None: run = self._runs.get(job_id) @@ -418,17 +661,40 @@ class JobManager: def running(self, job_id: str) -> bool: return self._alive(job_id) is not None - async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: - """停掉这个进程里在跑(或在排队等)的任务,等它收尾(最多 grace 秒)。不在这个进程里返回 False。""" + async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S, remote: bool = True) -> bool: + """停掉这个任务,等它收尾(最多 grace 秒)。在本进程里跑的直接停;remote 时在别的活着的进程里跑的给它发停止请求、 + 等它收完。都不是(不在跑、属主不在了)返回 False。""" run = self._alive(job_id) if run is None: - return False + return await self._stop_remote(job_id, reason, grace) if remote else False if run.stop_reason is None: run.stop_reason = reason run.task.cancel() # type: ignore[union-attr] await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] return True + async def _stop_remote(self, job_id: str, reason: str, grace: float) -> bool: + from app.cluster import identity, requests, workers + + model = self.model() + if model is None or not hasattr(model, "owner"): + return False + async with SessionLocal() as session: + row = await session.get(model, job_id) + owner, status = (row.owner, row.status) if row is not None else (None, None) + if status not in ACTIVE or not owner or owner == identity.owner() or not await workers.alive(owner): + return False + await requests.send(owner, "stop", self.name, job_id, payload={"reason": reason, "grace": grace}) + loop = asyncio.get_running_loop() + deadline = loop.time() + grace + workers.heartbeat_s() + while loop.time() < deadline: + await asyncio.sleep(0.1) + async with SessionLocal() as session: + status = await session.scalar(select(model.status).where(model.id == job_id)) + if status not in ACTIVE: + break + return True + async def wait(self, job_id: str, timeout: float | None = None) -> None: """等这个任务做完(测试、调试用)。不在跑直接返回。""" run = self._alive(job_id) @@ -449,4 +715,4 @@ class JobManager: __all__ = ["ACTIVE", "FINISHED", "JobFailed", "JobManager", "JobRun", "LaneBusy", "LaneTicket", "OPEN", "RETRYABLE", "SHUTDOWN_GRACE_S", "STOP_GRACE_S", "SerialWriter", "SourceLanes", "TASK_DONE", "TASK_STATUSES", "cancel_queued", - "claim", "close_job", "interrupt_on_startup", "lane_kinds", "lane_label", "lanes", "register_lane_kind"] + "claim", "close_job", "interrupt_on_startup", "lane_kinds", "lane_label", "lanes", "register_lane_kind", "stale"] diff --git a/backend/app/data/capability_review.py b/backend/app/data/capability_review.py index 3db355c0..9751f4e3 100644 --- a/backend/app/data/capability_review.py +++ b/backend/app/data/capability_review.py @@ -144,10 +144,18 @@ def normalize_fields(existing: Mapping[str, Any] | None, submitted: Any, *, at: async def run_counts(session: AsyncSession) -> dict[str, int]: """每项能力被几次运行用到:{能力名: 运行次数},没用过的不出现。按运行发起时记下的能力治理快照数 - (Run.options["capabilities"],一次运行里一项能力只记一次)。只用 SQLite 的 json_each,一次查询。""" - rows = await session.execute(text( - "SELECT j.key, COUNT(*) FROM runs, json_each(runs.options, '$.capabilities') AS j " - "WHERE json_valid(runs.options) GROUP BY j.key")) + (Run.options["capabilities"],一次运行里一项能力只记一次)。一次查询:两种库都用 json_each(PostgreSQL 的碰到不是 + 对象的值会报错,先按类型挑出来)。""" + from app.db.portable import dialect_of + + if dialect_of(session) == "postgresql": + sql = ("SELECT j.key, COUNT(*) FROM runs CROSS JOIN LATERAL json_each(CASE WHEN " + "json_typeof(runs.options -> 'capabilities') = 'object' THEN runs.options -> 'capabilities' " + "ELSE '{}'::json END) AS j GROUP BY j.key") + else: + sql = ("SELECT j.key, COUNT(*) FROM runs, json_each(runs.options, '$.capabilities') AS j " + "WHERE json_valid(runs.options) GROUP BY j.key") + rows = await session.execute(text(sql)) return {str(name): int(count) for name, count in rows.all()} diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index 415b4e5d..9a0826b3 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -142,7 +142,7 @@ async def active_job(session: AsyncSession, source_id: str) -> CatalogDraftJob | """排队或进行中的那个(同一个源最多一个)。""" return (await session.execute( select(CatalogDraftJob).where(CatalogDraftJob.source_id == source_id, CatalogDraftJob.status.in_(ACTIVE)) - .order_by(CatalogDraftJob.created_at.desc()).limit(1) + .order_by(CatalogDraftJob.created_at.desc(), CatalogDraftJob.id.desc()).limit(1) )).scalar_one_or_none() @@ -150,7 +150,7 @@ async def latest_job(session: AsyncSession, source_id: str) -> CatalogDraftJob | """最近建的一个(任何状态)。""" return (await session.execute( select(CatalogDraftJob).where(CatalogDraftJob.source_id == source_id) - .order_by(CatalogDraftJob.created_at.desc()).limit(1) + .order_by(CatalogDraftJob.created_at.desc(), CatalogDraftJob.id.desc()).limit(1) )).scalar_one_or_none() @@ -246,7 +246,7 @@ async def waiting_for(session: AsyncSession, job: CatalogDraftJob) -> str | None 任务。""" if job.status != "queued": return None - ahead = runtime.lanes.ahead(job.source_id, job.id) + ahead = await runtime.lanes.ahead(job.source_id, job.id) if ahead is not None: if "profile" in ahead: return "profile" @@ -288,7 +288,7 @@ async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[s 中断了、而且没有写进新内容(parts_written 为 0)时照样带着,界面据此继续提示留空的列;之后有起草写进了新内容(停止 前已经写了几段、或者正在写)就是 null:以前的留空列数可能已经不对了。不用模型的起草不改模型留空的列,不算。""" recent = (select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == source_id) - .order_by(CatalogDraftJob.created_at.desc()).limit(LAST_DRAFT_JOBS)).scalar_subquery() + .order_by(CatalogDraftJob.created_at.desc(), CatalogDraftJob.id.desc()).limit(LAST_DRAFT_JOBS)).scalar_subquery() def ranked() -> Any: return func.row_number().over(partition_by=CatalogDraftTask.table_name, @@ -347,19 +347,22 @@ def import_busy_message(row: Any) -> str: return f"数据源「{getattr(row, 'name', '')}」正在导入数据目录,请等导入完成后再试" -#: 正在同步起草(POST /catalog/draft)的数据源。进程内登记:服务端是单进程部署,多进程部署时要换成库里的锁 +#: 本进程里正在同步起草(POST /catalog/draft)的数据源:先登记、再查库(见 sync_draft)。别的进程上的同步起草看同源队里 +#: kind=sync_draft 的那一位(队记在库里,多进程共用) _SYNC: set[str] = set() -def sync_drafting(source_id: str) -> bool: - """这个源正在同步起草。""" - return source_id in _SYNC +async def sync_drafting(source_id: str) -> bool: + """这个源正在同步起草(本进程里,或者别的进程占着同源队做同步起草)。""" + if source_id in _SYNC: + return True + return any(h.kind == "sync_draft" for h in await runtime.lanes.holders(source_id)) async def ensure_draft_free(session: AsyncSession, row: Any, *, then: str = THEN_DRAFT) -> None: """同种任务互斥:这个源没有在起草——没有排队或进行中的起草任务,也没有在同步起草。有就抛 DraftBusy。建起草任务、 重试之前调它。剖析任务不算:同源有剖析任务时起草任务照建,排在它后面。then 是说法最后一句要做的事。""" - if row.id in _SYNC: + if await sync_drafting(row.id): raise DraftBusy(sync_busy_message(row, then)) if await active_job(session, row.id) is not None: raise DraftBusy(busy_message(row, then)) @@ -375,7 +378,7 @@ async def ensure_idle(session: AsyncSession, row: Any, kind: str = "import") -> await catalog_profile_jobs.ensure_free(session, row, then=_then(kind, "profile")) -def _lane_busy(row: Any, holder: runtime.LaneTicket, kind: str) -> Exception: +def _lane_busy(row: Any, holder: runtime.LaneHolder, kind: str) -> Exception: """库里查过没有任务、这个源的队却被占着(同步起草、导入目录正在做,或者一个任务刚好在收尾):按占着的是谁说,最后 一句按这次要做的事(kind)说。""" if holder.kind == "profile": @@ -400,13 +403,13 @@ async def _sync_write(row: Any, kind: str, check: Callable[[], Awaitable[None]]) 再占队——别的同步写操作正在做时占不到,同样抛忙,什么都不做。占到了,执行期间新建的任务排在它后面,做完才开始。""" await check() try: - ticket = runtime.lanes.try_enter(row.id, kind) + ticket = await runtime.lanes.try_enter(row.id, kind) except runtime.LaneBusy as e: raise _lane_busy(row, e.holder, kind) from e try: yield finally: - ticket.release() + await ticket.release() async def _ensure_idle_but_sync(session: AsyncSession, row: Any) -> None: @@ -423,7 +426,7 @@ async def sync_draft(session: AsyncSession, row: Any) -> AsyncIterator[None]: """同步起草期间登记这个源、占着它的队:同一个源建起草任务、重试、导入、再开一次同步起草都抛 DraftBusy,期间新建的 剖析任务排在它后面;有排队或进行中的起草任务时这里抛 DraftBusy,有剖析任务时抛 ProfileBusy。先登记、再查库:查库要 让出事件循环,先登记才不会和同时建的任务互相看不见(建任务那边写库之后也再看一次,见 create_job)。""" - if row.id in _SYNC: + if await sync_drafting(row.id): raise DraftBusy(sync_busy_message(row)) _SYNC.add(row.id) try: @@ -484,7 +487,7 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li # 部分唯一索引:另一个请求同时给这个源建了任务 await session.rollback() raise DraftBusy(busy_message(row)) from e - if row.id in _SYNC: + if await sync_drafting(row.id): # 写库的时候同一个源开始了同步起草(它查库时还没看到这个任务):这个任务不要了 await session.execute(delete(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id)) await session.execute(delete(CatalogDraftJob).where(CatalogDraftJob.id == job.id)) @@ -541,12 +544,12 @@ async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: task_error) -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """服务启动时:排队、进行中的任务标成 interrupted,没结束的表记 cancelled、写明服务重启。不自动续跑。返回标了几个。""" async def interrupt(session: AsyncSession, job_id: str) -> None: await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) - n = await runtime.interrupt_on_startup(CatalogDraftJob, interrupt, running=manager.running) + n = await runtime.interrupt_on_startup(CatalogDraftJob, interrupt, running=manager.running, live=live) if n: logger.warning("服务重启:%d 个数据目录起草任务没有做完,已标为中断,可在数据目录页重试", n) return n diff --git a/backend/app/data/catalog_profile_jobs.py b/backend/app/data/catalog_profile_jobs.py index 37a4b097..6b25d9a7 100644 --- a/backend/app/data/catalog_profile_jobs.py +++ b/backend/app/data/catalog_profile_jobs.py @@ -48,7 +48,7 @@ import logging import time from typing import Any -from sqlalchemy import case, delete, func, or_, select, update +from sqlalchemy import case, delete, func, not_, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -60,6 +60,7 @@ from app.data.catalog_profile import ( ) from app.data.engine import SnapshotTampered from app.db.base import SessionLocal, new_id, utcnow +from app.db.portable import json_flag from app.db.models import CatalogDraftJob, CatalogProfileJob, CatalogProfileTask, DataSource logger = logging.getLogger(__name__) @@ -126,7 +127,7 @@ async def active_job(session: AsyncSession, source_id: str) -> CatalogProfileJob return (await session.execute( select(CatalogProfileJob).where(CatalogProfileJob.source_id == source_id, CatalogProfileJob.status.in_(ACTIVE)) - .order_by(CatalogProfileJob.created_at.desc()).limit(1) + .order_by(CatalogProfileJob.created_at.desc(), CatalogProfileJob.id.desc()).limit(1) )).scalar_one_or_none() @@ -134,7 +135,7 @@ async def latest_job(session: AsyncSession, source_id: str) -> CatalogProfileJob """最近建的一个(任何状态)。""" return (await session.execute( select(CatalogProfileJob).where(CatalogProfileJob.source_id == source_id) - .order_by(CatalogProfileJob.created_at.desc()).limit(1) + .order_by(CatalogProfileJob.created_at.desc(), CatalogProfileJob.id.desc()).limit(1) )).scalar_one_or_none() @@ -278,7 +279,7 @@ async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | No 任务,或者正在同步起草。""" if job.status != "queued": return None - ahead = runtime.lanes.ahead(job.source_id, job.id) + ahead = await runtime.lanes.ahead(job.source_id, job.id) if ahead is not None: if any(kind in ("draft", "sync_draft") for kind in ahead): return "draft" @@ -286,7 +287,7 @@ async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | No return "introspect" if "introspect" in ahead else None from app.data import catalog_jobs - if catalog_jobs.sync_drafting(job.source_id): + if await catalog_jobs.sync_drafting(job.source_id): return "draft" earlier = (await session.execute( select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == job.source_id, @@ -345,7 +346,7 @@ _SHOWN = (case((CatalogProfileTask.started_at.is_(None), 1), else_=0), *_NEWEST) #: 只做了取值观察的那一项(observe 范围、maintenance 范围里只到期观察的表)在逐表一行的 summary 里带这个标记:它不是一次 #: 剖析,不算「上次剖析」——不然表清单的「上次剖析」、给助手的剖析事实(行数、日期范围)会被一次只记了行数的观察顶掉 OBSERVE_ONLY = "observe_only" -_PROFILED = func.coalesce(func.json_extract(CatalogProfileTask.summary, f"$.{OBSERVE_ONLY}"), 0) == 0 +_PROFILED = not_(json_flag(CatalogProfileTask.summary, OBSERVE_ONLY)) def _observe_only(checks: list[Any] | None) -> bool: @@ -689,7 +690,7 @@ async def _behind_draft(session: AsyncSession, source_id: str) -> bool: """这个源有排队、进行中的起草任务,或者正在同步起草:新建的剖析任务要排在它后面。""" from app.data import catalog_jobs - return catalog_jobs.sync_drafting(source_id) or await catalog_jobs.active_job(session, source_id) is not None + return await catalog_jobs.sync_drafting(source_id) or await catalog_jobs.active_job(session, source_id) is not None async def nothing_reason(session: AsyncSession, row: Any, source: Any, scope: str) -> str | None: @@ -858,13 +859,13 @@ async def cancel(session: AsyncSession, job: CatalogProfileJob) -> None: await _close(session, job_id, "cancelled", None, "cancelled", None) -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """服务启动时:排队、进行中的剖析任务标成 interrupted,没结束的表记 cancelled、写明服务重启,不自动续跑;再把剖析设置的 旧键迁移掉。返回标了几个任务。""" async def interrupt(session: AsyncSession, job_id: str) -> None: await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) - n = await runtime.interrupt_on_startup(CatalogProfileJob, interrupt, running=manager.running) + n = await runtime.interrupt_on_startup(CatalogProfileJob, interrupt, running=manager.running, live=live) if n: logger.warning("服务重启:%d 个数据剖析任务没有做完,已标为中断,可在数据目录页重试", n) async with SessionLocal() as session: diff --git a/backend/app/data/catalog_profile_observe.py b/backend/app/data/catalog_profile_observe.py index 0663312e..ad716a1c 100644 --- a/backend/app/data/catalog_profile_observe.py +++ b/backend/app/data/catalog_profile_observe.py @@ -505,7 +505,7 @@ async def observed_facts(session: AsyncSession, source_id: str, query = query.where(TableObservation.table_name.in_(wanted)) out: dict[str, dict[str, Any]] = {} seen: set[tuple[str, str, str]] = set() - for r in (await session.execute(query.order_by(TableObservation.at.desc()))).scalars(): + for r in (await session.execute(query.order_by(TableObservation.at.desc(), TableObservation.id.desc()))).scalars(): key = (r.table_name, r.kind, r.column_name) if key in seen: continue diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index 417302ad..a5db6903 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -646,8 +646,13 @@ class EngineCache: if self._verifying.get(token) is done: del self._verifying[token] - async def invalidate(self, source_id: str) -> None: + async def invalidate(self, source_id: str, *, broadcast: bool = True) -> None: sid = str(source_id) + if broadcast: + # 多进程:别的进程手上这个源的连接池也要扔掉(换了地址、口令、只读开关),见 app/cluster/caches.py + from app.cluster import caches + + await caches.broadcast("datasource_engine", sid) # 正在进行的核对也撇开:失效之后来的请求重新核对,不搭失效之前那一次的便车 for token in [t for t in self._verifying if _source_of(t[0]) == sid]: del self._verifying[token] diff --git a/backend/app/data/introspect_jobs.py b/backend/app/data/introspect_jobs.py index 0f730765..29fad631 100644 --- a/backend/app/data/introspect_jobs.py +++ b/backend/app/data/introspect_jobs.py @@ -104,13 +104,13 @@ def _iso(value: Any) -> str | None: async def active_job(session: AsyncSession, source_id: str) -> IntrospectJob | None: return (await session.execute( select(IntrospectJob).where(IntrospectJob.source_id == source_id, IntrospectJob.status.in_(ACTIVE)) - .order_by(IntrospectJob.created_at.desc()).limit(1))).scalar_one_or_none() + .order_by(IntrospectJob.created_at.desc(), IntrospectJob.id.desc()).limit(1))).scalar_one_or_none() async def latest_job(session: AsyncSession, source_id: str) -> IntrospectJob | None: return (await session.execute( select(IntrospectJob).where(IntrospectJob.source_id == source_id) - .order_by(IntrospectJob.created_at.desc()).limit(1))).scalar_one_or_none() + .order_by(IntrospectJob.created_at.desc(), IntrospectJob.id.desc()).limit(1))).scalar_one_or_none() async def load(session: AsyncSession, source_id: str, job_id: str) -> IntrospectJob | None: @@ -132,7 +132,7 @@ async def waiting_for(session: AsyncSession, job: IntrospectJob) -> str | None: 起草);任务刚建、后台还没排进队时看库:同源有比它先建的、排队或进行中的起草、剖析任务。""" if job.status != "queued": return None - ahead = runtime.lanes.ahead(job.source_id, job.id) + ahead = await runtime.lanes.ahead(job.source_id, job.id) if ahead is not None: return ahead[0] if ahead else None from app.db.models import CatalogDraftJob, CatalogProfileJob @@ -437,7 +437,7 @@ async def _record_scope_changes(session: AsyncSession, row: Any, job: Introspect select(IntrospectJob).where(IntrospectJob.source_id == job.source_id, IntrospectJob.id != job.id, IntrospectJob.started_at.is_not(None), IntrospectJob.created_at <= job.created_at) - .order_by(IntrospectJob.created_at.desc()).limit(1))).scalar_one_or_none() + .order_by(IntrospectJob.created_at.desc(), IntrospectJob.id.desc()).limit(1))).scalar_one_or_none() before = (previous.scope or {}).get("rules") if previous is not None else None after = (job.scope or {}).get("rules") if isinstance(before, dict) and isinstance(after, dict) and before != after: @@ -512,12 +512,12 @@ async def cancel(session: AsyncSession, job: IntrospectJob) -> None: await _close(session, job_id, "cancelled", None, STOPPED_ERROR) -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """服务启动时:排队、进行中的探查任务标成 interrupted,没读的表 cancelled,读完的物化进去;不自动续跑。""" async def interrupt(session: AsyncSession, job_id: str) -> None: await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, INTERRUPTED_ERROR) - n = await runtime.interrupt_on_startup(IntrospectJob, interrupt, running=manager.running) + n = await runtime.interrupt_on_startup(IntrospectJob, interrupt, running=manager.running, live=live) if n: logger.warning("服务重启:%d 个探查任务没有做完,已标为中断,可在数据源卡片上重新探查", n) return n diff --git a/backend/app/data/recipe_imports.py b/backend/app/data/recipe_imports.py index ee8d338c..59d29f1c 100644 --- a/backend/app/data/recipe_imports.py +++ b/backend/app/data/recipe_imports.py @@ -90,6 +90,7 @@ from app.data.recipe_types import ( from app.data.xlsx_scan import WorkbookScan from app.db.base import new_id, utcnow from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableBuild, TableImport, TableRecipe +from app.db.portable import unique_violation logger = logging.getLogger(__name__) @@ -1702,8 +1703,9 @@ def _adopt_ai_work(p: Pipeline, book: _Book, snap: _Snap, work: Any) -> _Eval: return _evaluate(p, book, snap, work) -#: 正在 AI 起草的暂存区(进程内):同一暂存区不并发起草,否则两次发送各花一份钱,同意记录、用量还会互相覆盖 -#: (SE-11)。事件循环单线程,查和加之间没有 await,不会有两个请求同时通过;进程重启后自然清空 +#: 正在 AI 起草的暂存区(本进程里):同一暂存区不并发起草,否则两次发送各花一份钱,同意记录、用量还会互相覆盖 +#: (SE-11)。事件循环单线程,查和加之间没有 await,不会有两个请求同时通过;进程重启后自然清空。多进程时另取跨进程的锁 +#: (run_ai_draft) _AI_RUNNING: set[str] = set() @@ -1715,9 +1717,18 @@ async def run_ai_draft(session: AsyncSession, staging: ImportStaging, *, consent raise ImportRefused(422, "ai_consent_required", "需要先查看并同意发送的内容,才能让 AI 起草") if staging.id in _AI_RUNNING: raise ImportRefused(409, "ai_in_progress", "这次导入正在由 AI 起草,请等它完成后再试") + # 多进程:另一个进程上正在起草同一个暂存区的,跨进程的锁占不到,同样回 409(app/cluster/mutex) + from app.cluster import mutex + + lock = mutex.lock(f"recipe-ai:{staging.id}") _AI_RUNNING.add(staging.id) try: - return await _run_ai_draft(session, staging, preview_sha256=preview_sha256, signed_by=signed_by) + if not await lock.try_take(): + raise ImportRefused(409, "ai_in_progress", "这次导入正在由 AI 起草,请等它完成后再试") + try: + return await _run_ai_draft(session, staging, preview_sha256=preview_sha256, signed_by=signed_by) + finally: + await lock.give_back() finally: _AI_RUNNING.discard(staging.id) @@ -2747,7 +2758,7 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial raise _publish_refused(e) from e except IntegrityError as e: await _back_to_drafting(session, staging_id) - if "data_sources.name" in str(e.orig if e.orig is not None else e): + if unique_violation(e, "data_sources", "name"): raise ImportRefused(409, "name_taken", f"名称「{name}」已被占用,请放弃这次导入后换一个名称") from e raise ImportRefused(500, "publish_failed", "导入失败:写入版本记录时出错,当前版本未受影响") from e except BaseException: diff --git a/backend/app/data/source_changes.py b/backend/app/data/source_changes.py index 5b625baf..adba3f79 100644 --- a/backend/app/data/source_changes.py +++ b/backend/app/data/source_changes.py @@ -29,7 +29,7 @@ from dataclasses import dataclass from datetime import datetime from typing import Any -from sqlalchemy import func, or_, select +from sqlalchemy import func, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -37,6 +37,7 @@ from app.data import schema_diff as sd from app.data.names import name_key from app.db.base import utcnow from app.db.models_sources import SourceChange +from app.db.portable import json_eq logger = logging.getLogger(__name__) @@ -150,13 +151,21 @@ async def record(session: AsyncSession, source: Any, change: NewChange, detected raise ValueError(f"不认识的严重程度:{change.severity}") fp = fingerprint(change.kind, change.table, change.column, change.key) now = utcnow() + # 已经有了的:先按条件 UPDATE 记下又见到一次(同时占住那一行:SQLite 上拿到写锁、PostgreSQL 上锁住这一行),再读出来改 + # 影响面——口径试算同时往 affects 里记没试算的那几版(ledger/recall._probe_skipped,同样先占住再读改写),谁也不盖掉谁 + seen = await session.execute(update(SourceChange).where( + SourceChange.source_id == source.id, SourceChange.fingerprint == fp) + .values(last_seen_at=now, seen_count=func.coalesce(SourceChange.seen_count, 0) + 1) + .execution_options(synchronize_session=False)) found = (await session.execute(select(SourceChange).where( - SourceChange.source_id == source.id, SourceChange.fingerprint == fp))).scalar_one_or_none() + SourceChange.source_id == source.id, SourceChange.fingerprint == fp) + .execution_options(populate_existing=True))).scalar_one_or_none() if seen.rowcount else None if found is not None: - found.last_seen_at = now - found.seen_count = int(found.seen_count or 0) + 1 if found.status == "open": - found.affects, found.before, found.after = change.affects, change.before, change.after + # 影响按这一次刷新;表结构变了之后口径试算的情况(ledger/recall 记的 probes)不是探查算的,留着 + probes = (found.affects or {}).get("probes") if isinstance(found.affects, dict) else None + found.affects = {**(change.affects or {}), "probes": probes} if probes else change.affects + found.before, found.after = change.before, change.after if SEVERITY_RANK[change.severity] > SEVERITY_RANK.get(found.severity, 0): found.severity = change.severity return found, False @@ -227,7 +236,8 @@ def change_out(row: SourceChange) -> dict[str, Any]: "severity": row.severity, "summary": summary(row), "detected_at": _iso(row.detected_at), "last_seen_at": _iso(row.last_seen_at), "seen_count": row.seen_count or 1, "detected_by": dict(row.detected_by or {}), "basis": row.basis, "before": row.before, "after": row.after, - "affects": row.affects or {"catalog": [], "templates": [], "rules_touched": False}, + # 缺的几项补上空的:只记了口径试算情况(ledger/recall 的 probes)的那种没有 catalog、templates + "affects": {"catalog": [], "templates": [], "rules_touched": False, **(row.affects or {})}, "struct_version_before": row.struct_version_before, "struct_version_after": row.struct_version_after, "status": row.status, "resolution": row.resolution, } @@ -337,7 +347,7 @@ async def notify_after_job(source_id: str, job_id: str) -> int: n = int((await session.execute(select(func.count()).select_from(SourceChange).where( SourceChange.source_id == source_id, SourceChange.severity == "action", SourceChange.status == "open", - func.json_extract(SourceChange.detected_by, "$.job_id") == job_id))).scalar() or 0) + json_eq(SourceChange.detected_by, "job_id", job_id)))).scalar() or 0) if n: from app.notify import service diff --git a/backend/app/data/source_maintenance.py b/backend/app/data/source_maintenance.py index aa8597bf..7c61dcc5 100644 --- a/backend/app/data/source_maintenance.py +++ b/backend/app/data/source_maintenance.py @@ -121,7 +121,7 @@ async def _entered(source_id: str, job_id: str) -> None: loop = asyncio.get_running_loop() deadline = loop.time() + LANE_WAIT_S while loop.time() < deadline: - if runtime.lanes.ahead(source_id, job_id) is not None or not introspect_jobs.manager.running(job_id): + if await runtime.lanes.ahead(source_id, job_id) is not None or not introspect_jobs.manager.running(job_id): return await asyncio.sleep(0.01) logger.warning("定时维护:探查任务 %s 过了 %.0f 秒还没进同源队,剖析任务照建", job_id, LANE_WAIT_S) @@ -241,7 +241,7 @@ async def schedules_for(session: Any, source_id: str) -> list[dict[str, Any]]: rows = (await session.execute(select(Schedule).where( Schedule.target_kind == "source_maintenance", Schedule.target_id == source_id) - .order_by(Schedule.created_at))).scalars() + .order_by(Schedule.created_at, Schedule.id))).scalars() return [{"id": r.id, "name": r.name, "cron": r.cron, "timezone": r.timezone, "enabled": r.enabled, "next_run_at": r.next_run_at.isoformat() if r.next_run_at else None} for r in rows] diff --git a/backend/app/data/source_objects.py b/backend/app/data/source_objects.py index b938a7d4..9ff8392c 100644 --- a/backend/app/data/source_objects.py +++ b/backend/app/data/source_objects.py @@ -47,6 +47,7 @@ from app.core.errors import describe_exception, first_line from app.data.names import name_key from app.db.base import SessionLocal, utcnow from app.db.models_sources import SourceObject +from app.db.portable import contains_ci logger = logging.getLogger(__name__) @@ -405,20 +406,13 @@ def target_schema(row: Any) -> str | None: return index_schema(row) or None -_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Lock]]" = weakref.WeakKeyDictionary() +def source_lock(source_id: str) -> Any: + """写一个源的对象索引和 schema_cache 时持的锁:后台探查的开头和收尾、按需加入、加入范围都持它。跨进程的锁 + (app/cluster/mutex:本进程按事件循环各一把 asyncio 锁,PostgreSQL 时另取咨询锁):多进程时按需探查、加入范围也是 + 一个接一个,schema_cache、结构版本不会被两个进程各写一半。""" + from app.cluster import mutex - -def source_lock(source_id: str) -> asyncio.Lock: - """写一个源的对象索引和 schema_cache 时持的锁(进程内,按事件循环各一份:asyncio.Lock 不能跨循环用)。后台探查的开头 - 和收尾、按需加入、加入范围都持它。""" - loop = asyncio.get_running_loop() - by_source = _locks.get(loop) - if by_source is None: - by_source = _locks[loop] = {} - lock = by_source.get(source_id) - if lock is None: - lock = by_source[source_id] = asyncio.Lock() - return lock + return mutex.lock(f"source-objects:{source_id}") def _scope_filter(source_id: str, schema: str) -> tuple[Any, ...]: @@ -886,8 +880,8 @@ async def search_index(session: AsyncSession, row: Any, query: str, *, limit: in return [], 0 key = name_key(text.rsplit(".", 1)[-1]) filt = (*_scope_filter(row.id, index_schema(row)), SourceObject.removed_at.is_(None), - or_(SourceObject.name_key.contains(key, autoescape=True), - SourceObject.comment.contains(text, autoescape=True))) + # 名字比的是 name_key(ASCII 已转小写);注释要不分大小写(db/portable.contains_ci:两种库都折 Unicode 的大小写) + or_(SourceObject.name_key.contains(key, autoescape=True), contains_ci(SourceObject.comment, text))) total = int((await session.execute(select(func.count()).select_from(SourceObject).where(*filt))).scalar() or 0) rows = (await session.execute( select(SourceObject.name, SourceObject.name_key, SourceObject.kind, SourceObject.comment, diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index 0d1f81a7..dd3ae597 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -9387,6 +9387,30 @@ def one_row(sql: str, dialect: str | None = None) -> bool: return _aggregating(root) +def ordered(sql: str, dialect: str | None = None) -> bool: + """这条查询最外层带 ORDER BY(一段 SELECT 或 UNION 整体的):结果的行序由查询定,按行号取的那一格重跑时还是那一行(排序键 + 相同的几行之间的先后另说,按位置认)。认不出的为假。""" + from sqlglot import exp + + read = _read(dialect) + try: + root, _ = _parse(_mysql_dashes(sql) if read == "mysql" and isinstance(sql, str) else sql, read) + except Exception: # noqa: BLE001 - 解析不了就是认不出 + return False + if not isinstance(root, (exp.Select, exp.Union)): + return False + order = root.args.get("order") + return order is not None and bool(order.expressions) + + +def row_stable(sql: str, row: int, dialect: str | None = None) -> bool: + """按行号取这条查询的第 row 行,重跑、核对时还是不是同一行:最外层带 ORDER BY 的按位置认;一定只出一行的(one_row)第 0 行 + 认得住;别的(多行、又没有 ORDER BY)行序不定——库可以按任何顺序交回这些行,第 2 行重跑时可能是另一行。""" + if ordered(sql, dialect): + return True + return row == 0 and one_row(sql, dialect) + + def output_names(sql: str, dialect: str | None = None, *, template: bool = False) -> list[str] | None: """最外层选择列表各项的输出名(按位置;同 _result_columns,不展开 *:有 * 的为 None)。template 为真时 sql 是口径 模板,按哨兵渲染之后认。解析不了、不是一段 SELECT 的为 None。""" diff --git a/backend/app/data/table_versions.py b/backend/app/data/table_versions.py index ccf212cf..cdb4e2dd 100644 --- a/backend/app/data/table_versions.py +++ b/backend/app/data/table_versions.py @@ -540,10 +540,13 @@ _DB_SIDE_FILES = ("", "-wal", "-shm", "-journal") def _app_db_files() -> list[Path]: - """应用自己的库:元数据库、运行检查点库,以及实际连着的元数据库(测试里和 data_dir 下的不是同一个)。""" + """应用自己的库:元数据库、运行检查点库,以及实际连着的元数据库(测试里和 data_dir 下的不是同一个)。 + + 应用库是 PostgreSQL 时,数据目录里的 agentlab.db、checkpoints.db 照样算:导入 PostgreSQL 之前的那份还留在那里, + 里面是同一批元数据。连着的库这时是库名不是文件,不算进来。""" dbs = [settings.db_path, settings.checkpoint_path] try: - live = db_base.engine.url.database + live = db_base.engine.url.database if db_base.engine.dialect.name == "sqlite" else None except Exception: # noqa: BLE001 - 取不到就只按配置里的路径判断 live = None if live and live != ":memory:": @@ -678,21 +681,17 @@ async def resolve_source( # 锁:发布、回收、启动清理彼此串行 # -------------------------------------------------------------------------- -_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() - - -def store_lock() -> asyncio.Lock: - """版本存储的写锁,按事件循环各一把。不可重入:持着它时别再调会自己取锁的函数。 +def store_lock() -> Any: + """版本存储的写锁(app/cluster/mutex:本进程一把 asyncio 锁,按事件循环各一把;应用库是 PostgreSQL 时另取一把会话级咨询锁, + 几个进程之间也串行)。不可重入:持着它时别再调会自己取锁的函数。用法同 asyncio.Lock(async with)。 回收要先看「哪些文件有记录」再删:发布正处在「文件已 link、记录未提交」之间时,回收 会把它当孤儿删掉;复用一个刚被判为无人引用的构建,也会撞上正在删它的回收。上传不频繁, 全部串行最省心。按循环分开是因为 asyncio.Lock 一旦在某个循环里等过,就不能换循环用。 """ - loop = asyncio.get_running_loop() - lock = _locks.get(loop) - if lock is None: - lock = _locks[loop] = asyncio.Lock() - return lock + from app.cluster import mutex + + return mutex.lock("upload-store") _store_lock = store_lock @@ -744,6 +743,10 @@ def acquire_store_guard() -> bool: 同一进程里重复调用是幂等的;数据目录换了就放掉旧的、对新文件重新加锁。上次没拿到的,再调一次会重试 (占着它的进程退出后就能拿到)。没有 fcntl 的平台照常可写,只记一条警告。 """ + if settings.db_dialect == "postgresql": + # 应用库是 PostgreSQL 时可以多进程:发布、回收、启动清理之间靠跨进程的锁(store_lock 的咨询锁)互斥,不再靠文件锁 + # 独占数据目录(flock 在网络文件系统上也靠不住) + return True path = _guard_path() if _guard.path == path and _guard.fd is not None: return True @@ -773,7 +776,10 @@ def acquire_store_guard() -> bool: def store_writable() -> bool: - """本进程能不能写版本存储。没对当前数据目录取过守卫(没调过 startup)的按能写处理:守卫只在启动时取。""" + """本进程能不能写版本存储。没对当前数据目录取过守卫(没调过 startup)的按能写处理:守卫只在启动时取。PostgreSQL 时总能写 + (见 acquire_store_guard)。""" + if settings.db_dialect == "postgresql": + return True return _guard.writable or _guard.path != _guard_path() @@ -1990,7 +1996,7 @@ async def _release_raws_locked(session: AsyncSession, shas: set[str]) -> set[str async def _expire_stagings_locked(session: AsyncSession, now: datetime) -> list[str]: """(持锁)过期的暂存区 → expired、瘦身、删试运行库、放掉没人要的原件;结束超过 90 天的行删掉(提交)。""" expired = list((await session.execute( - select(ImportStaging).where(_expiry_due(now)).order_by(ImportStaging.created_at) + select(ImportStaging).where(_expiry_due(now)).order_by(ImportStaging.created_at, ImportStaging.id) )).scalars()) ids, shas = [s.id for s in expired], {s.raw_sha256 for s in expired} trials = [close_staging(s, "expired", now=now) for s in expired] @@ -2171,7 +2177,7 @@ async def purge_import_raw( stagings = list((await session.execute( select(ImportStaging).where( ImportStaging.raw_sha256 == sha, ImportStaging.status.in_(STAGING_OPEN), - ).order_by(ImportStaging.created_at) + ).order_by(ImportStaging.created_at, ImportStaging.id) )).scalars()) if sha else [] now = utcnow() record = { @@ -2442,16 +2448,37 @@ def _same_path(a: str | os.PathLike[str], b: str | os.PathLike[str]) -> bool: return False -def _sweep_tmp() -> int: - """删掉上次崩溃留下的 *.tmp-* 半成品(库和原件都是先写临时文件再 link)。""" +#: 多进程时只清多久以前的半成品(秒):别的进程正在解析、写的临时文件不在锁里(解析在锁外做),新的不能删 +TMP_STALE_S = 6 * 3600.0 + + +async def _tmp_min_age() -> float: + """启动清理半成品时只清多久以前的:SQLite(守卫保证只有本进程)、或者 PostgreSQL 上现在没有别的活着的进程,一个都不会有人 + 在写,全清;有别的进程活着,只清 TMP_STALE_S 以前的。""" + if settings.db_dialect != "postgresql": + return 0.0 + from app.cluster import state + + return TMP_STALE_S if (await state.current(fresh=True)).others else 0.0 + + +def _sweep_tmp(min_age_s: float = 0.0) -> int: + """删掉上次崩溃留下的 *.tmp-* 半成品(库和原件都是先写临时文件再 link)。min_age_s:只删改动时间早于这么多秒以前的 + (多进程时别的进程可能正在写,见 startup)。""" + import time as _time + removed = 0 + floor = _time.time() - min_age_s for root in (tables_root(), raw_store.raw_root()): if not root.is_dir(): continue for path in root.rglob(f"*{_TMP}*"): - if path.is_file(): - path.unlink(missing_ok=True) - removed += 1 + try: + if path.is_file() and (not min_age_s or path.stat().st_mtime < floor): + path.unlink(missing_ok=True) + removed += 1 + except OSError: + continue return removed @@ -2674,7 +2701,8 @@ async def startup(session: AsyncSession) -> None: logger.error("上传表格的版本存储被另一个进程占用,本进程跳过启动清理、迁移和回收") return async with _store_lock(): - swept = await asyncio.to_thread(_sweep_tmp) + # 半成品:没有别的进程在跑时一个都不会有人在写,全清;多进程时别的进程正在解析的临时文件写在锁外,只清旧的(_tmp_min_age) + swept = await asyncio.to_thread(_sweep_tmp, await _tmp_min_age()) if swept: logger.info("清理了 %d 个上次未完成的上传临时文件", swept) migrated = await _migrate_legacy(session) diff --git a/backend/app/db/__main__.py b/backend/app/db/__main__.py new file mode 100644 index 00000000..661cc3b9 --- /dev/null +++ b/backend/app/db/__main__.py @@ -0,0 +1,31 @@ +"""应用库的维护命令:python -m app.db <命令>。 + + copy-to-postgres 把 SQLite 数据目录里的应用库和断点导入 PostgreSQL(db/to_postgres.py;先停服务) +""" + +from __future__ import annotations + +import sys + +COMMANDS = {"copy-to-postgres": "把 SQLite 数据目录里的应用库和断点导入 PostgreSQL(先停服务)"} + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if not args or args[0] in ("-h", "--help"): + print("用法:python -m app.db <命令> [参数]\n") + for name, what in COMMANDS.items(): + print(f" {name:<18}{what}") + print("\n各命令加 --help 看参数") + return 0 if args else 2 + command, rest = args[0], args[1:] + if command == "copy-to-postgres": + from app.db.to_postgres import main as copy_main + + return copy_main(rest) + print(f"认不出的命令:{command}(可用的:{'、'.join(COMMANDS)})", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/app/db/base.py b/backend/app/db/base.py index 24a40182..5369bd15 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -1,16 +1,24 @@ from __future__ import annotations +import json +import logging +import math +import re import uuid from datetime import datetime, timezone from typing import Any, AsyncIterator -from sqlalchemy import JSON, DateTime, MetaData, event -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy import JSON, BigInteger, Column, DateTime, Integer, MetaData, event +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from sqlalchemy.pool import NullPool +from sqlalchemy.engine import make_url from sqlalchemy.types import TypeDecorator from app.core.config import settings +logger = logging.getLogger(__name__) + NAMING_CONVENTION = { "ix": "ix_%(column_0_label)s", "uq": "uq_%(table_name)s_%(column_0_name)s", @@ -27,6 +35,14 @@ def new_id() -> str: return uuid.uuid4().hex +def current_owner() -> str: + """认领一行时记的属主:这个工作进程(app/cluster/identity.OWNER)。运行、问数据的一轮、各种后台任务表的 owner 列缺省取它: + 建这一行的进程就是认领它的进程,和这一行同一个事务写下,不会有「行建好了、还没人认领」的空档(多进程,W2-b)。""" + from app.cluster.identity import owner + + return owner() + + class UTCDateTime(TypeDecorator): """存 UTC、读出来也是带时区的 UTC。 @@ -57,11 +73,31 @@ class UTCDateTime(TypeDecorator): return value.astimezone(timezone.utc) +#: 应用库的 JSON 列:两种库上都是存原文的 JSON(PostgreSQL 的 json,不是 JSONB)。JSONB 会把对象里的键重新排序: +#: 应用里不少地方按存进去的键的先后出东西(快照的列、表的先后、复核结论各段的先后),换成 JSONB 读回来就变了样, +#: 测试当场抓到好几处。json 的代价是没有相等运算符:== 比较、DISTINCT、GROUP BY、ORDER BY 不能直接作用在整列上 +#: (应用里没有这种写法,test_db_portable 守着);按里面某个键筛选走 db/portable 的 json_eq 等,只把取出来的那一小段 +#: 转成 JSONB 比较 +PortableJSON = JSON() + + class Base(DeclarativeBase): metadata = MetaData(naming_convention=NAMING_CONVENTION) - # 图定义、节点配置、事件负载都是任意形状的 JSON,统一映射到 SQLite 的 JSON 列 - type_annotation_map = {dict[str, Any]: JSON, list[Any]: JSON, datetime: UTCDateTime} + # 图定义、节点配置、事件负载都是任意形状的 JSON,统一映射到 JSON 列(见 PortableJSON) + type_annotation_map = {dict[str, Any]: PortableJSON, list[Any]: PortableJSON, datetime: UTCDateTime} + + +@event.listens_for(Column, "after_parent_attach") +def _portable_integer(column: Column, table: Any) -> None: + """应用库的 Integer 列在 PostgreSQL 上换成 BIGINT(SQLite 照旧是 INTEGER,主键照旧是 rowid)。 + + SQLite 的 INTEGER 是 64 位,PostgreSQL 的只有 32 位:行数估计、字节数、累计的计数都可能过 21 亿。在列挂到表上 + 的那一刻换(定义模型时,任何查询之前):建表是 BIGINT(自增主键是 BIGSERIAL),和这一列比较、写进这一列的绑定 + 参数也按 BIGINT 转换。只管挂在应用 metadata 上、类型正好是 Integer 的列;业务数据源那边的表不受影响。""" + if getattr(table, "metadata", None) is Base.metadata and type(column.type) is Integer \ + and not column.type._variant_mapping: + column.type = Integer().with_variant(BigInteger(), "postgresql") class TimestampMixin: @@ -101,16 +137,10 @@ def _stop_aiosqlite_once() -> None: aiosqlite.Connection.stop = stop # type: ignore[method-assign] +# 应用库是 PostgreSQL 时也照样打:上传表格、手工登记的 SQLite 业务库(data/engine)还是走 aiosqlite 的连接池 _stop_aiosqlite_once() -# pool_pre_ping:取连接前先探一下活。取消异常打进 SQLAlchemy 的清理流程时 -# (_close_connection 记完日志会把 CancelledError 原样抛出,跳过了把连接记录 -# 清空的那一行),已经关掉的连接会留在池里,下一个取到它的人拿到的是 -# "no active connection"。探活失败会被当成断开,换一条新的——SQLite 本地一次 -# SELECT 1 的代价,换池子不被污染。 -engine = create_async_engine(settings.db_url, echo=False, future=True, pool_pre_ping=True) -@event.listens_for(engine.sync_engine, "connect") def _set_sqlite_pragma(dbapi_conn, _record): # pragma: no cover - 驱动回调 """WAL + 外键约束。WAL 让读写不互相阻塞,事件写入和前端轮询能并行。""" cur = dbapi_conn.cursor() @@ -120,8 +150,192 @@ def _set_sqlite_pragma(dbapi_conn, _record): # pragma: no cover - 驱动回调 cur.close() +def _finite(value: Any) -> Any: + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {k: _finite(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_finite(v) for v in value] + return value + + +def _no_nul(value: Any) -> Any: + if isinstance(value, str): + return value.replace("\x00", "") + if isinstance(value, dict): + return {_no_nul(k): _no_nul(v) for k, v in value.items()} + if isinstance(value, list): + return [_no_nul(v) for v in value] + return value + + +def pg_json_dumps(value: Any) -> str: + """PostgreSQL 上写 JSON 列。它不收 NaN / Infinity(SQLite 照存);字符串里的 \\u0000 json 列收,但一转成 JSONB + (按键筛选时,db/portable)、一取成文本就报错,那一整条查询跟着失败。 + + 非有限的浮点写成 null(同浏览器的 JSON.stringify),字符串里的 NUL 字符去掉;平常的值一次 dumps 就过, + 只有碰上了才多走一遍。""" + try: + text = json.dumps(value, allow_nan=False) + except ValueError: + text = json.dumps(_finite(value), allow_nan=False) + if "\\u0000" in text: + text = json.dumps(_no_nul(json.loads(text)), allow_nan=False) + return text + + +#: 应用连 PostgreSQL 时报的 application_name。copy-to-postgres 按它认目标库上还有没有服务连着(它自己报的是 agentlab-copy) +APPLICATION_NAME = "agentlab" + + +def pg_engine_kwargs(*, application_name: str = APPLICATION_NAME, pgbouncer: bool = False, + pool_size: int | None = None) -> dict[str, Any]: + """PostgreSQL 引擎的连接池和连接参数。pgbouncer:这个地址是 PgBouncer 的事务模式(AGENTLAB_DB_PGBOUNCER)—— + + - 不用客户端的连接池(NullPool,连接池交给 PgBouncer;SQLAlchemy 的说明里同样这么建议); + - SQLAlchemy 的预备语句缓存(prepared_statement_cache_size)和 asyncpg 自己的(statement_cache_size)都关掉:事务模式下同一个 + 客户端连接的下一条语句可能落在另一条服务端连接上,缓存着的预备语句在那边不存在; + - asyncpg 每条语句仍要先预备再执行(SQLAlchemy 的 asyncpg 方言一律这么做,同一个事务里,落在同一条服务端连接上),名字用随机的 + (prepared_statement_name_func),不和别的客户端在同一条服务端连接上留下的撞名。PgBouncer 要配 server_reset_query=DISCARD ALL + (且 server_reset_query_always=1)收掉留下的预备语句,或用 1.21 以上开 max_prepared_statements。 + 事务级的咨询锁(pg_advisory_xact_lock,发名额、同源队、建表时用)在事务模式下照常成立:取锁和放锁都在同一个事务里。""" + connect_args: dict[str, Any] = { + # 会话时区定成 UTC:时间列是不带时区的 UTC(UTCDateTime),库里 now() 之类的也按 UTC 算 + "server_settings": {"application_name": application_name, "timezone": "UTC"}} + size = settings.db_pool_size if pool_size is None else pool_size + if pgbouncer: + connect_args.update(statement_cache_size=0, prepared_statement_cache_size=0, + prepared_statement_name_func=_unique_statement_name) + return {"poolclass": NullPool, "connect_args": connect_args} + if size <= 0: + return {"poolclass": NullPool, "connect_args": connect_args} + return {"pool_size": size, "max_overflow": max(settings.db_max_overflow, 0), "pool_timeout": settings.db_pool_timeout, + "pool_recycle": settings.db_pool_recycle, "connect_args": connect_args} + + +def _unique_statement_name() -> str: + return f"__agentlab_{uuid.uuid4().hex}__" + + +def make_engine(url: str | None = None, *, application_name: str = APPLICATION_NAME, pgbouncer: bool | None = None, + pool_size: int | None = None) -> AsyncEngine: + """应用库的引擎。SQLite:WAL、外键、忙等,单个文件;PostgreSQL:连接池、方言调整(db/postgres.py)。 + + copy-to-postgres 建目标库的引擎也走这里,两边的列类型、排序规则是同一套。 + """ + url = url or settings.db_url + if pgbouncer is None: + # 只有应用库的普通地址按 AGENTLAB_DB_PGBOUNCER 处理:直连地址、copy-to-postgres 给的目标库照常 + pgbouncer = bool(settings.db_pgbouncer) and url == settings.database_url + if url.startswith("sqlite"): + # pool_pre_ping:取连接前先探一下活。取消异常打进 SQLAlchemy 的清理流程时 + # (_close_connection 记完日志会把 CancelledError 原样抛出,跳过了把连接记录 + # 清空的那一行),已经关掉的连接会留在池里,下一个取到它的人拿到的是 + # "no active connection"。探活失败会被当成断开,换一条新的——SQLite 本地一次 + # SELECT 1 的代价,换池子不被污染。 + made = create_async_engine(url, echo=False, future=True, pool_pre_ping=True) + event.listen(made.sync_engine, "connect", _set_sqlite_pragma) + return made + from app.db import postgres + + _require_postgres_driver("asyncpg") + made = create_async_engine(url, echo=False, future=True, pool_pre_ping=True, json_serializer=pg_json_dumps, + **pg_engine_kwargs(application_name=application_name, pgbouncer=pgbouncer, + pool_size=pool_size)) + postgres.install(made.sync_engine.dialect) + event.listen(made.sync_engine, "before_cursor_execute", postgres.strip_nul, retval=True) + return made + + +def _require_postgres_driver(*modules: str) -> None: + """设了 AGENTLAB_DATABASE_URL 却没装 [postgres]:说清楚要装什么,不丢一个光秃秃的 ModuleNotFoundError。""" + import importlib.util + + missing = [m for m in modules if importlib.util.find_spec(m) is None] + if missing: + raise RuntimeError( + f"应用库设成了 PostgreSQL(AGENTLAB_DATABASE_URL),但没装它要的包({'、'.join(missing)}):" + "pip install -e 'backend[postgres]' -c backend/requirements.lock") + + +def describe_database() -> str: + """启动日志里的一句:用的是哪种库、在哪(不写密码)。""" + from app.db.urls import describe + + if settings.checkpoint_url: + points = f"断点在 PostgreSQL {describe(settings.checkpoint_url)}" + elif settings.db_dialect == "sqlite": + points = f"断点 {settings.checkpoint_path}" + else: + points = "断点在同一个库" + if settings.db_dialect == "sqlite": + return f"SQLite({settings.db_path},{points})" + return f"PostgreSQL({describe(settings.db_url)},{points})" + + +engine = make_engine() + SessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession) +_session_engine: AsyncEngine | None = None + + +def session_engine() -> AsyncEngine: + """依赖会话状态的连接用的引擎(跨进程锁的会话级咨询锁,app/cluster/mutex):设了 AGENTLAB_DATABASE_DIRECT_URL 时是直连那个库的 + 一个小池子,否则就是应用库的引擎。""" + global _session_engine + if not settings.database_direct_url or settings.database_direct_url == settings.database_url: + return engine + if _session_engine is None: + _session_engine = make_engine(settings.database_direct_url, pgbouncer=False, pool_size=max(2, min(settings.db_pool_size or 5, 10))) + return _session_engine + + +async def dispose_session_engine() -> None: + """关停时关掉直连那个小池子(没另建就什么都不做)。""" + global _session_engine + if _session_engine is not None: + await _session_engine.dispose() + _session_engine = None + + +def check_pgbouncer() -> None: + """AGENTLAB_DB_PGBOUNCER=1 时直连地址要真的是直连:没给、或者给的和应用库地址指着同一个主机端口(还是那个 PgBouncer),都拒绝 + 启动(事务模式下 LISTEN、会话级咨询锁不成立,跨进程的锁会锁不住、放不掉;复核 R-b12)。""" + if not (settings.db_pgbouncer and settings.db_dialect == "postgresql"): + return + if not settings.database_direct_url: + raise RuntimeError("AGENTLAB_DB_PGBOUNCER=1(应用库地址是 PgBouncer 的事务模式)时要另给 AGENTLAB_DATABASE_DIRECT_URL:" + "进程间通知(LISTEN)、跨进程的锁(会话级咨询锁)、断点库要直连 PostgreSQL(或 PgBouncer 的会话模式)。" + "见 README「多进程」里的「直连与 PgBouncer」") + if _endpoint(settings.database_direct_url) == _endpoint(settings.database_url): + from app.db.urls import describe + + raise RuntimeError(f"AGENTLAB_DATABASE_DIRECT_URL 和 AGENTLAB_DATABASE_URL 指着同一个地址({describe(settings.database_url)}):" + "AGENTLAB_DB_PGBOUNCER=1 说应用库地址是 PgBouncer 的事务模式,直连地址要绕过它、直接连 PostgreSQL" + "(或者 PgBouncer 的会话模式那个端口)。见 README「多进程」里的「直连与 PgBouncer」") + + +def _endpoint(url: str) -> tuple[str, int]: + """地址连的是哪个主机端口(端口不写按 5432)。主机名规范化:localhost、127.x.x.x、::1 都算本机回环;IP 地址按地址比 + (::ffff:10.0.0.5 和 10.0.0.5 相同);解析不成 IP 的按字面(小写)比,不查 DNS。""" + import ipaddress + + parsed = make_url(url) + host = (parsed.host or "").strip().strip("[]").lower().rstrip(".") + try: + addr = ipaddress.ip_address(host) + except ValueError: + addr = None + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None: + addr = addr.ipv4_mapped + if host == "localhost" or host.endswith(".localhost") or (addr is not None and addr.is_loopback): + host = "loopback" + elif addr is not None: + host = str(addr) + return host, int(parsed.port or 5432) + async def get_session() -> AsyncIterator[AsyncSession]: """FastAPI 依赖注入用的会话。""" @@ -129,8 +343,10 @@ async def get_session() -> AsyncIterator[AsyncSession]: yield session -# 轻量迁移:create_all 只建新表,不给已有表加列。SQLite 的 ADD COLUMN -# 便宜且带默认值回填,够用;等真需要改列类型再考虑 alembic。 +# 轻量迁移:create_all 只建新表,不给已有表加列。ADD COLUMN 便宜且带默认值回填,够用;等真需要改列类型再考虑 alembic。 +# 第三项照 SQLite 写(类型和缺省值的字面量)。PostgreSQL 上类型取模型里这一列的(按应用库的方言调整编译:String(n) +# 是 VARCHAR、Integer 是 BIGINT、JSON 是 json、时间是 TIMESTAMP),缺省值取这里写的,布尔的 0 / 1 换成 false / true +# (_column_ddl)。所以这里加的每一列模型里都得有:test_db_portable 守着 _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("runs", "run_class", "TEXT DEFAULT 'exploratory'"), ("runs", "version", "INTEGER"), @@ -247,12 +463,30 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("schedule_fires", "eval_run_id", "VARCHAR(32)"), # 编排完善(docs/design/orchestration-round/00-总方案.md §4):定时任务的触发方式。存量行都是 cron ("schedules", "trigger_kind", "VARCHAR(20) DEFAULT 'cron'"), - ("schedules", "trigger", "JSON"), + # 第一版补这一列时没带缺省值:在那之前建过定时任务的库里这一列是 NULL(读时 or {} 兜着),模型是 NOT NULL。 + # 现在带上缺省值;已经补过、留着 NULL 的由 _NOT_NULL_BACKFILLS 填上 + ("schedules", "trigger", "JSON DEFAULT '{}'"), # 第 2 波:服务账号(users.kind);剖析上传表格源时的快照 ("users", "kind", "VARCHAR(20) DEFAULT 'person'"), ("catalog_profile_tasks", "snapshot_id", "VARCHAR(32)"), # 第 3 波:发布时这一版引用的表的结构版本 ("workflow_versions", "schema_versions", "JSON"), + # 编排完善第 2 波 W2-b(多进程协调):认领这一行的工作进程(app/cluster)。存量行为空:升级前的进程认领的,按「属主不在了」 + # 处理(升级要停掉全部进程再起) + ("runs", "owner", "VARCHAR(120)"), + ("chat_turn_jobs", "owner", "VARCHAR(120)"), + ("catalog_draft_jobs", "owner", "VARCHAR(120)"), + ("catalog_profile_jobs", "owner", "VARCHAR(120)"), + ("introspect_jobs", "owner", "VARCHAR(120)"), + ("eval_runs", "owner", "VARCHAR(120)"), + ("eval_catalog_snapshots", "owner", "VARCHAR(120)"), + ("documents", "owner", "VARCHAR(120)"), + ("publish_plans", "owner", "VARCHAR(120)"), + # 问数据:等不及的那次复核还在后台跑(实时流在它回来之前不关;多进程时别的进程上的流也看得到) + ("chat_turn_jobs", "late_review_at", "DATETIME"), + # 工作进程登记的代码版本、库结构版本(app/cluster/version.py:不许新旧版本混跑) + ("cluster_workers", "code_version", "VARCHAR(200)"), + ("cluster_workers", "schema_version", "VARCHAR(64)"), ] #: 迁移补上的列要建的索引:(索引名, 表, 列或多列)。create_all 不碰已有的表,表上新列的索引也就不会建。 @@ -272,46 +506,286 @@ _INDEX_MIGRATIONS: list[tuple[str, str, str | tuple[str, ...]]] = [ ] +#: 补列时没带缺省值、存量行留着 NULL 的 NOT NULL 列:每次启动按模型的缺省值填上(UPDATE … WHERE 列 IS NULL,填完再跑 +#: 什么都不改)。两种库存的才一样,导入 PostgreSQL 时这一列也不会撞 NOT NULL。以后新加的 NOT NULL 列在迁移表里一律带 +#: DEFAULT(test_db_portable 守着),就不用往这里加 +_NOT_NULL_BACKFILLS: list[tuple[str, str]] = [("schedules", "trigger")] + + +def model_default(column: Any) -> tuple[bool, Any]: + """模型列的缺省值:(有没有, 值)。标量原样,无参的可调用(dict、list、utcnow 之类)调一次;要按执行上下文算的不算。""" + default = getattr(column, "default", None) + if default is None: + return False, None + if default.is_scalar: + return True, default.arg + if default.is_callable: + try: + return True, default.arg(None) + except Exception: # noqa: BLE001 - 要读执行上下文的缺省值:这里给不出 + return False, None + return False, None + + +def _backfill_not_null(conn: Any, columns: dict[str, set[str]]) -> None: + from sqlalchemy import Table, update + + for table_name, column_name in _NOT_NULL_BACKFILLS: + if column_name not in (columns.get(table_name) or set()): + continue + model = Base.metadata.tables[table_name].c[column_name] + known, value = model_default(model) + if not known: + continue + # 只带这一列的表:模型表上 updated_at 的 onupdate 会跟着改(这是补数据,不是改了这一行),老库也未必有那一列 + bare = Table(table_name, MetaData(), Column(column_name, model.type)) + conn.execute(update(bare).where(bare.c[column_name].is_(None)).values({column_name: value})) + + #: 迁移补建的唯一索引:(索引名, 表, 列, 部分索引的条件, 建之前的去重语句)。只在索引还不存在时先去重再建——存量库里可能 #: 已经有并发留下的重复行,直接建会失败。去重的规矩写在各自的语句里:多出的钩子行删掉(只留最早的那一行,它的密钥照旧 -#: 能用),多出的助手对话归档(不删,历史还在) +#: 能用),多出的助手对话归档(不删,历史还在)。条件和语句里的 {false} / {true} 按方言换成布尔的字面量(SQLite 是 0 / 1) _UNIQUE_INDEX_MIGRATIONS: list[tuple[str, str, tuple[str, ...], str | None, str]] = [ ("ux_app_hooks_app_name", "app_hooks", ("app_id", "name"), None, "DELETE FROM app_hooks WHERE id NOT IN (SELECT id FROM (SELECT id, ROW_NUMBER() OVER " - "(PARTITION BY app_id, name ORDER BY created_at, id) AS rn FROM app_hooks) WHERE rn = 1)"), - ("ux_conversations_app_owner", "conversations", ("app_id", "created_by"), "kind = 'app' AND archived = 0", - "UPDATE conversations SET archived = 1 WHERE kind = 'app' AND archived = 0 AND created_by IS NOT NULL " + "(PARTITION BY app_id, name ORDER BY created_at, id) AS rn FROM app_hooks) AS ranked WHERE rn = 1)"), + ("ux_conversations_app_owner", "conversations", ("app_id", "created_by"), "kind = 'app' AND archived = {false}", + "UPDATE conversations SET archived = {true} WHERE kind = 'app' AND archived = {false} AND created_by IS NOT NULL " "AND id NOT IN (SELECT id FROM (SELECT id, ROW_NUMBER() OVER (PARTITION BY app_id, created_by " - "ORDER BY last_active_at DESC, id) AS rn FROM conversations WHERE kind = 'app' AND archived = 0 " - "AND created_by IS NOT NULL) WHERE rn = 1)"), + "ORDER BY last_active_at DESC, id) AS rn FROM conversations WHERE kind = 'app' AND archived = {false} " + "AND created_by IS NOT NULL) AS ranked WHERE rn = 1)"), ] +#: 布尔字面量:SQLite 的布尔列存 0 / 1 +_BOOL_LITERALS = {"sqlite": {"false": "0", "true": "1"}, "postgresql": {"false": "false", "true": "true"}} + +_DEFAULT = re.compile(r"\s+DEFAULT\s+(.+)$", re.I | re.S) + + +def _column_ddl(dialect: Any, table: str, column: str, ddl: str) -> str: + """ADD COLUMN 里列名后面那一段。SQLite 原样用迁移表里写的;PostgreSQL 类型取模型里这一列的,缺省值取迁移表里的。""" + if dialect.name == "sqlite": + return ddl + from sqlalchemy import Boolean + + from app.db.postgres import AppTypeCompiler + + col = Base.metadata.tables[table].c[column] + out = AppTypeCompiler(dialect).process(col.type, type_expression=col) + if match := _DEFAULT.search(ddl): + default = match.group(1).strip() + if isinstance(col.type, Boolean): + default = {"0": "false", "1": "true"}.get(default, default) + out += f" DEFAULT {default}" + return out + def _migrate(conn) -> None: # pragma: no cover - 同步回调 - from sqlalchemy import text + """给老库补列、补索引。已有的列和索引按 inspector 认(两种库同一套),再跑一遍什么都不做。""" + from sqlalchemy import inspect, text + dialect = conn.dialect + literals = _BOOL_LITERALS.get(dialect.name, _BOOL_LITERALS["postgresql"]) + inspector = inspect(conn) + tables = set(inspector.get_table_names()) + columns = {t: {c["name"] for c in inspector.get_columns(t)} for t in tables} + # IF NOT EXISTS:PostgreSQL 上多个进程同时启动,各自的检查和补列之间别人可能已经补上了(init_db 另外拿着锁) + add = "ADD COLUMN IF NOT EXISTS" if dialect.name == "postgresql" else "ADD COLUMN" for table, column, ddl in _COLUMN_MIGRATIONS: - cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))} + cols = columns.get(table) if cols and column not in cols: - conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}")) - for name, table, columns in _INDEX_MIGRATIONS: - wanted = (columns,) if isinstance(columns, str) else columns - cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))} + conn.execute(text(f"ALTER TABLE {table} {add} {column} {_column_ddl(dialect, table, column, ddl)}")) + cols.add(column) + for name, table, wanted in _INDEX_MIGRATIONS: + wanted = (wanted,) if isinstance(wanted, str) else wanted + cols = columns.get(table) or set() if all(c in cols for c in wanted): conn.execute(text(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({', '.join(wanted)})")) - for name, table, columns, where, dedupe in _UNIQUE_INDEX_MIGRATIONS: - cols = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))} - exists = conn.execute(text("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = :n"), {"n": name}).first() - if exists or not all(c in cols for c in columns): + _unique_run_event_seq(conn, columns) + for name, table, wanted, where, dedupe in _UNIQUE_INDEX_MIGRATIONS: + cols = columns.get(table) or set() + if not all(c in cols for c in wanted): continue - conn.execute(text(dedupe)) - conn.execute(text(f"CREATE UNIQUE INDEX {name} ON {table} ({', '.join(columns)})" - + (f" WHERE {where}" if where else ""))) + if any(ix["name"] == name for ix in inspect(conn).get_indexes(table)): + continue + conn.execute(text(dedupe.format(**literals))) + conn.execute(text(f"CREATE UNIQUE INDEX {name} ON {table} ({', '.join(wanted)})" + + (f" WHERE {where.format(**literals)}" if where else ""))) + _backfill_not_null(conn, columns) + + +#: 存量库里 run_events 撞了号、这一次没建唯一索引时记下的结果(settings 表里;cluster: 开头的不是用户设置,api/settings 不列) +SEQ_DUPLICATES_KEY = "cluster:run_events_seq_duplicates" +#: 记下的撞号最多几处(之后启动只核对这几处还在不在) +SEQ_DUPLICATES_KEPT = 200 + + +def _unique_run_event_seq(conn: Any, columns: dict[str, set[str]]) -> None: # pragma: no cover - 同步回调 + """run_events 的 (run_id, seq) 补唯一索引(ux_run_events_run_seq,换掉原来不唯一的 ix_run_events_run_seq)。存量库里可能已经 + 有撞了号的(多进程之前的进程内计数、按需裁判的老写法):先数一数,有就照实记日志、这一次不建(照旧用不唯一的索引),不让启动 + 失败——那几次运行的封存核对本来就对不上,删哪一条都不对,要人看。 + + 数一次就记下结果(settings 表的 SEQ_DUPLICATES_KEY:撞了号的那几处),之后启动不再整表 GROUP BY(复核 R-b13):只按索引核对 + 记下的那几处还撞不撞,还撞着就一句话带过;都清掉了才再整表数一遍、建索引、删掉这条记录。""" + from sqlalchemy import inspect, text + + if "run_events" not in columns: + return + names = {ix["name"] for ix in inspect(conn).get_indexes("run_events")} + if "ux_run_events_run_seq" in names: + return + known = _known_seq_duplicates(conn, columns) + if known: + still = _still_duplicated(conn, known["pairs"]) + if still: + logger.info("run_events 里还有撞了号的事件(%s 记下的 %d 处里还剩 %d 处):(run_id, seq) 的唯一索引照旧不建," + "处理掉之后下次启动会再核对一遍", known.get("found_at") or "之前", len(known["pairs"]), still) + return + dupes = conn.execute(text("SELECT run_id, seq FROM run_events GROUP BY run_id, seq HAVING COUNT(*) > 1 LIMIT :n"), + {"n": SEQ_DUPLICATES_KEPT + 1}).all() + if dupes: + runs = sorted({str(r[0]) for r in dupes}) + logger.warning("run_events 里有撞了号的事件(%d 处%s,涉及运行 %s):这一次不建 (run_id, seq) 的唯一索引。" + "这几次运行的封存核对对不上,请人工核对后处理(之后启动只核对这几处,不再整表查)", + min(len(dupes), SEQ_DUPLICATES_KEPT), "以上" if len(dupes) > SEQ_DUPLICATES_KEPT else "", + "、".join(runs[:5])) + _remember_seq_duplicates(conn, [[str(r[0]), int(r[1])] for r in dupes[:SEQ_DUPLICATES_KEPT]], + more=len(dupes) > SEQ_DUPLICATES_KEPT) + return + conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS ux_run_events_run_seq ON run_events (run_id, seq)")) + if "ix_run_events_run_seq" in names: + conn.execute(text("DROP INDEX IF EXISTS ix_run_events_run_seq")) + if known is not None: + _forget_seq_duplicates(conn) + logger.info("run_events 里的撞号都处理掉了:补上了 (run_id, seq) 的唯一索引") + + +def _known_seq_duplicates(conn: Any, columns: dict[str, set[str]]) -> dict[str, Any] | None: # pragma: no cover + if "settings" not in columns: + return None + from sqlalchemy import select + + from app.db.models import Setting + + row = conn.execute(select(Setting.value).where(Setting.key == SEQ_DUPLICATES_KEY)).first() + value = row[0] if row else None + if not isinstance(value, dict) or not isinstance(value.get("pairs"), list): + return None + return value + + +def _still_duplicated(conn: Any, pairs: list[Any]) -> int: # pragma: no cover + """记下的那几处里还撞着的有几处(每处按 (run_id, seq) 索引查一次)。""" + from sqlalchemy import func, select + + from app.db.models import RunEvent + + left = 0 + for pair in pairs: + if not (isinstance(pair, list) and len(pair) == 2): + continue + n = conn.execute(select(func.count()).select_from(RunEvent).where( + RunEvent.run_id == str(pair[0]), RunEvent.seq == int(pair[1]))).scalar() + left += int((n or 0) > 1) + return left + + +def _remember_seq_duplicates(conn: Any, pairs: list[list[Any]], *, more: bool) -> None: # pragma: no cover + from app.db.models import Setting + + _forget_seq_duplicates(conn) + conn.execute(Setting.__table__.insert().values( + key=SEQ_DUPLICATES_KEY, value={"pairs": pairs, "more": more, "found_at": utcnow().isoformat(timespec="seconds")})) + + +def _forget_seq_duplicates(conn: Any) -> None: # pragma: no cover + from sqlalchemy import delete + + from app.db.models import Setting + + conn.execute(delete(Setting).where(Setting.key == SEQ_DUPLICATES_KEY)) + + +#: init_db 在 PostgreSQL 上拿的会话级咨询锁的号:几个进程同时启动时,建表和迁移一个一个来(同一个号别处别用) +_INIT_LOCK = 0x4167656E744C6162 # "AgentLab" + + +async def create_schema(target: AsyncEngine) -> None: + """建表、补列补索引(create_all 只建新表,老库的新列靠 _migrate)。两种库同一套;copy-to-postgres 建目标库也用它。""" + from app.db import models # noqa: F401 确保模型已注册到 metadata + + async with target.begin() as conn: + if conn.dialect.name == "postgresql": + # 事务结束自动放锁。CREATE TABLE IF NOT EXISTS 在 PostgreSQL 上并发执行也会撞(系统表的唯一约束) + from sqlalchemy import text + + await conn.execute(text("SELECT pg_advisory_xact_lock(:k)"), {"k": _INIT_LOCK}) + await conn.run_sync(Base.metadata.create_all) + await conn.run_sync(_migrate) + + +#: 连不上时等也没用的错误(SQLSTATE 的类):28 认证失败、3D 库不存在、42 语法或权限。别的(拒绝连接、库正在启动 57P03、 +#: 超时、网络断了)按「还没就绪」退避重试 +_NOT_TRANSIENT_SQLSTATES = ("28", "3D", "42") + + +def _not_transient(error: BaseException) -> bool: + from app.db.portable import _driver_error + + seen: Any = error + for _ in range(4): + state = str(getattr(seen, "sqlstate", "") or "") + if state and state != "57P03": + return state.startswith(_NOT_TRANSIENT_SQLSTATES) + seen = getattr(seen, "__cause__", None) or getattr(seen, "orig", None) + if seen is None: + break + cause = _driver_error(error) + return bool(cause is not None and str(cause.sqlstate).startswith(_NOT_TRANSIENT_SQLSTATES)) + + +async def wait_for_database(target: AsyncEngine | None = None, *, timeout: float | None = None, + sleep: Any = None, clock: Any = None) -> None: + """PostgreSQL 还连不上时按退避(1、2、4、5、5… 秒)重试,每次照实打一条日志,等够 timeout 秒(缺省 + AGENTLAB_DB_STARTUP_TIMEOUT)还不行就把最后一次的错抛出去。和数据库容器一起起、库还在初始化时,应用不至于一上来就崩。 + 密码不对、库不存在这种等也没用的立刻抛出。SQLite 不等。""" + import asyncio + import time + + from sqlalchemy import text + + from app.core.errors import first_line + + target = target or engine + if target.dialect.name != "postgresql": + return + budget = settings.db_startup_timeout if timeout is None else timeout + pause = sleep or asyncio.sleep + now = clock or time.monotonic + started = now() + delay = 1.0 + attempt = 0 + while True: + attempt += 1 + try: + async with target.connect() as conn: + await conn.execute(text("SELECT 1")) + if attempt > 1: + logger.info("应用库连上了(第 %d 次,等了 %.0f 秒)", attempt, now() - started) + return + except Exception as e: # noqa: BLE001 - 判断完是不是该等再决定抛不抛 + waited = now() - started + if _not_transient(e) or waited + delay > budget: + logger.error("应用库连不上,不再等了(等了 %.0f 秒):%s", waited, first_line(e)) + raise + logger.warning("应用库还没就绪(%s),%.0f 秒后重试(已等 %.0f 秒,最多 %d 秒)", + first_line(e), delay, waited, budget) + await pause(delay) + delay = min(delay * 2, 5.0) async def init_db() -> None: - from app.db import models # noqa: F401 确保模型已注册到 metadata - - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - await conn.run_sync(_migrate) + logger.info("应用库:%s", describe_database()) + # 库还没起来(和 PostgreSQL 容器一起起)时先等它;建表、补列、补索引在库里的咨询锁里做,几个工作进程同时启动只做一次 + await wait_for_database() + await create_schema(engine) diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 860819a1..bf7f41ce 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -8,7 +8,7 @@ from sqlalchemy import ( ) from sqlalchemy.orm import Mapped, mapped_column, relationship -from app.db.base import Base, TimestampMixin, UTCDateTime, new_id, utcnow +from app.db.base import Base, current_owner, TimestampMixin, UTCDateTime, new_id, utcnow # -------------------------------------------------------------------------- # 设置:模型供应商 / 用户偏好 / MCP / 自定义工具 @@ -362,13 +362,16 @@ class CatalogDraftJob(Base, TimestampMixin): __table_args__ = ( Index("ix_catalog_draft_jobs_source_created", "source_id", "created_at"), Index("ux_catalog_draft_jobs_active", "source_id", unique=True, - sqlite_where=text("status IN ('queued', 'running')")), + sqlite_where=text("status IN ('queued', 'running')"), + postgresql_where=text("status IN ('queued', 'running')")), ) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) # queued | running | succeeded | partial | failed | cancelled | interrupted status: Mapped[str] = mapped_column(String(20), default="queued") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) use_model: Mapped[bool] = mapped_column(default=False) # 实际用的模型 id(用模型时开始起草后写上) model: Mapped[str | None] = mapped_column(String(200), default=None) @@ -450,13 +453,16 @@ class CatalogProfileJob(Base, TimestampMixin): __table_args__ = ( Index("ix_catalog_profile_jobs_source_created", "source_id", "created_at"), Index("ux_catalog_profile_jobs_active", "source_id", unique=True, - sqlite_where=text("status IN ('queued', 'running')")), + sqlite_where=text("status IN ('queued', 'running')"), + postgresql_where=text("status IN ('queued', 'running')")), ) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) # queued | running | succeeded | partial | failed | cancelled | interrupted status: Mapped[str] = mapped_column(String(20), default="queued") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) # selected 指定的表 | needed 需要剖析的表 | all 全部表 scope: Mapped[str] = mapped_column(String(20), default="selected") # 建任务时的剖析设置快照(catalog_profile.ProfileSettings.to_dict) @@ -655,6 +661,8 @@ class Run(Base, TimestampMixin): thread_id: Mapped[str] = mapped_column(String(64), index=True, default=new_id) # queued | running | interrupted | succeeded | failed | cancelled status: Mapped[str] = mapped_column(String(20), default="queued", index=True) + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) graph: Mapped[dict[str, Any]] = mapped_column(default=dict) # 执行时的图快照 input: Mapped[dict[str, Any]] = mapped_column(default=dict) output: Mapped[dict[str, Any]] = mapped_column(default=dict) @@ -713,7 +721,9 @@ class RunEvent(Base): """完整事件流落库 —— 这既是 trace,也是页面刷新后重建时间线的依据。""" __tablename__ = "run_events" - __table_args__ = (Index("ix_run_events_run_seq", "run_id", "seq"),) + # (run_id, seq) 唯一(编排完善第 2 波 W2-b):撞号时报错,不悄悄写两条——封存按 seq 排序算哈希,撞了号核对时对时不对。 + # 存量库由 db/base._migrate 补建(先查有没有重复,有就照实记日志、不建) + __table_args__ = (Index("ux_run_events_run_seq", "run_id", "seq", unique=True),) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) run_id: Mapped[str] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE")) @@ -809,6 +819,8 @@ class Document(Base, TimestampMixin): # 压在 HTTP 请求里必然超时——而后端其实还在跑,界面显示失败、数据其实成功, # 比直接拒绝还糟。ready | processing | failed status: Mapped[str] = mapped_column(String(20), default="ready") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) error: Mapped[str] = mapped_column(Text, default="") chunks: Mapped[list["Chunk"]] = relationship( @@ -904,7 +916,8 @@ class Conversation(Base, TimestampMixin): # 一个人在一个应用里只有一段(未归档的)助手对话(D-e)。存量库由 db/base.py 的 _UNIQUE_INDEX_MIGRATIONS 补建。 # created_by 为空(关登录又没填署名)的不受约束:SQLite 的唯一索引里 NULL 互不相等,那种情况靠接口里的进程内锁 __table_args__ = (Index("ux_conversations_app_owner", "app_id", "created_by", unique=True, - sqlite_where=text("kind = 'app' AND archived = 0")),) + sqlite_where=text("kind = 'app' AND archived = 0"), + postgresql_where=text("kind = 'app' AND archived = false")),) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) title: Mapped[str] = mapped_column(String(200), default="") @@ -1096,3 +1109,6 @@ from app.db.models_blueprints import ( # noqa: E402,F401 from app.db import models_chat, models_eval, models_sources # noqa: E402,F401 # 编排完善第 1 波(docs/design/orchestration-round/00-总方案.md §4):数据事件触发的待派发事件和去抖等待 from app.db import models_triggers # noqa: E402,F401 +# 编排完善第 2 波 W2-b(docs/design/orchestration-round/00-总方案.md §5):多进程协调——工作进程的登记和租约、调度当班、 +# 跨进程的请求、运行的并发名额、同源队、按时间窗的计数 +from app.db import models_cluster # noqa: E402,F401 diff --git a/backend/app/db/models_blueprints.py b/backend/app/db/models_blueprints.py index 5f057620..e31e8fc5 100644 --- a/backend/app/db/models_blueprints.py +++ b/backend/app/db/models_blueprints.py @@ -88,6 +88,8 @@ class PublishPlan(Base, TimestampMixin): summary: Mapped[list[Any]] = mapped_column(default=list) problems: Mapped[list[Any]] = mapped_column(default=list) status: Mapped[str] = mapped_column(String(20), default="planned") + #: 执行这份发布单的工作进程(status=running 时,app/cluster):属主不在了的才按「执行中断」收拾 + owner: Mapped[str | None] = mapped_column(String(120), default=None) class StoredFile(Base): diff --git a/backend/app/db/models_chat.py b/backend/app/db/models_chat.py index f13542f7..600f33ec 100644 --- a/backend/app/db/models_chat.py +++ b/backend/app/db/models_chat.py @@ -24,7 +24,7 @@ from typing import Any from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column -from app.db.base import Base, TimestampMixin, UTCDateTime, new_id +from app.db.base import Base, current_owner, TimestampMixin, UTCDateTime, new_id #: 这一轮现在在哪一步 STAGES = ("build", "run", "waiting", "review") @@ -40,6 +40,8 @@ class ChatTurnJob(Base, TimestampMixin): conversation_id: Mapped[str] = mapped_column(ForeignKey("conversations.id", ondelete="CASCADE"), index=True) #: queued | running | succeeded | failed | cancelled | interrupted(core/job_runtime 的状态词) status: Mapped[str] = mapped_column(String(20), default="queued", index=True) + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) stage: Mapped[str | None] = mapped_column(String(20), default=None) #: ask | retry | rerun | max_steps | run | continue | resume(app/chat/job.py 的 MODES) mode: Mapped[str] = mapped_column(String(20), default="ask") @@ -54,6 +56,9 @@ class ChatTurnJob(Base, TimestampMixin): error: Mapped[str | None] = mapped_column(Text, default=None) started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + #: 等不及的那次复核还在后台跑(从什么时候起):实时流在它回来之前不关。多进程时别的进程上的流也按它判断(属主不在了 + #: 就不再等) + late_review_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) class ChatTurnEvent(Base): diff --git a/backend/app/db/models_cluster.py b/backend/app/db/models_cluster.py new file mode 100644 index 00000000..6e2fe63f --- /dev/null +++ b/backend/app/db/models_cluster.py @@ -0,0 +1,121 @@ +"""多进程协调用的表(app/cluster,docs/design/orchestration-round/00-总方案.md §5 W2-b)。 + +由 app/db/models.py 末尾整模块导入,create_all 建表(都是新表)。认领用的 owner 列加在各自的表上(runs、chat_turn_jobs、 +各种后台任务表……,见 db/base 的迁移),这里只放协调本身的几张: + +- ClusterWorker:一个工作进程一行(id 是 app/cluster/identity.OWNER:主机 + pid + 启动随机串)。进程活着就隔一拍心跳续租约 + (lease_until);租约过了期就算这个进程没了,它认领的行由别的进程按各自的规矩接手。各行的认领只记 owner,不各自续租: + 租约按进程算,一个进程一拍只写一行,进程一没,它认领的全部一起到期。 +- ClusterLease:全站只许一个进程做的事(调度当班)的租约:name 一行,owner 是当班的进程,epoch 每换一次当班加一。 +- ClusterRequest:发给别的进程的请求(停止它在跑的运行、问数据、后台任务;往它在执行的运行里记一条事件)。owner 是收件的 + 进程:它隔一拍心跳(PostgreSQL 上另有 NOTIFY 叫醒)取走自己的请求照办,办完删掉。 +- RunSlot:运行的并发名额(app/cluster/slots.py):一次运行每段执行要一个名额,按类别(交互、应用、定时、系统)各有全站上限 + 和优先级,按库里的计数发。 +- LaneTicket:同源队(core/job_runtime 的 SourceLanes):同一个数据源上的起草、剖析、探查、同步起草、导入先来先到、一个 + 接一个;id 自增,先来的号小。 +- RateCounter:几个进程共用的计数(访问密钥每分钟限流的令牌桶、登录失败的退避),到期的由当班进程清掉。只在 PostgreSQL + (可以多进程)时用;SQLite 只有一个进程,计数照旧在进程里(app/cluster/counters.py)。 +""" +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import Boolean, Float, Index, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, UTCDateTime, new_id, utcnow + + +class ClusterWorker(Base): + __tablename__ = "cluster_workers" + + id: Mapped[str] = mapped_column(String(120), primary_key=True) + host: Mapped[str] = mapped_column(String(255), default="") + pid: Mapped[int] = mapped_column(Integer, default=0) + started_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + heartbeat_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + lease_until: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, index=True) + #: 版本、数据目录之类给人看的信息 + info: Mapped[dict[str, Any]] = mapped_column(default=dict) + #: 代码版本、库结构版本(app/cluster/version.py):库里有活着的、版本不同的进程时新进程不许加入 + code_version: Mapped[str | None] = mapped_column(String(200), default=None) + schema_version: Mapped[str | None] = mapped_column(String(64), default=None) + + +class ClusterLease(Base): + __tablename__ = "cluster_leases" + + name: Mapped[str] = mapped_column(String(64), primary_key=True) + owner: Mapped[str | None] = mapped_column(String(120), default=None) + lease_until: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + acquired_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + epoch: Mapped[int] = mapped_column(Integer, default=0) + + +class ClusterRequest(Base): + __tablename__ = "cluster_requests" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + #: 收件的进程 + owner: Mapped[str] = mapped_column(String(120), index=True) + #: stop 停止 | note 往运行里记一条事件 + kind: Mapped[str] = mapped_column(String(20)) + #: 对什么:run | chat_turn | 后台任务的种类(draft、profile、introspect、eval、shadow_draft) + target: Mapped[str] = mapped_column(String(40)) + target_id: Mapped[str] = mapped_column(String(64)) + actor: Mapped[str | None] = mapped_column(String(100), default=None) + payload: Mapped[dict[str, Any]] = mapped_column(default=dict) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + + +class RunSlot(Base): + __tablename__ = "run_slots" + + run_id: Mapped[str] = mapped_column(String(32), primary_key=True) + #: interactive | app | schedule | system(app/cluster/slots.CLASSES) + klass: Mapped[str] = mapped_column(String(20)) + priority: Mapped[int] = mapped_column(Integer, default=0) + owner: Mapped[str] = mapped_column(String(120), index=True) + granted: Mapped[bool] = mapped_column(Boolean, default=False, index=True) + requested_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + granted_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +class LaneTicket(Base): + __tablename__ = "lane_tickets" + __table_args__ = (Index("ix_lane_tickets_source_id_id", "source_id", "id"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + source_id: Mapped[str] = mapped_column(String(64)) + #: 登记过的一种事(core/job_runtime.register_lane_kind) + kind: Mapped[str] = mapped_column(String(40)) + job_id: Mapped[str | None] = mapped_column(String(64), default=None) + owner: Mapped[str] = mapped_column(String(120), index=True) + granted: Mapped[bool] = mapped_column(Boolean, default=False) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + + +class RateCounter(Base): + __tablename__ = "rate_counters" + + key: Mapped[str] = mapped_column(String(255), primary_key=True) + count: Mapped[int] = mapped_column(Integer, default=0) + #: 令牌桶里还剩几个(访问密钥限流)/ 要等到什么时刻(登录失败,Unix 秒) + value: Mapped[float] = mapped_column(Float, default=0.0) + #: 上次补令牌的时刻(Unix 秒) + at: Mapped[float] = mapped_column(Float, default=0.0) + expires_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, index=True) + + +class CacheEvent(Base): + """「某个进程里缓存着的东西变了」(app/cluster/caches.py):改的进程写一条,别的进程隔一拍心跳(或收到 NOTIFY)按 id + 往后读,照着清掉、重读自己那一份。id 自增;当班进程清掉一个钟头以前的。""" + + __tablename__ = "cache_events" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(64)) + key: Mapped[str | None] = mapped_column(String(255), default=None) + owner: Mapped[str] = mapped_column(String(120)) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, index=True) diff --git a/backend/app/db/models_eval.py b/backend/app/db/models_eval.py index 9d8e4f63..38d364cf 100644 --- a/backend/app/db/models_eval.py +++ b/backend/app/db/models_eval.py @@ -29,7 +29,7 @@ from typing import Any from sqlalchemy import ForeignKey, Index, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, synonym -from app.db.base import Base, TimestampMixin, UTCDateTime, new_id +from app.db.base import Base, current_owner, TimestampMixin, UTCDateTime, new_id #: 题目的种类:ask 问数据题(走问数据同一条路作答)、workflow 钉版本的工作流题 CASE_KINDS = ("ask", "workflow") @@ -128,6 +128,8 @@ class EvalCatalogSnapshot(Base, TimestampMixin): origin: Mapped[str] = mapped_column(String(20), default="live") # queued | running | succeeded | partial | failed | cancelled | interrupted status: Mapped[str] = mapped_column(String(20), default="succeeded") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) artifact_id: Mapped[str | None] = mapped_column(String(64), default=None) # 拍了哪些数据源:[{id, name}](名字是拍的那一刻的) sources: Mapped[list[Any]] = mapped_column(default=list) @@ -160,6 +162,8 @@ class EvalRun(Base, TimestampMixin): suite_id: Mapped[str] = mapped_column(ForeignKey("eval_suites.id", ondelete="CASCADE")) # queued | running | succeeded | partial | failed | cancelled | interrupted(core/job_runtime 的状态词) status: Mapped[str] = mapped_column(String(20), default="queued") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) trigger: Mapped[str] = mapped_column(String(20), default="manual") # 定时发起的那一条(ScheduleFire.eval_run_id 反过来指向这里) schedule_id: Mapped[str | None] = mapped_column(String(32), default=None) diff --git a/backend/app/db/models_sources.py b/backend/app/db/models_sources.py index 5495855e..9b23d379 100644 --- a/backend/app/db/models_sources.py +++ b/backend/app/db/models_sources.py @@ -19,10 +19,10 @@ from __future__ import annotations from datetime import datetime from typing import Any -from sqlalchemy import JSON, Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint, text +from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint, text from sqlalchemy.orm import Mapped, mapped_column -from app.db.base import Base, UTCDateTime, new_id, utcnow +from app.db.base import Base, current_owner, PortableJSON, UTCDateTime, new_id, utcnow class SourceObject(Base): @@ -92,13 +92,16 @@ class IntrospectJob(Base): __table_args__ = ( Index("ix_introspect_jobs_source_created", "source_id", "created_at"), Index("ux_introspect_jobs_active", "source_id", unique=True, - sqlite_where=text("status IN ('queued', 'running')")), + sqlite_where=text("status IN ('queued', 'running')"), + postgresql_where=text("status IN ('queued', 'running')")), ) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) # queued | running | succeeded | partial | failed | cancelled | interrupted status: Mapped[str] = mapped_column(String(20), default="queued") + #: 认领这一行的工作进程(app/cluster;缺省是建它的进程)。多进程时启动整理、接手只碰属主不在了的行 + owner: Mapped[str | None] = mapped_column(String(120), default=current_owner) #: manual 人点的 | schedule 定时维护(第 3 波) trigger: Mapped[str] = mapped_column(String(20), default="manual") #: 这次探查的是哪个 schema("" 是默认 schema) @@ -219,8 +222,8 @@ class SourceChange(Base): seen_count: Mapped[int] = mapped_column(Integer, default=1) detected_by: Mapped[dict[str, Any]] = mapped_column(default=dict) basis: Mapped[dict[str, Any] | None] = mapped_column(default=None) - before: Mapped[Any] = mapped_column(JSON, nullable=True, default=None) - after: Mapped[Any] = mapped_column(JSON, nullable=True, default=None) + before: Mapped[Any] = mapped_column(PortableJSON, nullable=True, default=None) + after: Mapped[Any] = mapped_column(PortableJSON, nullable=True, default=None) affects: Mapped[dict[str, Any] | None] = mapped_column(default=None) struct_version_before: Mapped[int | None] = mapped_column(Integer, default=None) struct_version_after: Mapped[int | None] = mapped_column(Integer, default=None) diff --git a/backend/app/db/portable.py b/backend/app/db/portable.py new file mode 100644 index 00000000..054df487 --- /dev/null +++ b/backend/app/db/portable.py @@ -0,0 +1,296 @@ +"""应用库上 SQLite、PostgreSQL 写法不一样的那几样,收成按方言编译的表达式。 + +只给应用库(agentlab.db / AGENTLAB_DATABASE_URL)用;发给业务数据源的 SQL 不走这里。 + +**按 JSON 列里的键筛选。** 两种库同一套语义,按 JSON 的类型比(SQLite 的 json_extract 单独用时不分:true 取出来 +是 1,和 1 比相等;PostgreSQL 的 JSON 列是存原文的 json,db/base.PortableJSON,比较时只把取出来的那一小段转成 JSONB): + +- json_eq(列, 键, 值):字符串只等于同样的字符串;数只等于数(1 和 1.0 相等);True / False 只等于 JSON 的 true / false + (不等于 1 / 0);None 是「没有这个键或值为 null」。只收这几种标量; +- json_text:这个键的值当文本——字符串是它本身,数、true / false、数组、对象是它的 JSON 写法(同 PostgreSQL 的 ->>, + SQLite 上用 json_type 分开、非字符串取 -> 的写法),没有这个键或值为 null 是 NULL。所以数 1 的 json_text 是 '1', + 两种库都是; +- json_flag:标记位「真」(true、非零的数):没有这个键、null、false、0 都算假; +- json_is_string:这个键的值是不是字符串(SQLite 的 json_type 叫 text,PostgreSQL 的 jsonb_typeof 叫 string)。 + +键名只收标识符(字母、数字、下划线),直接写进 SQL:键都是代码里的常量,写进去语句缓存也按它区分。 + +**插入,撞了唯一键就什么都不做。** insert_ignore 按会话连着的库挑 sqlite / postgresql 的 insert,两边的 +on_conflict_do_nothing 是同一个接口。 + +**不分大小写地找子串。** contains_ci:SQLite 上是 lower(列) LIKE lower(模式)(同 SQLAlchemy 的 ilike;大小写怎么折 +看 SQLite 的构建,本机 conda 的 3.53 连 É/é、全角A/a 都认)。PostgreSQL 上应用库的文本列是 COLLATE "C"(db/postgres.py), +列上的 ILIKE、lower() 只认 ASCII 的大小写;所以模式带 COLLATE "default",按建库时的区域设置折大小写(en_US.UTF-8、 +C.UTF-8 都认 Unicode;建库用的是纯 C 区域的话只认 ASCII)。用户输入里的 %、_、反斜杠按字面匹配。 + +**撞的是哪个唯一约束。** unique_violation 认 IntegrityError:SQLite 的报错是「UNIQUE constraint failed: 表.列」, +PostgreSQL(asyncpg)的是 23505,表名、DETAIL 里的「Key (列)=」分别带着。不靠约束名(命名规则改了也认得出)。 +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from sqlalchemy import Boolean, String, bindparam, literal_column +from sqlalchemy.ext.compiler import compiles +from sqlalchemy.sql.functions import FunctionElement +from sqlalchemy.types import TypeDecorator + +_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") + + +def _key(name: str) -> Any: + if not _KEY.match(name): + raise ValueError(f"JSON 键名只能是标识符:{name!r}") + return literal_column(name) + + +class _JsonScalar(TypeDecorator): + """json_eq 右边的值:SQLite 原样绑定(和 json_extract 取出的标量比),PostgreSQL 绑定成 JSON 文本再转 JSONB。""" + + impl = String + cache_ok = True + + def process_bind_param(self, value: Any, dialect: Any) -> Any: + if dialect.name == "postgresql": + return json.dumps(value, ensure_ascii=False) + return value + + +class json_text(FunctionElement[str]): + """JSON 列顶层键的值,当文本用。""" + + type = String() + inherit_cache = True + name = "json_text" + + def __init__(self, column: Any, key: str) -> None: + super().__init__(column, _key(key)) + + +class _JsonEq(FunctionElement[bool]): + """kind 是 text / number / true / false,写进语句(语句缓存按它区分);true / false 不用绑定参数。""" + + type = Boolean() + inherit_cache = True + name = "json_eq" + + def __init__(self, column: Any, key: str, kind: str, value: Any = None) -> None: + extra = [] if kind in ("true", "false") else [bindparam(None, value, type_=_JsonScalar())] + super().__init__(column, _key(key), literal_column(kind), *extra) + + +class _JsonNull(FunctionElement[bool]): + type = Boolean() + inherit_cache = True + name = "json_null" + + def __init__(self, column: Any, key: str) -> None: + super().__init__(column, _key(key)) + + +def json_eq(column: Any, key: str, value: Any) -> Any: + """JSON 列顶层键的值等于 value(按 JSON 的类型比,见模块说明)。value 为 None 时是「没有这个键或值为 null」。 + + 值的种类写进语句:语句缓存不按绑定参数的值区分,同一个表达式不能因为值的种类不同编译成两样。""" + if value is None: + return _JsonNull(column, key) + if isinstance(value, bool): + return _JsonEq(column, key, "true" if value else "false") + if isinstance(value, (int, float)): + return _JsonEq(column, key, "number", value) + if isinstance(value, str): + return _JsonEq(column, key, "text", value) + raise TypeError(f"json_eq 只比标量(字符串、数、布尔、None),不收 {type(value).__name__}") + + +class json_flag(FunctionElement[bool]): + """JSON 列顶层键是「真」的标记位:true 或非零的数(字符串也算真,同 SQLite)。没有这个键、null、false、0 是假。""" + + type = Boolean() + inherit_cache = True + name = "json_flag" + + def __init__(self, column: Any, key: str) -> None: + super().__init__(column, _key(key)) + + +class json_is_string(FunctionElement[bool]): + """JSON 列顶层键的值是字符串。""" + + type = Boolean() + inherit_cache = True + name = "json_is_string" + + def __init__(self, column: Any, key: str) -> None: + super().__init__(column, _key(key)) + + +def _parts(element: Any, compiler: Any, **kw: Any) -> tuple[str, str, list[Any]]: + column, key, *rest = list(element.clauses) + return compiler.process(column, **kw), key.name, rest + + +# ---- SQLite(缺省)---- + + +@compiles(json_text) +def _json_text_sqlite(element: Any, compiler: Any, **kw: Any) -> str: + # 同 PostgreSQL 的 ->>:字符串取它本身,别的取 JSON 写法(-> 要 SQLite 3.38 以上) + col, key, _ = _parts(element, compiler, **kw) + path = f"'$.{key}'" + return (f"(CASE json_type({col}, {path}) WHEN 'text' THEN json_extract({col}, {path}) WHEN 'null' THEN NULL " + f"ELSE ({col} -> {path}) END)") + + +_SQLITE_KIND = {"text": "= 'text'", "number": "IN ('integer', 'real')", "true": "= 'true'", "false": "= 'false'"} + + +@compiles(_JsonEq) +def _json_eq_sqlite(element: Any, compiler: Any, **kw: Any) -> str: + col, key, (kind, *value) = _parts(element, compiler, **kw) + typed = f"json_type({col}, '$.{key}') {_SQLITE_KIND[kind.name]}" + if not value: + return f"({typed})" + return f"({typed} AND json_extract({col}, '$.{key}') = {compiler.process(value[0], **kw)})" + + +@compiles(_JsonNull) +def _json_null_sqlite(element: Any, compiler: Any, **kw: Any) -> str: + col, key, _ = _parts(element, compiler, **kw) + return f"(json_extract({col}, '$.{key}') IS NULL)" + + +@compiles(json_flag) +def _json_flag_sqlite(element: Any, compiler: Any, **kw: Any) -> str: + col, key, _ = _parts(element, compiler, **kw) + return f"(coalesce(json_extract({col}, '$.{key}'), 0) != 0)" + + +@compiles(json_is_string) +def _json_is_string_sqlite(element: Any, compiler: Any, **kw: Any) -> str: + col, key, _ = _parts(element, compiler, **kw) + return f"(json_type({col}, '$.{key}') = 'text')" + + +# ---- PostgreSQL(应用库的 JSON 列是 json)---- + + +def _got(col: str, key: str) -> str: + """取出的那一段转成 JSONB:json 没有相等运算符。列本身是 JSONB 也照样成立。""" + return f"CAST(({col} -> '{key}') AS JSONB)" + + +@compiles(json_text, "postgresql") +def _json_text_pg(element: Any, compiler: Any, **kw: Any) -> str: + col, key, _ = _parts(element, compiler, **kw) + return f"({col} ->> '{key}')" + + +@compiles(_JsonEq, "postgresql") +def _json_eq_pg(element: Any, compiler: Any, **kw: Any) -> str: + # JSONB 的相等本来就按类型:'1' = '1.0','1' ≠ 'true','"1"' ≠ '1' + col, key, (kind, *value) = _parts(element, compiler, **kw) + right = f"'{kind.name}'::jsonb" if not value else f"CAST({compiler.process(value[0], **kw)} AS JSONB)" + return f"({_got(col, key)} = {right})" + + +@compiles(_JsonNull, "postgresql") +def _json_null_pg(element: Any, compiler: Any, **kw: Any) -> str: + # json_extract 对「没有这个键」和「值为 null」都给 NULL;-> 对后者给的是 JSON 的 null + col, key, _ = _parts(element, compiler, **kw) + return f"(coalesce({_got(col, key)}, 'null'::jsonb) = 'null'::jsonb)" + + +@compiles(json_flag, "postgresql") +def _json_flag_pg(element: Any, compiler: Any, **kw: Any) -> str: + col, key, _ = _parts(element, compiler, **kw) + return f"(coalesce({_got(col, key)}, 'null'::jsonb) NOT IN ('null'::jsonb, 'false'::jsonb, '0'::jsonb))" + + +@compiles(json_is_string, "postgresql") +def _json_is_string_pg(element: Any, compiler: Any, **kw: Any) -> str: + col, key, _ = _parts(element, compiler, **kw) + return f"(jsonb_typeof({_got(col, key)}) = 'string')" + + +# ---- 不分大小写的子串 ---- + + +def _escape_like(text: str) -> str: + return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +class _ContainsCI(FunctionElement[bool]): + type = Boolean() + inherit_cache = True + name = "contains_ci" + + +def contains_ci(column: Any, text: str) -> Any: + """列里含有 text,不分大小写(见模块说明)。""" + return _ContainsCI(column, bindparam(None, f"%{_escape_like(text)}%", type_=String())) + + +@compiles(_ContainsCI) +def _contains_ci_sqlite(element: Any, compiler: Any, **kw: Any) -> str: + column, pattern = list(element.clauses) + return (f"(lower({compiler.process(column, **kw)}) LIKE lower({compiler.process(pattern, **kw)}) " + "ESCAPE '\\')") + + +@compiles(_ContainsCI, "postgresql") +def _contains_ci_pg(element: Any, compiler: Any, **kw: Any) -> str: + column, pattern = list(element.clauses) + return (f"({compiler.process(column, **kw)} ILIKE ({compiler.process(pattern, **kw)} COLLATE \"default\") " + "ESCAPE '\\')") + + +# ---- 插入,撞了唯一键就跳过 ---- + + +def dialect_of(session: Any) -> str: + """会话(AsyncSession / Session)连着的库的方言名。""" + return session.get_bind().dialect.name + + +def insert_ignore(session: Any, table: Any) -> Any: + """按会话连着的库挑方言的 insert,调用方接着 .values(...).on_conflict_do_nothing(index_elements=[...])。""" + if dialect_of(session) == "postgresql": + from sqlalchemy.dialects.postgresql import insert + else: + from sqlalchemy.dialects.sqlite import insert + return insert(table) + + +# ---- 撞的是哪个唯一约束 ---- + +_SQLITE_UNIQUE = "UNIQUE constraint failed:" +_PG_KEY = re.compile(r"Key \(([^)]*)\)=") + + +def _driver_error(error: BaseException) -> Any: + """SQLAlchemy 的异常 → asyncpg 自己的异常(在 orig 的 __cause__ 上;orig 只抄了 sqlstate,没有表名和 DETAIL)。""" + seen: Any = getattr(error, "orig", None) or error + for _ in range(4): + if getattr(seen, "sqlstate", None) and hasattr(seen, "table_name"): + return seen + seen = getattr(seen, "__cause__", None) + if seen is None: + break + return None + + +def unique_violation(error: BaseException, table: str, *columns: str) -> bool: + """这个 IntegrityError 是不是撞了 table 上正好这几列的唯一约束(主键、唯一索引都算)。""" + text = str(getattr(error, "orig", None) or error) + if _SQLITE_UNIQUE in text: + failed = text.split(_SQLITE_UNIQUE, 1)[1].splitlines()[0] + return {part.strip() for part in failed.split(",")} == {f"{table}.{c}" for c in columns} + cause = _driver_error(error) + if cause is None or getattr(cause, "sqlstate", None) != "23505" or getattr(cause, "table_name", None) != table: + return False + match = _PG_KEY.search(getattr(cause, "detail", None) or "") + return bool(match) and [c.strip().strip('"') for c in match.group(1).split(",")] == list(columns) diff --git a/backend/app/db/postgres.py b/backend/app/db/postgres.py new file mode 100644 index 00000000..3921c60b --- /dev/null +++ b/backend/app/db/postgres.py @@ -0,0 +1,120 @@ +"""应用库是 PostgreSQL 时,让它在 SQLite 上写成的那些假设下照样成立。 + +只装在应用库那个引擎的方言实例上(db/base.make_engine),业务数据源的 PostgreSQL 引擎不受影响。 + +**建表的列类型。** 应用是在 SQLite 上写成的:SQLite 不查 VARCHAR 的长度,INTEGER 是 64 位。PostgreSQL 上 +VARCHAR(n) 超长直接报错、INTEGER 只有 32 位。模型里的 String(n) 是写给人看的「大致多长」,从来没有被库约束过: +逐列核对只核得到测试走到的值,漏一处就是线上的写入失败,从 SQLite 导过来的存量数据里也可能已经有超长的。所以 +应用库上 String(n) 建成不限长的 VARCHAR(PostgreSQL 里 varchar、varchar(n)、text 存储和性能一样,限长只多一次 +检查)。CAST(x AS VARCHAR(n)) 这种查询里的写法不动——那是截断的意思。Integer 列在 PostgreSQL 上是 BIGINT: +那是在模型这一层换的(db/base._portable_integer),绑定参数的类型转换跟着列走,大数不在半路溢出。 + +**文本怎么比大小、怎么排序。** SQLite 缺省按字节比(BINARY);PostgreSQL 按建库时的区域设置(常见的 en_US.UTF-8 不分 +大小写地先比字母、忽略标点),同一列排出来的先后、`key < '…'` 这种比较的结果都会变。应用库上的文本列建表时一律带 +COLLATE "C"(按字节比),和 SQLite 一致,不看建库时选了什么区域;LIKE 'xx%' 这种前缀匹配也因此用得上普通索引。 +代价是列上的 ILIKE、lower() 也按 C 的规则,只认 ASCII 的大小写(É/é、全角A/a 不认):要不分大小写地找,用 +db/portable.contains_ci,模式带 COLLATE "default",按建库时的区域设置折大小写。 + +**ORDER BY 里 NULL 的位置。** SQLite 升序 NULL 在前、降序 NULL 在后,PostgreSQL 正好相反。应用里两百多处排序, +可空的列(开始时间、到期时间、发布时间……)排出来的先后要和 SQLite 一致:可空列上没写 nulls_first / nulls_last +的 ASC 补 NULLS FIRST、DESC 补 NULLS LAST,ORDER BY 里直接写列名(隐含 ASC)的同样补上。不可空的列不动—— +补了会让索引用不上,又没有 NULL 可排。窗口函数 OVER (ORDER BY …) 里同样补。 + +只认直接写的列(Column,含它的 asc() / desc())。这几种不补,两种库排出来的先后可能不同,要紧的地方自己写明 +nulls_first / nulls_last:label 出来的聚合(max(...) 之类)、coalesce / case 这类表达式、外连接进来的表上的列(模型里 +不可空,外连接没连上时却是 NULL)、原生 text() 里的 ORDER BY(db/base 的去重语句里写明了)。 +""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import Column +from sqlalchemy.dialects.postgresql.asyncpg import PGCompiler_asyncpg +from sqlalchemy.dialects.postgresql.base import PGTypeCompiler +from sqlalchemy.sql import operators +from sqlalchemy.sql.elements import ClauseList, UnaryExpression + +#: 外层已经写了 nulls_first / nulls_last:里面那一层 ASC / DESC 不再补 +_EXPLICIT = "_agentlab_explicit_nulls" + + +def _nullable(element: Any) -> bool: + return isinstance(element, Column) and bool(element.nullable) and not element.primary_key + + +#: 建表时文本列的排序规则:按字节比,同 SQLite 的 BINARY(见模块说明) +_COLLATE = ' COLLATE "C"' + + +class AppTypeCompiler(PGTypeCompiler): + def visit_string(self, type_: Any, **kw: Any) -> str: + if isinstance(kw.get("type_expression"), Column): + return ("VARCHAR" if type_.length else super().visit_string(type_, **kw)) + _COLLATE + return super().visit_string(type_, **kw) + + def visit_text(self, type_: Any, **kw: Any) -> str: + text = super().visit_text(type_, **kw) + return text + _COLLATE if isinstance(kw.get("type_expression"), Column) else text + + +def _ordered(clauses: Any) -> list[Any]: + """ORDER BY 里直接写的可空列补成 ASC(visit_unary 再给它补 NULLS FIRST)。""" + return [c.asc() if _nullable(c) else c for c in clauses] + + +class AppCompiler(PGCompiler_asyncpg): + def order_by_clause(self, select: Any, **kw: Any) -> str: + order_by = self._generate_delimited_list(_ordered(select._order_by_clauses), ", ", **kw) + return " ORDER BY " + order_by if order_by else "" + + def visit_over(self, over: Any, **kw: Any) -> str: + clauses = list(over.order_by) if over.order_by is not None else [] + if any(_nullable(c) for c in clauses): + # 在副本上换:语句对象可能被缓存、被别处共用,不能改原件 + over = over._clone() + over.order_by = ClauseList(*_ordered(clauses)) + return super().visit_over(over, **kw) + + def visit_unary(self, unary: UnaryExpression, **kw: Any) -> str: + modifier = unary.modifier + if modifier in (operators.nulls_first_op, operators.nulls_last_op): + return super().visit_unary(unary, **{**kw, _EXPLICIT: True}) + explicit = kw.pop(_EXPLICIT, False) + text = super().visit_unary(unary, **kw) + if not explicit and modifier in (operators.asc_op, operators.desc_op) and _nullable(unary.element): + text += " NULLS FIRST" if modifier is operators.asc_op else " NULLS LAST" + return text + + +def _has_nul(value: Any) -> bool: + return isinstance(value, str) and "\x00" in value + + +def _clean_row(params: Any) -> Any: + if isinstance(params, dict): + return {k: v.replace("\x00", "") if _has_nul(v) else v for k, v in params.items()} \ + if any(_has_nul(v) for v in params.values()) else params + if isinstance(params, (list, tuple)) and any(_has_nul(v) for v in params): + return type(params)(v.replace("\x00", "") if _has_nul(v) else v for v in params) + return params + + +def strip_nul(conn: Any, cursor: Any, statement: str, parameters: Any, context: Any, + executemany: bool) -> tuple[str, Any]: + """写进 PostgreSQL 之前去掉文本参数里的 NUL 字符(before_cursor_execute,retval=True)。 + + PostgreSQL 的 text / varchar 不收 \\x00(报 invalid byte sequence),SQLite 照存:一段带 NUL 的模型输出、一个从二进制 + 文件里粘出来的问题,在 SQLite 上写得进去,在 PostgreSQL 上整条写入失败。所以和 JSON 列(db/base.pg_json_dumps)一样 + 去掉。只看参数里的字符串,bytes(LargeBinary)不动;没有 NUL 的参数原样交回,不多一次拷贝。""" + if executemany: + if any(_clean_row(row) is not row for row in parameters): + return statement, [_clean_row(row) for row in parameters] + return statement, parameters + return statement, _clean_row(parameters) + + +def install(dialect: Any) -> None: + """装到应用库引擎的方言实例上。""" + dialect.statement_compiler = AppCompiler + dialect.type_compiler_instance = dialect.type_compiler = AppTypeCompiler(dialect) diff --git a/backend/app/db/to_postgres.py b/backend/app/db/to_postgres.py new file mode 100644 index 00000000..6e94cec6 --- /dev/null +++ b/backend/app/db/to_postgres.py @@ -0,0 +1,616 @@ +"""把 SQLite 数据目录里的应用库和断点导入 PostgreSQL:`python -m app.db copy-to-postgres`。 + + python -m app.db copy-to-postgres <数据目录或 agentlab.db> [目标地址] [--checkpoint-url 地址] [--dry-run] + +目标地址缺省取 AGENTLAB_DATABASE_URL;断点缺省进同一个库(AGENTLAB_CHECKPOINT_URL 或 --checkpoint-url 另给)。 + +**先停服务。** 来源库旁边有 agentlab.db-wal(WAL 模式下还有连接开着,或者上次是强杀),或者目标库上有 application_name +是 agentlab 的连接(服务已经连着 PostgreSQL 跑起来了),就拒绝导入。确认是强杀留下的 -wal 可以加 --assume-stopped; +这只是能查到的两样,查不到的(比如服务连着别的数据目录的拷贝、另一台机器上的服务)要自己保证。 + +**读的是一份快照。** 先用 SQLite 的备份接口把 agentlab.db、checkpoints.db 拷到临时目录(拷贝过程中别人在写也拿到 +一致的一份;放在 --work-dir,缺省系统临时目录,要留出和两个库一样大的空间),在拷贝上跑一遍和启动时一样的建表补列 +(老版本的库升级到当前的表结构),从拷贝读,来源的内容不改。 + +**幂等。** 目标上建表(同启动时的 create_schema),按外键先后逐表、按主键分批插入,`ON CONFLICT DO NOTHING`: +目标上已经有同一主键(或撞了别的唯一约束)的行一律不动,不覆盖、不合并——重跑只补上还没有的行,不重复。所以 +导入之后又在 SQLite 上改过的行,重跑不会带过去;要整份重来就清空目标库再导。断点同理:目标上已有的断点不重写, +pending writes 按(断点、任务、序号)插入,已有的不动。 + +**核对。** 每张表按主键核对:来源的每一行,目标上有没有同一主键的行(导入前就有的、这次补上的分开数),断点和 +pending writes 按条数核对,逐项打印;有来源的行目标上没有(撞了约束没插进去的会列出主键和原因),退出码非零。 +目标上另有来源里没有的行不算错(同一个目标导过别的、或者服务已经在上面跑过)。 + +**值的转换。** 读写都按模型的列类型走:时间是不带时区的 UTC、布尔是真布尔、JSON 原文照搬(SQL 的 NULL 和 JSON 的 +null 分开,不混)。SQLite 不查类型、PostgreSQL 查:整数列里的整数值的浮点、数字字符串换成整数,文本列里的非文本换成 +字符串;文本里的 NUL 字符(PostgreSQL 不收)去掉,JSON 里的 NaN / Infinity 写成 null;模型里 NOT NULL、有缺省值的列, +老库补列时没带缺省值留下的 NULL 填成缺省值(启动时的迁移在 SQLite 上也这么填);各自计数打印。 + +**孤儿行。** 外键指向已经不在的行(SQLite 当时没强制外键留下的):外键是 SET NULL 的置空、计入转换;别的从快照里拿掉、 +不导入,按表列出条数,不算核对失败(settle_orphans)。 + +**自增主键。** 导入后序列接在已用过的最大号后面:目标表的 MAX、SQLite 的 AUTOINCREMENT 计数器(删过的号不复用)取大的。 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sqlite3 +import sys +import tempfile +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from sqlalchemy import JSON, Boolean, Float, Integer, LargeBinary, String, Text, bindparam, cast, func, select, text +from sqlalchemy import create_engine as create_sync_engine +from sqlalchemy.dialects import postgresql +from sqlalchemy.exc import IntegrityError + +from app.db.urls import describe, libpq_conninfo, normalize_checkpoint_url, normalize_database_url + +COPY_APPLICATION_NAME = "agentlab-copy" +DEFAULT_BATCH = 1000 + + +@dataclass +class TableResult: + name: str + #: 来源的行数(不含孤儿行) + source: int = 0 + #: 外键指向已不存在的行、又不能置空的(外键是 CASCADE 或不可空):SQLite 当时没强制外键留下的,没导入 + orphans: int = 0 + #: 来源的这些主键,目标上导入前已经有的(--dry-run 时就是「不用补的」)、导入后有的 + present_before: int = 0 + present_after: int = 0 + #: 目标整张表的行数(导入前、导入后):比来源多出来的是目标上原有的 + target_before: int = 0 + target_after: int = 0 + refused: list[str] = field(default_factory=list) + + @property + def missing(self) -> int: + """来源有、目标上还没有的行(按主键数)。""" + return self.source - self.present_after + + +@dataclass +class Report: + tables: list[TableResult] = field(default_factory=list) + checkpoints: tuple[int, int] = (0, 0) # 来源(断点、pending writes) + checkpoints_target: tuple[int, int] = (0, 0) # 目标上同一批线程的 + checkpoints_written: int = 0 + threads: int = 0 + fixed: dict[str, int] = field(default_factory=dict) + #: 没导入的孤儿行,按表(见 settle_orphans) + orphans: dict[str, int] = field(default_factory=dict) + dry_run: bool = False + + @property + def ok(self) -> bool: + if self.dry_run: + return True + tables_ok = all(t.present_after == t.source and not t.refused for t in self.tables) + return tables_ok and self.checkpoints_target[0] >= self.checkpoints[0] \ + and self.checkpoints_target[1] >= self.checkpoints[1] + + +class CopyRefused(RuntimeError): + """导入前的检查没过(服务还开着、来源不对):照实说明,什么都没写。""" + + +# -------------------------------------------------------------------------- +# 来源 +# -------------------------------------------------------------------------- + + +def resolve_source(raw: str | Path) -> tuple[Path, Path | None]: + """数据目录或 agentlab.db → (应用库文件, 断点库文件或 None)。""" + path = Path(raw).expanduser() + db = path / "agentlab.db" if path.is_dir() else path + if not db.is_file(): + raise CopyRefused(f"找不到应用库:{db}(给数据目录,或者直接给 agentlab.db 的路径)") + with open(db, "rb") as f: + if f.read(16) != b"SQLite format 3\x00": + raise CopyRefused(f"{db} 不是 SQLite 库文件") + ckpt = db.parent / "checkpoints.db" + return db, (ckpt if ckpt.is_file() else None) + + +def source_in_use(db: Path, ckpt: Path | None = None) -> Path | None: + """来源库是不是还有连接开着:WAL 模式下最后一个连接正常关掉时 -wal 会被删掉,还在就是有人开着(或者上次是强杀)。 + 交回还在的那个 -wal(应用库、断点库各看一个),都没有是 None。""" + for path in (db, ckpt): + if path is not None and (wal := Path(f"{path}-wal")).exists(): + return wal + return None + + +def snapshot(db: Path, into: Path) -> Path: + """用 SQLite 的备份接口拷一份一致的快照(别人正在写也是某一刻的完整一份)。 + + 来源按读写打开、只许读(query_only):WAL 模式的库用 mode=ro 打开,关掉时删不了它自己建出来的 -wal、-shm, + 下一次导入就会当成服务还开着;读写打开的连接是最后一个关的,关掉时 SQLite 把日志并回主库、删掉这两个文件 + (内容不变)。""" + out = into / db.name + src = sqlite3.connect(db) + src.execute("PRAGMA query_only = 1") + try: + dst = sqlite3.connect(out) + try: + src.backup(dst) + finally: + dst.close() + finally: + src.close() + return out + + +def settle_orphans(path: Path, *, fixed: dict[str, int]) -> dict[str, int]: + """快照上按模型的外键找孤儿行:外键列有值、指向的行却不在(SQLite 当时没强制外键:老版本、或者那时外键还没加上)。 + PostgreSQL 一律拒收,重跑也补不上,所以导入前在快照上(来源不动)先处理: + + - 外键是 ON DELETE SET NULL、列可空的:置空,计入「转换」——父行被删时本来就该这样; + - 别的(CASCADE、不可空):从快照里拿掉、不导入,按表计数交回——父行被删时本来就该跟着删。拿掉一行可能让下一层又成了 + 孤儿,反复扫到没有为止。 + + 这些不算核对失败:报告里照实列出哪张表、几行。""" + from sqlalchemy import and_, delete, exists, update + + from app.db import models # noqa: F401 + from app.db.base import Base + + orphans: dict[str, int] = {} + engine = create_sync_engine(f"sqlite:///{path}") + try: + while True: + removed = 0 + with engine.begin() as conn: + for table in Base.metadata.sorted_tables: + for fk in table.foreign_key_constraints: + parent = fk.referred_table.alias("parent") + pairs = [(el.parent, parent.c[el.column.name]) for el in fk.elements] + lost = and_(*(child.is_not(None) for child, _ in pairs), + ~exists(select(1).select_from(parent).where( + and_(*(p == child for child, p in pairs))))) + n = int(conn.execute(select(func.count()).select_from(table).where(lost)).scalar() or 0) + if not n: + continue + if (fk.ondelete or "").upper() == "SET NULL" and all(c.nullable for c, _ in pairs): + conn.execute(update(table).where(lost).values({c.name: None for c, _ in pairs})) + fixed["外键指向已不存在的行,置空"] = fixed.get("外键指向已不存在的行,置空", 0) + n + else: + conn.execute(delete(table).where(lost)) + orphans[table.name] = orphans.get(table.name, 0) + n + removed += n + if not removed: + return orphans + finally: + engine.dispose() + + +def upgrade_snapshot(path: Path) -> None: + """在快照上跑一遍启动时的建表补列:老版本的库按当前的表结构读。""" + from app.db import models # noqa: F401 + from app.db.base import Base, _migrate + + engine = create_sync_engine(f"sqlite:///{path}") + try: + with engine.begin() as conn: + Base.metadata.create_all(conn) + _migrate(conn) + finally: + engine.dispose() + + +# -------------------------------------------------------------------------- +# 值 +# -------------------------------------------------------------------------- + + +def _is_json(column: Any) -> bool: + return isinstance(column.type, JSON) + + +class _Fixer: + """SQLite 里存得进、PostgreSQL 不收的值,换掉并计数。""" + + def __init__(self, fixed: dict[str, int]) -> None: + self.fixed = fixed + + def _count(self, what: str) -> None: + self.fixed[what] = self.fixed.get(what, 0) + 1 + + def value(self, column: Any, value: Any) -> Any: + if value is None: + return self.missing(column) + if _is_json(column): + return self.json(value) + kind = column.type + if isinstance(kind, Boolean): + return bool(value) + if isinstance(kind, Integer): + if isinstance(value, bool) or isinstance(value, int): + return int(value) + if isinstance(value, float) and value.is_integer(): + self._count("整数列里的浮点写法") + return int(value) + if isinstance(value, str) and value.strip().lstrip("-").isdigit(): + self._count("整数列里的数字字符串") + return int(value.strip()) + return value + if isinstance(kind, Float): + return float(value) if isinstance(value, (int, float)) else value + if isinstance(kind, LargeBinary): + return bytes(value) if isinstance(value, (bytes, bytearray, memoryview)) else str(value).encode() + if isinstance(kind, (String, Text)) and not isinstance(value, str): + self._count("文本列里的非文本值") + value = str(value) + if isinstance(value, str) and "\x00" in value: + self._count("文本里的 NUL 字符") + value = value.replace("\x00", "") + return value + + def missing(self, column: Any) -> Any: + """SQL 的 NULL。模型列 NOT NULL、又有缺省值的(老版本补列时没带缺省值,存量行留着 NULL;SQLite 的 ADD COLUMN + 不带 NOT NULL,照存)填成缺省值:PostgreSQL 的表照模型建,带 NOT NULL,不填就整行插不进。""" + from app.db.base import model_default, pg_json_dumps + + if column.nullable or column.primary_key: + return None + known, default = model_default(column) + if not known: + return None + self._count("NOT NULL 列的空值填成缺省值") + return pg_json_dumps(default) if _is_json(column) else default + + def json(self, raw: Any) -> str: + """JSON 原文 → 写进 PostgreSQL 的原文。平常的原样照搬(键的先后不变);有 NaN / NUL 的才重写一遍。""" + from app.db.base import pg_json_dumps + + textual = raw if isinstance(raw, str) else json.dumps(raw) + if "NaN" in textual or "Infinity" in textual or "\\u0000" in textual: + redone = pg_json_dumps(json.loads(textual)) + if redone != textual: + self._count("JSON 里的 NaN / Infinity / NUL") + return redone + return textual + + +# -------------------------------------------------------------------------- +# 应用库 +# -------------------------------------------------------------------------- + + +def _pk(table: Any) -> list[Any]: + return list(table.primary_key.columns) or list(table.columns) + + +def _insert(table: Any, columns: list[Any]) -> Any: + """目标上的插入:JSON 列按原文绑定再转 json(SQL 的 NULL 还是 NULL),撞了任何唯一约束都不动目标上的行。""" + values = {c.name: (cast(bindparam(c.name, type_=Text()), c.type) if _is_json(c) else bindparam(c.name, type_=c.type)) + for c in columns} + return postgresql.insert(table).values(values).on_conflict_do_nothing() + + +async def _count(conn: Any, table: Any) -> int: + return int((await conn.execute(select(func.count()).select_from(table))).scalar() or 0) + + +async def _write_batch(target: Any, statement: Any, rows: list[dict[str, Any]], table: Any, + refused: list[str]) -> None: + """一批一个事务。整批撞了约束(外键对不上之类)就逐行重试,插不进的记下主键和原因、跳过。""" + try: + async with target.begin() as conn: + await conn.execute(statement, rows) + return + except IntegrityError: + pass + keys = [c.name for c in _pk(table)] + for row in rows: + try: + async with target.begin() as conn: + await conn.execute(statement, [row]) + except IntegrityError as e: + from app.core.errors import first_line + + ident = ", ".join(f"{k}={row.get(k)!r}" for k in keys) + refused.append(f"{ident}:{first_line(e)}") + + +def _present(table: Any, keys: list[tuple[Any, ...]]) -> Any: + """目标上这些主键有几行。""" + from sqlalchemy import tuple_ + + pk = _pk(table) + where = pk[0].in_([k[0] for k in keys]) if len(pk) == 1 else tuple_(*pk).in_(keys) + return select(func.count()).select_from(table).where(where) + + +async def copy_tables(snapshot_db: Path, target: Any, *, batch: int, dry_run: bool, report: Report, + progress: Callable[[str], None]) -> None: + """按外键先后逐表、按主键分批:插入(目标上已有的不动),每批插完按主键数一遍目标上有了几行。""" + from sqlalchemy import type_coerce + + from app.db import models # noqa: F401 + from app.db.base import Base + + fixer = _Fixer(report.fixed) + source = create_sync_engine(f"sqlite:///{snapshot_db}") + try: + with source.connect() as src: + # AUTOINCREMENT 表的计数器(删过最后几行时比 MAX 大):PostgreSQL 的序列要接在它后面,不复用删掉的号—— + # 召回按 source_changes.seq 当游标往后拉,复用了就跳过新变化 + counters = dict(src.execute(text("SELECT name, seq FROM sqlite_sequence")).all()) if src.execute(text( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'sqlite_sequence'")).first() else {} + for table in Base.metadata.sorted_tables: + result = TableResult(table.name, orphans=report.orphans.get(table.name, 0)) + report.tables.append(result) + result.source = int(src.execute(select(func.count()).select_from(table)).scalar() or 0) + async with target.connect() as conn: + exists = await conn.run_sync(lambda sync, name=table.name: _has_table(sync, name)) + result.target_before = await _count(conn, table) if exists else 0 + columns = list(table.columns) + pk_at = [columns.index(c) for c in _pk(table)] + # JSON 列按原文取(type_coerce 成 Text:SQL 的 NULL 是 None,JSON 的 null 是 'null'),别的按列类型取 + reads = [type_coerce(c, Text()).label(c.name) if _is_json(c) else c for c in columns] + statement = _insert(table, columns) + rows = src.execution_options(yield_per=batch).execute(select(*reads).order_by(*_pk(table))) + done = 0 + for n, part in enumerate(rows.partitions(batch), 1): + keys = [tuple(row[i] for i in pk_at) for row in part] + if exists: + async with target.connect() as conn: + result.present_before += int((await conn.execute(_present(table, keys))).scalar() or 0) + if not dry_run: + payload = [{c.name: fixer.value(c, row[i]) for i, c in enumerate(columns)} for row in part] + await _write_batch(target, statement, payload, table, result.refused) + async with target.connect() as conn: + result.present_after += int((await conn.execute(_present(table, keys))).scalar() or 0) + done += len(part) + if n % 20 == 0: + progress(f" {table.name}:{done}/{result.source}") + if dry_run: + result.present_after, result.target_after = result.present_before, result.target_before + continue + await _reset_sequences(target, table, int(counters.get(table.name) or 0)) + async with target.connect() as conn: + result.target_after = await _count(conn, table) + finally: + source.dispose() + + +def _has_table(sync: Any, name: str) -> bool: + from sqlalchemy import inspect + + return inspect(sync).has_table(name) + + +async def _reset_sequences(target: Any, table: Any, floor: int = 0) -> None: + """自增主键的序列接到已用过的最大号后面:导入时写的是来源的主键值,序列自己没动过,不改的话下一条撞主键。 + 已用过的取三者最大:目标表里的 MAX、SQLite 的 AUTOINCREMENT 计数器(floor)、序列现在的值(重跑时不往回拨)。""" + for column in table.primary_key.columns: + if column is not table.autoincrement_column: + continue + async with target.begin() as conn: + name = (await conn.execute(text("SELECT pg_get_serial_sequence(:t, :c)"), + {"t": table.name, "c": column.name})).scalar() + if not name: + continue + top = int((await conn.execute(select(func.max(column)))).scalar() or 0) + last, called = (await conn.execute(text(f"SELECT last_value, is_called FROM {name}"))).one() + used = max(top, floor, int(last) if called else int(last) - 1) + await conn.execute(text("SELECT setval(:n, :v, :called)"), + {"n": name, "v": max(used, 1), "called": used >= 1}) + + +async def services_on(target: Any) -> int: + """目标库上服务的连接数(application_name 是 agentlab 的,不算本命令自己的)。""" + from app.db.base import APPLICATION_NAME + + async with target.connect() as conn: + return int((await conn.execute(text( + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() AND application_name = :n " + "AND pid <> pg_backend_pid()"), {"n": APPLICATION_NAME})).scalar() or 0) + + +# -------------------------------------------------------------------------- +# 断点 +# -------------------------------------------------------------------------- + + +def _split_writes(writes: list[tuple[str, str, Any]]) -> dict[str, tuple[list[tuple[str, Any]], list[tuple[str, Any]]]]: + """pending writes 按任务分组,各分成「特殊通道」(错误、中断、恢复……,序号是固定的负数)和普通的两拨:普通的分开 + 单独写,序号才和来源里的一样从 0 排起(来源里一个任务在一个断点上的普通写入是一次写下的)。""" + from langgraph.checkpoint.base import WRITES_IDX_MAP + + out: dict[str, tuple[list[tuple[str, Any]], list[tuple[str, Any]]]] = {} + for task_id, channel, value in writes: + special, regular = out.setdefault(task_id, ([], [])) + (special if channel in WRITES_IDX_MAP else regular).append((channel, value)) + return out + + +async def copy_checkpoints(snapshot_ckpt: Path, conninfo: str, *, dry_run: bool, report: Report, + progress: Callable[[str], None]) -> None: + """逐条读出来源的断点(saver 的公开接口,含 pending writes),写进 PostgreSQL 的断点库,再按线程核对条数。""" + from app.engine.checkpoints import count, open_postgres_saver, open_sqlite_saver + + threads: set[str] = set() + async with open_sqlite_saver(snapshot_ckpt) as src: + checkpoints = writes = 0 + async for tup in src.alist(None): + checkpoints += 1 + writes += len(tup.pending_writes or ()) + threads.add(str(tup.config["configurable"]["thread_id"])) + report.checkpoints, report.threads = (checkpoints, writes), len(threads) + if dry_run: + return + async with open_postgres_saver(conninfo, max_size=4, application_name=COPY_APPLICATION_NAME) as dst: + done = 0 + async for tup in src.alist(None): + conf = tup.config["configurable"] + if await dst.aget_tuple(tup.config) is None: + parent = (tup.parent_config or {}).get("configurable", {}).get("checkpoint_id") + put_config = {"configurable": {"thread_id": conf["thread_id"], + "checkpoint_ns": conf.get("checkpoint_ns", ""), + **({"checkpoint_id": parent} if parent else {})}} + await dst.aput(put_config, tup.checkpoint, tup.metadata, + dict(tup.checkpoint.get("channel_versions") or {})) + report.checkpoints_written += 1 + for task_id, (special, regular) in _split_writes(list(tup.pending_writes or ())).items(): + for group in (special, regular): + if group: + await dst.aput_writes(tup.config, group, task_id) + done += 1 + if done % 500 == 0: + progress(f" 断点:{done}/{checkpoints}") + target = [0, 0] + for thread in sorted(threads): + c, w = await count(dst, thread) + target[0] += c + target[1] += w + report.checkpoints_target = (target[0], target[1]) + + +# -------------------------------------------------------------------------- +# 入口 +# -------------------------------------------------------------------------- + + +def checkpoint_conninfo(target_url: str, *, checkpoint_url: str | None = None, direct_url: str | None = None) -> str: + """断点搬到哪儿(psycopg 的连接串):给了断点库地址用它;否则进目标库——给了直连地址就走直连(复核 R-b12:目标地址是 + PgBouncer 的事务模式时,psycopg 的预备语句、建表时取的会话级锁都会出问题)。""" + if checkpoint_url: + return normalize_checkpoint_url(checkpoint_url) + return libpq_conninfo(normalize_database_url(direct_url) if direct_url else normalize_database_url(target_url)) + + +async def copy_to_postgres(source: str | Path, target_url: str, *, checkpoint_url: str | None = None, + direct_url: str | None = None, dry_run: bool = False, assume_stopped: bool = False, + batch: int = DEFAULT_BATCH, work_dir: str | Path | None = None, + progress: Callable[[str], None] = print) -> Report: + """direct_url:目标库的直连地址(目标地址是 PgBouncer 时给;断点走它)。""" + from app.db.base import create_schema, make_engine + + target_url = normalize_database_url(target_url) + conninfo = checkpoint_conninfo(target_url, checkpoint_url=checkpoint_url, direct_url=direct_url) + db, ckpt = resolve_source(source) + if (wal := source_in_use(db, ckpt)) and not assume_stopped: + raise CopyRefused(f"{wal} 还在:服务看起来还开着这个库。先停服务再导入;确认服务已停(上次是强杀,留下了 " + "-wal)就加 --assume-stopped") + report = Report(dry_run=dry_run) + target = make_engine(target_url, application_name=COPY_APPLICATION_NAME) + try: + if not assume_stopped and (on := await services_on(target)): + raise CopyRefused(f"目标库上已经有 {on} 条服务的连接(application_name=agentlab):先停掉连着它的服务再导入") + progress(f"来源:{db}{f',断点 {ckpt}' if ckpt else '(没有断点库)'}") + progress(f"目标:PostgreSQL {describe(target_url)}{'(只核对,不写)' if dry_run else ''}") + with tempfile.TemporaryDirectory(prefix="agentlab-copy-", dir=work_dir) as tmp: + snap = snapshot(db, Path(tmp)) + upgrade_snapshot(snap) + report.orphans = settle_orphans(snap, fixed=report.fixed) + if not dry_run: + await create_schema(target) + await copy_tables(snap, target, batch=batch, dry_run=dry_run, report=report, progress=progress) + if ckpt is not None: + ckpt_snap = snapshot(ckpt, Path(tmp)) + await copy_checkpoints(ckpt_snap, conninfo, dry_run=dry_run, report=report, progress=progress) + finally: + await target.dispose() + return report + + +def render(report: Report) -> list[str]: + lines = ["", f"{'表':<28}{'来源':>10}{'导入前已有':>10}{'导入后已有':>10}{'目标总行数':>10} 结论"] + for t in report.tables: + lacking = t.source - t.present_after + if not t.source: + verdict = "空表" + elif report.dry_run: + verdict = f"要补 {lacking} 行" if lacking else "都已经有了" + elif t.refused: + verdict = f"少 {lacking} 行:撞了约束,没插进去(见下)" + elif lacking: + verdict = f"少 {lacking} 行" + else: + verdict = "一致" + (f"(这次补了 {t.present_after - t.present_before} 行)" + if t.present_after > t.present_before else "(都已经有了)") + if t.orphans: + verdict += f";另有 {t.orphans} 行孤儿行没导入" + lines.append(f"{t.name:<28}{t.source:>10}{t.present_before:>10}{t.present_after:>10}{t.target_after:>10}" + f" {verdict}") + if report.orphans: + total = sum(report.orphans.values()) + lines.append(f"孤儿行(SQLite 当时没强制外键,外键指向的行已经不在):{total} 行没导入,不算核对失败——" + + "、".join(f"{name} {n} 行" for name, n in report.orphans.items())) + for t in report.tables: + for item in t.refused[:20]: + lines.append(f" {t.name} 没插进去:{item}") + if len(t.refused) > 20: + lines.append(f" {t.name} 另有 {len(t.refused) - 20} 行没插进去") + c, w = report.checkpoints + if report.dry_run: + lines.append(f"断点:来源 {report.threads} 次运行、{c} 个断点、{w} 条 pending writes") + else: + tc, tw = report.checkpoints_target + lines.append(f"断点:来源 {report.threads} 次运行、{c} 个断点、{w} 条 pending writes;目标上这些运行有 {tc} 个断点、" + f"{tw} 条 pending writes(这次新写 {report.checkpoints_written} 个断点)" + + ("" if tc >= c and tw >= w else "——对不上")) + for what, n in report.fixed.items(): + lines.append(f"转换:{what} {n} 处") + if report.dry_run: + lines.append("只核对,什么都没写") + else: + lines.append("核对通过" if report.ok else "核对没过:见上面标出的表和断点") + return lines + + +def main(argv: list[str] | None = None) -> int: + from app.core.config import settings + + parser = argparse.ArgumentParser(prog="python -m app.db copy-to-postgres", + description="把 SQLite 数据目录里的应用库和断点导入 PostgreSQL(先停服务)。") + parser.add_argument("source", help="数据目录,或者 agentlab.db 的路径(断点库 checkpoints.db 在同一个目录里找)") + parser.add_argument("target", nargs="?", help="目标地址 postgresql+asyncpg://…;不给就用 AGENTLAB_DATABASE_URL") + parser.add_argument("--checkpoint-url", help="断点放到另一个 PostgreSQL 库;不给就用 AGENTLAB_CHECKPOINT_URL,再没有就进目标库") + parser.add_argument("--direct-url", help="目标库的直连地址(目标地址是 PgBouncer 时给,断点走它);不给、目标又是 " + "AGENTLAB_DATABASE_URL 时用 AGENTLAB_DATABASE_DIRECT_URL") + parser.add_argument("--dry-run", action="store_true", help="只核对:数来源和目标各有多少行,不建表、不写") + parser.add_argument("--assume-stopped", action="store_true", + help="不查服务停没停(来源旁边留着强杀后的 -wal 时用)") + parser.add_argument("--batch", type=int, default=DEFAULT_BATCH, help=f"每批插入的行数,缺省 {DEFAULT_BATCH}") + parser.add_argument("--work-dir", help="放快照的目录(要留出和两个库一样大的空间),缺省系统临时目录") + args = parser.parse_args(argv) + target = args.target or settings.database_url + if not target: + parser.error("没有目标地址:给第二个参数,或者设 AGENTLAB_DATABASE_URL") + direct = args.direct_url or None + if direct is None and settings.database_direct_url: + try: + same = normalize_database_url(target) == settings.database_url + except ValueError: + same = False + direct = settings.database_direct_url if same else None + try: + report = asyncio.run(copy_to_postgres( + args.source, target, checkpoint_url=args.checkpoint_url or settings.checkpoint_url or None, direct_url=direct, + dry_run=args.dry_run, assume_stopped=args.assume_stopped, batch=max(args.batch, 1), + work_dir=args.work_dir)) + except (CopyRefused, ValueError) as e: + print(f"没有导入:{e}", file=sys.stderr) + return 2 + except Exception as e: # noqa: BLE001 - 连不上、库报错:照实说一句,不甩一屏调用栈 + from app.core.errors import first_line + + what = "核对" if args.dry_run else "导入" + tail = "什么都没写" if args.dry_run else "已经写进目标库的行留着,处理好之后重新运行只补缺的" + print(f"没有{what}完:{type(e).__name__}:{first_line(e) or '(没有说明)'}。{tail}", file=sys.stderr) + return 1 + for line in render(report): + print(line) + return 0 if report.ok else 1 + diff --git a/backend/app/db/urls.py b/backend/app/db/urls.py new file mode 100644 index 00000000..cf7f16e1 --- /dev/null +++ b/backend/app/db/urls.py @@ -0,0 +1,83 @@ +"""应用库、断点库的地址:规范写法、换算成 psycopg 用的连接串、写进日志的样子(不带密码)。 + +不依赖 app.core.config:配置的校验器要调这里,这里再导 config 就成了循环导入。 +""" + +from __future__ import annotations + +from sqlalchemy.engine import URL, make_url +from sqlalchemy.exc import ArgumentError + +#: 应用库认的写法。asyncpg 是 SQLAlchemy 这边的驱动;postgresql:// / postgres:// 是常见的通用写法,一律换成 asyncpg +_APP_SCHEMES = {"postgresql+asyncpg", "postgresql", "postgres"} +#: 断点库认的写法:psycopg 只认 libpq 的 postgresql:// / postgres://;带驱动名的 SQLAlchemy 写法也收,去掉驱动名 +_CHECKPOINT_SCHEMES = {"postgresql", "postgres", "postgresql+psycopg", "postgresql+asyncpg"} + +#: asyncpg 和 libpq 对 SSL 的参数名不一样:asyncpg 是 ssl=require,libpq 是 sslmode=require(取值同一套) +_ASYNCPG_TO_LIBPQ = {"ssl": "sslmode"} +#: libpq 写法的 SSL 证书参数:asyncpg 不收这几个连接参数(SQLAlchemy 原样交过去就是 TypeError),改用同名的环境变量 +_LIBPQ_SSL_FILES = {"sslrootcert": "PGSSLROOTCERT", "sslcert": "PGSSLCERT", "sslkey": "PGSSLKEY", + "sslcrl": "PGSSLCRL", "sslpassword": "PGSSLPASSWORD"} +_SSL_MODES = {"disable", "allow", "prefer", "require", "verify-ca", "verify-full"} +#: 应用库地址里认的参数。SQLAlchemy 把地址里的参数原样交给 asyncpg.connect,它不认的(application_name、connect_timeout +#: 这类 libpq 写法)连的时候才报 TypeError;这里在启动时就说清楚 +_APP_PARAMS = {"ssl", "prepared_statement_cache_size", "target_session_attrs", "async_fallback"} +#: 只给 SQLAlchemy / asyncpg 看的参数,libpq 不认(psycopg 连的时候会报「invalid connection option」) +_ASYNCPG_ONLY = {"prepared_statement_cache_size", "prepared_statement_name_func", "command_timeout", "async_fallback"} + + +def _parse(raw: str, schemes: set[str], example: str) -> URL: + try: + url = make_url(raw) + except ArgumentError: + raise ValueError(f"认不出这个地址,应当写成 {example}") from None + if url.drivername not in schemes: + raise ValueError(f"只认 PostgreSQL({example}),不认 {url.drivername}://;用 SQLite 就不要设这一项") + if not url.database: + raise ValueError(f"地址里要写库名,例如 {example}") + return url + + +def normalize_database_url(raw: str) -> str: + """AGENTLAB_DATABASE_URL → postgresql+asyncpg://…(密码原样保留)。认不出的写法抛 ValueError。 + + SSL:libpq 的 sslmode= 换成 asyncpg 的 ssl=(取值同一套:disable / allow / prefer / require / verify-ca / verify-full); + 证书文件(sslrootcert 之类)asyncpg 不收连接参数,要用 PGSSLROOTCERT 等环境变量(asyncpg 和断点库的 libpq 都认)。""" + url = _parse(raw, _APP_SCHEMES, "postgresql+asyncpg://用户:密码@主机:5432/库名") + query = dict(url.query) + if files := [k for k in query if k in _LIBPQ_SSL_FILES]: + raise ValueError("地址里不能写 " + "、".join(files) + ":asyncpg 不收这几个参数,改设环境变量 " + + "、".join(_LIBPQ_SSL_FILES[k] for k in files)) + if "sslmode" in query: + if "ssl" in query: + raise ValueError("ssl= 和 sslmode= 只写一个") + mode = str(query.pop("sslmode")) + if mode not in _SSL_MODES: + raise ValueError(f"认不出 sslmode={mode}(可选 {' / '.join(sorted(_SSL_MODES))})") + query["ssl"] = mode + if other := sorted(k for k in query if k not in _APP_PARAMS): + raise ValueError("地址里的参数 " + "、".join(other) + " 不认(asyncpg 不收);能写的只有 " + + "、".join(sorted(_APP_PARAMS | {"sslmode"}))) + return url.set(drivername="postgresql+asyncpg", query=query).render_as_string(hide_password=False) + + +def normalize_checkpoint_url(raw: str) -> str: + """AGENTLAB_CHECKPOINT_URL → libpq 认的 postgresql://…。认不出的写法抛 ValueError。""" + url = _parse(raw, _CHECKPOINT_SCHEMES, "postgresql://用户:密码@主机:5432/库名") + return libpq_conninfo(url.render_as_string(hide_password=False)) + + +def libpq_conninfo(database_url: str) -> str: + """SQLAlchemy 的地址 → psycopg 的连接串:去掉驱动名,asyncpg 写法的参数换成 libpq 的。""" + url = make_url(database_url) + query = {_ASYNCPG_TO_LIBPQ.get(k, k): v for k, v in url.query.items() if k not in _ASYNCPG_ONLY} + return url.set(drivername="postgresql", query=query).render_as_string(hide_password=False) + + +def describe(database_url: str) -> str: + """写进日志的样子:主机、端口、库名、用户,不写密码。""" + url = make_url(database_url) + host = url.host or "本机套接字" + port = f":{url.port}" if url.port else "" + user = f",用户 {url.username}" if url.username else "" + return f"{host}{port}/{url.database}{user}" diff --git a/backend/app/engine/checkpoints.py b/backend/app/engine/checkpoints.py new file mode 100644 index 00000000..919ec46c --- /dev/null +++ b/backend/app/engine/checkpoints.py @@ -0,0 +1,85 @@ +"""LangGraph 的断点库:应用库是 SQLite 时用 AsyncSqliteSaver(数据目录里的 checkpoints.db,和业务表分开免得抢写锁), +是 PostgreSQL 时用官方的 AsyncPostgresSaver(psycopg 3 + 连接池),放在同一个库里,AGENTLAB_CHECKPOINT_URL 另给就放那里。 + +它建的四张表(checkpoints、checkpoint_blobs、checkpoint_writes、checkpoint_migrations)和应用的表不重名 +(test_db_portable 守着),所以同一个库、同一个 schema 就行。 + +应用里读断点一律走 saver 的公开接口(aget_state / aget_state_history / aget_tuple / alist),不直接查这几张表: +两种库的表结构完全不一样(SQLite 整份断点是一个 msgpack,PostgreSQL 拆成 JSONB 加 blob)。 +""" + +from __future__ import annotations + +import contextlib +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +from langgraph.checkpoint.base import BaseCheckpointSaver + +from app.core.config import settings + +#: 建断点表时拿的会话级咨询锁的号:几个进程同时启动,各自的 setup(建表、记迁移版本)一个一个来 +_SETUP_LOCK = 0x4167656E74436B70 # "AgentCkp" + + +def _pool_size() -> int: + # 运行里每个超步写一次断点、每个任务写一次 pending writes,都是一来一回的短操作。并发运行多时多给几条 + return max(4, min(settings.max_concurrent_runs, 20)) + + +@contextlib.asynccontextmanager +async def open_postgres_saver(conninfo: str, *, max_size: int | None = None, + application_name: str = "agentlab") -> AsyncIterator[BaseCheckpointSaver]: + """PostgreSQL 的断点库:开连接池、建表(幂等),退出时关池子。application_name 同应用库(db/base.APPLICATION_NAME)。""" + from app.db.base import _require_postgres_driver + + _require_postgres_driver("psycopg", "psycopg_pool", "langgraph.checkpoint.postgres") + from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver + from psycopg.rows import dict_row + from psycopg_pool import AsyncConnectionPool + + # 连接参数照 AsyncPostgresSaver.from_conn_string:自动提交(它不自己管事务)、语句第一次执行就预备、行按 dict 取 + pool = AsyncConnectionPool( + conninfo, min_size=1, max_size=max_size or _pool_size(), open=False, + kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row, + "application_name": application_name}, + check=AsyncConnectionPool.check_connection, name="agentlab-checkpoints") + await pool.open(wait=True) + try: + saver = AsyncPostgresSaver(pool) # type: ignore[arg-type] + async with pool.connection() as conn: + await conn.execute("SELECT pg_advisory_lock(%s)", (_SETUP_LOCK,)) + try: + await saver.setup() + finally: + await conn.execute("SELECT pg_advisory_unlock(%s)", (_SETUP_LOCK,)) + yield saver + finally: + await pool.close() + + +@contextlib.asynccontextmanager +async def open_sqlite_saver(path: Path | str) -> AsyncIterator[BaseCheckpointSaver]: + from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver + + async with AsyncSqliteSaver.from_conn_string(str(path)) as saver: + yield saver + + +def open_saver() -> contextlib.AbstractAsyncContextManager[BaseCheckpointSaver]: + """按配置开这个进程的断点库。""" + conninfo = settings.checkpoint_conninfo + if conninfo: + return open_postgres_saver(conninfo) + return open_sqlite_saver(settings.checkpoint_path) + + +async def count(saver: Any, thread_id: str | None = None) -> tuple[int, int]: + """(断点条数, pending writes 条数)。thread_id 为空数全部。copy-to-postgres 核对用,走公开接口。""" + config = {"configurable": {"thread_id": thread_id}} if thread_id else None + checkpoints = writes = 0 + async for tup in saver.alist(config): + checkpoints += 1 + writes += len(tup.pending_writes or ()) + return checkpoints, writes diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index 49839bf1..bbc278ae 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -183,8 +183,9 @@ class Side: 没有的,就是读不出来或者被改过 - load:工件 id → 内容,只认封存范围内的;文件不在返回 None,内容和哈希对不上抛 ValueError - masked:查询快照 → 要遮的列(小写):查询当时记下的加数据源现在设的 - - sources:口径卡那次执行开始的事件序号 → 推演重算还原输入要用的 Sources(节点产出、跑完的时刻都按那一刻)。 - 不给就按 load、masked 拼一个最小的:nodes.* 这类容器输入还原不出来 + - sources:(口径卡那次执行开始的事件序号, 卡在这次运行里的节点 id) → 推演重算还原输入要用的 Sources(节点产出、跑完的 + 时刻都按那一刻,按卡所在那一层的 id 记:子工作流里的卡只看同一张子图里的节点)。不给就按 load、masked 拼一个最小的: + nodes.* 这类容器输入还原不出来 - dialects:数据源名 → 类型(sqlite / postgres …),规范化 SQL 时按它的方言解析 - inputs:运行输入(Run.input) """ @@ -195,7 +196,7 @@ class Side: cards: list[tuple[str, str, dict[str, Any]]] = field(default_factory=list) load: Callable[[str], Any] = lambda _: None masked: Mapping[str, frozenset[str]] = field(default_factory=dict) - sources: Callable[[int], recompute.Sources] | None = None + sources: Callable[[int, str], recompute.Sources] | None = None dialects: Mapping[str, str | None] = field(default_factory=dict) inputs: dict[str, Any] = field(default_factory=dict) @@ -454,7 +455,7 @@ def _plan_card(side: Side, card: _Card, queries: list[_Query]) -> None: """按推演重算的规则还原卡上每个指标的输入:落在哪份快照上、是不是遮罩列、容器的完整值。""" if card.content is None: return - sources = side.sources(card.started) if side.sources is not None else recompute.Sources( + sources = side.sources(card.started, card.node_id) if side.sources is not None else recompute.Sources( load=side.load, queries=tuple(dict.fromkeys(q.artifact for q in queries)), masked=side.masked) store = recompute._Store(sources) card.sources = sources diff --git a/backend/app/engine/compiler.py b/backend/app/engine/compiler.py index bfc48df8..25dd3c17 100644 --- a/backend/app/engine/compiler.py +++ b/backend/app/engine/compiler.py @@ -136,7 +136,8 @@ def _wrap( "attempt": attempt + 1, "ts": time.time(), } - return {**updates, "trail": [trail]} + # 节点自己交回的 trail(子工作流里走了驳回出口的那几步,multi._rejected_steps)排在这一步前面 + return {**updates, "trail": [*(updates.get("trail") or []), trail]} except GraphBubbleUp: # interrupt() / Command 靠异常向上传递控制流,不是执行失败: # 必须原样放行,否则人工介入会被当成节点报错吞掉。 @@ -230,7 +231,10 @@ async def _rejected_exit(node: GraphNode, ctx: NodeContext, error: NodeRejected, # 节点在驳回之前已经算出的(NodeRejected.exit_updates:双盲复算的出具缺口、台账条目)照样交出;驳回的那几个键在上面 payload = {**(extra.get("payload") or {}), **error.exit_payload()} nodes = run_ctx.spec.node_map() - to = [{"node_id": run_ctx.global_id(t), "label": nodes[t].title if t in nodes else t} for t in dict.fromkeys(targets)] + # 名字按这次运行的说法记(子工作流里的写「子工作流 › x」,RunContext.global_label):父运行的图上没有子工作流里的节点, + # 实例页、通知、待办、Webhook 都照这个名字说 + to = [{"node_id": run_ctx.global_id(t), "label": run_ctx.global_label(nodes[t].title if t in nodes else t)} + for t in dict.fromkeys(targets)] try: artifact_id: str | None = await put_json(payload, kind="node_output", run_id=run_ctx.run_id, node_id=run_ctx.global_id(node.id), @@ -244,7 +248,7 @@ async def _rejected_exit(node: GraphNode, ctx: NodeContext, error: NodeRejected, outcome="rejected", **said, to=to, **({"evidence": ledger} if ledger else {})) updates: dict[str, Any] = {"nodes": {node.id: payload}, "trail": [{ "node_id": node.id, "type": str(node.type), "duration_ms": elapsed, "attempt": attempt, "ts": time.time(), - "outcome": "rejected", "label": node.title, **said, "to": to}]} + "outcome": "rejected", "label": run_ctx.global_label(node.title), **said, "to": to}]} if ledger: updates["evidence"] = ledger if extra.get("usage"): diff --git a/backend/app/engine/context.py b/backend/app/engine/context.py index fb85a203..6aff018f 100644 --- a/backend/app/engine/context.py +++ b/backend/app/engine/context.py @@ -43,9 +43,16 @@ class RunContext: """这张图里的节点 id → 这次运行里的 id(事件、工件上记的那个):最外层原样,子工作流里加前缀。""" return f"{self.prefix}{node_id}" + def global_label(self, title: str) -> str: + """这张图里的节点标题 → 这次运行里给人看的名字:最外层原样,子工作流里接上子工作流的名字(「子工作流 › x」,嵌套的 + 一层层接上,同 multi.run_subgraph 记在异常上的 label)。父运行的图上没有子工作流里的节点,各处照这个名字说。""" + return f"{self.extra.get(LABEL_PREFIX_KEY) or ''}{title}" + #: 子工作流里的节点 id 前缀(RunContext.extra 里的键,engine/nodes/multi.run_subgraph 放) PREFIX_KEY = "node_prefix" +#: 子工作流里的节点名前缀(「子工作流 › 」,同上):RunContext.global_label +LABEL_PREFIX_KEY = "node_label_prefix" #: 父运行的事件流(RunContext.extra 里的键,同上)。子工作流在父节点里用 ainvoke 跑,LangGraph 不把它的 custom 事件交给 #: 父图的流(ainvoke 不开 custom 模式,get_stream_writer 拿到的是空的那个):子工作流里每一步以前在运行的事件里一条都 #: 没有,时间线看不到、封存也不含它查过的数。所以 run_subgraph 把父图那一刻的写入口交下来,子工作流的节点经它发 diff --git a/backend/app/engine/nodes/challenge.py b/backend/app/engine/nodes/challenge.py index 4b8f21ce..f44c98ad 100644 --- a/backend/app/engine/nodes/challenge.py +++ b/backend/app/engine/nodes/challenge.py @@ -811,8 +811,10 @@ class _Review: # 没能反查的句数:没派出去、派出去却没查成的(unreviewed 里各有原因),运行时长上限将至没开始的,加上触顶时没能给出 # 结论的。都记为无法判断,可「无法判断 2 句」既可能是查了、数据不够,也可能是根本没查:审批卡要分开说 missed = len(unreviewed) + overdue + sum(skipped for _, skipped in hit.values()) - artifact = await self._store({"report": self.report_id, "doc_artifact": self.doc_artifact, "claims": claims, - "model": model}) + # 报告按这次运行里的 id 记(子工作流里的写「子工作流节点 id/报告」,同 report.checked、报告文档上的 node_id): + # 证据面板、可验证报告按它认审的是哪份报告 + artifact = await self._store({"report": self.ctx.run.global_id(self.report_id), "doc_artifact": self.doc_artifact, + "claims": claims, "model": model}) for limit, (touched, skipped) in hit.items(): words = self._limit_words(limit) message = f"已达上限({words}),{skipped} 句未反查" if limit == "daily_max_usd" else \ @@ -872,7 +874,8 @@ class _Review: async def _store(self, record: dict[str, Any]) -> str | None: try: return await artifact_store.put_json(record, kind="challenge", run_id=self.ctx.run.run_id, - node_id=self.ctx.node.id, meta={"report": self.report_id}) + node_id=self.ctx.run.global_id(self.ctx.node.id), + meta={"report": self.ctx.run.global_id(self.report_id)}) except Exception as e: # noqa: BLE001 - 记录写失败不毁掉运行:结论照常处置,只是理由和查询点不开 self.ctx.emit(EventType.LOG, level="warn", code="evidence_store_failed", message=f"反方审稿记录保存失败({describe_exception(e)}),审稿的理由和查询将无法查看") diff --git a/backend/app/engine/nodes/human.py b/backend/app/engine/nodes/human.py index b112a39f..cf2639ee 100644 --- a/backend/app/engine/nodes/human.py +++ b/backend/app/engine/nodes/human.py @@ -227,7 +227,8 @@ def _reviewed_reports(state: GraphState, ctx: NodeContext) -> list[dict[str, Any or not isinstance(result.get("stats"), dict): continue item: dict[str, Any] = { - "node_id": rid, "label": node.title, + # 按这次运行里的 id 记(子工作流里的写「子工作流节点 id/报告」):审批卡据此打开证据面板(?report=) + "node_id": ctx.run.global_id(rid), "label": node.title, "stats": {k: v for k, v in result["stats"].items() if isinstance(v, int) and not isinstance(v, bool)}, "claims": result.get("claims") or "off", } diff --git a/backend/app/engine/nodes/multi.py b/backend/app/engine/nodes/multi.py index 80b20064..f3ff3002 100644 --- a/backend/app/engine/nodes/multi.py +++ b/backend/app/engine/nodes/multi.py @@ -14,7 +14,9 @@ from app.core.errors import describe_exception, raw_detail from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import Workflow -from app.engine.context import PREFIX_KEY, STREAM_KEY, NodeContext, NodeError, stream_writer +from app.engine.context import ( + LABEL_PREFIX_KEY, PREFIX_KEY, STREAM_KEY, NodeContext, NodeError, stream_writer, +) from app.engine.lineage import whole_ref from app.engine.state import GraphState, message_text, template_context from app.engine.toolcalls import ( @@ -691,8 +693,15 @@ async def run_subgraph(state: GraphState, ctx: NodeContext) -> dict[str, Any]: # 恢复重放的标记和签批人只属于父图的节点,不能漏进子图(节点 id 可能重名)。前缀和写入口见 engine/context: # 子图里的节点经父运行的写入口发事件、节点 id 带前缀;嵌套的子图拿到的是同一个写入口、更长的前缀 extra={**{k: v for k, v in ctx.run.extra.items() if k not in ("resumed", "actors")}, - PREFIX_KEY: f"{ctx.run.global_id(ctx.node.id)}/", STREAM_KEY: stream_writer(ctx.run)}, + PREFIX_KEY: f"{ctx.run.global_id(ctx.node.id)}/", LABEL_PREFIX_KEY: f"{ctx.run.global_label(ctx.node.title)} › ", + STREAM_KEY: stream_writer(ctx.run)}, ) + # 这个子工作流节点之前几次执行里,子图各节点跑过的记录(交回父图 trail 的,见下):子图从它们接着数第几次执行 + # (engine/evidence.next_exec)。以前每次都从空的 trail 起步,循环里第二轮的子图节点又记成第 1 次:台账分不出是哪一次, + # 写外部系统的幂等键(按第几次执行算)和第一轮撞上,第二轮的写入交回第一轮的回执、不发送 + prefix = f"{ctx.node.id}/" + seed = [{**t, "node_id": str(t.get("node_id"))[len(prefix):]} for t in state.get("trail") or [] + if isinstance(t, dict) and str(t.get("node_id") or "").startswith(prefix)] ctx.emit(EventType.LOG, level="info", message=f"进入子工作流「{workflow.name}」({len(sub_spec.nodes)} 个节点)") @@ -710,7 +719,7 @@ async def run_subgraph(state: GraphState, ctx: NodeContext) -> dict[str, Any]: "output": {}, "messages": [], "usage": {}, - "trail": [], + "trail": seed, "loops": {}, }, {"recursion_limit": limit}, @@ -731,6 +740,9 @@ async def run_subgraph(state: GraphState, ctx: NodeContext) -> dict[str, Any]: raise output = result_state.get("output") or {} + # 这一次执行新跑的那几步(trail 只追加:前面是交进去的 seed) + ran = [t for t in (result_state.get("trail") or [])[len(seed):] if isinstance(t, dict)] + result_state = {**result_state, "trail": ran} result = { "output": output, "workflow": workflow.name, @@ -753,6 +765,12 @@ async def run_subgraph(state: GraphState, ctx: NodeContext) -> dict[str, Any]: } if ledger := [_forwarded(e, ctx.node.id) for e in result_state.get("evidence") or [] if isinstance(e, dict)]: updates["evidence"] = ledger + if ran: + # 子图这一次跑过的每一步交回父图的 trail,节点 id 接上前缀(「子工作流节点 id/…」):同一个子工作流节点下一次执行时 + # 从这里接着数(上面的 seed);走了驳回出口的那几步(outcome=rejected)运行收尾据此在终态事件上照实写「途经驳回」 + # (runner._rejected_via),实例页、通知、定时任务的 Webhook 说得出「子工作流 › 把关」被驳回、转到哪(名字、转到哪几步 + # 在子图里已经按这次运行的说法记好:global_label、global_id)。父图自己的节点按自己的 id 数,不受影响 + updates["trail"] = _forwarded_trail(ran, prefix) var_name = ctx.cfg("assign_to", "") if var_name: updates["vars"] = {var_name: output} @@ -763,6 +781,11 @@ async def run_subgraph(state: GraphState, ctx: NodeContext) -> dict[str, Any]: _LEDGER_NODE_KEYS = ("node_id", "card") +def _forwarded_trail(ran: list[dict[str, Any]], prefix: str) -> list[dict[str, Any]]: + """子图这一次跑过的步骤交回父图 trail 的样子:节点 id 接上「子工作流节点 id/」前缀。""" + return [{**t, "node_id": f"{prefix}{t.get('node_id')}"} for t in ran] + + def _forwarded(entry: dict[str, Any], node_id: str) -> dict[str, Any]: """子图台账里的一条交回父图:节点 id 加上「子工作流节点 id/」前缀,别的照抄。""" out = dict(entry) diff --git a/backend/app/engine/nodes/tools.py b/backend/app/engine/nodes/tools.py index 7c299fcb..17b3e257 100644 --- a/backend/app/engine/nodes/tools.py +++ b/backend/app/engine/nodes/tools.py @@ -274,9 +274,14 @@ async def _run_write(state: GraphState, ctx: NodeContext, name: str, args: dict[ from app.process import actions exec_no = next_exec(state, ctx.node.id) + # 幂等键按这次运行里的节点 id 算(子工作流里的写「sub/pay」):以前按子图里的 id(pay)算,父图里同名的节点、套同一个 + # 工作流的两个子工作流节点撞同一个键,后发的那一笔交回先发的回执、不发送。升级前子工作流里的写入按子图里的 id 记过键: + # 写入节点本身重新执行时(结果未知核实之后、失败后接着跑;审批恢复时子图从自己的断点接着跑,不走到这里)照旧认那个键 + # (legacy_node_id),是不是这个节点的那一笔、是不是这一次的内容,由 actions.prepare 按那条记录的事件和内容指纹认 + node_id, legacy = _action_ids(ctx) try: - prep = await actions.prepare(run_id=ctx.run.run_id, node_id=ctx.node.id, exec_no=exec_no, - capability=capability, args=args) + prep = await actions.prepare(run_id=ctx.run.run_id, node_id=node_id, exec_no=exec_no, + capability=capability, args=args, legacy_node_id=legacy) except actions.ActionFailed as e: raise NodeError(ctx.node.id, str(e)) from e if prep.receipt is not None: @@ -317,13 +322,19 @@ async def _run_write(state: GraphState, ctx: NodeContext, name: str, args: dict[ # 外部系统会把审批人当成申请人;无人值守的写入没有人确认,不写 act on_behalf = {"sub": prep.run.get("started_by"), "act": confirmer} try: - result = await actions.send(prep, capability, run_id=ctx.run.run_id, node_id=ctx.node.id, - exec_no=exec_no, emit=ctx.emit, on_behalf=on_behalf) + result = await actions.send(prep, capability, run_id=ctx.run.run_id, node_id=node_id, + exec_no=exec_no, emit=ctx.emit, on_behalf=on_behalf, confirm_node=ctx.node.id) except actions.ActionFailed as e: raise NodeError(ctx.node.id, str(e)) from e return _write_updates(ctx, result) +def _action_ids(ctx: NodeContext) -> tuple[str, str | None]: + """写入的幂等键按哪个节点 id 算:(这次运行里的 id, 升级前按子图里的 id 记过的那种;最外层的为 None)。""" + node_id = ctx.run.global_id(ctx.node.id) + return node_id, ctx.node.id if node_id != ctx.node.id else None + + async def _confirmed_by(run_id: str, node_id: str) -> str | None: """确认这次发送的人:节点重新执行(结果未知核实后接着跑、重启后恢复)时 ctx.actor() 已经不是确认的那次恢复, 按待办上记的处理人认(会签时是凑齐的那一票)。""" @@ -334,7 +345,7 @@ async def _confirmed_by(run_id: str, node_id: str) -> str | None: async with SessionLocal() as session: return await session.scalar(select(Task.resolved_by).where( Task.run_id == run_id, Task.node_id == node_id, Task.kind == "action_confirm", - Task.status == "resolved").order_by(Task.created_at.desc()).limit(1)) + Task.status == "resolved").order_by(Task.created_at.desc(), Task.id.desc()).limit(1)) def _write_updates(ctx: NodeContext, result: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/app/engine/replay.py b/backend/app/engine/replay.py index 3d4d1078..e95e2446 100644 --- a/backend/app/engine/replay.py +++ b/backend/app/engine/replay.py @@ -75,4 +75,32 @@ async def ask(ctx: NodeContext, payload: dict[str, Any], **request: Any) -> Any: 答复真正到达的那次执行里出现。 """ await once(ctx, EventType.HUMAN_REQUESTED, **request) - return interrupt(payload) + route = reject_route(ctx) if isinstance(payload, dict) else None + return interrupt({**payload, "reject_route": route} if route else payload) + + +def reject_route(ctx: NodeContext) -> dict[str, Any] | None: + """驳回之后去哪(总方案 §3 的驳回出口),放进中断内容:待办卡的提示、驳回之后的那句、处理过的结果照它说,不再一律写 + 「转入驳回分支」或「本次运行终止」。 + + - {"to": [{node_id, label}]}:接了驳回出口,驳回后转到这几步(节点 id、名字按这次运行的说法:子工作流里的写 + 「子工作流 › x」,同 compiler._rejected_exit 记在 node.finished 上的 to); + - {"ends": "run"}:运行就此结束、记为被驳回(没接驳回出口的关卡;审批设了驳回即终止时驳回出口走不到,也是这样); + - {"ends": "path"}:审批模式没接驳回出口、没设驳回即终止:这一路到审批为止,不再往下走(engine/nodes/human)。 + + 不是关卡的节点(agent 自己调工具时的审批:驳回了 agent 照常往下想)为 None,中断内容和以前一字不差。""" + from app.engine.schema import GATE_TYPES, approve_mode, reject_targets + + node, spec = ctx.node, ctx.run.spec + if node.type not in GATE_TYPES: + return None + approving = approve_mode(spec, node) + if approving and bool(ctx.cfg("stop_on_reject", False)): + return {"ends": "run"} + targets = reject_targets(spec, node) + if targets: + nodes = spec.node_map() + return {"to": [{"node_id": ctx.run.global_id(t), + "label": ctx.run.global_label(nodes[t].title if t in nodes else t)} + for t in dict.fromkeys(targets)]} + return {"ends": "path" if approving else "run"} diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 98f349ee..26b10939 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -206,6 +206,8 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list elif etype == EventType.TOOL_END and tool and tool in failed_at: recovered.add(tool) + #: 报过的节点失败 (节点 id, 原话):子工作流节点跟着子图里那一步失败时认得出是同一件事 + failures: list[tuple[str, str]] = [] for e in rows: etype = str(_event_field(e, "type", "")) data = _event_field(e, "data") or {} @@ -224,11 +226,18 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list # 失败」;也不进 RETRYABLE——重跑只会把同一件事再问一遍人。答案没走完是缺口,记 DEGRADED(没有成果时 # empty_output 另按 BROKEN 说) err = str(data.get("error") or "").strip()[:200] - add("node_rejected", f"节点「{titles.get(node_id) or node_id}」被驳回:{err}", DEGRADED) + if any(inner.startswith(f"{node_id}/") and said == err for inner, said in failures): + continue # 子工作流里那一步被驳回、子工作流节点跟着收尾:同一件事,只报那一步(同下面的执行失败) + failures.append((str(node_id), err)) + add("node_rejected", f"节点「{_step_name(str(node_id), titles)}」被驳回:{err}", DEGRADED) elif etype == EventType.NODE_FAILED: err = str(data.get("error") or "").strip()[:200] - add("node_failed", f"节点「{titles.get(node_id) or node_id}」执行失败:{err}", BROKEN) + if any(inner.startswith(f"{node_id}/") and said == err for inner, said in failures): + # 子工作流里那一步失败了,子工作流节点(「sub」)跟着失败、原话相同:同一件事,只报更具体的那一步(上面已报) + continue + failures.append((str(node_id), err)) + add("node_failed", f"节点「{_step_name(str(node_id), titles)}」执行失败:{err}", BROKEN) elif etype == EventType.RUN_FAILED: err = str(data.get("error") or "").strip()[:200] @@ -264,6 +273,15 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list return signals +def _step_name(node_id: str, titles: Mapping[str, str]) -> str: + """失败的那一步给人看的名字。子工作流里的(「子工作流节点 id/…」)一层层接上 node.started 记的名字:「复核子流程 › 算一下」, + 不写内部 id;没记名字的那一层照写 id 的那一段。""" + if "/" not in node_id: + return titles.get(node_id) or node_id + parts = node_id.split("/") + return " › ".join(titles.get("/".join(parts[:i + 1])) or parts[i] for i in range(len(parts))) + + def worst(signals: Iterable[Signal]) -> str: """一组信号里最重的那档。没有信号时返回空串。""" ranked = sorted(signals, key=lambda s: _SEVERITY_RANK.get(s.severity, 1), reverse=True) diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index 1b3d020c..3688318a 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -10,11 +10,13 @@ from collections.abc import Callable, Sequence from datetime import datetime, timezone from typing import Any -from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver +from langgraph.checkpoint.base import BaseCheckpointSaver from langgraph.errors import GraphRecursionError from langgraph.types import Command from sqlalchemy import case, func, select, update +from app.cluster import fence, identity +from app.cluster.fence import Fenced from app.core.bus import bus from app.core.config import settings from app.core.errors import describe_exception, raw_detail @@ -56,6 +58,9 @@ REJECTED_CONTINUE = "被驳回的运行不能继续运行:改好内容后重 REJECTED_RESUME = "这次运行已被驳回,没有待处理的审批,无法恢复:改好内容后重新发起" #: 服务重启时还在排队的运行改成失败时写的话(run.error 和补发的 run.failed 是同一句) QUEUED_AT_RESTART = "服务重启时这次运行仍在排队,没有可恢复的进度,请重新发起运行" +#: 本进程隔离(续不上租约)之后没能重新登记,这一段执行没有开始(cluster/fence.ready 等不到) +NOT_REREGISTERED_QUEUED = "本进程隔离后没能重新登记,这次运行没有开始执行,请重新发起运行" +NOT_REREGISTERED_RESUMABLE = "本进程隔离后没能重新登记,这次运行没有接着执行,已挂起,可从断点继续运行" class ContinueRefused(ValueError): @@ -71,9 +76,10 @@ class RunManager: def __init__(self) -> None: self._tasks: dict[str, asyncio.Task[None]] = {} - self._checkpointer: AsyncSqliteSaver | None = None + self._checkpointer: BaseCheckpointSaver | None = None self._cm: Any = None - self._semaphore = asyncio.Semaphore(settings.max_concurrent_runs) + # 并发上限以前是这里一把进程内的信号量(max_concurrent_runs);现在是库里按类别发的名额(app/cluster/slots.py), + # 几个进程合起来算,评测、召回这些系统运行不再挤占问数据和试跑 #: 正在关停。此时收到的取消来自关停而不是用户,运行要记成可恢复的中断 self._closing = False #: 已经跑完、正在落状态和封存的运行。关停不去打断它们 @@ -86,8 +92,11 @@ class RunManager: async def setup(self) -> None: self._closing = False settings.ensure_dirs() - self._cm = AsyncSqliteSaver.from_conn_string(str(settings.checkpoint_path)) - self._checkpointer = await self._cm.__aenter__() + # SQLite:数据目录里的 checkpoints.db;PostgreSQL:同一个库(或 AGENTLAB_CHECKPOINT_URL),见 engine/checkpoints.py + from app.engine.checkpoints import open_saver + + self._cm = open_saver() + self._checkpointer = _fenced_saver(await self._cm.__aenter__()) # 上次进程被强杀时留下的记录。running 和 queued 要分开处理: # running 至少跑过一步、有 checkpoint,可以续;queued 还在排队, # 一个 checkpoint 都没写过,"恢复"它只会从 START 用默认值重跑一遍, @@ -102,44 +111,68 @@ class RunManager: update(Run).where(func.coalesce(Run.last_seq, 0) < latest) .values(last_seq=latest, updated_at=Run.updated_at) .execution_options(synchronize_session=False)) - stranded = list((await session.execute( - select(Run.id, Run.last_seq).where(Run.status == "running") - )).all()) - queued = list((await session.execute( - select(Run.id, Run.last_seq).where(Run.status == "queued") - )).all()) - await session.execute( - update(Run) - .where(Run.status == "running") - .values(status="interrupted", error="服务重启,运行已挂起,可从断点继续运行") - ) - await session.execute( - update(Run) - .where(Run.status == "queued") - .values(status="failed", error=QUEUED_AT_RESTART) - ) await session.commit() - # 进程被强杀时连 server_shutdown 日志都来不及发,事件流停在半路。补上这一条, - # 回放它的客户端才知道它是挂起了,而不是还在跑 - for run_id, last_seq in stranded: - bus.set_seq(run_id, last_seq or 0) - await self._emit(run_id, EventType.LOG, data={ - "level": "warn", "code": "server_shutdown", - "message": "服务上次未正常关闭,这次运行已中断;可从断点继续运行", - }) - # 排队中的改成了失败,也补一条 run.failed:以前只改状态,时间线里没有失败原因这一行(05-已知遗留 (h))。 - # 不写结束时间、不封存:它从没开始执行,同以前(定时通知按「没有开始时间」认出它,scheduler/webhook.py) - for run_id, last_seq in queued: - bus.set_seq(run_id, last_seq or 0) - await self._emit(run_id, EventType.RUN_FAILED, data={"error": QUEUED_AT_RESTART}) + # 上次进程被强杀时留下的记录:只碰属主不在了的(多进程时别的活着的进程在跑的一概不碰,见 reap) + from app.cluster import state + + await self.reap(await state.current(fresh=True)) # 口径账本:最近几天封存了、一次都没比过的探索运行,在后台补比一次(ledger/match.py)。只读工件和版本表,不查业务库、 # 不调模型;和调度循环同一个开关,拷来的数据目录、测试不补 from app.ledger import match as ledger_match from app.ledger import recall as ledger_recall - ledger_match.start_backfill() - # 口径召回(ledger/recall.py):最近发布、自动那一批没发完的接着发,结束了还没收尾的召回运行补上收尾。同一个开关 - ledger_recall.start_resume() + # 多进程时只由调度当班的进程做(app/cluster;单进程时本进程启动就当班):几个进程各补一遍会重复比对、重复召回 + from app.cluster.service import leading + + if leading(): + ledger_match.start_backfill() + # 口径召回(ledger/recall.py):最近发布、自动那一批没发完的接着发,结束了还没收尾的召回运行补上收尾。同一个开关 + ledger_recall.start_resume() + + async def reap(self, live: Any) -> int: + """接手属主不在了的运行(启动时;多进程时当班进程隔一会儿再来一次,app/cluster/reaper.py)。规矩同以前的启动整理: + + - running 至少跑过一步、有断点:标 interrupted(可从断点继续运行),补一条 server_shutdown 日志——进程被强杀时连这一条 + 都来不及发,事件流停在半路,回放它的客户端才知道它是挂起了而不是还在跑; + - queued 还在排队,一个断点都没写过,「恢复」它只会从 START 用默认值重跑一遍、还丢掉 run.input,那不是恢复是伪造: + 标 failed,补一条 run.failed(不写结束时间、不封存:它从没开始执行,同以前)。 + + 不自动重跑有副作用的步骤:人看了再点「继续运行」。属主还活着的(别的进程正在跑、本进程里正在跑)一概不碰;认领那一下是 + 带条件的 UPDATE(状态、属主都还是刚才看到的),几个进程同时接手同一次运行只有一个改得动。交回整理了几次运行。""" + from app.cluster.claims import same + + async with SessionLocal() as session: + rows = (await session.execute(select(Run.id, Run.owner, Run.status).where( + Run.status.in_(("running", "queued"))))).all() + done = 0 + for run_id, owner, status in rows: + if live.alive(owner, local=self.is_active(run_id)): + continue + event: RunEventModel | None = None + async with SessionLocal() as session: + values: dict[str, Any] = ( + {"status": "interrupted", "error": "服务重启,运行已挂起,可从断点继续运行"} if status == "running" + else {"status": "failed", "error": QUEUED_AT_RESTART}) + claimed = await session.execute(update(Run).where(Run.id == run_id, Run.status == status, + same(Run.owner, owner)) + .values(owner=None, **values).execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + await session.rollback() + continue + run = await session.get(Run, run_id) + bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq if run else 0)) + event = self._event(run_id, EventType.LOG, data={ + "level": "warn", "code": "server_shutdown", + "message": "服务上次未正常关闭,这次运行已中断;可从断点继续运行", + }) if status == "running" else self._event(run_id, EventType.RUN_FAILED, data={"error": QUEUED_AT_RESTART}) + await _stage(session, event, run) + await session.commit() + await bus.publish(event) + await bus.close(run_id) + done += 1 + if done: + logger.warning("接手了 %d 次属主已经不在的运行(在跑的标中断、排队的标失败)", done) + return done async def shutdown(self) -> None: self._closing = True @@ -163,7 +196,7 @@ class RunManager: await self._cm.__aexit__(None, None, None) @property - def checkpointer(self) -> AsyncSqliteSaver: + def checkpointer(self) -> BaseCheckpointSaver: if self._checkpointer is None: raise RuntimeError("RunManager 还没初始化") return self._checkpointer @@ -180,6 +213,8 @@ class RunManager: persist: bool = True, ts: float | None = None, ) -> None: + # 执行这次运行的那段(cluster/fence 绑定过世代的)写之前问一句:租约续晚了就不写(可能已经被别的进程接手) + fence.check() event = self._event(run_id, event_type, node_id=node_id, data=data, ts=ts) # 先落库,再广播。顺序反过来会漏事件,而且是结构性的漏: # @@ -221,7 +256,20 @@ class RunManager: self, run_id: str, event_type: EventType | str, *, node_id: str | None = None, **data: Any, ) -> None: - """给运行之外的代码往运行里记一条事件。发起时就定下的事件不走这里,交给 start 的 notes。""" + """给运行之外的代码往运行里记一条事件。发起时就定下的事件不走这里,交给 start 的 notes。 + + 执行中的运行,序号由执行它的进程发(core/bus 的进程内计数):运行在别的活着的进程里执行时交给它记(cluster/requests), + 不在这里取号——这里的计数和那边的不是同一个,直接写会撞号。""" + if not self.is_active(run_id): + from app.cluster import requests, workers + + async with SessionLocal() as session: + row = (await session.execute(select(Run.status, Run.owner).where(Run.id == run_id))).first() + if row is not None and row.status in ("running", "queued") and row.owner \ + and row.owner != identity.owner() and await workers.alive(row.owner): + await requests.send(row.owner, "note", "run", run_id, payload={ + "type": str(event_type), "node_id": node_id, "data": data}) + return await self._emit(run_id, event_type, node_id=node_id, data=data) # ---------------- 启动 ---------------- @@ -331,6 +379,7 @@ class RunManager: agent_limits=agent_limits, data_versions=data_versions, options=options, + origin_kind=origin_kind, ) ) return run @@ -641,6 +690,19 @@ class RunManager: self._stopped_by[run_id] = actor task.cancel() return "stopping" + # 不在本进程里执行(多进程,app/cluster):在别的活着的进程里跑(或排队)的给它发停止请求,它照本进程的规矩停、发 + # run.cancelled(带上是谁停的),界面上同样是「正在停止」;属主不在了的照租约过期接手(标中断),再按停在断点上的放弃 + from app.cluster import requests, workers + + async with SessionLocal() as session: + row = (await session.execute(select(Run.status, Run.owner).where(Run.id == run_id))).first() + if row is not None and row.status in ("running", "queued") and row.owner and row.owner != identity.owner(): + if await workers.alive(row.owner): + await requests.send(row.owner, "stop", "run", run_id, actor=actor) + return "stopping" + from app.cluster import state + + await self.reap((await state.current(fresh=True)).trusting()) return "cancelled" if await self._abandon(run_id, actor) else None async def _abandon(self, run_id: str, actor: str | None) -> bool: @@ -714,7 +776,10 @@ class RunManager: options: dict[str, Any] | None = None, resumed: list[str] | tuple[str, ...] = (), actors: dict[str, str | None] | None = None, + origin_kind: str | None = None, ) -> None: + from app.cluster import slots + started = time.perf_counter() status = "succeeded" error: str | None = None @@ -732,12 +797,32 @@ class RunManager: # 等待从 run.interrupted 算起——以前执行一直计到 finally,收尾那一截执行和等待两边都算, # active + wait 比墙钟还长(机器一忙就差出几十毫秒) paused: float | None = None - async with self._semaphore: + # 这段执行属于现在这一世代(cluster/fence.py):本进程隔离之后,它不再写。隔离之后新身份还没登记上时先等一等 + try: + await fence.ready() + except Fenced: + await self._not_reregistered(run_id) + return + fence.bind() + mine = identity.owner() + #: 被隔离了(续不上租约):停手,不写终态(这次运行由接手的进程按「属主不在了」处理) + fenced = False + # 先要并发名额(app/cluster/slots.py)。等名额的时候被停止(或者关停)也要照实收尾,见 _stopped_in_queue + slot = slots.hold(run_id, origin_kind) + try: + await slot.__aenter__() + except asyncio.CancelledError: + await self._stopped_in_queue(run_id) + raise + try: + # 执行时长从拿到名额算起:排队等名额的那一段不算执行(墙钟从 started_at 算,started_at 也是拿到名额之后才写) + started = time.perf_counter() try: async with SessionLocal() as session: row = await session.get(Run, run_id) if row is not None: row.status = "running" + row.owner = mine # started_at 只在第一次开始时写。以前每段 _drive 都覆盖它, # 审批恢复、接着跑之后,运行的"开始时间"就成了最后一段的开始 if row.started_at is None: @@ -822,7 +907,13 @@ class RunManager: elif snapshot.next: status = "interrupted" + except Fenced: + fenced = True + logger.error("运行 %s:本进程续不上租约,停手、不写终态(由接手的进程按「属主不在了」处理)", run_id) except asyncio.CancelledError: + if fence.fenced_here(): + fenced = True + raise if self._closing: # 服务关停(dev.sh 的 --reload 改一个文件就是一次)不是用户取消。 # checkpoint 完好,记成中断、重启后从断点接着跑——和强杀后启动 @@ -887,8 +978,18 @@ class RunManager: elapsed = int((stop - started) * 1000) self._finalizing.add(run_id) try: - await self._finalize(run_id, status, error, final_state, elapsed, - terminal=terminal, error_node=error_node, end_reason=end_reason) + if not fenced: + await self._finalize(run_id, status, error, final_state, elapsed, terminal=terminal, + error_node=error_node, end_reason=end_reason, owner=mine) + # 记 action.unknown、安排核实也算收尾:做完才把自己摘掉。以前摘掉之后才做,等运行「不在跑了」 + # 再去读事件的一方(接口、测试)有一小段读不到这一条;库慢一点(PostgreSQL 满载时)就读不到 + if pending_action is not None and not fenced: + from app.process.actions import on_unknown + + try: + await on_unknown(pending_action) + except Exception: # noqa: BLE001 - 运行已经收成挂起;没安排上的核实,下次启动时 actions.startup 补上 + logger.exception("run %s 的动作 %s 结果未知,安排核实时出错", run_id, pending_action) finally: # 收尾被打断(关停时的取消、库连接出错)也得把自己摘掉。以前写在 # _finalize 后面,一被打断就漏掉,留下的任务让下一次 shutdown 的 @@ -896,22 +997,74 @@ class RunManager: self._finalizing.discard(run_id) self._tasks.pop(run_id, None) self._stopped_by.pop(run_id, None) - if pending_action is not None: - from app.process.actions import on_unknown - - try: - await on_unknown(pending_action) - except Exception: # noqa: BLE001 - 运行已经收成挂起;没安排上的核实,下次启动时 actions.startup 补上 - logger.exception("run %s 的动作 %s 结果未知,安排核实时出错", run_id, pending_action) # 运行到终态就把沙箱会话收掉。thread_id 每个 run 都是新的, # 不收的话每跑一次带代码节点的图就多一台常驻 microVM(几百 MB), # 而且没有任何自动回收路径会碰它——闲置回收只在"下次有人执行 # 代码"时才触发。interrupted 要留着,人回来还得接着跑。 - if status in ("succeeded", "failed", "cancelled"): + if status in ("succeeded", "failed", "cancelled") and not fenced: with contextlib.suppress(Exception): from app.sandbox.manager import sandbox_manager await sandbox_manager.cleanup(run_id) + finally: + await slot.__aexit__(None, None, None) + + async def _not_reregistered(self, run_id: str) -> None: + """隔离之后新身份一直没登记上、这一段执行不开始了(fence.ready 等不到,复核 R-b15):照实收尾,不留给别的进程按「服务 + 重启时还在排队」去标。排队中的(一步都没执行过)收成失败;已经执行过、停在断点上又要接着执行的收成中断(断点完好,可以 + 继续运行)。条件更新:状态、属主还是刚才看到的才改(别的进程先接手了就不动)。""" + from app.cluster.claims import same + + logger.error("运行 %s:本进程隔离之后一直没重新登记上,这一段执行不开始", run_id) + try: + event: RunEventModel | None = None + async with SessionLocal() as session: + seen = (await session.execute(select(Run.status, Run.owner).where(Run.id == run_id))).first() + if seen is None or seen.status not in ("queued", "running"): + return + if seen.status == "queued": + values: dict[str, Any] = {"status": "failed", "error": NOT_REREGISTERED_QUEUED} + else: + values = {"status": "interrupted", "error": NOT_REREGISTERED_RESUMABLE} + claimed = await session.execute(update(Run).where( + Run.id == run_id, Run.status == seen.status, same(Run.owner, seen.owner)) + .values(owner=None, **values).execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + await session.rollback() + return + run = await session.get(Run, run_id) + bus.set_seq(run_id, await _seq_floor(session, run_id, run.last_seq if run else 0)) + event = (self._event(run_id, EventType.RUN_FAILED, data={"error": NOT_REREGISTERED_QUEUED}) + if seen.status == "queued" else + self._event(run_id, EventType.LOG, data={"level": "warn", "code": "not_reregistered", + "message": NOT_REREGISTERED_RESUMABLE})) + await _stage(session, event, run) + await session.commit() + await bus.publish(event) + await bus.close(run_id) + except Exception: # noqa: BLE001 - 收不了尾的留给别的进程按「属主不在了」处理 + logger.exception("运行 %s:收尾时出错,留给别的进程按属主不在了处理", run_id) + finally: + self._tasks.pop(run_id, None) + self._stopped_by.pop(run_id, None) + + async def _stopped_in_queue(self, run_id: str) -> None: + """还在等并发名额时被取消。关停:什么都不写(还是排队中 / 刚认领的样子),照「属主不在了」的规矩由下次启动或别的 + 进程接手(排队的标失败、认领了的标中断);人点了停止:收成 cancelled,同执行中停止(run.cancelled 带上是谁停的、封存)。""" + try: + if self._closing: + return + if fence.fenced_here(): + return + actor = self._stopped_by.pop(run_id, None) + self._finalizing.add(run_id) + await self._finalize(run_id, "cancelled", "用户取消", {}, 0, + terminal=(EventType.RUN_CANCELLED, {"reason": END_STOPPED, "actor": actor}), + end_reason=END_STOPPED, owner=identity.owner()) + finally: + self._finalizing.discard(run_id) + self._tasks.pop(run_id, None) + self._stopped_by.pop(run_id, None) async def _handle_custom(self, run_id: str, chunk: Any) -> None: """节点通过 get_stream_writer 发来的事件。""" @@ -932,6 +1085,7 @@ class RunManager: interrupts = chunk.get("__interrupt__") if not interrupts: return False + fence.check() for item in interrupts: value = getattr(item, "value", item) @@ -993,7 +1147,9 @@ class RunManager: terminal: tuple[EventType, dict[str, Any]] | None = None, error_node: str | None = None, end_reason: str | None = None, + owner: str | None = None, ) -> None: + """收尾。owner:执行这一段的属主——运行那一行的属主已经不是它(被别的进程当成没了接手了,cluster/fence.py)就什么都不写。""" usage = dict(state.get("usage") or {}) output = dict(state.get("output") or {}) now = datetime.now(timezone.utc) @@ -1011,7 +1167,13 @@ class RunManager: run_class: str | None = None origin_kind: str | None = None async with SessionLocal() as session: - run = await session.get(Run, run_id) + # 先占住这一行再看属主(PostgreSQL 上 FOR UPDATE;SQLite 只有一个进程):接手的进程改属主用的是带条件的 UPDATE, + # 两边一个接一个,被接手了的这边不写 + run = (await session.execute(select(Run).where(Run.id == run_id).with_for_update())).scalar_one_or_none() + if run is not None and owner is not None and run.owner != owner: + logger.error("运行 %s 已经被别的进程接手(属主 %s),本进程不写终态", run_id, identity.describe(run.owner)) + await session.rollback() + return if run: app_id = run.app_id run_class, origin_kind = run.run_class, run.origin_kind @@ -1056,6 +1218,10 @@ class RunManager: run.end_reason = end_reason if status != "interrupted": run.finished_at = now + else: + # 停在审批上(或挂起)的运行不属于任何进程:谁收到决定谁接着跑(resume / continue_failed 的条件 UPDATE + # 认领,同一个决定只会被一个进程执行) + run.owner = None if status in ("failed", "cancelled"): # 并行支路上已经停下来等人的那张审批卡,运行结束了它也就没有意义了; # 留着的话它一直挂在全局待办里,批了也只会被告知运行不在断点上 @@ -1242,6 +1408,8 @@ def _carried(run: Run) -> dict[str, Any]: "data_versions": run.data_versions, # 运行选项同理:照发起时的快照,不重新读设置 "options": run.options or {}, + # 并发名额按来源归类(app/cluster/slots.py):恢复、接着跑的那一段和第一段同一类 + "origin_kind": run.origin_kind, } @@ -1353,7 +1521,8 @@ def _rejected_via(state: dict[str, Any]) -> dict[str, Any]: """这次运行途经的驳回出口(engine/compiler._rejected_exit 记在 trail 上的 outcome=rejected):{rejected_via: [{node_id, label, kind, decision, actor, note, to}]},按先后。终态事件带上它,运行照它的路径结束(成功结束也一样)照实说「途经 驳回」;没有途经的不带这个键,终态事件和以前一字不差。读的是收尾时的状态:节点出错、被驳回结束的那几种收尾拿不到 - 状态(final_state 为空),不带;子工作流里的驳回出口不在这里(它的 trail 是子工作流自己的状态)。""" + 状态(final_state 为空),不带。子工作流里走了驳回出口的那几步也在这里:子图跑过的步骤交回父图的 trail(节点 id 带 + 「子工作流节点 id/」前缀、名字写「子工作流 › x」,engine/nodes/multi.run_subgraph)。""" via = [{k: t.get(k) for k in ("node_id", "label", "kind", "decision", "actor", "note", "to")} for t in state.get("trail") or [] if isinstance(t, dict) and t.get("outcome") == "rejected"] return {"rejected_via": via} if via else {} @@ -1489,7 +1658,7 @@ async def _claim( """只在状态仍是 expected 时把运行改成 running;已经被别人改过就回滚、说明原因。""" claimed = await session.execute( update(Run).where(Run.id == run_id, Run.status.in_(expected)) - .values(status="running", error=None, **values) + .values(status="running", error=None, owner=identity.owner(), **values) ) if claimed.rowcount != 1: await session.rollback() @@ -1661,3 +1830,58 @@ async def _timing(session: Any, run: Run, elapsed_ms: int, now: datetime) -> dic run_manager = RunManager() + + +def _fenced_saver(saver: BaseCheckpointSaver) -> BaseCheckpointSaver: + """断点库的写也要过 cluster/fence:本进程续不上租约之后,手上的执行不再写断点(接手的进程会从断点接着跑,两边都写就分叉了)。 + + 包成协程、在协程里查(复核 R-b8):LangGraph 在节点的完成回调里调 aput_writes 拿协程交给后台任务——同步包装在回调里就抛了, + 每次隔离 asyncio 都记一条「Exception in callback … FuturesDict.on_done」;在协程里抛,异常留在写断点的那个任务里,由 LangGraph + 照常收走。""" + for name in ("aput", "aput_writes"): + original = getattr(saver, name, None) + if original is None: + continue + + async def guarded(*args: Any, __original: Any = original, **kwargs: Any) -> Any: + fence.check() + return await __original(*args, **kwargs) + + setattr(saver, name, guarded) + return saver + + +def _stop_fenced() -> None: + """隔离了:停掉本进程在执行的运行(停的时候不写终态,见 _drive 的 fenced)。""" + for task in list(run_manager._tasks.values()): + if not task.done(): + task.cancel() + + +fence.on_trip(_stop_fenced) + + +def _register_requests() -> None: + """别的进程发来的:停止本进程在执行的运行、往它里面记一条事件(app/cluster/requests.py)。""" + from app.cluster import requests + + @requests.handle("stop", "run") + async def _stop(run_id: str, actor: str | None, _payload: dict[str, Any]) -> None: + # 不在本进程执行的也照停止的规矩办(cancel 自己看属主):在别的活着的进程里执行就转给它,属主不在了就接手、放弃,停在 + # 断点上的收成 cancelled,已经结束的什么都不做。隔离之后转到新身份名下的停止请求就是这样落下的(复核 R-b10) + await run_manager.cancel(run_id, actor=actor) + + @requests.handle("note", "run") + async def _note(run_id: str, _actor: str | None, payload: dict[str, Any]) -> None: + kind, node_id, data = payload.get("type") or EventType.LOG, payload.get("node_id"), dict(payload.get("data") or {}) + if run_manager.is_active(run_id): + await run_manager._emit(run_id, kind, node_id=node_id, data=data) + return + # 收到时运行已经停下了:停在审批上的接在库里最后一条后面记(同 process/events.record),别的不记 + from app.engine.run_append import append_events + + if await append_events(run_id, [(kind, node_id, data)], require="interrupted") is None: + logger.info("运行 %s 已经不在执行,别的进程要记的一条 %s 没有记", run_id, kind) + + +_register_requests() diff --git a/backend/app/engine/spend.py b/backend/app/engine/spend.py index 87510ba7..a9d2eaf1 100644 --- a/backend/app/engine/spend.py +++ b/backend/app/engine/spend.py @@ -15,6 +15,10 @@ 各管各的上限,互不挤占。三本按同一个本地日期滚动(_today)。 目录里没有价格的模型估不出金额:只计次数(unpriced_calls),金额上限对它不起作用,由调用方照实说明。 + +**多进程。** 先读后写以前只靠进程内的锁:两个进程各读到同一个余额,会一起越过上限。现在每次记账先占住设置表里这本账的那一行 +(_locked:PostgreSQL 上是行锁,SQLite 上拿到写锁),读、判断、写在同一个事务里,几个进程一个接一个;这本账还没有那一行时 +插一行,撞了说明别的进程刚插上,重来一次。进程内的锁留着:同一个进程里的几个并行核查先在本进程排队,少占库的锁。 """ from __future__ import annotations @@ -43,6 +47,15 @@ def _lock(key: str) -> asyncio.Lock: return lock +async def _locked(session: Any, key: str) -> Any: + """占住设置表里 key 这一行再读它(见模块说明「多进程」):没有这一行交回 None。updated_at 不动。""" + from sqlalchemy import update + + await session.execute(update(Setting).where(Setting.key == key).values(updated_at=Setting.updated_at) + .execution_options(synchronize_session=False)) + return await session.get(Setting, key, populate_existing=True) + + def _today() -> str: """本地日期:每日上限按它滚动。""" return datetime.now().astimezone().date().isoformat() @@ -80,21 +93,31 @@ class Ledger: limit 为 None 表示不限。估不出金额的调用(priced=False)照记次数,金额记 0,调用方应传 limit=None: 拿 0 去比上限,等于说它不花钱。 """ + from sqlalchemy.exc import IntegrityError + async with _lock(self.key): - async with SessionLocal() as session: - row = await session.get(Setting, self.key) - spend = self._spend_of(row) - if limit is not None and spend["usd"] + amount > limit + 1e-12: - return None - spend["usd"] = round(spend["usd"] + amount, 8) - spend["calls"] += 1 - spend["unpriced_calls"] += 0 if priced else 1 - if row is None: - session.add(Setting(key=self.key, value=spend)) - else: - row.value = spend - await session.commit() - return spend["date"] + for _ in range(3): + async with SessionLocal() as session: + row = await _locked(session, self.key) + spend = self._spend_of(row) + if limit is not None and spend["usd"] + amount > limit + 1e-12: + await session.rollback() + return None + spend["usd"] = round(spend["usd"] + amount, 8) + spend["calls"] += 1 + spend["unpriced_calls"] += 0 if priced else 1 + if row is None: + session.add(Setting(key=self.key, value=spend)) + else: + row.value = spend + try: + await session.commit() + except IntegrityError: + # 这本账的那一行刚被别的进程插上:重来一次,这回占得住它 + await session.rollback() + continue + return spend["date"] + raise RuntimeError(f"记账「{self.key}」失败:设置表里这一行一直插不进去") async def settle(self, date: str, delta: float) -> None: """调用结束后按实际用量找补预留的差额。跨了天就不找补:预留记在前一天,今天从零算。""" @@ -102,9 +125,10 @@ class Ledger: return async with _lock(self.key): async with SessionLocal() as session: - row = await session.get(Setting, self.key) + row = await _locked(session, self.key) spend = self._spend_of(row) if row is None or spend["date"] != date: + await session.rollback() return spend["usd"] = max(0.0, round(spend["usd"] + delta, 8)) row.value = spend diff --git a/backend/app/eval/answer.py b/backend/app/eval/answer.py index 6acc25cf..2842b27e 100644 --- a/backend/app/eval/answer.py +++ b/backend/app/eval/answer.py @@ -24,7 +24,7 @@ app/chat/job.py),生成过程是 api/copilot.answer_ops,几个「决定」 用量按角色记(做法 3、11):助手 = 生成(answer_ops 末尾的 usage)+ 复核(review_of_run 的 usage);写作 = 答案运行里的 llm.end(按模型分)。重来的话两次都算——那都是这一题花的钱。 -时限、停止由调用方管(runner.py 用 asyncio.wait_for 包住 drive):这里随时把正在跑的那次运行记在 current_run,调用方 +时限、停止由调用方管(runner.py 用 within 包住 drive,排队等名额的那一段不算):这里随时把正在跑的那次运行记在 current_run,调用方 超时、停止时据此取消它(stop_run)。 """ from __future__ import annotations diff --git a/backend/app/eval/cleanup.py b/backend/app/eval/cleanup.py index 44a22f69..956b37f2 100644 --- a/backend/app/eval/cleanup.py +++ b/backend/app/eval/cleanup.py @@ -83,7 +83,7 @@ async def clean_suite(suite_id: str, *, keep: int | None = None) -> dict[str, An if keep is None: keep = int((await eval_settings(session))["keep_runs"]) runs = list((await session.execute(select(EvalRun).where(EvalRun.suite_id == suite_id) - .order_by(EvalRun.created_at.desc()))).scalars()) + .order_by(EvalRun.created_at.desc(), EvalRun.id.desc()))).scalars()) kept = keep_set(runs, keep) targets = [r for r in runs if r.id not in kept and r.cleaned_at is None] results = removed = 0 @@ -119,8 +119,10 @@ def spawn(suite_id: str) -> None: async def drain() -> None: """等后台的清理做完(测试、关停用)。""" - while _background: - await asyncio.gather(*list(_background), return_exceptions=True) + from app.core.background import drain as drain_tasks + + # 不写成 while _background: await gather(...):别的事件循环上做完、还没摘掉的任务会让那样的循环空转(core/background) + await drain_tasks(_background) __all__ = ["clean_suite", "drain", "keep_set", "spawn"] diff --git a/backend/app/eval/compare.py b/backend/app/eval/compare.py index 35d97996..bbf83118 100644 --- a/backend/app/eval/compare.py +++ b/backend/app/eval/compare.py @@ -53,7 +53,7 @@ async def pick_baseline(session: AsyncSession, run: Any, *, explicit: str | None rows = (await session.execute( select(EvalRun).where(EvalRun.suite_id == run.suite_id, EvalRun.id != run.id, EvalRun.created_at < run.created_at, EvalRun.status.in_(BASELINE_STATUSES)) - .order_by(EvalRun.created_at.desc()).limit(200))).scalars().all() + .order_by(EvalRun.created_at.desc(), EvalRun.id.desc()).limit(200))).scalars().all() same = [r for r in rows if (r.fingerprint or {}).get("variant_key") == key] pinned = next((r for r in same if r.pinned), None) return pinned or (same[0] if same else None) diff --git a/backend/app/eval/gold.py b/backend/app/eval/gold.py index b0219398..4b584f1b 100644 --- a/backend/app/eval/gold.py +++ b/backend/app/eval/gold.py @@ -209,11 +209,19 @@ async def _launch_and_wait(graph: dict[str, Any], payload: dict[str, Any], *, na run_id = run.id except LaunchRefused as e: return _Ran("failed", None, f"算期望的运行没有发起:{e.message}", GOLD_FAILED) + from app.cluster import slots + loop = asyncio.get_running_loop() deadline = loop.time() + max(0.0, float(ctx.timeout_s)) + last = loop.time() while True: # 执行任务收完尾(含封存)才回来;任务已经不在了(刚建好还没排上、或者已经结束)就立刻回来,再按库里的状态判 - await run_manager.wait_idle(run_id, timeout=max(0.05, min(5.0, deadline - loop.time()))) + await run_manager.wait_idle(run_id, timeout=max(0.05, min(1.0, deadline - loop.time()))) + now = loop.time() + if await slots.waiting(run_id, here=run_manager.is_active(run_id)): + # 排队等名额的那一段不算时限(复核:满载时排队不该算进每题时限) + deadline += now - last + last = now async with SessionLocal() as session: row = await session.get(Run, run_id) status, error = (row.status, row.error) if row is not None else ("failed", "运行记录不见了") @@ -241,7 +249,7 @@ async def _artifacts(run_id: str, kind: str) -> list[tuple[str | None, tuple[str async with SessionLocal() as session: rows = (await session.execute(select(Artifact.id, Artifact.node_id).where( - Artifact.run_id == run_id, Artifact.kind == kind).order_by(Artifact.created_at))).all() + Artifact.run_id == run_id, Artifact.kind == kind).order_by(Artifact.created_at, Artifact.id))).all() out: list[tuple[str | None, tuple[str, Any]]] = [] for artifact_id, node_id in rows: try: diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index dd340d9f..a552a528 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -12,9 +12,10 @@ EvalResult:建评测时就全部建好(pending),进度、「正在做什 snapshot 视图):起草、剖析照常进行,也不影响这次评测的可比性。各数据源的上传表格版本、表结构哈希开始和结束各记一次, 变了在指纹里标出(sources_changed)。 -**并发。** 同时最多 max_parallel 题,且所有臂合起来不超过全局运行并发的四分之一(全局只有一个信号量、没有优先级,评测不能 -挤占真实用户的运行)。A/B 的同一题几臂同时开跑。每题时限 per_case_timeout_s:作答一臂超时就取消它正在跑的运行、记超时; -期望探针每次也按这个时限。 +**并发。** 同时最多 max_parallel 题,且所有臂合起来不超过全局运行并发的四分之一。评测发起的作答、算期望的运行归「系统」类 +(app/cluster/slots.py):全站的名额按类别有上限和优先级,评测再多也只占得到系统那一类的名额,问数据、试跑优先,不被挤占。 +A/B 的同一题几臂同时开跑。每题时限 per_case_timeout_s:作答一臂超时就取消它正在跑的运行、记超时; +期望探针每次也按这个时限。排队等运行名额的那一段不算时限(within、gold._launch_and_wait):满载时排队不该被记成超时、触发止损。 **预算**(评测-D9;单独一本账 engine/spend.eval_spend,不挤占裁判、核查的额度):开跑前按上一次的每题平均花费估算(estimate)。 每题开评前按估算预留——加上会超过这次评测的金额上限、或评测每日上限就不再开新题,剩下的记「已到上限」;评完按实际找补。 @@ -32,6 +33,7 @@ snapshot 视图):起草、剖析照常进行,也不影响这次评测的 from __future__ import annotations import asyncio +import contextlib import copy import logging from collections.abc import Mapping, Sequence @@ -43,7 +45,7 @@ from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession from app.core import job_runtime as runtime -from app.db.base import SessionLocal, utcnow +from app.db.base import current_owner, SessionLocal, utcnow from app.db.models_eval import EvalCase, EvalResult, EvalRun, EvalSuite from app.eval.cases import CaseInvalid @@ -131,7 +133,7 @@ async def estimate(session: AsyncSession, suite_id: str, items: int) -> dict[str 平均记成 0、却有估不出的调用的,也按估不出算。""" row = (await session.execute(select(EvalRun).where( EvalRun.suite_id == suite_id, EvalRun.status.in_(("succeeded", "partial"))) - .order_by(EvalRun.created_at.desc()).limit(1))).scalar_one_or_none() + .order_by(EvalRun.created_at.desc(), EvalRun.id.desc()).limit(1))).scalar_one_or_none() if row is None: return {"items": items, "per_item_usd": None, "total_usd": None, "basis": None, "note": "这个题集还没有评完过,估不出花费:按每题开评前的实际花费累计,到了上限就停"} @@ -321,7 +323,7 @@ class EvalExecution(runtime.JobRun): async with SessionLocal() as session: leader = await session.get(EvalRun, self.job_id) runs = [leader] if not leader.pair_id else list((await session.execute( - select(EvalRun).where(EvalRun.pair_id == leader.pair_id).order_by(EvalRun.created_at))).scalars()) + select(EvalRun).where(EvalRun.pair_id == leader.pair_id).order_by(EvalRun.created_at, EvalRun.id))).scalars()) suite = await session.get(EvalSuite, leader.suite_id) if suite is None: raise runtime.JobFailed("题集不存在,可能已被删除") @@ -397,7 +399,7 @@ class EvalExecution(runtime.JobRun): unrestricted = True if unrestricted: ids += list((await session.execute(select(DataSource.id).where(DataSource.enabled.is_(True)) - .order_by(DataSource.created_at))).scalars()) + .order_by(DataSource.created_at, DataSource.id))).scalars()) return list(dict.fromkeys(ids)) # ---------------------------------------------------------------- 主循环 @@ -518,9 +520,9 @@ class EvalExecution(runtime.JobRun): try: if case.get("kind") == "workflow": graph, inputs = await self._workflow_graph(case, values) - await asyncio.wait_for(driver.drive_workflow(graph, inputs), timeout=timeout) + await within(driver.drive_workflow(graph, inputs), timeout, lambda: _queued(driver)) else: - await asyncio.wait_for(driver.drive(), timeout=timeout) + await within(driver.drive(), timeout, lambda: _queued(driver)) except asyncio.TimeoutError: turn = driver.turn turn.timed_out = True @@ -963,22 +965,69 @@ def running(run: Any) -> bool: return manager.running(leader_of(run)) +async def _queued(driver: Any) -> bool: + """这一题的作答运行正排着队等名额(app/cluster/slots):这一段不算每题的时限。""" + from app.cluster import slots + from app.engine.runner import run_manager + + run_id = getattr(driver.turn, "current_run", None) + return bool(run_id) and await slots.waiting(run_id, here=run_manager.is_active(run_id)) + + +async def within(coro: Any, budget: float, paused: Any, *, tick: float = 0.5) -> Any: + """同 asyncio.wait_for(coro, budget),只是 paused() 为真的那几段不算时间(复核:满载时排队等名额不该算进每题时限、触发止损)。 + 超时取消 coro、抛 TimeoutError。每隔一小段(tick,且不超过时限的 1/20)看一次;一段的头尾有一头在暂停就整段不算—— + 宁可少算一点,不把排队的时间算进去。""" + loop = asyncio.get_running_loop() + task = asyncio.ensure_future(coro) + step_s = max(0.005, min(tick, budget / 20)) + spent, last = 0.0, loop.time() + was_paused = await paused() + try: + while True: + done, _ = await asyncio.wait({task}, timeout=max(0.005, min(step_s, budget - spent))) + if task in done: + return task.result() + now = loop.time() + step, last = now - last, now + is_paused = await paused() + if not (was_paused or is_paused): + spent += step + was_paused = is_paused + if spent >= budget: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + raise asyncio.TimeoutError() + except asyncio.CancelledError: + task.cancel() + raise + + +def leader_of_id(rows: Sequence[EvalRun], job_id: str) -> str: + """rows 里 id 是 job_id 的那一臂由哪一臂的任务在跑(A/B 几臂共用领头那一臂的后台任务)。""" + row = next((r for r in rows if r.id == job_id), None) + return leader_of(row) if row is not None else job_id + + async def reconcile_stale(session: AsyncSession, rows: Sequence[EvalRun]) -> None: """读接口当场整理:记着排队、进行中,这个进程里却没在跑(服务重启过)的,照实标中断。""" - stale = [r for r in rows if r.status in runtime.ACTIVE and not running(r)] + # 属主不在了才算:多进程时别的进程正在跑的评测,这里看不到它的任务,却不能标中断(runtime.stale 按租约判断) + stale = await runtime.stale(list(rows), running=lambda job_id: manager.running(leader_of_id(rows, job_id))) for row in stale: await runtime.close_job(session, EvalRun, EvalResult, row.id, "interrupted", None, "cancelled", INTERRUPTED, finish=_closing(row.id, INTERRUPTED)) await session.refresh(row) -async def startup() -> int: - """服务启动时:没结束的评测标中断(没结束的项记停止),不自动续跑。""" +async def startup(*, live: Any = None) -> int: + """服务启动时(多进程时当班进程隔一会儿再来一次):属主不在了的评测标中断(没结束的项记停止),不自动续跑。""" async def close(session: AsyncSession, job_id: str) -> None: await runtime.close_job(session, EvalRun, EvalResult, job_id, "interrupted", None, "cancelled", INTERRUPTED, finish=_closing(job_id, INTERRUPTED)) - return await runtime.interrupt_on_startup(EvalRun, close, running=manager.running) + # 几臂共用领头那一臂的后台任务,各臂的属主是同一个进程:按属主判断就够了(running 只在属主是本进程时看) + return await runtime.interrupt_on_startup(EvalRun, close, running=manager.running, live=live) async def stop(run_id: str) -> bool: @@ -1035,6 +1084,7 @@ async def retry(run_id: str) -> list[str]: setattr(r, key, value) for arm in arms: arm.status, arm.error, arm.stop_reason, arm.finished_at = "queued", None, None, None + arm.owner = current_owner() # 重新排队的这一次由本进程认领(多进程:别把它当成原来那个进程的) await session.flush() for arm in arms: await _recount(session, arm.id) diff --git a/backend/app/eval/snapshots.py b/backend/app/eval/snapshots.py index 3aa29f8c..93a0db47 100644 --- a/backend/app/eval/snapshots.py +++ b/backend/app/eval/snapshots.py @@ -562,8 +562,8 @@ class ShadowDrafts(runtime.JobManager): manager = ShadowDrafts() -async def startup() -> int: - """服务启动时:没结束的影子起草标中断(不自动续跑)。""" +async def startup(*, live: Any = None) -> int: + """服务启动时(多进程时当班进程隔一会儿再来一次):属主不在了的影子起草标中断(不自动续跑)。""" async def close(session: AsyncSession, job_id: str) -> None: await session.execute(update(EvalCatalogSnapshot).where( EvalCatalogSnapshot.id == job_id, EvalCatalogSnapshot.status.in_(runtime.ACTIVE)) @@ -571,12 +571,13 @@ async def startup() -> int: .execution_options(synchronize_session=False)) await session.commit() - return await runtime.interrupt_on_startup(EvalCatalogSnapshot, close, running=manager.running) + return await runtime.interrupt_on_startup(EvalCatalogSnapshot, close, running=manager.running, live=live) async def reconcile_stale(session: AsyncSession, rows: Iterable[EvalCatalogSnapshot]) -> None: """读接口当场整理:记着排队、进行中,这个进程里却没在跑(服务重启过)的,照实标中断。""" - stale = [r for r in rows if r.status in runtime.ACTIVE and not manager.running(r.id)] + # 属主不在了才算:多进程时别的进程正在跑的,这里看不到它的任务,却不能标中断(runtime.stale 按租约判断) + stale = await runtime.stale(list(rows), running=manager.running) for row in stale: row.status, row.error, row.finished_at = "interrupted", INTERRUPTED, utcnow() if stale: diff --git a/backend/app/files/store.py b/backend/app/files/store.py index f3f74aa5..4b466005 100644 --- a/backend/app/files/store.py +++ b/backend/app/files/store.py @@ -294,7 +294,7 @@ def accept_problem(setting: dict[str, Any] | None, *, name: str, mime: str, size # -------------------------------------------------------------------------- #: 还没绑到记录上:JSON 列显式写过 None 的存的是 JSON 的 null,不是 SQL 的 NULL,两种都算 -_NO_RECORD = text("(files.bound_record IS NULL OR files.bound_record = 'null')") +_NO_RECORD = text("(files.bound_record IS NULL OR CAST(files.bound_record AS TEXT) = 'null')") def file_fields(form: Any) -> list[str]: diff --git a/backend/app/files/text.py b/backend/app/files/text.py index dae3e4ef..5c901d72 100644 --- a/backend/app/files/text.py +++ b/backend/app/files/text.py @@ -20,6 +20,7 @@ import logging from pathlib import Path from typing import Any +from app.core import atomic from app.files.store import as_ref, bound_files, files_root, read_blob logger = logging.getLogger(__name__) @@ -73,10 +74,8 @@ def extract_text(sha: str, name: str, mime: str) -> tuple[str | None, str | None if text is not None and not text.strip(): text, problem = None, "没有抽出文字(可能是扫描件)" try: - cache.parent.mkdir(parents=True, exist_ok=True) - tmp = cache.with_suffix(".tmp") - tmp.write_text(json.dumps({"text": text, "problem": problem}, ensure_ascii=False), encoding="utf-8") - tmp.replace(cache) + # 临时文件名带随机后缀(core/atomic):两个进程同时抽同一个附件,不会一个把另一个写到一半的改名过去 + atomic.write_text(cache, json.dumps({"text": text, "problem": problem}, ensure_ascii=False)) except OSError: logger.warning("附件 %s 抽出的文字没能缓存", name, exc_info=True) return text, problem diff --git a/backend/app/ledger/certify.py b/backend/app/ledger/certify.py index 3f1d454c..7da2ea32 100644 --- a/backend/app/ledger/certify.py +++ b/backend/app/ledger/certify.py @@ -309,7 +309,7 @@ async def queue(session: Any, *, principal: Any) -> dict[str, Any]: kinds = {name: kind for name, kind in (await session.execute(select(DataSource.name, DataSource.kind))).all()} runs = list((await session.execute(select(Run).where( Run.origin_kind == "chat", Run.status == "succeeded", Run.run_class != "formal", counted_runs()) - .order_by(Run.created_at.desc()).limit(RUN_LIMIT))).scalars()) + .order_by(Run.created_at.desc(), Run.id.desc()).limit(RUN_LIMIT))).scalars()) turns = await _turns(session, runs) by_run = {r.id: r for r in runs} # 每次运行最新的一份比对(store.latest 按事件认这一次封存的;这里只要内容,按工件引用的时间取最新) diff --git a/backend/app/ledger/issue.py b/backend/app/ledger/issue.py index 66f701bd..ff742d6c 100644 --- a/backend/app/ledger/issue.py +++ b/backend/app/ledger/issue.py @@ -44,6 +44,7 @@ from sqlalchemy import func, select from app.db.base import SessionLocal from app.db.models import Run, Workflow +from app.db.portable import json_eq logger = logging.getLogger(__name__) @@ -150,10 +151,10 @@ async def issued_today(session: Any) -> int: async def issued_runs(session: Any, run_id: str) -> list[Run]: """这次运行按口径出具过的正式运行(origin.source_run 是它),新的在前。origin_kind 有索引,先按它筛再看 origin - (json_extract,同 auth/keys 认幂等键的写法)。""" + (db/portable.json_eq,同 auth/keys 认幂等键的写法)。""" rows = (await session.execute(select(Run).where( - Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.source_run") == run_id) - .order_by(Run.created_at.desc()))).scalars() + Run.origin_kind == ORIGIN, json_eq(Run.origin, "source_run", run_id)) + .order_by(Run.created_at.desc(), Run.id.desc()))).scalars() return list(rows) @@ -318,7 +319,7 @@ async def _conversation_of(session: Any, run: Run) -> dict[str, str]: if origin.get("conversation_id") and origin.get("turn_id"): return {"conversation_id": str(origin["conversation_id"]), "turn_id": str(origin["turn_id"])} turn = (await session.execute(select(ConversationTurn).where(ConversationTurn.run_id == run.id) - .order_by(ConversationTurn.created_at.desc()))).scalars().first() + .order_by(ConversationTurn.created_at.desc(), ConversationTurn.id.desc()))).scalars().first() return {"conversation_id": turn.conversation_id, "turn_id": turn.id} if turn is not None else {} @@ -327,9 +328,17 @@ _locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[tuple[str, st weakref.WeakKeyDictionary() -def _lock(key: tuple[str, str, int]) -> asyncio.Lock: - group = _locks.setdefault(asyncio.get_running_loop(), {}) - return group.setdefault(key, asyncio.Lock()) +def _lock(key: tuple[str, str, int]) -> Any: + """同一(运行、工作流、版本)的出具一个接一个(app/cluster/mutex,多进程之间也是一把):管的是「这一次出具过没有」的去重, + 同时点了几下只发起一次。每日上限不靠它:发起之前先在库里占一份额度(app/cluster/quota),几份不同的出具可以同时发起,上限 + 照样数得准。""" + from app.cluster import mutex + + return mutex.lock(f"ledger-issue:{key[0]}:{key[1]}:{key[2]}") + + +#: 每日出具额度在 cluster/quota 里的名字 +QUOTA = "ledger_issue" async def _hider(run_id: str) -> Any: @@ -365,12 +374,24 @@ async def issue(run_id: str, workflow_id: str, version: int, *, principal: Any, "how": plan.how, "binding_hash": input_key(plan.binding), "match": plan.match_artifact, "auto": auto, **({"granted_by_app": plan.granted_by_app} if plan.granted_by_app else {})} actor = run.started_by if auto else principal.username + # 先占一份今天的额度(库里带条件的加一,几个进程同时出具也正好到上限为止),发起不成退回 + from app.api.settings import ledger_settings + from app.cluster import quota + + limit = int((await ledger_settings(session))["issue_daily_limit"]) + day = await quota.take(QUOTA, limit, seed=issued_today) + if day is None: + raise IssueRefused(429, OVER_LIMIT.format(limit=limit), DAILY_LIMIT) try: launched = await launch_run(session, workflow_id=workflow_id, run_class="formal", version=int(version), input_payload=plan.inputs, actor=actor, origin_kind=ORIGIN, origin=origin, expect_hash=plan.expect_hash) except LaunchRefused as e: + await quota.give_back(QUOTA, day=day) raise IssueRefused(e.status_code, e.message, e.code or REFUSED) from e + except BaseException: + await quota.give_back(QUOTA, day=day) + raise return launched, True diff --git a/backend/app/ledger/labels.py b/backend/app/ledger/labels.py index e876f6c2..f53861e4 100644 --- a/backend/app/ledger/labels.py +++ b/backend/app/ledger/labels.py @@ -47,6 +47,9 @@ METRIC_GONE = "指标集里没有这个指标,没有比对" UNFINISHED = "比对没有做完" #: related 的几种说明 FORMULA_DIFFERS = "取数和口径等价,但指标的算法和口径卡上的不同" +#: 指标按「同一行」认来自口径,可那一行重跑时不一定还是它:多行结果、查询又没有 ORDER BY(sql_semantics.row_stable) +ROW_ORDER_UNSTABLE = ("指标取的是多行结果里按行号的一格,查询没有 ORDER BY,行序不定:重新执行时这一行可能是另一行," + "不能认定对应口径的同一格") INPUTS_ELSEWHERE = "指标有的输入不来自查询(运行输入、整形或代码),不能算来自口径" MIXED_CALIBERS = "指标的几个输入分别对上了不同的口径" BINDING_CONFLICT = "指标的几个输入对上了同一份口径,但参数不一致" @@ -143,6 +146,15 @@ def cell_label(artifact: str, column: Any, queries: Mapping[str, Mapping[str, An return {**_unknown(str(query.get("reason") or UNFINISHED)), "query": artifact} +def _row_stable(query: Mapping[str, Any], item: Mapping[str, Any]) -> bool: + """这个输入取的那一行重跑时还认得住(match 记的 ordered / one_row,sql_semantics.row_stable 的认法):最外层带 ORDER BY, + 或者一定只出一行、取的是第 0 行。老的比对记录没记 ordered 的照旧(当时不判)。""" + if "ordered" not in query: + return True + where = item.get("locator") if isinstance(item.get("locator"), dict) else {} + return bool(query.get("ordered")) or (where.get("row") in (0, None) and bool(query.get("one_row"))) + + def _leaf_token(node_id: Any, row: Any, column: Any) -> str: """把指标式里的一个输入换成「口径的哪个取数节点、第几行、哪一列」:两边的式子按这个换完再逐字比。""" return f"⟦{node_id}|{row}|{column}⟧" @@ -307,6 +319,10 @@ def metric_label(card: Mapping[str, Any] | None, metric_id: Any, queries: Mappin return {"kind": "related", "related": nearby, "reason": MIXED_CALIBERS, **queried} related = next((q for q in queries_found if q.get("status") == "related"), None) return {**_related(related), **queried} if related is not None else {"kind": "reasoning", **queried} + if unstable := [i for i, q in zip(from_queries, queries_found) if not _row_stable(q, i)]: + # 按「同一行」认来自口径(_leaf_token 记的是行号):多行结果又没有 ORDER BY 的,重跑时第几行不一定是哪一行, + # 照实标相关、写明行序不定,不认成来自口径 + return {"kind": "related", "related": nearby, "reason": ROW_ORDER_UNSTABLE, **queried} reason = FORMULA_DIFFERS for key in sorted(common): chosen = [next(c for c in _proven(q) if c.get("caliber") == key) for q in queries_found] diff --git a/backend/app/ledger/match.py b/backend/app/ledger/match.py index f5d72135..7a1b2fa3 100644 --- a/backend/app/ledger/match.py +++ b/backend/app/ledger/match.py @@ -98,6 +98,8 @@ SNAPSHOT_TAMPERED = "查询快照与哈希不一致,疑似被修改,不比 SNAPSHOT_MISSING = "查询快照读不出来,不比对" NO_SQL = "查询快照里没有 SQL,不比对" PARSE_FAILED = "这次的 SQL 无法解析,没有比对" +#: 指标按「同一行」认来自口径,可那一行重跑时不一定还是它(说法在 labels,这里给调用方一个名字) +ROW_ORDER_UNSTABLE = ledger_labels.ROW_ORDER_UNSTABLE MERGED = "合并查询的格不和口径比对(它的数来自各个输入的查询)" COMPOSED_UNPROVEN = "这条查询出自助手选用的口径,但和口径的取数没能证明一致(可能被改过)" MORE_CANDIDATES = "另有 {n} 份相关口径没有比对(每条查询最多比 {max} 份)" @@ -546,7 +548,7 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: 等价)/ none(没有相关的口径)/ skipped(没比完:超时、解析失败、快照读不出)。composed 是 composed_nodes 的结果, 不给就没有助手选用的。referenced 是报告引用了哪条查询的哪几列(labels.referenced_columns:{查询快照: {列名}}): 给了的,整条比不等价时按引用的列投影再比(_projected);不给就只整条比。纯计算,从不抛异常。评测逐题比 SQL 也调它。""" - from app.data.sql_semantics import match_template, normalize_sql, one_row + from app.data.sql_semantics import match_template, normalize_sql, one_row, ordered out: list[dict[str, Any]] = [] for query in queries: @@ -567,6 +569,9 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: if normalize_sql(query.sql, query.dialect).parse_error: out.append({**row, "status": "skipped", "reason": PARSE_FAILED, "candidates": []}) continue + # 按行号取的那一格重跑时还是不是同一行(sql_semantics.row_stable):指标标签按「同一行」认来自口径时要看它 + row["ordered"] = ordered(query.sql, query.dialect) + row["one_row"] = one_row(query.sql, query.dialect) candidates: list[dict[str, Any]] = [] for caliber, fetch in pool[:MAX_CANDIDATES]: if deadline is not None and time.monotonic() > deadline: @@ -600,7 +605,6 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: if wanted is not None and len(wanted) >= 2 and status == "related": # 整条(按全部引用列投影)没证明:被引用的几列逐格各比一次,每个数按它自己那一格的结论标(labels.cell_label) row["cells"] = _cells(query, wanted, pool[:MAX_CANDIDATES], candidates, chosen, deadline) - row["one_row"] = one_row(query.sql, query.dialect) out.append(row) return out @@ -717,6 +721,18 @@ async def match_run(run_id: str, *, trigger: str = "sealed", timeout: float | No 运行一封存就比的(trigger=sealed)比完接着看能不能按口径自动正式出具(ledger/issue.auto_issue,拍板 L1):事后补比、 按需比的不自动出具——那是按现在的口径回看老答案,不该替人补发正式运行。""" + from app.cluster import mutex + + # 多进程:封存时比(执行它的进程)和补比(当班进程)可能同时来;跨进程的锁里一次一个,补比、封存时比先看比过没有 + async with mutex.lock(f"ledger-match:{run_id}"): + if trigger in ("sealed", "backfill"): + async with SessionLocal() as session: + if await store.matched_run_ids(session, [run_id]): + return None + return await _match_run(run_id, trigger=trigger, timeout=timeout) + + +async def _match_run(run_id: str, *, trigger: str, timeout: float | None) -> str | None: from app.api.evidence import _sealed from app.api.settings import ledger_settings @@ -782,7 +798,7 @@ async def backfill(*, days: int = BACKFILL_DAYS, limit: int = BACKFILL_LIMIT) -> select(Run.id).where(Run.status == "succeeded", Run.run_class != "formal", Run.origin_kind.in_(EXPLORE_ORIGINS), Run.manifest_hash.is_not(None), Run.finished_at >= since) - .order_by(Run.finished_at.desc()).limit(limit))).all()] + .order_by(Run.finished_at.desc(), Run.id.desc()).limit(limit))).all()] done = await store.matched_run_ids(session, ids) count = 0 for run_id in ids: diff --git a/backend/app/ledger/recall.py b/backend/app/ledger/recall.py index 62bf2dbe..5d87fb28 100644 --- a/backend/app/ledger/recall.py +++ b/backend/app/ledger/recall.py @@ -66,11 +66,12 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any -from sqlalchemy import func, select +from sqlalchemy import func, not_, select, update from app.core.events import EventType from app.db.base import SessionLocal, utcnow from app.db.models import Run, RunEvent, Workflow +from app.db.portable import json_eq, json_flag, json_text from app.ledger import uses as ledger_uses logger = logging.getLogger(__name__) @@ -82,7 +83,7 @@ REASONS = ("caliber", "catalog", "structure") #: 发起的批次:发布后自动的、版本历史里手动接着算的 BATCHES = ("auto", "manual") #: 一组的结论 -OUTCOMES = ("changed", "unchanged", "failed", "running", "pending", "not_recomputable") +OUTCOMES = ("changed", "unchanged", "row_order", "failed", "running", "pending", "not_recomputable") #: 启动时接着做最近几天发布的 RESUME_DAYS = 7 #: 目录、表结构变了时最多看多少次探索运行(新的在前) @@ -109,6 +110,8 @@ SUPERSEDED = "已有更新的发布版 v{latest},召回在 v{latest} 那一版 NOT_CURRENT = "v{version} 已不是当前发布版,召回不再接着算" #: 收尾 CHANGED_TITLE = "口径「{name}」升到 v{version},答案的数变了" +#: 全部指标行序不定、核对不了时的待办标题(settle 的 row_order) +ROW_ORDER_TITLE = "口径「{name}」升到 v{version},答案里的数行序不定、没能核对,请人工核对" CATALOG_TITLE = "答案依据的数据目录有变:按新目录检查出了新的问题" STRUCTURE_TITLE = "口径「{name}」v{version} 依据的表结构变了,试算没有算出来" #: 依据有变的四种(ledger.basis_changed 的 kind) @@ -225,11 +228,10 @@ def source_cards(graph: Any) -> list[str]: async def recall_runs(session: Any, workflow_id: str, new_version: int | None = None) -> list[Run]: """以 W(新版本 N)发起过的召回运行(试算的不算),新的在前。origin_kind 有索引,先按它筛再看 origin。""" stmt = select(Run).where(Run.origin_kind == ORIGIN, - func.json_extract(Run.origin, "$.workflow_id") == workflow_id, - func.coalesce(func.json_extract(Run.origin, "$.probe"), 0) == 0) + json_eq(Run.origin, "workflow_id", workflow_id), not_(json_flag(Run.origin, "probe"))) if new_version is not None: - stmt = stmt.where(func.json_extract(Run.origin, "$.to_version") == int(new_version)) - return list((await session.execute(stmt.order_by(Run.created_at.desc()))).scalars()) + stmt = stmt.where(json_eq(Run.origin, "to_version", int(new_version))) + return list((await session.execute(stmt.order_by(Run.created_at.desc(), Run.id.desc()))).scalars()) async def build_plan(session: Any, workflow_id: str, new_version: int) -> Plan: @@ -364,15 +366,48 @@ def recall_card(pairs: Sequence[Pair]) -> tuple[dict[str, Any], list[dict[str, A _metric(f"new_{k}", f"{name} · 新版 v{pair.new.version}", new_expr, nm), _metric(f"diff_{k}", f"{name} · 两版之差", f"({new_expr}) - ({old_expr})", nm)] # 旧版的式子(定义,不是数)一并记下:问数回答当时的值要按它找到答案里对应的那一格(original_values) - meta.append({"k": k, "card": pair.old.node_id, "metric": m["id"], "name": name, - "unit": str(nm.get("unit") or ""), "decimals": nm.get("decimals"), "format": nm.get("format"), - "expression": str(m.get("expression") or "")}) + item = {"k": k, "card": pair.old.node_id, "metric": m["id"], "name": name, + "unit": str(nm.get("unit") or ""), "decimals": nm.get("decimals"), "format": nm.get("format"), + "expression": str(m.get("expression") or "")} + if not (_rows_stable(pair.old, m) and _rows_stable(pair.new, nm)): + # 两版都按行号取多行结果里的一格、查询又没有 ORDER BY:两次查询交回的行序可能不同,第几行不一定是同一行, + # 按位置比出来的「变了 / 没变」都不可信(compare 照实记行序不定) + item["row_order"] = "unstable" + meta.append(item) k += 1 node = derive._node(derive.RECALL_CARD, "metrics", "召回卡", caliber="口径召回", caliber_version="v1", on_missing="null", metrics=metrics) return node, meta +def _rows_stable(pinned: Any, metric: Mapping[str, Any]) -> bool: + """这个指标按行号取的每一格(cell(nodes.<取数>, 行, '列')),在那一版的取数 SQL 上重跑时还是同一行(sql_semantics.row_stable)。 + 行号、SQL 认不出来的照旧当认得住(以前就是按位置比的,不凭猜测改结论)。""" + import ast + + from app.data.sql_semantics import row_stable + from app.engine.expressions import ExpressionError, cell_parts, leaf_refs, parse_expression + + sql_of = {str(n.get("id")): ((n.get("data") or {}).get("config") or {}).get("args", {}).get("sql") + for n in pinned.fetches if isinstance(n, dict)} + try: + leaves = leaf_refs(parse_expression(str(metric.get("expression") or ""))[0]) + except ExpressionError: + return True + for leaf in leaves: + parts = cell_parts(str(leaf)) + if parts is None or not parts[0].startswith("nodes."): + continue + sql = sql_of.get(parts[0].removeprefix("nodes.")) + try: + row = ast.literal_eval(parts[1].strip()) + except (ValueError, SyntaxError): + continue + if isinstance(sql, str) and isinstance(row, int) and not row_stable(sql, row): + return False + return True + + def build_graph(group: Group, pairs: Sequence[Pair]) -> tuple[Any, list[dict[str, Any]]]: """一组的召回图(derive.recall_graph)和比较说明。同样的输入永远得到同一张图。""" from app.ledger import derive @@ -413,9 +448,11 @@ _locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[tuple[str, in weakref.WeakKeyDictionary() -def _lock(key: tuple[str, int]) -> asyncio.Lock: - group = _locks.setdefault(asyncio.get_running_loop(), {}) - return group.setdefault(key, asyncio.Lock()) +def _lock(key: tuple[str, int]) -> Any: + """跨进程的锁(app/cluster/mutex):同一(W、N)的计划和发起多进程时也串行,「已经自动发过几次」读得准。""" + from app.cluster import mutex + + return mutex.lock(f"ledger-recall:{key[0]}:{key[1]}") async def launch(workflow_id: str, new_version: int, *, batch: str = "auto", actor: str | None = None) -> list[str]: @@ -498,11 +535,12 @@ def _reasons(plan: Plan) -> list[dict[str, Any]]: async def outcome_of(session: Any, run: Run) -> tuple[str, dict[str, Any]]: - """一次召回运行的结论:(changed / unchanged / failed / running, 收尾时记下的那一条 ledger.recalled 的内容)。""" + """一次召回运行的结论:(changed / unchanged / row_order(全部指标行序不定、核对不了)/ failed / running, 收尾时记下的那一条 + ledger.recalled 的内容)。""" data = await _settled(session, run.id) if data is not None: - if data.get("outcome") == "failed": - return "failed", data + if data.get("outcome") in ("failed", "row_order"): + return str(data["outcome"]), data return ("changed" if data.get("changed") else "unchanged"), data if run.status in ("failed", "cancelled"): return "failed", {"error": run.error} @@ -567,9 +605,8 @@ async def version_recall_counts(session: Any, workflow_id: str, version: int) -> state, _ = await outcome_of(session, run) changed += state == "changed" as_old = int(await session.scalar(select(func.count()).select_from(Run).where( - Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.workflow_id") == workflow_id, - func.json_extract(Run.origin, "$.from_version") == int(version), - func.coalesce(func.json_extract(Run.origin, "$.probe"), 0) == 0)) or 0) + Run.origin_kind == ORIGIN, json_eq(Run.origin, "workflow_id", workflow_id), + json_eq(Run.origin, "from_version", int(version)), not_(json_flag(Run.origin, "probe")))) or 0) return {"groups": len(as_new), "changed": changed, "recalled_from": as_old} @@ -607,11 +644,14 @@ def compare(meta: Sequence[Mapping[str, Any]], values: Mapping[str, Mapping[str, for item in meta: k = item.get("k") old, new, diff = (values.get(f"{p}_{k}") or {} for p in ("old", "new", "diff")) - changed = str(old.get("rendered")) != str(new.get("rendered")) + unstable = item.get("row_order") == "unstable" + # 行序不定的(recall_card 记的):按位置比不出变没变,changed 为空,照实记 row_order,不假装对上 + changed = None if unstable else str(old.get("rendered")) != str(new.get("rendered")) out.append({"k": k, "card": item.get("card"), "metric": item.get("metric"), "name": item.get("name"), "unit": item.get("unit"), "old": old.get("value"), "new": new.get("value"), "diff": diff.get("value"), "old_text": old.get("rendered"), "new_text": new.get("rendered"), - "diff_text": diff.get("rendered"), "changed": changed}) + "diff_text": diff.get("rendered"), "changed": changed, + **({"row_order": "unstable"} if unstable else {})}) return out @@ -654,48 +694,65 @@ async def settle(run_id: str) -> str | None: return None _settling.add(run_id) try: - async with SessionLocal() as session: - run = await session.get(Run, run_id) - if run is None or run.origin_kind != ORIGIN or run.status not in ("succeeded", "failed", "cancelled") \ - or not run.manifest_hash: - return None - if await _settled(session, run_id) is not None: - return None - origin = run.origin if isinstance(run.origin, dict) else {} - if origin.get("probe"): - return await _settle_probe(run) - base = {"role": "recall", "workflow_id": origin.get("workflow_id"), "from_version": origin.get("from_version"), - "to_version": origin.get("to_version")} - if run.status != "succeeded": - await _mark(run_id, {**base, "outcome": "failed", "changed": None, - "error": str(run.error or "运行没有成功结束")[:300]}) - return "failed" - meta = origin.get("metrics") or [] - values = await card_values(run_id) - # 读不出召回卡、少了哪个比较的旧值或新值:两边都是空,按渲染比会被当成「没变」,给每个原答案盖上「已核对,不影响」 - # ——照实记重算失败,原答案一个不动(收过尾就不再核对,盖错了收不回来) - if not meta or any(f"{side}_{m.get('k')}" not in values for m in meta for side in ("old", "new")): - await _mark(run_id, {**base, "outcome": "failed", "changed": None, "error": CARD_UNREADABLE}) - return "failed" - results = compare(meta, values) - changed = any(r["changed"] for r in results) - answers = list(dict.fromkeys(str(u.get("run_id")) for u in origin.get("uses") or [] if u.get("run_id"))) - tasks: list[str] = [] - if changed: - tasks = await open_tasks(run, reason="caliber") - else: - for answer in answers: - await _mark_answer(answer, run) - await _mark(run_id, {**base, "outcome": "changed" if changed else "unchanged", "changed": changed, - "answers": len(answers), "tasks": tasks, - "metrics": [{"k": r["k"], "name": r["name"], "changed": r["changed"]} for r in results]}) - return "changed" if changed else "unchanged" + # 多进程:收尾的进程(runner 收尾时)和当班进程启动时的「补上收尾」可能同时来,跨进程的锁里先看收过没有 + from app.cluster import mutex + + async with mutex.lock(f"ledger-settle:{run_id}"): + return await _settle(run_id) finally: _settling.discard(run_id) -async def _mark_answer(answer_id: str, recall_run: Run) -> None: - """数没变:在原答案的运行上记一条 ledger.recalled(已按 vN 核对,不影响)。这次召回记过的不再记。""" +async def _settle(run_id: str) -> str | None: + async with SessionLocal() as session: + run = await session.get(Run, run_id) + if run is None or run.origin_kind != ORIGIN or run.status not in ("succeeded", "failed", "cancelled") \ + or not run.manifest_hash: + return None + if await _settled(session, run_id) is not None: + return None + origin = run.origin if isinstance(run.origin, dict) else {} + if origin.get("probe"): + return await _settle_probe(run) + base = {"role": "recall", "workflow_id": origin.get("workflow_id"), "from_version": origin.get("from_version"), + "to_version": origin.get("to_version")} + if run.status != "succeeded": + await _mark(run_id, {**base, "outcome": "failed", "changed": None, + "error": str(run.error or "运行没有成功结束")[:300]}) + return "failed" + meta = origin.get("metrics") or [] + values = await card_values(run_id) + # 读不出召回卡、少了哪个比较的旧值或新值:两边都是空,按渲染比会被当成「没变」,给每个原答案盖上「已核对,不影响」 + # ——照实记重算失败,原答案一个不动(收过尾就不再核对,盖错了收不回来) + if not meta or any(f"{side}_{m.get('k')}" not in values for m in meta for side in ("old", "new")): + await _mark(run_id, {**base, "outcome": "failed", "changed": None, "error": CARD_UNREADABLE}) + return "failed" + results = compare(meta, values) + # 行序不定的(recall_card 记的 row_order)比不出变没变:不算「没变」。全部都行序不定的结论写核对不了、开待办请人核对; + # 部分行序不定的,已判定的照常判,行序不定的那几项在结论、待办、原答案的记账里照实列出 + unstable = [r["name"] for r in results if r.get("row_order") == "unstable"] + decided = [r for r in results if r.get("row_order") != "unstable"] + changed = any(r["changed"] for r in decided) + outcome = "row_order" if not decided else "changed" if changed else "unchanged" + answers = list(dict.fromkeys(str(u.get("run_id")) for u in origin.get("uses") or [] if u.get("run_id"))) + tasks: list[str] = [] + if outcome in ("changed", "row_order"): + tasks = await open_tasks(run, reason="caliber", row_order=outcome == "row_order") + else: + for answer in answers: + await _mark_answer(answer, run, row_order=unstable) + await _mark(run_id, {**base, "outcome": outcome, "changed": changed if decided else None, + "answers": len(answers), "tasks": tasks, + **({"row_order_metrics": unstable} if unstable else {}), + "metrics": [{"k": r["k"], "name": r["name"], "changed": r["changed"], + **({"row_order": "unstable"} if r.get("row_order") else {})} + for r in results]}) + return outcome + + +async def _mark_answer(answer_id: str, recall_run: Run, *, row_order: Sequence[str] = ()) -> None: + """数没变:在原答案的运行上记一条 ledger.recalled(已按 vN 核对,不影响)。这次召回记过的不再记。row_order 是行序不定、 + 没能核对的那几项(照实列出,不算进「不影响」)。""" origin = recall_run.origin if isinstance(recall_run.origin, dict) else {} async with SessionLocal() as session: rows = (await session.execute(select(RunEvent.data).where( @@ -704,7 +761,8 @@ async def _mark_answer(answer_id: str, recall_run: Run) -> None: return await _mark(answer_id, {"role": "answer", "changed": False, "recall_run": recall_run.id, "workflow_id": origin.get("workflow_id"), "from_version": origin.get("from_version"), - "to_version": origin.get("to_version")}) + "to_version": origin.get("to_version"), + **({"row_order_metrics": list(row_order)} if row_order else {})}) # ========================================================================== @@ -739,7 +797,7 @@ def correlation(run_id: str, owner: tuple[str, str], reason: str = "caliber", ex async def open_tasks(run: Run, *, reason: str, owners: Iterable[tuple[str, str]] | None = None, - title: str | None = None, extra: str = "") -> list[str]: + title: str | None = None, extra: str = "", row_order: bool = False) -> list[str]: """给答案的主人开 recall 待办(挂在 run 上)。owners 不给就按召回运行 origin 里各答案的主人。按 correlation_key 认同一条, 开过的不再开。提交之后推通知。返回这次新开的待办 id。""" from app.process import tasks as process_tasks @@ -751,7 +809,8 @@ async def open_tasks(run: Run, *, reason: str, owners: Iterable[tuple[str, str]] targets = await _owner_targets(session, owners) name = await ledger_uses.workflow_name(session, str(origin.get("workflow_id") or "")) if title is None: - title = CHANGED_TITLE.format(name=name or "口径", version=origin.get("to_version")) + title = (ROW_ORDER_TITLE if row_order else CHANGED_TITLE).format(name=name or "口径", + version=origin.get("to_version")) opened: list[str] = [] for owner in targets: key = correlation(run.id, owner, reason, extra) @@ -810,6 +869,7 @@ async def task_payload(session: Any, task: Any, principal: Any) -> dict[str, Any cells[side] = {"value": None if masked else value, "text": "已遮罩" if masked else text, "metric": f"{side}_{item['k']}", "masked": masked} metrics.append({"k": item["k"], "name": item["name"], "unit": item["unit"], "changed": item["changed"], + **({"row_order": item["row_order"]} if item.get("row_order") else {}), "card": item["card"], "metric": item["metric"], **cells}) answers = [] for use in _uses_of(origin, _owner_of_key(key)): @@ -970,7 +1030,10 @@ async def answer_marks(session: Any, run_id: str, principal: Any) -> dict[str, A data = data if isinstance(data, dict) else {} if etype == EventType.LEDGER_RECALLED and data.get("role") == "answer": checked.append({k: data.get(k) for k in ("workflow_id", "from_version", "to_version", "recall_run")} - | {"changed": bool(data.get("changed")), "at": at(ts)}) + | {"changed": bool(data.get("changed")), "at": at(ts)} + # 行序不定、没能核对的那几项(只列名字):不算进「不影响」 + | ({"row_order_metrics": [str(x) for x in data["row_order_metrics"]]} + if isinstance(data.get("row_order_metrics"), list) and data["row_order_metrics"] else {})) workflows.add(str(data.get("workflow_id") or "")) elif etype == EventType.LEDGER_BASIS_CHANGED: items = data.get("items") @@ -986,7 +1049,7 @@ async def answer_marks(session: Any, run_id: str, principal: Any) -> dict[str, A select(Task, Run.origin).join(Run, Run.id == Task.run_id) .where(Task.kind == TASK_KIND, Task.status == "pending", (Task.run_id == run_id) | ((Run.origin_kind == ORIGIN) - & (func.instr(func.json_extract(Run.origin, "$.uses"), run_id) > 0))) + & json_text(Run.origin, "uses").contains(run_id, autoescape=True))) .order_by(Task.created_at, Task.id))).all() mine = await _assignments(session, [t.id for t, _ in candidates]) tasks: list[dict[str, Any]] = [] @@ -1134,8 +1197,8 @@ async def _explore_snapshots(session: Any, source: Any, tables: Iterable[str], * stmt = (select(Artifact.id, Artifact.meta, Run.id, Run.finished_at).join(Run, Run.id == Artifact.run_id) .where(Artifact.kind == QUERY_SNAPSHOT_KIND, Run.run_class == "exploratory", Run.status == "succeeded", Run.manifest_hash.is_not(None), counted_runs(), - or_(func.json_extract(Artifact.meta, "$.source_id") == source.id, - func.json_extract(Artifact.meta, "$.source") == source.name)) + or_(json_eq(Artifact.meta, "source_id", source.id), + json_eq(Artifact.meta, "source", source.name))) .order_by(Run.finished_at.desc(), Run.id, Artifact.id)) cut = _aware(before) offset = 0 @@ -1263,8 +1326,10 @@ _catalog_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, weakref.WeakKeyDictionary() -def _catalog_lock(source_id: str) -> asyncio.Lock: - return _catalog_locks.setdefault(asyncio.get_running_loop(), {}).setdefault(source_id, asyncio.Lock()) +def _catalog_lock(source_id: str) -> Any: + from app.cluster import mutex + + return mutex.lock(f"ledger-catalog:{source_id}") async def catalog_pass(source_id: str, edits: Sequence[CatalogEdit]) -> int: @@ -1473,15 +1538,15 @@ async def _probe_calibers(change: Mapping[str, Any]) -> list[str]: continue async with SessionLocal() as session: done = await session.scalar(select(func.count()).select_from(Run).where( - Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.probe") == 1, - func.json_extract(Run.origin, "$.workflow_id") == caliber.workflow_id, - func.json_extract(Run.origin, "$.version") == caliber.version, - func.json_extract(Run.origin, "$.change") == change.get("id"))) + Run.origin_kind == ORIGIN, json_flag(Run.origin, "probe"), + json_eq(Run.origin, "workflow_id", caliber.workflow_id), + json_eq(Run.origin, "version", caliber.version), + json_eq(Run.origin, "change", change.get("id")))) if done: continue latest = (await session.execute(select(Run.input).where( Run.workflow_id == caliber.workflow_id, Run.version == caliber.version, Run.run_class == "formal", - Run.status == "succeeded").order_by(Run.finished_at.desc()).limit(1))).scalar_one_or_none() + Run.status == "succeeded").order_by(Run.finished_at.desc(), Run.id.desc()).limit(1))).scalar_one_or_none() inputs_from = "formal" if latest else "defaults" if not latest: # 这一版还没有正式运行:退回最近一次等价于它的问数回答绑定出的参数,再没有才只用入口的缺省值 @@ -1490,11 +1555,16 @@ async def _probe_calibers(change: Mapping[str, Any]) -> list[str]: if answer is not None: latest, inputs_from = dict(answer.binding), "answer" for card in caliber.cards[:1]: + pinned = None try: pinned = await derive.load_pinned(session, caliber.workflow_id, caliber.version, card) names = {str(f.get("name")) for f in pinned.entry_fields} - inputs = compose.check_inputs(pinned, {k: v for k, v in (latest or {}).items() if k in names}) - except (derive.NotDerivable, compose.ComposeRefused): + given = {k: v for k, v in (latest or {}).items() if k in names} + inputs = compose.check_inputs(pinned, given) + except (derive.NotDerivable, compose.ComposeRefused) as e: + # 凑不齐参数(这一版没有正式运行、也没有等价的问数回答,必填的入口字段又没有缺省值)或者派生不了:不试算, + # 照实记一条缺哪几个参数——日志里一条,变化记录的影响里一条(变化页照它写),不再一声不响地跳过 + await _probe_skipped(change, caliber, pinned, latest, inputs_from, e.message) continue probe = derive.caliber_probe(pinned) origin = {"probe": 1, "workflow_id": caliber.workflow_id, "version": caliber.version, @@ -1514,6 +1584,52 @@ async def _probe_calibers(change: Mapping[str, Any]) -> list[str]: return launched +#: 变化记录 affects 里试算情况的键(data/source_changes 再次探查到同一条变化时保留它) +PROBES_KEY = "probes" + + +def missing_inputs(pinned: Any, given: Mapping[str, Any] | None) -> list[str]: + """口径入口里必填、又没给值也没有缺省值的字段(按入口字段的先后)。""" + out = [] + for f in pinned.entry_fields if pinned is not None else (): + name = str(f.get("name") or "").strip() + if not name or not f.get("required") or f.get("default") not in (None, ""): + continue + if (given or {}).get(name) in (None, ""): + out.append(name) + return out + + +async def _probe_skipped(change: Mapping[str, Any], caliber: Any, pinned: Any, given: Mapping[str, Any] | None, + inputs_from: str, reason: str) -> None: + """表结构变了、这一版口径没能试算:记一条日志,写进变化记录的 affects.probes(同一版只记一条,换成最新的说法)。""" + from app.db.models_sources import SourceChange + + missing = missing_inputs(pinned, given) + name = getattr(pinned, "workflow_name", None) or caliber.workflow_id + said = f"缺少参数 {'、'.join(missing)}" if missing else reason + logger.info("表结构变了之后口径「%s」v%s 没有试算:%s(参数来自%s)", name, caliber.version, said, + {"formal": "最近一次正式运行", "answer": "等价的问数回答", "defaults": "入口的缺省值"}.get(inputs_from, inputs_from)) + note = {"workflow_id": caliber.workflow_id, "version": caliber.version, "name": str(name), "outcome": "skipped", + "missing": missing, "reason": reason, "inputs_from": inputs_from} + async with SessionLocal() as session: + # 先占住那一行再读改写(SQLite 上拿到写锁、PostgreSQL 上锁住这一行):几版口径同时记、探查同时刷新影响 + # (source_changes.record)时一个接一个来,谁也不盖掉谁 + await session.execute(update(SourceChange).where(SourceChange.id == change.get("id")) + .values(seen_count=SourceChange.seen_count).execution_options(synchronize_session=False)) + row = (await session.execute(select(SourceChange).where(SourceChange.id == change.get("id")) + .execution_options(populate_existing=True))).scalar_one_or_none() + if row is None: + await session.rollback() + return + affects = dict(row.affects or {}) + kept = [p for p in affects.get(PROBES_KEY) or [] if isinstance(p, dict) + and (p.get("workflow_id"), p.get("version")) != (caliber.workflow_id, caliber.version)] + affects[PROBES_KEY] = [*kept, note] + row.affects = affects + await session.commit() + + async def _settle_probe(run: Run) -> str: """试算运行结束:算出来了只记结论;没算出来给那一版口径的发布人开 recall 待办(reason=structure)。""" from app.apps import versions as app_versions @@ -1568,26 +1684,34 @@ async def consume_source_changes() -> int: return 0 _consuming.add(loop) try: - count = 0 - while True: - async with SessionLocal() as session: - since = await _cursor(session) - batch = await source_changes.list_changes(session, since_seq=since, limit=200) - if not batch: - return count - for change in batch: - try: - if basis_kind(change) is not None: - await process_change(change) - count += 1 - except Exception: # noqa: BLE001 - 一条出错不挡后面的 - logger.exception("召回处理变化记录 #%s 出错", change.get("seq")) - async with SessionLocal() as session: - await _set_cursor(session, int(batch[-1]["seq"])) + # 多进程:几个进程的探查、剖析结束时都会来拉,游标是设置表里先读后写的一行——跨进程的锁里一份接一份,后来的接着游标往下拉 + from app.cluster import mutex + + async with mutex.lock("ledger-consume-changes"): + return await _consume(source_changes) finally: _consuming.discard(loop) +async def _consume(source_changes: Any) -> int: + count = 0 + while True: + async with SessionLocal() as session: + since = await _cursor(session) + batch = await source_changes.list_changes(session, since_seq=since, limit=200) + if not batch: + return count + for change in batch: + try: + if basis_kind(change) is not None: + await process_change(change) + count += 1 + except Exception: # noqa: BLE001 - 一条出错不挡后面的 + logger.exception("召回处理变化记录 #%s 出错", change.get("seq")) + async with SessionLocal() as session: + await _set_cursor(session, int(batch[-1]["seq"])) + + # ========================================================================== # 发布钩子、启动时接着做、测试用的 drain # ========================================================================== diff --git a/backend/app/ledger/reconcile.py b/backend/app/ledger/reconcile.py index db2edfff..4de86bf5 100644 --- a/backend/app/ledger/reconcile.py +++ b/backend/app/ledger/reconcile.py @@ -143,11 +143,11 @@ def actions_of(content: dict[str, Any], names: dict[str, str], hide: Any = None) async def reconciled_runs(session: Any, run_id: str) -> list[Run]: """对这次运行对过的账(origin.source_run 是它),新的在前。""" - from sqlalchemy import func + from app.db.portable import json_eq rows = (await session.execute(select(Run).where( - Run.origin_kind == ORIGIN, func.json_extract(Run.origin, "$.source_run") == run_id) - .order_by(Run.created_at.desc()))).scalars() + Run.origin_kind == ORIGIN, json_eq(Run.origin, "source_run", run_id)) + .order_by(Run.created_at.desc(), Run.id.desc()))).scalars() return list(rows) @@ -296,15 +296,37 @@ def tail_nodes(plan: Plan, *, card_id: str, fetch_id: str) -> list[dict[str, Any sql=align_sql(plan.keys, plan.pairs, plan.key_values))) for i, row in enumerate(plan.key_values): where = ",".join(f"{ours}={v}" for (ours, _), v in zip(plan.keys, row)) + # 先核对第 i 行就是这一组的维度:重跑时这次的结果缺了一行、某个维度多出一行(口径那边重复),合并结果会错位, + # 按位置取到的是别的维度的数。对不上的记空值(对账卡 on_missing=null),不拿别的维度的数冒充 + guard = _row_guard(i, plan.keys, row) for j, pair in enumerate(plan.pairs): - metrics += _four(f"{i}_{j}", f"{where} · {pair.name or pair.ours}", - f"cell(nodes.{ALIGN_ID}, {i}, 'ours_{j}')", f"cell(nodes.{ALIGN_ID}, {i}, 'cal_{j}')", - pair) + ours = f"cell(nodes.{ALIGN_ID}, {i}, 'ours_{j}')" + theirs = f"cell(nodes.{ALIGN_ID}, {i}, 'cal_{j}')" + if guard: + ours, theirs = f"({ours} if {guard} else None)", f"({theirs} if {guard} else None)" + metrics += _four(f"{i}_{j}", f"{where} · {pair.name or pair.ours}", ours, theirs, pair) nodes.append(_node(RECONCILE_CARD, "metrics", "对账卡", caliber="口径对账", caliber_version="v1", on_missing="null", metrics=metrics)) return nodes +def _row_guard(i: int, keys: list[tuple[str, str]], values: tuple[Any, ...]) -> str: + """对齐结果第 i 行是不是这一组(维度列的值都对得上)的条件式;维度值是空值的那一列不核对(表达式里没有空值的写法)。""" + import ast + + from app.engine.expressions import _literal + + parts = [f"cell(nodes.{_align_id()}, {i}, {ours!r}) == {ast.unparse(_literal(v))}" + for (ours, _), v in zip(keys, values) if v is not None and not isinstance(v, (list, dict))] + return " and ".join(parts) + + +def _align_id() -> str: + from app.ledger.derive import ALIGN_ID + + return ALIGN_ID + + def _numeric(rows: list[Any], index: int) -> bool: values = [r[index] for r in rows if isinstance(r, (list, tuple)) and index < len(r) and r[index] is not None] return bool(values) and all(isinstance(v, (int, float)) and not isinstance(v, bool) for v in values) @@ -328,9 +350,11 @@ def _plain_cell(expression: str, fetch: str) -> str | None: def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str], caliber_columns: list[str] | None, metrics: list[dict[str, Any]], fetch: str, card: str, fallback: bool, chosen: Any, - referenced: list[tuple[int, str]], projected: Any = None) -> Plan: + referenced: list[tuple[int, str]], projected: Any = None, cells: Any = None, + caliber: str | None = None) -> Plan: """配列、(多行时)定行和键:交回 Plan,配不了的抛 ReconcileRefused(见模块说明)。projected 是比对候选的 projection - (按引用的列投影过的:{ours: [这次留下的列], caliber: [口径留下的列]},按位置一一对应)。纯计算。""" + (按引用的列投影过的:{ours: [这次留下的列], caliber: [口径留下的列]},按位置一一对应)。cells 是这条查询逐格比的结论 + (ledger/match._cells),caliber 是这份口径的键:要人选对照时按它预选(_suggested)。纯计算。""" def check_column(column: str) -> None: if column not in ours_columns: raise ReconcileRefused(422, PAIR_COLUMN.format(column=column)) @@ -343,7 +367,8 @@ def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str numeric = [c for i, c in enumerate(ours_columns) if c.lower() not in hidden and _numeric(ours_rows, i)] options = {"options": {"columns": numeric, "metrics": [{"id": str(m.get("id")), "name": str(m.get("name") or m.get("id")), - "unit": str(m.get("unit") or "")} for m in metrics if m.get("id")]}} + "unit": str(m.get("unit") or "")} for m in metrics if m.get("id")], + "suggested": _suggested(cells, caliber, fetch, numeric, metrics)}} if multi: return _plan_rows(ours_columns, ours_rows, hidden, caliber_columns, referenced, check_column) if chosen is not None: @@ -387,6 +412,40 @@ def plan_pairs(*, ours_columns: list[str], ours_rows: list[Any], hidden: set[str return Plan(pairs=pairs) +def _suggested(cells: Any, caliber: str | None, fetch: str, columns: list[str], + metrics: list[dict[str, Any]]) -> list[dict[str, str]]: + """要人选对照时预选哪几对:[{ours, metric}]。按逐格比的结论(ledger/match._cells)——这次结果的这一列单独比,和这份口径 + 取数节点的某一列等价(逐格证明了,projection.caliber 是那一列),口径卡上只读这一列这一格的指标就是它最可能的对照。只认 + 证明了的:只是「相关」的那一格比到的是第一个试的列,不是最像的,不替人猜;一个口径列对上几个指标的也不预选。界面照它 + 预选、人可以改。""" + from app.engine.expressions import ExpressionError, cell_parts, leaf_refs, parse_expression + + if not isinstance(cells, dict) or not caliber: + return [] + reads: dict[str, list[str]] = {} + for m in metrics: + try: + leaves = leaf_refs(parse_expression(str(m.get("expression") or ""))[0]) + except ExpressionError: + continue + cols = {_plain_cell(str(leaf), fetch) for leaf in leaves if cell_parts(str(leaf)) is not None} + if len(leaves) == 1 and len(cols) == 1 and None not in cols and m.get("id"): + reads.setdefault(str(next(iter(cols))).lower(), []).append(str(m["id"])) + out: list[dict[str, str]] = [] + for col in columns: + cell = cells.get(col) + cands = cell.get("candidates") if isinstance(cell, dict) else None + proven = next((c for c in cands or [] if isinstance(c, dict) and c.get("status") == "proven" + and c.get("caliber") == caliber and c.get("node_id") == fetch), None) + theirs = ((proven or {}).get("projection") or {}).get("caliber") if proven else None + if not isinstance(theirs, list) or len(theirs) != 1 or not isinstance(theirs[0], str): + continue + hits = reads.get(theirs[0].lower()) or [] + if len(hits) == 1: + out.append({"ours": col, "metric": hits[0]}) + return out + + def _projected_pairs(projected: Any, ours_columns: list[str]) -> list[tuple[str, str]] | None: """比对候选的 projection → [(这次的列, 口径的列)],按位置一一对应;不是投影过的、两边列数不一样、这次的列不在结果里的 为 None(照旧按整条的位置配)。""" @@ -561,7 +620,8 @@ async def reconcile(run_id: str, req: Request, *, principal: Any) -> Run: caliber_columns=_caliber_columns(template, inputs, list(pinned.entry_fields), kind, schema), metrics=pinned.metrics, fetch=str(fetch["id"]), card=pinned.card_title, fallback=bool((cand.get("diff") or {}).get("fallback")), chosen=req.pairs, - referenced=_referenced(sealed, req.query), projected=cand.get("projection")) + referenced=_referenced(sealed, req.query), projected=cand.get("projection"), + cells=query.get("cells"), caliber=key) graph = build(pinned, sql=snap["sql"], tool=f"{QUERY_PREFIX}{source}", plan=plan, inputs=inputs, fetch=str(fetch["id"])) diff --git a/backend/app/ledger/uses.py b/backend/app/ledger/uses.py index b473e110..523720b0 100644 --- a/backend/app/ledger/uses.py +++ b/backend/app/ledger/uses.py @@ -116,7 +116,7 @@ async def _answers(session: AsyncSession, workflow_id: str, below: int | None, o rows = (await session.execute( select(Artifact.run_id, Artifact.meta, Run.finished_at).join(Run, Run.id == Artifact.run_id) .where(Artifact.kind == store.KIND, Run.status == "succeeded", counted_runs()) - .order_by(Run.finished_at.desc()))).all() + .order_by(Run.finished_at.desc(), Run.id.desc()))).all() wanted: dict[str, datetime | None] = {} for run_id, meta, at in rows: keys = [k for k in ((meta or {}).get("calibers") or []) if isinstance(k, str) and k.startswith(prefix)] @@ -179,7 +179,7 @@ async def _chat_owners(session: AsyncSession, found: list[Use]) -> None: if missing: for turn_id, conv, rid in (await session.execute( select(ConversationTurn.id, ConversationTurn.conversation_id, ConversationTurn.run_id) - .where(ConversationTurn.run_id.in_(missing)).order_by(ConversationTurn.created_at))).all(): + .where(ConversationTurn.run_id.in_(missing)).order_by(ConversationTurn.created_at, ConversationTurn.id))).all(): turns.setdefault(rid, (conv, turn_id)) convs = sorted({c for c, _ in turns.values()}) owners = dict((await session.execute(select(Conversation.id, Conversation.created_by) @@ -219,7 +219,7 @@ async def _formal(session: AsyncSession, clause: Any, *, kind_of: Any) -> tuple[ rows = list((await session.execute( select(Run).where(clause, Run.run_class == "formal", Run.status == "succeeded", Run.manifest_hash.is_not(None), counted_runs()) - .order_by(Run.finished_at.desc()).limit(MAX_FORMAL + 1))).scalars()) + .order_by(Run.finished_at.desc(), Run.id.desc()).limit(MAX_FORMAL + 1))).scalars()) truncated = len(rows) > MAX_FORMAL per: dict[tuple[str, str], int] = {} out: list[Use] = [] @@ -325,7 +325,7 @@ async def _exporters(session: AsyncSession, uses: list[Use]) -> None: by: dict[str, list[str]] = {} for rid, actor in (await session.execute(select(AuditEvent.subject_id, AuditEvent.actor).where( AuditEvent.action == EXPORTED, AuditEvent.subject_kind == "run", - AuditEvent.subject_id.in_(ids)).order_by(AuditEvent.at))).all(): + AuditEvent.subject_id.in_(ids)).order_by(AuditEvent.at, AuditEvent.id))).all(): if actor and actor not in by.setdefault(rid, []): by[rid].append(actor) for use in uses: diff --git a/backend/app/look/calibers.py b/backend/app/look/calibers.py index 17cbc5c4..8bae0ca2 100644 --- a/backend/app/look/calibers.py +++ b/backend/app/look/calibers.py @@ -193,7 +193,7 @@ class CaliberKind: from app.db.models import Workflow items: list[CaliberView] = [] - for wf in (await session.execute(select(Workflow).order_by(Workflow.name))).scalars().all(): + for wf in (await session.execute(select(Workflow).order_by(Workflow.name, Workflow.id))).scalars().all(): latest = await app_versions.latest_published(session, wf.id) if latest is not None: items += views_of(wf.id, wf.name, latest) diff --git a/backend/app/look/versions.py b/backend/app/look/versions.py index f24d25de..b2191010 100644 --- a/backend/app/look/versions.py +++ b/backend/app/look/versions.py @@ -70,7 +70,7 @@ class VersionKind: from app.db.models import Workflow items: list[VersionView] = [] - for wf in (await session.execute(select(Workflow).order_by(Workflow.name))).scalars().all(): + for wf in (await session.execute(select(Workflow).order_by(Workflow.name, Workflow.id))).scalars().all(): for v in await app_versions.published_versions(session, wf.id): items.append(VersionView(workflow_id=wf.id, workflow_name=wf.name, description=wf.description or "", version=v)) diff --git a/backend/app/main.py b/backend/app/main.py index c96663a3..026a85a3 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from contextlib import asynccontextmanager -from typing import AsyncIterator +from typing import Any, AsyncIterator from fastapi import Depends, FastAPI, Request from fastapi.middleware.cors import CORSMiddleware @@ -49,27 +49,64 @@ logger = logging.getLogger("agentlab") @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: + from app import cluster + from app.cluster import guard, mutex from app.data.engine import sqlite_dqs_self_check sqlite_dqs_self_check() # 写错的双引号列名必须报错;做不到就记日志并让启动失败 settings.ensure_dirs() + # SQLite 只支持单进程:同一个数据目录已经有一个进程在用(uvicorn --workers、两个服务指着同一个目录)就拒绝启动、照实说 + # (app/cluster/guard.py)。PostgreSQL 可以多进程,什么都不做 + guard.acquire() + # PgBouncer 事务模式(AGENTLAB_DB_PGBOUNCER=1)要另给直连地址,没给就拒绝启动(db/base.check_pgbouncer) + from app.db.base import check_pgbouncer, wait_for_database + + check_pgbouncer() + # 库还没起来(和 PostgreSQL 容器一起起、库还在初始化)时先按退避等它:下面的版本检查就要连库,不等的话一上来就崩 + await wait_for_database() + # 不许新旧版本混跑(app/cluster/version.py):库里还有活着的、版本不同的进程就不启动——在补列之前看 + from app.cluster import version + + await version.preflight() await init_db() - await seed_defaults() + # 多进程协调(app/cluster):登记这个工作进程、开始心跳和进程间通知、争调度当班。要在下面的启动整理之前:整理只碰属主不在了的 + # 行,要先知道别的进程谁还活着 + await cluster.start() + from app.cluster.relay import relay - # 记级别之前发布过的老版本:有正式运行凭据的补记为已发布,免得子工作流、口径卡钉住它们的正式运行被误判成「没发布过」 - from app.apps.versions import backfill_legacy_levels - from app.db.base import SessionLocal as _Sessions + await relay.start() + _register_reapers() + _register_cluster_hooks() - async with _Sessions() as _s: - if levelled := await backfill_legacy_levels(_s): - logger.info("给 %d 个老的发布版本补记了级别(有正式运行凭据)", levelled) - await _s.commit() - await _platform_startup() + # 下面这一段是全站的一次性整理(补默认数据、补记级别、存量密钥加密、上传版本的清理和迁移……),各项都是幂等的;多进程同时 + # 启动时一个接一个做(跨进程的锁),后起的看得到先起的做完的样子,不会重复插、互相删 + async with mutex.lock("startup"): + await seed_defaults() + + # 记级别之前发布过的老版本:有正式运行凭据的补记为已发布,免得子工作流、口径卡钉住它们的正式运行被误判成「没发布过」 + from app.apps.versions import backfill_legacy_levels + from app.db.base import SessionLocal as _Sessions + + async with _Sessions() as _s: + if levelled := await backfill_legacy_levels(_s): + logger.info("给 %d 个老的发布版本补记了级别(有正式运行凭据)", levelled) + await _s.commit() + await _platform_startup() + + # 上传表格的版本底座:清理上次没发布完的文件、给老上传补初始版本、回收没人用的版本。 + # 出错只记日志:最坏是留下几个多余的文件,不该让服务起不来 + from app.data import table_versions + from app.db.base import SessionLocal + + async with SessionLocal() as _s: + try: + await table_versions.startup(_s) + except Exception: # noqa: BLE001 + logger.exception("上传表格的版本整理失败,服务照常启动") # 把存的 embedding 配置读进来。不读的话重启后会悄悄退回本地哈希向量, # 而库里的向量是上次那个模型建的——检索会整体退回关键词, # 用户只看到"重启之后搜得不准了" - from app.db.base import SessionLocal from app.memory.embeddings import load_setting async with SessionLocal() as _session: @@ -78,88 +115,20 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: except Exception as e: # noqa: BLE001 - key 没了不该拦住整个服务起不来 logger.warning("embedding 配置没能生效,暂用本地向量:%s", e) - # 上次被强杀时留下的"处理中"文档。后台任务不会自己回来,留着它只会在 - # 界面上永远转圈——和 runner 对 running 的 run 是同一个道理 - from sqlalchemy import update as _update - - from app.db.models import Document as _Doc - - async with SessionLocal() as _s: - await _s.execute( - _update(_Doc).where(_Doc.status == "processing") - .values(status="failed", error="服务重启,这份文档没有处理完,请重新上传") - ) - await _s.commit() - - # 上传表格的版本底座:清理上次没发布完的文件、给老上传补初始版本、回收没人用的版本。 - # 出错只记日志:最坏是留下几个多余的文件,不该让服务起不来 - from app.data import table_versions - - async with SessionLocal() as _s: - try: - await table_versions.startup(_s) - except Exception: # noqa: BLE001 - logger.exception("上传表格的版本整理失败,服务照常启动") - - # 上次没做完的数据目录起草、剖析任务:标成中断(没结束的表记停止),不自动续跑——拷来的数据目录(检查脚本、验收副本) - # 启动时不能替人去调模型、查业务库。界面上点「重试」接着做没做完的表。剖析设置里按整次剖析算的旧预算顺带迁移掉 - from app.data import catalog_jobs, catalog_profile_jobs - - try: - await catalog_jobs.startup() - except Exception: # noqa: BLE001 - logger.exception("数据目录起草任务的启动整理失败,服务照常启动") - try: - await catalog_profile_jobs.startup() - except Exception: # noqa: BLE001 - logger.exception("数据剖析任务的启动整理失败,服务照常启动") - # 后台探查任务(next-round 03 A1)同一个规矩:没做完的标中断,读完的表物化进表结构,不自动续跑 - from app.data import introspect_jobs - - try: - await introspect_jobs.startup() - except Exception: # noqa: BLE001 - logger.exception("探查任务的启动整理失败,服务照常启动") - # 评测(next-round 01 做法 6、9):上次没做完的评测、影子起草同一个规矩——标中断(没做完的题记停止),不自动续跑: - # 拷来的数据目录启动时不能替人去调模型、查业务库。界面上点「重试」只把没做完的题接着做 - from app.eval import runner as eval_runner, snapshots as eval_snapshots - - try: - await eval_snapshots.startup() - except Exception: # noqa: BLE001 - logger.exception("影子起草的启动整理失败,服务照常启动") - try: - await eval_runner.startup() - except Exception: # noqa: BLE001 - logger.exception("评测任务的启动整理失败,服务照常启动") - + # 打开断点库;上次被强杀时留下的运行(只碰属主不在了的:多进程时别的活着的进程在跑的一概不碰)——在跑的标中断、排队的标失败 await run_manager.setup() - # 问数据的一轮(app/chat/job.py):在 run_manager 之后整理——服务重启时在跑的运行这时已经标成挂起。还在建图的标中断; - # 已经发起了运行的接着等它、照常复核收尾(和调度循环同一个开关:拷来的数据目录不替人调模型,等人打开对话时再接着收尾) - from app.chat import job as chat_job - - try: - await chat_job.startup() - except Exception: # noqa: BLE001 - logger.exception("问数据的启动整理失败,服务照常启动") - # 流程:存量的待审批补建待办(派给任何人,幂等);上次进程没了时已发出、没拿到结果的写入收成结果未知、安排核实 - from app.process import actions as _actions, tasks as _tasks - - await _tasks.startup() - await _actions.startup() - # 发布单:上次进程没了时还在执行的单改成停下(停在没做完的那一步、写明执行中断),这样能接着做 - from app.blueprints import publish_plan as _plans - - try: - if stopped := await _plans.startup(): - logger.info("%d 份发布单执行中断,已改成停下,可以接着做", stopped) - except Exception: # noqa: BLE001 - logger.exception("发布单的启动整理失败,服务照常启动") + # 接手属主不在了的:处理中的文档标失败;数据目录起草、剖析、探查、评测、影子起草标中断(不自动续跑:拷来的数据目录启动时不能 + # 替人去调模型、查业务库,界面上点「重试」接着做);问数据的一轮(在 run_manager 之后:在跑的运行这时已经标成挂起)还在建图的 + # 标中断,发起了运行的接着等它、照常复核收尾;待审批补建待办、停在 resolving 的收拾;结果未知的写入安排核实;执行中断的 + # 发布单改成停下。各项的规矩见 _register_reapers 登记的各模块。启动时每个进程都做一次(同样一个接一个),之后当班进程隔一会儿 + # 再做一次(别的进程没了的时候) + async with mutex.lock("startup"): + await cluster.service.startup_reap() from app.sandbox.manager import sandbox_manager as _sbm await _sbm.start_reaper() # 定时任务的调度循环放在 run_manager 之后起:它到点要发起运行。拷来的数据目录(检查脚本的 - # --stacks)用 AGENTLAB_SCHEDULER=0 关掉,免得替原库把定时任务再执行一遍 + # --stacks)用 AGENTLAB_SCHEDULER=0 关掉,免得替原库把定时任务再执行一遍。每个进程都起,只有调度当班的进程真的做 from app import scheduler if scheduler.scheduler_enabled(): @@ -170,10 +139,14 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: finally: # 先停调度,再停运行:停运行的途中不该再有新运行被发起。起草、剖析任务停下来标成中断,下次启动不会再续 await scheduler.stop() + from app.data import catalog_jobs, catalog_profile_jobs, introspect_jobs + from app.chat import job as chat_job + from app.eval import runner as eval_runner, snapshots as eval_snapshots + await catalog_jobs.manager.shutdown() await catalog_profile_jobs.manager.shutdown() await introspect_jobs.manager.shutdown() - # 问数据的一轮停下不改那几行(下次启动照重启的规矩整理),也要在运行之前停 + # 问数据的一轮停下不改那几行(下次启动、或者别的进程照「属主不在了」的规矩接手),也要在运行之前停 await chat_job.shutdown() # 评测要在运行之前停:它停下时会取消自己发起的作答、算期望的运行,再标中断 await eval_runner.shutdown() @@ -182,9 +155,142 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: from app.sandbox.manager import sandbox_manager await sandbox_manager.close() + # 交班、删掉本进程的登记:别的进程马上就知道本进程不在了,接手它留下的行(要用库,在关连接之前) + await relay.stop() + await cluster.stop() + # 应用库的连接全关掉:SQLite 最后一个连接关掉时才把 -wal 并回、删掉(copy-to-postgres 按它认服务停没停), + # PostgreSQL 的连接池也正常关 + from app.db.base import dispose_session_engine, engine as db_engine + + await db_engine.dispose() + await dispose_session_engine() logger.info("已停止") +def _register_cluster_hooks() -> None: + """心跳里本进程自己的活、接班之后要补做的事(app/cluster/service.py)。""" + from app import cluster + from app.auth import keys + + # 访问密钥的请求记录攒在每个进程自己的内存里(每个请求同步写一行会和运行抢 SQLite 的写锁):每个进程隔一拍心跳写自己的。 + # 过了保留期的清理只在当班进程(调度循环的那一拍) + cluster.on_beat("访问密钥的请求记录", keys.flush) + + def on_lead(now_leading: bool) -> None: + if not now_leading: + return + from app.ledger import match as ledger_match, recall as ledger_recall + from app.scheduler.loop import runner + + # 接了调度班:马上补一拍;口径比对的补比、自动召回的续发也归当班进程(同 run_manager.setup 里启动时的那一次) + runner.poke() + ledger_match.start_backfill() + ledger_recall.start_resume() + + cluster.on_leadership(on_lead) + _register_cache_handlers() + + +def _register_cache_handlers() -> None: + """别的进程改了、本进程缓存着的东西怎么重读(app/cluster/caches.py:改的进程 broadcast,这里照着清掉、重读)。""" + from app.cluster import caches + + async def auth_settings(_key: str | None) -> None: + from app.auth import load_settings + + await load_settings(force=True) + + async def public_address(_key: str | None) -> None: + from app.core import public_url + + await public_url.refresh() + + async def embeddings(_key: str | None) -> None: + from app.db.base import SessionLocal + from app.memory.embeddings import load_setting + + async with SessionLocal() as session: + await load_setting(session) + + async def capabilities(_key: str | None) -> None: + from app.capabilities import registry + from app.db.base import SessionLocal + + async with SessionLocal() as session: + await registry.refresh_cache(session, broadcast=False) + + async def datasource_engine(key: str | None) -> None: + from app.data.engine import engines + + if key: + await engines.invalidate(key, broadcast=False) + + def app_pages(_key: str | None) -> None: + from app.apps.resolve import clear_cache + + clear_cache() + + for name, fn in (("auth_settings", auth_settings), ("public_url", public_address), ("embeddings", embeddings), + ("capabilities", capabilities), ("datasource_engine", datasource_engine), ("app_pages", app_pages)): + caches.on(name, fn) + + +def _register_reapers() -> None: + """接手属主不在了的行的各种规矩(app/cluster/reaper.py),按这个先后:运行在问数据之前(问数据接手时要看运行的状态)。""" + from app.blueprints import publish_plan + from app.chat import job as chat_job + from app.cluster import reaper + from app.data import catalog_jobs, catalog_profile_jobs, introspect_jobs + from app.eval import runner as eval_runner, snapshots as eval_snapshots + from app.process import actions, tasks + + async def documents(live: Any) -> int: + return await _reap_documents(live) + + for name, fn in ( + ("运行", run_manager.reap), + ("处理中的文档", documents), + ("数据目录起草任务", lambda live: catalog_jobs.startup(live=live)), + ("数据剖析任务", lambda live: catalog_profile_jobs.startup(live=live)), + ("探查任务", lambda live: introspect_jobs.startup(live=live)), + ("影子起草", lambda live: eval_snapshots.startup(live=live)), + ("评测任务", lambda live: eval_runner.startup(live=live)), + ("问数据的一轮", _chat_reaper(chat_job)), + ("待办", lambda live: tasks.startup(live=live)), + ("结果未知的写入", lambda live: actions.startup(live=live)), + ("发布单", lambda live: publish_plan.startup(live=live)), + ): + reaper.register(name, fn) + + +def _chat_reaper(chat_job: Any) -> Any: + async def reap(live: Any) -> Any: + counts = await chat_job.startup(live=live) + return {k: v for k, v in counts.items() if v} or 0 + return reap + + +async def _reap_documents(live: Any) -> int: + """上次被强杀时留下的「处理中」文档(只看属主不在了的)。后台任务不会自己回来,留着它只会在界面上永远转圈——和 runner + 对 running 的 run 是同一个道理。""" + from sqlalchemy import update as _update + + from app.cluster import claims + from app.db.base import SessionLocal + from app.db.models import Document as _Doc + + ids = [doc_id for doc_id, _ in await claims.orphans(_Doc, live, running=lambda _id: False, + statuses=("processing",))] + if ids: + async with SessionLocal() as _s: + await _s.execute( + _update(_Doc).where(_Doc.id.in_(ids), _Doc.status == "processing") + .values(status="failed", error="服务重启,这份文档没有处理完,请重新上传") + ) + await _s.commit() + return len(ids) + + async def _platform_startup() -> None: """应用平台的启动整理(BE-1):登录设置、对外访问地址读进缓存;自定义工具请求头、MCP env 和请求头里的存量明文加密 一遍(幂等);存量的内网 MCP 服务改为绑定登记端点(只在升级后第一次启动时做);能力登记同步一次。后三样出错只记 diff --git a/backend/app/memory/kb.py b/backend/app/memory/kb.py index a324d973..9c976e63 100644 --- a/backend/app/memory/kb.py +++ b/backend/app/memory/kb.py @@ -399,7 +399,8 @@ async def list_collections(session: AsyncSession) -> list[dict[str, Any]]: func.sum(Document.chunk_count), ).group_by(Document.collection) ) + # sum 在 PostgreSQL 上给的是 numeric(驱动交回 Decimal),SQLite 给整数:一律转成 int return [ - {"collection": name, "documents": docs, "chunks": chunks or 0} + {"collection": name, "documents": int(docs), "chunks": int(chunks or 0)} for name, docs, chunks in rows ] diff --git a/backend/app/memory/store.py b/backend/app/memory/store.py index 44524cd3..534d6f65 100644 --- a/backend/app/memory/store.py +++ b/backend/app/memory/store.py @@ -215,7 +215,7 @@ async def clear_scope(session: AsyncSession, scope: str) -> int: async def list_memories( session: AsyncSession, *, scope: str | None = None, limit: int = 200 ) -> list[MemoryItem]: - stmt = select(MemoryItem).order_by(MemoryItem.created_at.desc()).limit(limit) + stmt = select(MemoryItem).order_by(MemoryItem.created_at.desc(), MemoryItem.id.desc()).limit(limit) if scope: stmt = stmt.where(MemoryItem.scope == scope) return list((await session.execute(stmt)).scalars()) diff --git a/backend/app/notify/service.py b/backend/app/notify/service.py index 4c401397..211c35ff 100644 --- a/backend/app/notify/service.py +++ b/backend/app/notify/service.py @@ -121,8 +121,10 @@ def spawn(coro: Coroutine[Any, Any, Any]) -> None: async def drain() -> None: """等后台的推送都推完(测试、关停用)。""" - while _background: - await asyncio.gather(*list(_background), return_exceptions=True) + from app.core.background import drain as drain_tasks + + # 不写成 while _background: await gather(...):别的事件循环上做完、还没摘掉的任务会让那样的循环空转(core/background) + await drain_tasks(_background) async def personal_url_problem(url: str) -> str | None: @@ -262,7 +264,11 @@ async def _where(session: Any, run: Run | None) -> str: #: 邮件渠道:设置表里一行 {用户 id: 建的时间}(通知自己记的账,键的开头同 notify/digest.STATE_PREFIX,设置接口不列)。只在 #: 第一次派到待办时建一次——本人删掉了就是不要,不再替他建回来 MAIL_AUTO_KEY = "notify_state:mail_auto" -_mail_auto_lock = asyncio.Lock() +#: 先读后写那一行:跨进程的锁(app/cluster/mutex) +def _mail_auto_lock() -> Any: + from app.cluster import mutex + + return mutex.lock("notify-mail-auto") async def ensure_assignee_mail(users: list[str]) -> int: @@ -276,7 +282,7 @@ async def ensure_assignee_mail(users: list[str]) -> int: if not users or current_mode() == "off": return 0 - async with _mail_auto_lock, SessionLocal() as session: + async with _mail_auto_lock(), SessionLocal() as session: cfg = await mail.load_config(session) if not (cfg.enabled and cfg.default_for_assignees): return 0 @@ -405,7 +411,9 @@ async def notify_recall(task_id: str, users: list[str]) -> int: def message(answers: str, tasks: int, link: tuple[str, str] | None) -> dict[str, Any]: return {"event": event, "status": "pending", "ok": False, - "title": recall.CHANGED_TITLE.format(name=name[:80] or "口径", version=origin.get("to_version")), + # 待办开出来时按结论起的标题(数变了 / 行序不定、没能核对):通知和待办说同一句 + "title": (recall.ROW_ORDER_TITLE if "行序不定" in (task.title or "") else recall.CHANGED_TITLE) + .format(name=name[:80] or "口径", version=origin.get("to_version")), "lines": [*head, ("答案", answers), *([("待办", f"{tasks} 条")] if tasks > 1 else []), *tail], "links": [link] if base and link else [], "subject": {"task_id": task_id, "run_id": task.run_id, "workflow_id": origin.get("workflow_id")}} diff --git a/backend/app/ops.py b/backend/app/ops.py new file mode 100644 index 00000000..00be80cf --- /dev/null +++ b/backend/app/ops.py @@ -0,0 +1,205 @@ +"""部署脚本(deploy/deploy.sh、scripts/dev.sh)在容器里调的几件小事:`python -m app.ops <命令>`。 + +放在镜像里用 Python 做,不在宿主机上用 shell 做:数据目录归容器里的用户(uid 10001),宿主机上跑 deploy.sh 的人不一定写得 +进去;生成密码、改权限、拷 SQLite、删旧备份在容器里以同一个用户做,属主和权限都对。只用标准库,不连应用库、不读设置。 + + postgres-secrets 内置 PostgreSQL 的密码:<数据目录>/.postgres/password(给 PostgreSQL 容器建库时读)和 pgpass(给应用 + 连库时读,PGPASSFILE 指过来)。已有就不动;密码不打印、不进任何参数和日志 + state 数据目录用的是哪种库:没切过是 sqlite;--set builtin 记成「已切到内置 PostgreSQL」(deploy.sh 只在导入、 + 核对都通过之后才记,记下之前出错就还是 sqlite);migrating 是「开始往内置库里导了、还没切成」—— + 用的仍是 SQLite,只是让下一次 migrate-to-postgres 认得出数据卷里那半份是自己上次留下的 + backup-sqlite 拷一份 agentlab.db、checkpoints.db 到 <数据目录>/backups/sqlite/<时间>/(SQLite 的备份接口,一致的一份), + 只留最近 --keep 份 + backup-dir 建(或收紧)备份目录:从 backups/ 起每一层 0700(pg_dump 写进去之前用) + prune 目录里按名字排序只留最近 --keep 个(pg_dump 的备份用) + +备份里是整个库(含加密存的密钥、业务数据),目录一律 0700、文件 0600,属主是容器里的用户(同数据目录);宿主机上要读用 sudo。 +""" + +from __future__ import annotations + +import argparse +import contextlib +import os +import secrets +import sqlite3 +import sys +from datetime import datetime, timezone +from pathlib import Path + +from app.core import atomic + +#: 内置 PostgreSQL 的东西放在数据目录的这个子目录里 +POSTGRES_DIR = ".postgres" +STATE_FILE = "state" +MODES = ("sqlite", "builtin", "migrating") +BACKUPS_DIR = "backups" +#: 备份的目录、文件的权限:只有属主(容器里的用户)能读 +BACKUP_DIR_MODE = 0o700 +BACKUP_FILE_MODE = 0o600 + + +def _stamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def postgres_secrets(data: Path, *, folder: str, host: str, port: int, database: str, user: str) -> bool: + """建密码文件和 pgpass(都是 0600)。已有密码就沿用(库是按它初始化的),pgpass 按现在的主机名重写。交回是不是新建的密码。""" + root = data / folder + root.mkdir(parents=True, exist_ok=True) + # 0711:别人列不出目录、读不到密码(两个文件 0600),但宿主机上跑 deploy.sh 的人能按名字读到 state(0644) + root.chmod(0o711) + password_file = root / "password" + made = False + if not password_file.exists(): + made = atomic.create_text(password_file, secrets.token_urlsafe(32) + "\n", mode=0o600) + password = password_file.read_text(encoding="utf-8").strip() + if not password: + raise SystemExit(f"{password_file} 是空的:删掉它再来会生成新的(库已经按旧密码初始化过的话,要连库一起重建)") + # pgpass 的格式:主机:端口:库:用户:密码,冒号和反斜杠要转义 + escaped = password.replace("\\", "\\\\").replace(":", "\\:") + atomic.write_text(root / "pgpass", f"{host}:{port}:{database}:{user}:{escaped}\n", mode=0o600) + return made + + +def read_state(data: Path) -> str: + """数据目录用的是哪种库:sqlite(没切过)、builtin(已切到内置 PostgreSQL)或 migrating(切到一半,用的仍是 SQLite)。""" + path = data / POSTGRES_DIR / STATE_FILE + try: + for line in path.read_text(encoding="utf-8").splitlines(): + if line.startswith("mode="): + mode = line.split("=", 1)[1].strip() + return mode if mode in MODES else "sqlite" + except FileNotFoundError: + pass + return "sqlite" + + +def write_state(data: Path, mode: str) -> None: + if mode not in MODES: + raise SystemExit(f"认不出的模式:{mode}({' / '.join(MODES)})") + root = data / POSTGRES_DIR + if not root.exists(): + root.mkdir(parents=True) + root.chmod(0o711) + # 不是密钥:宿主机上的 deploy.sh 直接读它,0644 + atomic.write_text(root / STATE_FILE, f"mode={mode}\nsince={datetime.now(timezone.utc).isoformat()}\n", mode=0o644) + + +@contextlib.contextmanager +def _private_umask(): + """这一段里新建的文件、目录别人读不到(SQLite 备份时建的 -journal 也一样),出来恢复原来的 umask。""" + old = os.umask(0o077) + try: + yield + finally: + os.umask(old) + + +def backup_dir(data: Path, *parts: str) -> Path: + """<数据目录>/backups/:从 backups/ 起每一层建出来(已有的收紧)成 0700,交回最里面那层。""" + folder = data / BACKUPS_DIR + levels = [folder] + for part in parts: + folder = folder / part + levels.append(folder) + for level in levels: + level.mkdir(exist_ok=True) + level.chmod(BACKUP_DIR_MODE) + return folder + + +def _copy_sqlite(src: Path, dst: Path) -> None: + # 读写打开、只许读:WAL 模式的库用 mode=ro 打开,关掉时会留下它自己建的 -wal(同 db/to_postgres.snapshot) + conn = sqlite3.connect(src) + conn.execute("PRAGMA query_only = 1") + try: + # 先以 0600 建出空文件再交给 SQLite 写:从有这个文件起就只有属主读得到 + os.close(os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, BACKUP_FILE_MODE)) + out = sqlite3.connect(dst) + try: + conn.backup(out) + finally: + out.close() + dst.chmod(BACKUP_FILE_MODE) + finally: + conn.close() + + +def prune(folder: Path, keep: int) -> list[Path]: + """folder 里按名字(名字里带时间,排序就是先后)只留最近 keep 个,交回删掉的。""" + if keep < 1 or not folder.is_dir(): + return [] + items = sorted((p for p in folder.iterdir() if not p.name.startswith(".")), key=lambda p: p.name) + gone = items[:-keep] if len(items) > keep else [] + for path in gone: + if path.is_dir(): + for child in sorted(path.rglob("*"), reverse=True): + child.unlink() if child.is_file() or child.is_symlink() else child.rmdir() + path.rmdir() + else: + path.unlink() + return gone + + +def backup_sqlite(data: Path, *, keep: int) -> Path | None: + """agentlab.db、checkpoints.db 拷到 backups/sqlite/<时间>/。两个都没有(库在 PostgreSQL)交回 None。""" + sources = [p for p in (data / "agentlab.db", data / "checkpoints.db") if p.is_file()] + if not sources: + return None + with _private_umask(): + root = backup_dir(data, "sqlite") + target = root / _stamp() + target.mkdir(exist_ok=True) + target.chmod(BACKUP_DIR_MODE) + for src in sources: + _copy_sqlite(src, target / src.name) + prune(root, keep) + return target + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m app.ops", description="部署脚本在容器里调的几件小事") + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("postgres-secrets") + p.add_argument("--data", default="/data") + p.add_argument("--dir", default=POSTGRES_DIR) + p.add_argument("--host", required=True) + p.add_argument("--port", type=int, default=5432) + p.add_argument("--db", default="agentlab") + p.add_argument("--user", default="agentlab") + s = sub.add_parser("state") + s.add_argument("--data", default="/data") + s.add_argument("--set", choices=MODES) + b = sub.add_parser("backup-sqlite") + b.add_argument("--data", default="/data") + b.add_argument("--keep", type=int, default=5) + d = sub.add_parser("backup-dir") + d.add_argument("--data", default="/data") + d.add_argument("parts", nargs="*", help="backups/ 下面的几层,比如 postgres") + r = sub.add_parser("prune") + r.add_argument("--dir", required=True) + r.add_argument("--keep", type=int, default=5) + args = parser.parse_args(argv) + + if args.command == "postgres-secrets": + made = postgres_secrets(Path(args.data), folder=args.dir, host=args.host, port=args.port, database=args.db, + user=args.user) + print("生成了新密码" if made else "沿用已有的密码") + elif args.command == "state": + if args.set: + write_state(Path(args.data), args.set) + print(read_state(Path(args.data))) + elif args.command == "backup-sqlite": + target = backup_sqlite(Path(args.data), keep=args.keep) + print(target if target is not None else "没有 SQLite 库,不用备份") + elif args.command == "backup-dir": + print(backup_dir(Path(args.data), *args.parts)) + elif args.command == "prune": + for gone in prune(Path(args.dir), args.keep): + print(f"删掉旧备份 {gone.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/app/process/actions.py b/backend/app/process/actions.py index 87ab7fca..f8c8e2b2 100644 --- a/backend/app/process/actions.py +++ b/backend/app/process/actions.py @@ -236,19 +236,69 @@ class Prepared: run: dict[str, Any] = field(default_factory=dict) #: 按参数定义收好类型的参数(typed_args):哈希、预览按它算,send 发的也是它 args: dict[str, Any] = field(default_factory=dict) + #: 记录上写的节点 id:认出升级前那种键(legacy_node_id)时照它记,同一个键下的记录 node_id 一致,下一次还认得出 + node_id: str | None = None @property def idem_key(self) -> str: return f"{self.key}:{self.attempt}" +#: 按自己的键找到别人的那一笔时换用的键的前缀(prepare):节点 id 里不会有这个字符 +OWN_KEY_MARK = "\x1e" +#: 带 action_id 的那几种事件:认一条记录是哪个节点写的(_owner) +_ACTION_EVENTS = (EventType.ACTION_PREPARED, EventType.ACTION_SENT, EventType.ACTION_CONFIRMED, EventType.ACTION_FAILED, + EventType.ACTION_UNKNOWN, EventType.ACTION_SIMULATED) + + +async def _owner(session: AsyncSession, rec: ActionRecord) -> str | None: + """这条记录是哪个节点写的(这次运行里的 id,子工作流里的带前缀):按它的 action.* 事件上记的 node_id。记录上的 node_id + 升级前子工作流里记的是子图里的 id,和父图同名的节点分不开;事件一向按这次运行里的 id 记。认不出(事件没写上)为 None。""" + from app.db.models import RunEvent + + rows = (await session.execute(select(RunEvent.node_id, RunEvent.data).where( + RunEvent.run_id == rec.run_id, RunEvent.type.in_([str(t) for t in _ACTION_EVENTS])))).all() + return next((str(node) for node, data in rows if node and isinstance(data, dict) and data.get("action_id") == rec.id), + None) + + +def _same_payload(rec: ActionRecord, capability: Any, args: dict[str, Any]) -> bool | None: + """这条记录是不是这一次要发的内容:能力相同、内容指纹等于这一次参数收好类型之后的指纹。收不成类型(参数定义改了)的 + 比不了,为 None。""" + name = capability.name + if rec.capability != name: + return False + try: + typed = typed_args(capability, args) + except ActionFailed: + return None + return rec.payload_hash == payload_hash(name, typed) + + +async def _belongs(session: AsyncSession, rec: ActionRecord, node_id: str, capability: Any, args: dict[str, Any], *, + strict: bool = False) -> bool: + """这条记录是不是这个节点(node_id,这次运行里的 id)的。事件认得出是谁写的就按事件;认不出的按内容(能力、内容指纹), + 比不了的当是(以前就是这样认的)。strict(认升级前的旧键时):事件说是这个节点的,内容也要对得上——循环跨了升级时, + 同一个节点第二轮按老算法同样是第 1 次执行,找到的是第一轮那一笔;比不了内容(参数定义改了)的照事件认。""" + owner = await _owner(session, rec) + same = _same_payload(rec, capability, args) + if owner is not None and owner != node_id: + return False + if owner is not None and not strict: + return True + return same is not False + + async def latest(session: AsyncSession, key: str) -> ActionRecord | None: return await session.scalar(select(ActionRecord).where(ActionRecord.action_key == key) .order_by(ActionRecord.attempt.desc()).limit(1)) -async def prepare(*, run_id: str, node_id: str, exec_no: int, capability: Any, args: dict[str, Any]) -> Prepared: - """节点每次执行都先走这里:按 action_key 查记录,决定发不发、第几次发、走哪个环境。 +async def prepare(*, run_id: str, node_id: str, exec_no: int, capability: Any, args: dict[str, Any], + legacy_node_id: str | None = None) -> Prepared: + """节点每次执行都先走这里:按 action_key 查记录,决定发不发、第几次发、走哪个环境。node_id 是这次运行里的节点 id + (子工作流里的带前缀);legacy_node_id 是升级前子工作流里按子图里的 id 记的那种(tools._run_write 交进来),按它也没查到 + 才算头一次。 sent / unknown 抛 ActionPending(不发);failed 且没有人核实为「重新发送」的抛 ActionFailed。先查记录再收类型:已生效 (演练过)的交回存下的回执、已发出 / 结果未知的挂起,都和这一次渲染出来的参数无关——发出之后能力的参数定义改了、手上的 @@ -263,13 +313,29 @@ async def prepare(*, run_id: str, node_id: str, exec_no: int, capability: Any, a run = await session.get(Run, run_id) endpoint = await _endpoint(session, capability) prev = await latest(session, key) + if prev is not None and not await _belongs(session, prev, node_id, capability, args): + # 按自己的键找到的是别的节点的那一笔:升级前子工作流里同名的节点按子图里的 id 记的键,和父图这个节点的一样。 + # 那一笔不是这个节点的,换一个只属于这个节点的键(同样的输入永远算出同一个) + key = action_key(run_id, f"{OWN_KEY_MARK}{node_id}", exec_no) + prev = await latest(session, key) + if prev is None and legacy_node_id: + # 升级前子工作流里的写入按子图里的节点 id 记幂等键(node_id 也记成它):写入节点本身重新执行时(结果未知核实之后、 + # 失败后接着跑)照那个键认出已经记下的那一笔,接着用它的键(不重发、第几次发送接着数)。只认这个节点的、这一次的: + # 那一笔的事件记着是这个节点写的(父图里同名的节点按同一个键写过),内容指纹也和这一次的一样(循环跨了升级时, + # 第二轮按老算法同样是第 1 次执行,找到的是第一轮那一笔) + old_key = action_key(run_id, legacy_node_id, exec_no) + old = await latest(session, old_key) + if old is not None and old.node_id == legacy_node_id \ + and await _belongs(session, old, node_id, capability, args, strict=True): + key, prev, node_id = old_key, old, legacy_node_id facts = {"app_id": run.app_id, "workflow_id": run.workflow_id, "workflow_version": run.version, "graph_hash": run.version_hash, "started_by": run.started_by} if run is not None else {} endpoint_name = endpoint.name if endpoint is not None else None if prev is not None and prev.state in ("confirmed", "simulated"): # 交回的是生效的那一次:指纹、预览、环境、第几次都照记录,不按这一次的参数重算 return Prepared(key=key, attempt=prev.attempt, payload_hash=prev.payload_hash, preview=prev.preview, - environment=prev.environment, endpoint=endpoint_name, receipt=_receipt_of(prev), run=facts) + environment=prev.environment, endpoint=endpoint_name, receipt=_receipt_of(prev), run=facts, + node_id=node_id) if prev is not None and prev.state in ("sent", "unknown"): raise ActionPending(prev.id) formal = run is not None and run.run_class == "formal" @@ -290,15 +356,18 @@ async def prepare(*, run_id: str, node_id: str, exec_no: int, capability: Any, a args = typed_args(capability, args) return Prepared(key=key, attempt=attempt, payload_hash=payload_hash(name, args), preview=masked(args, masked_paths(capability)), environment=environment, endpoint=endpoint_name, - run=facts, args=args) + run=facts, args=args, node_id=node_id) async def send(prep: Prepared, capability: Any, *, run_id: str, node_id: str, exec_no: int, - emit: Callable[..., None], on_behalf: str | dict[str, Any] | None = None) -> dict[str, Any]: + emit: Callable[..., None], on_behalf: str | dict[str, Any] | None = None, + confirm_node: str | None = None) -> dict[str, Any]: """先写记录再发送。返回节点的产出;结果未知抛 ActionPending,确定失败抛 ActionFailed。发的是 prepare 收好类型的 那一份参数(prep.args),和确认卡上的哈希是同一份。 - on_behalf 原样交给平台执行器写身份断言:{"sub": 运行的发起人, "act": 确认人}(capabilities.endpoints.on_behalf_parts)。""" + on_behalf 原样交给平台执行器写身份断言:{"sub": 运行的发起人, "act": 确认人}(capabilities.endpoints.on_behalf_parts)。 + 记录上的节点 id 照 prepare 认定的(prep.node_id);confirm_node 是动作确认待办上记的节点 id(确认卡上的,子工作流里是 + 子图里的 id),不给就按 node_id 找。""" from app.capabilities.registry import endpoint_request, masked_paths name = capability.name @@ -306,11 +375,12 @@ async def send(prep: Prepared, capability: Any, *, run_id: str, node_id: str, ex rec = await session.scalar(select(ActionRecord).where(ActionRecord.idem_key == prep.idem_key)) if rec is None: rec = ActionRecord( - action_key=prep.key, attempt=prep.attempt, idem_key=prep.idem_key, run_id=run_id, node_id=node_id, + action_key=prep.key, attempt=prep.attempt, idem_key=prep.idem_key, run_id=run_id, + node_id=prep.node_id or node_id, exec_no=exec_no, capability=name, capability_version=getattr(capability, "version", None), endpoint_id=getattr(capability, "endpoint_id", None), environment=prep.environment, payload_hash=prep.payload_hash, preview=prep.preview, state="pending", - confirm_task_id=await _confirm_task(session, run_id, node_id), + confirm_task_id=await _confirm_task(session, run_id, confirm_node or node_id), app_id=prep.run.get("app_id"), workflow_id=prep.run.get("workflow_id"), workflow_version=prep.run.get("workflow_version"), graph_hash=prep.run.get("graph_hash")) session.add(rec) @@ -329,6 +399,10 @@ async def send(prep: Prepared, capability: Any, *, run_id: str, node_id: str, ex else "演练:探索运行里端点没有测试环境,没有发送请求") return _output(action_id, prep, "simulated", None, receipt) + # 本进程续不上租约了(cluster/fence.py)就不发:这次运行可能已经被别的进程接手 + from app.cluster import fence + + fence.check() # 先记 sent 再发:发到一半进程没了,重启后看到的是 sent,只能核实,不会当成没发过再发一次 if not await _set(action_id, ("pending",), state="sent", request_at=utcnow()): raise ActionPending(action_id) @@ -449,7 +523,7 @@ async def _confirm_task(session: AsyncSession, run_id: str, node_id: str) -> str return await session.scalar(select(Task.id).where(Task.run_id == run_id, Task.node_id == node_id, Task.kind == "action_confirm") - .order_by(Task.created_at.desc()).limit(1)) + .order_by(Task.created_at.desc(), Task.id.desc()).limit(1)) # -------------------------------------------------------------------------- @@ -651,12 +725,33 @@ async def _lookup(action_id: str, claimed_at: datetime, now: datetime) -> int: return 1 -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """启动时:上次进程没了的时候已经发出(sent)、还没拿到结果的动作收成结果未知,安排核实;结果未知却还没安排核实的 - (挂起之后、安排之前进程没了)补上。返回处理的条数。""" + (挂起之后、安排之前进程没了)补上。返回处理的条数。 + + 多进程:所属的运行还在别的活着的进程里执行的不碰(那边正在发、正在等结果);当班进程隔一会儿来的那一次只收拾停了两个租约 + 时长以上的。""" + from app.cluster import state + from app.db.models import Run + from app.engine.runner import run_manager + + live = live if live is not None else await state.current(fresh=True) async with SessionLocal() as session: - ids = list((await session.execute(select(ActionRecord.id).where( - ActionRecord.state.in_(("sent", "unknown")), ActionRecord.resolution.is_(None)))).scalars()) + rows = (await session.execute(select(ActionRecord.id, ActionRecord.run_id, Run.status, Run.owner) + .outerjoin(Run, Run.id == ActionRecord.run_id).where( + ActionRecord.state.in_(("sent", "unknown")), ActionRecord.resolution.is_(None)))).all() + ids = [action_id for action_id, run_id, status, owner in rows + if not (status in ("running", "queued") and live.alive(owner, local=run_manager.is_active(run_id)))] + if live.trust_mine and ids: + # 当班进程隔一会儿来的那一次:刚挂起、执行它的进程正在安排核实的(收尾和安排核实之间)不碰,只收拾停了两个租约时长以上的 + from datetime import timedelta + + from app.cluster.workers import lease_s + + floor = utcnow() - timedelta(seconds=2 * lease_s()) + async with SessionLocal() as session: + ids = list((await session.execute(select(ActionRecord.id).where( + ActionRecord.id.in_(ids), ActionRecord.updated_at < floor))).scalars()) for action_id in ids: try: await on_unknown(action_id) diff --git a/backend/app/process/hooks.py b/backend/app/process/hooks.py index 8afee1e5..2973a6e8 100644 --- a/backend/app/process/hooks.py +++ b/backend/app/process/hooks.py @@ -251,7 +251,8 @@ async def _started_before(hook: AppHook, idem_key: str) -> str | None: async with SessionLocal() as session: rows = (await session.execute(select(Run.id, Run.origin).where( - Run.app_id == hook.app_id, Run.origin_kind == "hook").order_by(Run.created_at.desc()).limit(500))).all() + Run.app_id == hook.app_id, Run.origin_kind == "hook") + .order_by(Run.created_at.desc(), Run.id.desc()).limit(500))).all() for run_id, origin in rows: if isinstance(origin, dict) and origin.get("hook") == hook.name and origin.get("hook_key") == idem_key: return run_id @@ -267,12 +268,12 @@ async def _signal(hook: AppHook, request: HookRequest, header_key: str, *, recla async with SessionLocal() as session: task_id = await session.scalar(select(Task.id).where( Task.app_id == hook.app_id, Task.kind == "signal", Task.status == "pending", - Task.correlation_key == key).order_by(Task.created_at).limit(1)) + Task.correlation_key == key).order_by(Task.created_at, Task.id).limit(1)) if task_id is None and reclaimed: # 上一次处理到一半服务停了:在等的待办其实已经解决,交回它,不当成没人在等 done = (await session.execute(select(Task.id, Task.run_id).where( Task.app_id == hook.app_id, Task.kind == "signal", Task.status != "pending", - Task.correlation_key == key).order_by(Task.created_at.desc()).limit(1))).first() + Task.correlation_key == key).order_by(Task.created_at.desc(), Task.id.desc()).limit(1))).first() if done is not None: return {"task_id": done[0], "run_id": done[1]} if task_id is None: @@ -409,7 +410,7 @@ async def recent_receipts(session: AsyncSession, app_id: str, name: str, .scalars()) rows = list((await session.execute( select(HookReceipt).where(HookReceipt.hook_id.in_(ids or [""]), HookReceipt.status != "signature") - .order_by(HookReceipt.received_at.desc()).limit(max(1, min(limit, RECEIPTS_MAX))))).scalars()) + .order_by(HookReceipt.received_at.desc(), HookReceipt.id.desc()).limit(max(1, min(limit, RECEIPTS_MAX))))).scalars()) return {"items": [{"idem_key": r.idem_key, "status": r.status, "received_at": r.received_at.isoformat() if r.received_at else None, "run_id": r.run_id, "task_id": r.task_id, "detail": r.detail} for r in rows], diff --git a/backend/app/process/tasks.py b/backend/app/process/tasks.py index 0080edb5..a8bbbb4c 100644 --- a/backend/app/process/tasks.py +++ b/backend/app/process/tasks.py @@ -35,6 +35,7 @@ from app.db.base import SessionLocal, utcnow from app.db.models import ( App, Approval, AppVersion, AuditEvent, Group, GroupMember, Run, Task, TaskAssignment, User, ) +from app.db.portable import json_text from app.engine.lint_process import normalize_policy, parse_duration from app.process.events import record @@ -913,7 +914,7 @@ async def tick(now: datetime | None = None) -> list[str]: """调度循环每一拍:提醒、升级、超时驳回(永不自动批准)。交回这一拍处理过的待办 id(生效可查:办了哪几条, 不只是几条;测试按自己造的那条断言,不受同一个库里别的待办影响)。试运行的待办不处理(只显示时限)。""" now = now or utcnow() - action = func.json_extract(Task.on_timeout, "$.action") + action = json_text(Task.on_timeout, "action") trial = select(Run.id).where(Run.origin_kind == TRIAL_ORIGIN) async with SessionLocal() as session: due = list((await session.execute(select(Task.id).where( @@ -921,7 +922,7 @@ async def tick(now: datetime | None = None) -> list[str]: and_(Task.remind_at.is_not(None), Task.remind_at <= now), and_(Task.due_at.is_not(None), Task.due_at <= now, action == "reject"), and_(Task.due_at.is_not(None), Task.due_at <= now, action == "escalate", Task.escalation_level == 0), - )).order_by(Task.due_at))).scalars()) + )).order_by(Task.due_at, Task.id))).scalars()) done: list[str] = [] for task_id in due: try: @@ -1022,16 +1023,27 @@ def _aware(moment: datetime) -> datetime: # -------------------------------------------------------------------------- -async def startup() -> int: +async def startup(*, live: Any = None) -> int: """启动时:存量的待审批补建 Task(派给任何人,幂等);上次停在 resolving 的(判定之后、恢复之前进程没了)按 Approval - 的状态收拾——还在等的退回 pending、撤回那一票,已经交给引擎的记成已处理。返回补建的条数。""" + 的状态收拾——还在等的退回 pending、撤回那一票,已经交给引擎的记成已处理。返回补建的条数。 + + 多进程:别的进程活着时,resolving 可能正是它在判定、恢复(几秒的事),只收拾停在 resolving 超过两个租约时长的;当班进程隔一会儿 + 来的那一次(live.trust_mine)不论有没有别的进程都这样——本进程自己也可能正在判定。只有启动时、又没有别的进程的那一次全收拾。""" + from datetime import timedelta + + from app.cluster import state + from app.cluster.workers import lease_s + + live = live if live is not None else await state.current(fresh=True) + stale_before = utcnow() - timedelta(seconds=2 * lease_s()) if (live.others or live.trust_mine) else None async with SessionLocal() as session: orphans = list((await session.execute( select(Approval).outerjoin(Task, Task.approval_id == Approval.id) .where(Approval.status == "pending", Task.id.is_(None)))).scalars()) for approval in orphans: await open_task(session, approval) - stuck = list((await session.execute(select(Task).where(Task.status == "resolving"))).scalars()) + stuck = list((await session.execute(select(Task).where( + Task.status == "resolving", *((Task.updated_at < stale_before,) if stale_before is not None else ())))).scalars()) for task in stuck: waiting = task.approval_id and await session.scalar( select(Approval.status).where(Approval.id == task.approval_id)) == "pending" @@ -1093,7 +1105,8 @@ async def _assignments(session: AsyncSession, task_ids: list[str], *, fresh: boo ) -> dict[str, list[TaskAssignment]]: if not task_ids: return {} - stmt = select(TaskAssignment).where(TaskAssignment.task_id.in_(task_ids)).order_by(TaskAssignment.created_at) + stmt = (select(TaskAssignment).where(TaskAssignment.task_id.in_(task_ids)) + .order_by(TaskAssignment.created_at, TaskAssignment.id)) if fresh: stmt = stmt.execution_options(populate_existing=True) out: dict[str, list[TaskAssignment]] = {} @@ -1124,7 +1137,7 @@ def _mine(principal: Any) -> Any: async def list_tasks(principal: Any, *, assigned: str = "me", statuses: list[str] | None = None, app_id: str | None = None, run_id: str | None = None, limit: int = 100) -> list[dict[str, Any]]: async with SessionLocal() as session: - stmt = select(Task).order_by(Task.created_at.desc()).limit(limit) + stmt = select(Task).order_by(Task.created_at.desc(), Task.id.desc()).limit(limit) if statuses: stmt = stmt.where(Task.status.in_(statuses)) if app_id: diff --git a/backend/app/providers/factory.py b/backend/app/providers/factory.py index e7fbb29b..4389225e 100644 --- a/backend/app/providers/factory.py +++ b/backend/app/providers/factory.py @@ -53,7 +53,7 @@ class ProviderNotConfigured(RuntimeError): async def list_providers(session: AsyncSession) -> list[Provider]: - rows = await session.execute(select(Provider).order_by(Provider.created_at)) + rows = await session.execute(select(Provider).order_by(Provider.created_at, Provider.id)) return list(rows.scalars()) diff --git a/backend/app/records/actions.py b/backend/app/records/actions.py index 1b94218a..aa4fb6da 100644 --- a/backend/app/records/actions.py +++ b/backend/app/records/actions.py @@ -79,9 +79,11 @@ async def append_action(args: dict[str, Any], *, idem_key: str | None, run_id: s environment=environment) try: columns = columns_of(table) - done = await asyncio.to_thread( - store.append, path_of(table), columns, values, version=table.version, run_id=run_id, - actor=actor, action_key=action_key_of(idem_key), supersedes=supersedes) + # 多进程时几个进程会往同一个记录表文件里追加:库里的锁(store.write_lock)一个接一个,进程里的 _WRITE_LOCK 照旧 + async with store.write_lock(): + done = await asyncio.to_thread( + store.append, path_of(table), columns, values, version=table.version, run_id=run_id, + actor=actor, action_key=action_key_of(idem_key), supersedes=supersedes) except store.RecordsRefused as e: return _result(None, error=e.message, environment=environment) if not done.duplicate: diff --git a/backend/app/records/store.py b/backend/app/records/store.py index 1a370754..5cd17f62 100644 --- a/backend/app/records/store.py +++ b/backend/app/records/store.py @@ -4,7 +4,8 @@ 业务列(「填什么」卡片生成),外加一列不对外的 _action_key(幂等键,唯一)。查询走连接上的同名临时视图 (view_select),视图不露 _action_key。 -**写。** 只有 append 一条路:单进程锁(_WRITE_LOCK)里开一个短事务,先按 action_key 查——同一个动作第二次来交回 +**写。** 只有 append 一条路:写锁里开一个短事务(进程里的 _WRITE_LOCK;多进程时调用方另持库里的锁 write_lock(),几个进程 +往同一个文件里写一个接一个——SQLite 文件自己的文件锁在网络文件系统上靠不住),先按 action_key 查——同一个动作第二次来交回 第一次写下的那一行,不再写;更正是一次新的追加,带 _supersedes 指向被更正的行,原行不改。_seq 是 SQLite 的 rowid (INTEGER PRIMARY KEY),只增不减:从不删行,也就不会复用。 @@ -49,6 +50,14 @@ _BAD_NAME = re.compile(r"[\"`\[\]\x00-\x1f\x7f]") _WRITE_LOCK = threading.Lock() +def write_lock() -> Any: + """跨进程的写锁(app/cluster/mutex:PostgreSQL 时是咨询锁,SQLite 只有一个进程、就是本进程的锁)。异步的调用方在把写入 + 放进线程之前持着它(async with)。""" + from app.cluster import mutex + + return mutex.lock("records-store") + + class RecordsRefused(ValueError): """这一次写入不收:code 是问题编号(records.*,白话在 app/plain.py),message 直接给人看。""" diff --git a/backend/app/records/tables.py b/backend/app/records/tables.py index 878c7896..b42c805c 100644 --- a/backend/app/records/tables.py +++ b/backend/app/records/tables.py @@ -187,7 +187,8 @@ async def _free_name(session: AsyncSession, base: str) -> str: async def _register(session: AsyncSession, *, key: str, version: int, columns: list[Column], label: str, app_id: str | None, blueprint_id: str | None) -> RecordsTable: path = store.table_path(key, version) - await asyncio.to_thread(store.create_file, path, columns) + async with store.write_lock(): + await asyncio.to_thread(store.create_file, path, columns) base = f"{SOURCE_PREFIX}{re.sub(r'[^a-z0-9]', '', key.lower())[:12] or 'table'}" name = await _free_name(session, base if version == 1 else f"{base}_v{version}") source = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, origin=RECORDS_ORIGIN, @@ -371,7 +372,8 @@ async def _sync_registration(session: AsyncSession, row: RecordsTable, wanted: l have = columns_of(row) known = {h.name.lower() for h in have} added = [c for c in wanted if c.name.lower() not in known] - await asyncio.to_thread(store.add_columns, store.table_path(row.table_key, row.version), added) + async with store.write_lock(): + await asyncio.to_thread(store.add_columns, store.table_path(row.table_key, row.version), added) by_name = {w.name.lower(): w for w in wanted} # 列的顺序:原有的在前(照原来的顺序),新加的在后——和文件里的列序一致。删掉的列(只在不兼容时出现)这里不会有 kept = [h if draft else by_name.get(h.name.lower(), h) for h in have] diff --git a/backend/app/sandbox/bubblewrap_sandbox.py b/backend/app/sandbox/bubblewrap_sandbox.py index 941cecc8..09f13be8 100644 --- a/backend/app/sandbox/bubblewrap_sandbox.py +++ b/backend/app/sandbox/bubblewrap_sandbox.py @@ -11,6 +11,7 @@ import time from functools import lru_cache from pathlib import Path +from app.core import atomic from app.core.config import settings, sanitize_session from app.core.errors import describe_exception, raw from app.sandbox import interpreter @@ -131,9 +132,14 @@ class BubblewrapSandbox(Sandbox): return _apply - def _argv(self, workdir: Path, limits: SandboxLimits, cmd: list[str]) -> list[str]: + def _argv(self, workdir: Path, limits: SandboxLimits, cmd: list[str], + env: dict[str, str] | None = None) -> list[str]: argv = [ self._bwrap or "bwrap", + # 先清空环境变量,再一条条设:bwrap 按参数的先后处理,--clearenv 要在所有 --setenv 前面。以前不清,被执行的 + # 代码看得到后端进程的全部环境变量(模型的 API Key、带密码的 AGENTLAB_DATABASE_URL);local、seatbelt 一直只给 + # 最小的一套 + "--clearenv", "--unshare-all", # mount/pid/ipc/uts/cgroup/net 全部隔离 *(["--share-net"] if limits.network else []), "--die-with-parent", # 父进程没了,沙箱跟着走 @@ -146,8 +152,11 @@ class BubblewrapSandbox(Sandbox): "--setenv", "HOME", "/work", "--setenv", "TMPDIR", "/tmp", "--setenv", "PATH", "/usr/bin:/bin", + "--setenv", "LANG", "C.UTF-8", "--setenv", "PYTHONDONTWRITEBYTECODE", "1", ] + for key, value in (env or {}).items(): + argv += ["--setenv", key, value] for path in _RO_BINDS: if Path(path).exists(): argv += ["--ro-bind", path, path] @@ -176,22 +185,22 @@ class BubblewrapSandbox(Sandbox): entry = entry_name(ext) cmd = [*cmd_prefix, f"/work/{entry}"] workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}") - (workdir / entry).write_text(code) + atomic.write_text(workdir / entry, code) for rel, content in (files or {}).items(): target = (workdir / rel).resolve() if not str(target).startswith(str(workdir.resolve())): return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}") target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content) + atomic.write_text(target, content) - argv = self._argv(workdir, limits, cmd) - for key, value in (env or {}).items(): - argv[1:1] = ["--setenv", key, value] + argv = self._argv(workdir, limits, cmd, env) started = time.perf_counter() try: proc = await asyncio.create_subprocess_exec( *argv, + # bwrap 自己也只拿最小的环境:沙箱里的进程靠上面的 --setenv,不从这里继承 + env={"PATH": os.environ.get("PATH", "/usr/bin:/bin")}, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, preexec_fn=self._preexec(limits), diff --git a/backend/app/sandbox/local_sandbox.py b/backend/app/sandbox/local_sandbox.py index 2215fa03..378e6682 100644 --- a/backend/app/sandbox/local_sandbox.py +++ b/backend/app/sandbox/local_sandbox.py @@ -9,6 +9,7 @@ import sys import time from pathlib import Path +from app.core import atomic from app.core.config import settings, sanitize_session from app.core.errors import describe_exception, raw from app.sandbox import interpreter @@ -107,13 +108,13 @@ class LocalSandbox(Sandbox): ) workdir = self._session_dir(session_id or f"tmp-{int(time.time()*1000)}") - (workdir / entry).write_text(code) + atomic.write_text(workdir / entry, code) for rel, content in (files or {}).items(): target = (workdir / rel).resolve() if not str(target).startswith(str(workdir.resolve())): return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}") target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content) + atomic.write_text(target, content) # 只透传最小环境变量,避免把宿主机的 API key 泄漏给被执行的代码 child_env = { diff --git a/backend/app/sandbox/seatbelt_sandbox.py b/backend/app/sandbox/seatbelt_sandbox.py index 3dcb1c26..221a1499 100644 --- a/backend/app/sandbox/seatbelt_sandbox.py +++ b/backend/app/sandbox/seatbelt_sandbox.py @@ -9,6 +9,7 @@ import sys import time from pathlib import Path +from app.core import atomic from app.core.config import sanitize_session, settings from app.core.errors import describe_exception, raw from app.sandbox import interpreter @@ -194,18 +195,18 @@ class SeatbeltSandbox(Sandbox): ext, cmd_of = _LANG[language] entry = entry_name(ext) workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}") - (workdir / entry).write_text(code) + atomic.write_text(workdir / entry, code) for rel, content in (files or {}).items(): target = (workdir / rel).resolve() if not str(target).startswith(str(workdir.resolve())): return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}") target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content) + atomic.write_text(target, content) self._policies.mkdir(parents=True, exist_ok=True) policy_file = self._policies / f"{workdir.name}-{language}.sb" - policy_file.write_text(build_policy(workdir, language=language, network=limits.network)) + atomic.write_text(policy_file, build_policy(workdir, language=language, network=limits.network)) argv = [SANDBOX_EXEC, "-f", str(policy_file), *cmd_of(), entry] diff --git a/backend/app/scheduler/__init__.py b/backend/app/scheduler/__init__.py index d6a48183..3b4250e8 100644 --- a/backend/app/scheduler/__init__.py +++ b/backend/app/scheduler/__init__.py @@ -34,9 +34,9 @@ async def start(*, interval: float | None = None, config: Settings | None = None if not scheduler_enabled(config): logger.info("定时任务调度已关闭(AGENTLAB_SCHEDULER),不会自动发起运行") return False - from app.scheduler.loop import TICK_SECONDS, runner + from app.scheduler.loop import runner - await runner.start(interval if interval is not None else TICK_SECONDS) + await runner.start(interval if interval is not None else float((config or settings).scheduler_tick_s)) return True diff --git a/backend/app/scheduler/loop.py b/backend/app/scheduler/loop.py index 59948cb6..6c2a9f14 100644 --- a/backend/app/scheduler/loop.py +++ b/backend/app/scheduler/loop.py @@ -9,6 +9,10 @@ skipped_misfire;更早的积压只记一条跳过,不补。下一次一律 事件触发的任务(docs/design/orchestration-round/00-总方案.md §4)不在 fire_due 里:它们没有 next_run_at。每一拍发完到点的, 接着派待派发事件(scheduler/outbox.dispatch_due)。调度关着时这一拍不存在,事件留在表里。 + +**多进程只有一个进程当班**(app/cluster/leader.py):循环在每个进程里都起,每一拍先看本进程当不当班,不当班这一拍什么都 +不做(定时认领、事件派发、流程的时限和核实、Webhook 投递、各类周期清理都只在当班的进程里跑)。当班的进程没了,待命的接班 +之后马上补一拍。切换的那一会儿两边都以为自己当班也重复不了:认领是条件 UPDATE、触发记录和投递记录有唯一约束。 """ from __future__ import annotations @@ -141,16 +145,19 @@ async def process_due(now: datetime | None = None) -> None: """流程的一拍(app/process):待办的提醒、升级、超时驳回(永不自动批准);结果未知的写入按退避去查; 上传了却一直没提交的文件过了时限清掉(app/files,每小时一次);访问密钥的请求记录批量写库、过了保留期的清掉 (auth/keys.py,每小时清一次:每个请求同步写一行会和运行抢 SQLite 的写锁,所以攒到这一拍写);每天到点发待办汇总、 - 访问密钥到期前 7 天和 1 天各提醒一次(notify/digest.py,04 细案 B3:按日期认领、按审计去重,推送放在后台,不拖这一拍)。 + 访问密钥到期前 7 天和 1 天各提醒一次(notify/digest.py,04 细案 B3:按日期认领、按审计去重,推送放在后台,不拖这一拍); + 已结束超过保留期的问数据轮次整轮删事件日志(chat/log.purge_tick,每小时一次)。 各自出错只记日志:这一拍出错不该让调度循环停下,下一拍照常再查。""" from app.auth import keys + from app.chat import log as chat_log from app.files import store as files_store from app.notify import digest from app.process import actions, tasks for name, step in (("待办的时限处理", tasks.tick), ("写入结果的查询", actions.tick), ("没有提交的上传文件清理", files_store.cleanup_tick), ("访问密钥的请求记录", keys.tick), - ("每天的待办汇总", digest.digest_due), ("访问密钥快到期的提醒", digest.keys_due)): + ("每天的待办汇总", digest.digest_due), ("访问密钥快到期的提醒", digest.keys_due), + ("已结束的问数据轮次的事件日志清理", chat_log.purge_tick)): try: done = await step(now) if isinstance(done, list) and done: @@ -175,6 +182,7 @@ class _Runner: #: 通知推送另起任务:Webhook 超时加重试最长要几十秒,不能拖着下一拍的发起。同一时刻只跑一份 self.notify: asyncio.Task[None] | None = None self._stopping: asyncio.Event | None = None + self._poked: asyncio.Event | None = None def running(self) -> bool: return self.task is not None and not self.task.done() @@ -186,10 +194,22 @@ class _Runner: self.task = asyncio.create_task(self._loop(interval, self._stopping), name="agentlab-scheduler") logger.info("定时任务调度已启动,每 %s 秒检查一次", interval) + def poke(self) -> None: + """马上补一拍(刚接了调度班)。""" + if self._poked is not None: + self._poked.set() + async def _loop(self, interval: float, stopping: asyncio.Event) -> None: + from app.cluster import leading + from app.cluster.fence import Fenced from app.scheduler import webhook + self._poked = asyncio.Event() while not stopping.is_set(): + if not leading(): + # 不当班:这一拍什么都不做,等下一拍(或者接班时被叫醒) + await self._wait(stopping, interval) + continue try: await fire_due() await dispatch_events() @@ -199,10 +219,22 @@ class _Runner: self.notify = asyncio.create_task(_notify(webhook), name="agentlab-scheduler-notify") except asyncio.CancelledError: raise - except Exception: # noqa: BLE001 - 这一拍出错不该让循环停下 + except (Exception, Fenced): # noqa: BLE001 - 这一拍出错不该让循环停下(Fenced 是 BaseException:复核 R-b9) logger.exception("定时任务调度这一拍出错,下一拍照常进行") - with contextlib.suppress(TimeoutError): - await asyncio.wait_for(stopping.wait(), timeout=interval) + await self._wait(stopping, interval) + + async def _wait(self, stopping: asyncio.Event, interval: float) -> None: + poked = self._poked + waits = [asyncio.ensure_future(stopping.wait())] + if poked is not None: + waits.append(asyncio.ensure_future(poked.wait())) + try: + await asyncio.wait(waits, timeout=interval, return_when=asyncio.FIRST_COMPLETED) + finally: + for w in waits: + w.cancel() + if poked is not None: + poked.clear() async def stop(self) -> None: """让循环收完手上这一拍再停、正在推送的通知推完;各等一个宽限(STOP_GRACE_S、NOTIFY_GRACE_S),两件事并行 diff --git a/backend/app/scheduler/outbox.py b/backend/app/scheduler/outbox.py index 366051a5..5ae11922 100644 --- a/backend/app/scheduler/outbox.py +++ b/backend/app/scheduler/outbox.py @@ -273,12 +273,10 @@ async def route(now: datetime | None = None) -> int: .limit(ROUTE_BATCH))).scalars()) if not events: return 0 - rows = (await session.execute(select(Schedule.id, Schedule.trigger, Schedule.target_kind).where( - Schedule.trigger_kind == "event", Schedule.enabled.is_(True)))).all() - listeners = [(sid, trigger) for sid, trigger, kind in rows if (kind or "workflow") == "workflow"] + listening = await listeners(session, events) routed = 0 for ev in events: - matched = [sid for sid, trigger in listeners if triggers.matches(trigger, ev.event, ev.subject_id, ev.payload)] + matched = [sid for sid, trigger in listening if triggers.matches(trigger, ev.event, ev.subject_id, ev.payload)] try: async with SessionLocal() as session: # 条件 UPDATE 认领:两拍撞在一起(以后多开了进程)只有一拍改得动这一行 @@ -298,6 +296,28 @@ async def route(now: datetime | None = None) -> int: return routed +def _listening_to(pairs: Any) -> Any: + """触发条件监听着这几对(事件种类, 对象)的事件触发任务的 WHERE 条件:按触发条件里的事件种类和对象筛(SQLAlchemy 的通用 + JSON 访问,各库都能用),不把全部事件触发任务读出来。pairs 是 {事件种类: {对象 id…}}。""" + from sqlalchemy import and_, false, or_ + + conds = [and_(Schedule.trigger["event"].as_string() == kind, + Schedule.trigger[triggers.SUBJECT_KEY[kind]].as_string().in_(sorted(subjects))) + for kind, subjects in pairs.items() if kind in triggers.SUBJECT_KEY and subjects] + return or_(*conds) if conds else false() + + +async def listeners(session: Any, events: list[TriggerEvent]) -> list[tuple[str, dict[str, Any]]]: + """这几条事件可能派给的任务:启用着的、指向工作流的事件触发任务里,触发条件的事件种类和对象对得上的 [(任务 id, 触发条件)]。 + 过滤条件(应用、来源、期次……)由 triggers.matches 逐条再判。""" + pairs: dict[str, set[str]] = {} + for ev in events: + pairs.setdefault(ev.event, set()).add(ev.subject_id) + rows = (await session.execute(select(Schedule.id, Schedule.trigger, Schedule.target_kind).where( + Schedule.trigger_kind == "event", Schedule.enabled.is_(True), _listening_to(pairs)))).all() + return [(sid, trigger) for sid, trigger, kind in rows if (kind or "workflow") == "workflow"] + + async def fire_waiting(now: datetime | None = None) -> list[ScheduleFire]: """有等着的事件的任务,各把到点的去抖窗口处理完。""" now = now or utcnow() @@ -368,7 +388,7 @@ async def _drain(schedule_id: str, now: datetime) -> list[ScheduleFire]: select(TriggerDelivery.id, TriggerDelivery.created_at, TriggerEvent) .join(TriggerEvent, TriggerEvent.id == TriggerDelivery.event_id) .where(TriggerDelivery.schedule_id == schedule_id, TriggerDelivery.status == "pending") - .order_by(TriggerDelivery.occurred_at, TriggerEvent.created_at).limit(DRAIN_BATCH))).all() + .order_by(TriggerDelivery.occurred_at, TriggerEvent.created_at, TriggerEvent.id).limit(DRAIN_BATCH))).all() # 等着的时候改了触发条件(换了对象、加了过滤):已经不合条件的不再派发 stale = [did for did, _, ev in rows if not triggers.matches(trigger, ev.event, ev.subject_id, ev.payload)] if stale: @@ -382,6 +402,7 @@ async def _drain(schedule_id: str, now: datetime) -> list[ScheduleFire]: missed = [w for w in due if (now - w.due).total_seconds() > limit] timely = [w for w in due if w not in missed] outputs = {id(w): await _outputs_of(session, w.events[-1]) for w in timely} + await _clear_of_earlier_fires(schedule_id, timely) out: list[ScheduleFire] = [] if missed: fire = await _skip_missed(schedule_id, missed, tz, grace) @@ -396,6 +417,32 @@ async def _drain(schedule_id: str, now: datetime) -> list[ScheduleFire]: return out +async def _clear_of_earlier_fires(schedule_id: str, wins: list[_Window]) -> None: + """要发起的窗口的结束时刻(fire_at)和这个任务已经有的另一次事件触发撞在同一刻(别的期次的窗口早一拍发了,结束时刻相同) + 时往后挪 1 微秒,直到不撞:service.fire_once 按 (任务, fire_at, kind) 认「已经处理过」,撞上了就交回已有的那一条,这个 + 窗口的几条事件会被记到别人的触发上、不再发起。已有的那次触发里有这个窗口的事件(两拍撞在一起处理了同一个窗口)的不挪: + 那就是同一次,交回已有的那一条是对的。""" + from app.scheduler import service + + if not wins: + return + lo, hi = min(w.due for w in wins), max(w.due for w in wins) + timedelta(seconds=1) + async with SessionLocal() as session: + rows = (await session.execute(select(ScheduleFire.fire_at, ScheduleFire.inputs).where( + ScheduleFire.schedule_id == schedule_id, ScheduleFire.kind == "event", + ScheduleFire.fire_at >= lo, ScheduleFire.fire_at <= hi))).all() + taken: dict[datetime, set[str]] = {} + for at, inputs in rows: + brief = (inputs or {}).get("event") if isinstance(inputs, dict) else None + ids = brief.get("event_ids") if isinstance(brief, dict) else None + taken.setdefault(service.as_utc(at), set()).update(str(i) for i in ids or []) + for w in sorted(wins, key=lambda w: w.due): + mine = {ev.id for ev in w.events} + while w.due in taken and not taken[w.due] & mine: + w.due += timedelta(microseconds=1) + taken.setdefault(w.due, set()).update(mine) + + def _event_fire(events: list[TriggerEvent], tz: str, outputs: dict[str, Any]) -> EventFire: """几条合成一次的事件:内容取最新的那一条,深度取最深的那一条加 1。""" latest = events[-1] @@ -491,17 +538,69 @@ async def _clean(now: datetime) -> None: async def pending_for(session: Any, schedule: Schedule) -> dict[str, Any]: - """一个事件触发任务现在有几条事件在等:events 还没派发的(调度关着时一直留在这里)、waiting 已经派给它、在去抖窗口里的; - due_at 是窗口结束、会合成一次触发的时刻(没有在等的为空)。""" - trigger = schedule.trigger if isinstance(schedule.trigger, dict) else {} - subject, event = triggers.subject_of(trigger), trigger.get("event") or "" - rows = (await session.execute(select(TriggerEvent.payload).where( - TriggerEvent.status == "pending", TriggerEvent.event == event, TriggerEvent.subject_id == subject) - .limit(ROUTE_BATCH))).scalars() if subject else [] - unrouted = sum(1 for payload in rows if triggers.matches(trigger, event, subject, payload)) - waiting, first = (await session.execute(select(func.count(), func.min(TriggerDelivery.occurred_at)).where( - TriggerDelivery.schedule_id == schedule.id, TriggerDelivery.status == "pending"))).one() + """一个事件触发任务现在有几条事件在等(见 pending_for_many)。""" + return (await pending_for_many(session, [schedule]))[schedule.id] + + +async def pending_for_many(session: Any, schedules: list[Schedule]) -> dict[str, dict[str, Any]]: + """几个事件触发任务各有几条事件在等,一次查齐(定时任务列表每行都要,不再每行各查一次):任务 id → {events, waiting, due_at}。 + + events 是还没派发的(调度关着时一直留在这里);waiting 是已经派给它、在去抖窗口里的;due_at 是最早那个窗口结束、会合成 + 一次触发的时刻(没有在等的为空)。等着的时候任务改了条件、已经不合条件的(下一拍会标 dropped、不再派发)不算:窗口按剩下的 + 最早一条起算,和派发时(_drain)切的窗口一样。待派发数按任务分组一条 SQL 算;窗口要看事件内容,等着的那几条一次读出来。""" from app.scheduler import service - due = service.as_utc(first) + timedelta(seconds=triggers.debounce_of(trigger)) if first is not None else None - return {"events": unrouted, "waiting": int(waiting or 0), "due_at": due.isoformat() if due else None} + rows = [s for s in schedules if s is not None] + out = {s.id: {"events": 0, "waiting": 0, "due_at": None} for s in rows} + if not rows: + return out + trig = {s.id: s.trigger if isinstance(s.trigger, dict) else {} for s in rows} + ids = list(out) + counts = dict((await session.execute(select(TriggerDelivery.schedule_id, func.count()).where( + TriggerDelivery.schedule_id.in_(ids), TriggerDelivery.status == "pending") + .group_by(TriggerDelivery.schedule_id))).all()) + # 每个任务各读最早的 DRAIN_BATCH 条(同 _drain 一拍读的):合起来一个上限的话,积压多的任务会把别的挤掉,它们的 due_at 为空 + ranked = (select(TriggerDelivery.id.label("did"), func.row_number().over( + partition_by=TriggerDelivery.schedule_id, + order_by=(TriggerDelivery.occurred_at, TriggerDelivery.created_at, TriggerDelivery.id)).label("rn")) + .where(TriggerDelivery.schedule_id.in_([i for i in ids if counts.get(i)]), TriggerDelivery.status == "pending") + .subquery()) + waiting = (await session.execute( + select(TriggerDelivery.schedule_id, TriggerDelivery.id, TriggerDelivery.created_at, TriggerEvent) + .join(TriggerEvent, TriggerEvent.id == TriggerDelivery.event_id) + .join(ranked, ranked.c.did == TriggerDelivery.id).where(ranked.c.rn <= DRAIN_BATCH) + .order_by(TriggerDelivery.occurred_at, TriggerEvent.created_at))).all() if counts else [] + by: dict[str, list[tuple[str, TriggerEvent, datetime]]] = {} + stale: dict[str, int] = {} + for sid, did, routed, ev in waiting: + if triggers.matches(trig[sid], ev.event, ev.subject_id, ev.payload): + by.setdefault(sid, []).append((did, ev, service.as_utc(routed))) + else: + stale[sid] = stale.get(sid, 0) + 1 + for sid in ids: + out[sid]["waiting"] = max(int(counts.get(sid) or 0) - stale.get(sid, 0), 0) + wins = windows(by.get(sid, []), triggers.debounce_of(trig[sid])) + if wins: + out[sid]["due_at"] = min(w.due for w in wins).isoformat() + pairs: dict[str, set[str]] = {} + for sid in ids: + if subject := triggers.subject_of(trig[sid]): + pairs.setdefault(str(trig[sid].get("event") or ""), set()).add(subject) + if pairs: + from sqlalchemy import and_, or_ + + cond = or_(*[and_(TriggerEvent.event == kind, TriggerEvent.subject_id.in_(sorted(subs))) + for kind, subs in pairs.items()]) + # 每个(事件种类, 对象)各读前 ROUTE_BATCH 条(同改前每个任务各读的那么多):合起来一个上限会把别的对象挤掉、少算 + ranked = (select(TriggerEvent.id.label("eid"), func.row_number().over( + partition_by=(TriggerEvent.event, TriggerEvent.subject_id), + order_by=(TriggerEvent.created_at, TriggerEvent.id)).label("rn")) + .where(TriggerEvent.status == "pending", cond).subquery()) + unrouted = (await session.execute(select(TriggerEvent.event, TriggerEvent.subject_id, TriggerEvent.payload) + .join(ranked, ranked.c.eid == TriggerEvent.id) + .where(ranked.c.rn <= ROUTE_BATCH))).all() + for kind, subject, payload in unrouted: + for sid in ids: + if triggers.matches(trig[sid], kind, subject, payload): + out[sid]["events"] += 1 + return out diff --git a/backend/app/scheduler/service.py b/backend/app/scheduler/service.py index 4b0aa041..e3e0aa3f 100644 --- a/backend/app/scheduler/service.py +++ b/backend/app/scheduler/service.py @@ -380,26 +380,33 @@ def check_fields(template: dict[str, str], fields: list[dict[str, Any]], workflo # 发起一次触发 # -------------------------------------------------------------------------- -#: 每个事件循环一把锁:发起(查重叠 → 记触发 → 发起运行 → 改记录)和通知扫描查库互斥。单进程部署,进程内 -#: 互斥就够;按事件循环分开是因为测试每个用例一个新循环,跨循环用同一把 asyncio.Lock 会报错 -_LOCKS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() +def launch_lock() -> Any: + """发起(查重叠 → 记触发 → 发起运行 → 改记录)和通知扫描查库互斥。跨进程的锁(app/cluster/mutex:本进程按事件循环各一把, + PostgreSQL 时另取咨询锁):多进程时到点的发起在当班进程、「立即运行」在接到请求的进程,照样互斥。""" + from app.cluster import mutex - -def launch_lock() -> asyncio.Lock: - loop = asyncio.get_running_loop() - lock = _LOCKS.get(loop) - if lock is None: - lock = _LOCKS[loop] = asyncio.Lock() - return lock + return mutex.lock("schedule-launch") #: 正在发起的触发:占位记录已写进库、launch_run 还没返回。通知扫描有 launch_lock 挡着,接口的读(任务列表的 -#: last_fire、触发历史)没有:它们据此按「已发起」给,不把还在发起的说成「未能发起」(单进程部署,进程内的集合就够) +#: last_fire、触发历史)没有:它们据此按「已发起」给,不把还在发起的说成「未能发起」 _LAUNCHING: set[str] = set() -def launching(fire_id: str) -> bool: - return fire_id in _LAUNCHING +def launching(fire_id: str, fire: Any = None) -> bool: + """这一次触发是不是还在发起:本进程里在发起;多进程时也可能在别的进程里——占位记录(「未能发起:中断」、还没有运行)写下 + 不到一个租约时长的,按还在发起算(进程真没了的,过了租约就照实显示)。""" + if fire_id in _LAUNCHING: + return True + if fire is None or getattr(fire, "status", None) != "failed_to_start" or getattr(fire, "run_id", None) \ + or getattr(fire, "reason", None) != INTERRUPTED_REASON or getattr(fire, "created_at", None) is None: + return False + from datetime import timedelta + + from app.cluster.workers import lease_s + + created = fire.created_at if fire.created_at.tzinfo else fire.created_at.replace(tzinfo=timezone.utc) + return created > utcnow() - timedelta(seconds=lease_s()) def actor_label(name: str) -> str: @@ -526,7 +533,7 @@ async def _active_run(schedule_id: str) -> tuple[str, str, bool, bool] | None: async with SessionLocal() as session: row = (await session.execute( select(Run.id, Run.status).where(Run.schedule_id == schedule_id, Run.status.in_(ACTIVE_RUN)) - .order_by(Run.created_at.desc()).limit(1))).first() + .order_by(Run.created_at.desc(), Run.id.desc()).limit(1))).first() if row is None: return None run_id, status = row diff --git a/backend/app/scheduler/triggers.py b/backend/app/scheduler/triggers.py index 1a667cf3..f13a4b5e 100644 --- a/backend/app/scheduler/triggers.py +++ b/backend/app/scheduler/triggers.py @@ -248,29 +248,40 @@ async def find_cycle(session: Any, *, schedule_id: str | None, trigger: dict[str start, target = subject_of(trigger), target_workflow_id if start == target: return [start, start] - edges: dict[str, set[str]] = {} - rows = (await session.execute(select(Schedule.id, Schedule.trigger, Schedule.workflow_id, Schedule.target_kind) - .where(Schedule.trigger_kind == "event"))).all() - for sid, other, workflow_id, kind in rows: - if sid == schedule_id or (kind or "workflow") != "workflow" or not isinstance(other, dict): - continue - if other.get("event") == "workflow.finished" and subject_of(other) and workflow_id: - edges.setdefault(subject_of(other), set()).add(workflow_id) - # 从 B 出发找回 A 的路(广度优先,取最短的一条说给人听) - parent: dict[str, str | None] = {target: None} - queue = [target] - while queue: - node = queue.pop(0) - for nxt in sorted(edges.get(node, ())): - if nxt in parent: + + async def edges_from(nodes: list[str]) -> dict[str, set[str]]: + """这几个工作流跑完会触发哪些工作流:只读「上游工作流运行成功结束」、对象是它们的那几个任务(按触发条件里的事件种类和 + 对象筛,SQLAlchemy 的通用 JSON 访问),不把全部事件触发任务读出来。""" + rows = (await session.execute(select(Schedule.id, Schedule.trigger, Schedule.workflow_id, Schedule.target_kind) + .where(Schedule.trigger_kind == "event", + Schedule.trigger["event"].as_string() == "workflow.finished", + Schedule.trigger["workflow_id"].as_string().in_(nodes)))).all() + out: dict[str, set[str]] = {} + for sid, other, workflow_id, kind in rows: + if sid == schedule_id or (kind or "workflow") != "workflow" or not isinstance(other, dict): continue - parent[nxt] = node - if nxt == start: - back = [start] # A ← … ← B,倒着记 - while back[-1] != target: - back.append(parent[back[-1]] or target) - return [start, *reversed(back)] - queue.append(nxt) + if other.get("event") == "workflow.finished" and subject_of(other) and workflow_id: + out.setdefault(subject_of(other), set()).add(workflow_id) + return out + + # 从 B 出发找回 A 的路(广度优先,一层查一次,取最短的一条说给人听) + parent: dict[str, str | None] = {target: None} + frontier = [target] + while frontier: + edges = await edges_from(frontier) + nxt_level: list[str] = [] + for node in frontier: + for nxt in sorted(edges.get(node, ())): + if nxt in parent: + continue + parent[nxt] = node + if nxt == start: + back = [start] # A ← … ← B,倒着记 + while back[-1] != target: + back.append(parent[back[-1]] or target) + return [start, *reversed(back)] + nxt_level.append(nxt) + frontier = nxt_level return None @@ -536,7 +547,7 @@ async def sample(session: Any, trigger: dict[str, Any]) -> Sample: query = query.where(Run.app_id == trigger["app_id"]) if trigger.get("origin_kind"): query = query.where(Run.origin_kind == trigger["origin_kind"]) - candidates = list((await session.execute(query.order_by(Run.finished_at.desc()).limit(SAMPLE_SCAN))) + candidates = list((await session.execute(query.order_by(Run.finished_at.desc(), Run.id.desc()).limit(SAMPLE_SCAN))) .scalars()) if subject else [] # 和真派发同一套条件(matches):缺省不认的途经驳回、应用补算跳过,取合条件的最近一次 for run in candidates: diff --git a/backend/app/scheduler/webhook.py b/backend/app/scheduler/webhook.py index c911b118..2db4eb85 100644 --- a/backend/app/scheduler/webhook.py +++ b/backend/app/scheduler/webhook.py @@ -77,6 +77,8 @@ RERUN_ACTION = "如需重新发起,请在 AgentLab 的「定时任务」中点 REJECTED_HEADLINE = "未出具(被驳回)" REJECTED_STATUS = "被驳回" REJECTED_ACTION = "驳回的说明请在 AgentLab 的「记录」中查看" +#: 照驳回那条路成功结束的(驳回出口):标题、状态后面接这一句 +VIA_REJECTED = "(途经驳回)" # -------------------------------------------------------------------------- @@ -118,6 +120,9 @@ class Notice: error_node: str | None = None #: 运行结束在 cancelled 的原因(Run.end_reason):rejected 时按「未出具(被驳回)」说,error_node 是驳回的那一步 end_reason: str | None = None + #: 照驳回那条路成功结束的运行途经的驳回出口(run.finished 的 rejected_via,engine/runner._rejected_via):标题、状态写 + #: 途经驳回,另起一行写谁被驳回、转到哪一步(同实例页 apps/resolve.rejected_exit_text)。没途经驳回的为 None + rejected_via: list[dict[str, Any]] | None = None next_fire: bool = False #: 事件触发的那一次(fire_kind=event):事件的说法(scheduler/triggers.event_text),不含出口字段的值、变化的内容 trigger_text: str | None = None @@ -130,6 +135,11 @@ class Notice: def rejected(self) -> bool: return self.status == "cancelled" and self.end_reason == "rejected" + @property + def via_rejection(self) -> bool: + """成功结束、途经驳回出口(审批接了驳回出口、按驳回那条路走完):状态是 succeeded,但不能只说「运行完成」。""" + return self.status == "succeeded" and bool(self.rejected_via) + @property def never_started(self) -> bool: """失败的运行从没开始执行(没有开始时间):服务重启时还在排队、被改成失败的就是这种,没有断点可续。""" @@ -160,10 +170,18 @@ def message(notice: Notice) -> tuple[str, list[tuple[str, str]], str | None]: headline = REJECTED_HEADLINE else: headline = _HEADLINE.get(notice.event, notice.event) + if notice.via_rejection: + headline += VIA_REJECTED title = f"定时任务「{notice.schedule_name}」{headline}" lines: list[tuple[str, str]] = [] if notice.status: lines.append(("状态", _status_label(notice))) + if notice.via_rejection: + from app.apps.resolve import rejected_exit_text + + # 一个关卡一句:「「把关」被驳回,转到「退回申请人」」(名字是搭图时起的,不带查询到的数据) + lines.append(("驳回", ";".join(f"「{str(v.get('label') or v.get('node_id') or '')[:80]}」{rejected_exit_text(v)}" + for v in notice.rejected_via or [] if isinstance(v, dict)))) if notice.tier: gaps = f"(缺口 {notice.gaps} 条)" if notice.gaps is not None else "" lines.append(("出具档位", f"{ISSUANCE_LABEL.get(notice.tier, notice.tier)}{gaps}")) @@ -244,6 +262,8 @@ def _status_label(notice: Notice) -> str: return "已挂起,等待核实" if notice.action_pending else "已挂起,可继续运行" if notice.rejected: return REJECTED_STATUS + if notice.via_rejection: + return f"{_STATUS_LABEL['succeeded']}{VIA_REJECTED}" return _STATUS_LABEL.get(notice.status or "", notice.status or "") @@ -426,7 +446,7 @@ async def notify_due(now: datetime | None = None) -> int: and_(ScheduleFire.status == "started", Run.status.in_(_WATCHED), or_(ScheduleFire.notified_status.is_(None), ScheduleFire.notified_status != Run.status)), )) - .order_by(ScheduleFire.created_at) + .order_by(ScheduleFire.created_at, ScheduleFire.id) )).all() if rows: # 链接里的地址:全站同一个对外访问地址(环境变量优先,其次设置,core/public_url.py) @@ -554,12 +574,23 @@ async def _notice(session: Any, schedule: Schedule, fire: ScheduleFire, run: Run error_node=_failed_node(run, await _end_label(session, run)) if run is not None and (run.status == "failed" or run.end_reason == "rejected") else None, end_reason=run.end_reason if run is not None else None, + rejected_via=await _rejected_via_of(session, run) if run is not None and run.status == "succeeded" else None, # 事件触发的任务没有下一次「到点」,启用着就是下一次事件来了照常发起 next_fire=bool(schedule.enabled and (schedule.next_run_at is not None or schedule.trigger_kind == "event")), trigger_text=_trigger_text(fire), ) +async def _rejected_via_of(session: Any, run: Run) -> list[dict[str, Any]] | None: + """这次运行途经的驳回出口:最后一条 run.finished 上的 rejected_via(子工作流里的那几步名字写「子工作流 › x」),没有为 None。""" + data = await session.scalar(select(RunEvent.data).where(RunEvent.run_id == run.id, RunEvent.type == "run.finished") + .order_by(RunEvent.seq.desc()).limit(1)) + via = data.get("rejected_via") if isinstance(data, dict) else None + if not isinstance(via, list): + return None + return [v for v in via if isinstance(v, dict)] or None + + def _trigger_text(fire: ScheduleFire) -> str | None: event = fire.inputs.get("event") if isinstance(fire.inputs, dict) else None text = event.get("text") if isinstance(event, dict) else None diff --git a/backend/app/tools/builtin/files.py b/backend/app/tools/builtin/files.py index 0a7ab4f0..cd988987 100644 --- a/backend/app/tools/builtin/files.py +++ b/backend/app/tools/builtin/files.py @@ -5,6 +5,7 @@ from typing import Any from pydantic import BaseModel, Field +from app.core import atomic from app.core.config import settings from app.tools.registry import ToolContext, register @@ -74,9 +75,12 @@ async def file_write(ctx: ToolContext, **kwargs: Any) -> dict[str, Any]: target = _resolve(args.path, ctx) except ValueError as e: return {"error": str(e)} - target.parent.mkdir(parents=True, exist_ok=True) - with target.open("a" if args.append else "w", encoding="utf-8") as f: - f.write(args.content) + # 覆盖:先写临时文件再改名(core/atomic),读的一方看到的要么是旧的整份、要么是新的整份,权限照旧。追加:原地追加写 + # (O_APPEND),不整份重写——文件的 inode、权限不变,原有内容不动,读的一方最多看到半截新内容 + if args.append: + atomic.append_bytes(target, args.content.encode("utf-8")) + else: + atomic.write_bytes(target, args.content.encode("utf-8")) return {"path": args.path, "bytes": target.stat().st_size, "ok": True} diff --git a/backend/pyproject.toml b/backend/pyproject.toml index d7af0048..dc027c29 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -20,6 +20,9 @@ dependencies = [ # 所以上限卡到下一个小版本——升级要主动改这里、重新锁、跑一遍测试。 # 精确版本在 requirements.lock 里 "langgraph>=1.2.11,<1.3", + # 断点的序列化格式就在这个包里(langgraph 只要求 <5,上面那条卡不住它);SQLite、PostgreSQL 两种断点库 + # 共用这一份格式,copy-to-postgres 搬断点也靠它前后一致 + "langgraph-checkpoint>=4.2.0,<4.3", "langgraph-checkpoint-sqlite>=3.1.1,<3.2", "langchain-core>=1.6.3,<1.7", "langchain-anthropic>=1.7.2,<1.8", @@ -49,6 +52,15 @@ docs = ["pypdf>=5.0", "python-docx>=1.1", "python-pptx>=1.0", "openpyxl>=3.1"] # 数据库接入。三个驱动都是 asyncio 原生且免装数据库客户端—— # oracledb 走 thin 模式,省掉了 Oracle Instant Client 这个历来最劝退的一步。 db = ["aiomysql>=0.2", "oracledb>=2.0", "asyncpg>=0.30"] +# 应用自己的库用 PostgreSQL(AGENTLAB_DATABASE_URL=postgresql+asyncpg://…)。应用库走 asyncpg;LangGraph 的断点 +# 走官方的 langgraph-checkpoint-postgres,它只认 psycopg 3 和它的连接池。psycopg 带 binary:wheel 里自带 libpq, +# 镜像和本机都不用另装 PostgreSQL 客户端。断点格式归 LangGraph 一族管,上限同上卡到下一个小版本 +postgres = [ + "asyncpg>=0.30", + "psycopg[binary]>=3.2,<4", + "psycopg-pool>=3.2,<4", + "langgraph-checkpoint-postgres>=3.1.2,<3.2", +] [build-system] requires = ["hatchling"] diff --git a/backend/pytest.ini b/backend/pytest.ini index b30d93f3..facc6e1c 100644 --- a/backend/pytest.ini +++ b/backend/pytest.ini @@ -3,3 +3,5 @@ asyncio_mode = auto testpaths = tests addopts = -q --tb=short +markers = + sqlite_only: 只对 SQLite 应用库成立的测试(数据目录里的库文件、PRAGMA);AGENTLAB_TEST_DATABASE_URL 跑 PostgreSQL 时跳过 diff --git a/backend/requirements.lock b/backend/requirements.lock index ee31c969..20b8fb3d 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -136,9 +136,13 @@ langgraph==1.2.11 # via agentlab-backend (pyproject.toml) langgraph-checkpoint==4.2.0 # via + # agentlab-backend (pyproject.toml) # langgraph + # langgraph-checkpoint-postgres # langgraph-checkpoint-sqlite # langgraph-prebuilt +langgraph-checkpoint-postgres==3.1.2 + # via agentlab-backend (pyproject.toml) langgraph-checkpoint-sqlite==3.1.1 # via agentlab-backend (pyproject.toml) langgraph-prebuilt==1.1.0 @@ -167,6 +171,7 @@ oracledb==26.0.0 # via agentlab-backend (pyproject.toml) orjson==3.12.0 # via + # langgraph-checkpoint-postgres # langgraph-sdk # langsmith ormsgpack==1.12.2 @@ -180,6 +185,16 @@ pillow==12.3.0 # via python-pptx pluggy==1.6.0 # via pytest +psycopg==3.3.6 + # via + # agentlab-backend (pyproject.toml) + # langgraph-checkpoint-postgres +psycopg-binary==3.3.6 ; implementation_name != 'pypy' + # via psycopg +psycopg-pool==3.3.3 + # via + # agentlab-backend (pyproject.toml) + # langgraph-checkpoint-postgres pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' # via cffi pydantic==2.13.5 @@ -294,6 +309,8 @@ typing-extensions==4.16.0 # mcp # openai # oracledb + # psycopg + # psycopg-pool # pydantic # pydantic-core # pytest-asyncio @@ -309,6 +326,8 @@ typing-inspection==0.4.4 # mcp # pydantic # pydantic-settings +tzdata==2026.5 ; sys_platform == 'win32' + # via psycopg urllib3==2.7.0 # via requests uuid-utils==0.17.1 diff --git a/backend/tests/appdb.py b/backend/tests/appdb.py new file mode 100644 index 00000000..fa3c643a --- /dev/null +++ b/backend/tests/appdb.py @@ -0,0 +1,102 @@ +"""测试里另开一条连接直接改应用库:模拟别处(另一个进程、恢复、继续运行)抢在中间提交。 + +应用库是 SQLite 时用 sqlite3 连数据目录里的 agentlab.db(同以前各处的写法);是 PostgreSQL 时(AGENTLAB_TEST_DATABASE_URL, +见 conftest)用 psycopg 连同一个库。SQL 一律用 ? 占位,两种库写成同一句;锁等 5 秒(同以前 sqlite3 的 timeout=5): +另一条连接要等的锁正好被调用方拿着的话,等满报错,不会把测试卡死。 +""" + +from __future__ import annotations + +import contextlib +import re +from typing import Any + +from app.core.config import settings + + +def is_postgres() -> bool: + return settings.db_dialect == "postgresql" + + +def _pg_conninfo() -> str: + from app.db.urls import libpq_conninfo + + return libpq_conninfo(settings.db_url) + + +def execute(*statements: str | tuple[str, tuple[Any, ...]]) -> None: + """在另一条连接上依次执行、一起提交。每一项是 SQL,或者 (SQL, 参数)。""" + items = [(s, ()) if isinstance(s, str) else (s[0], tuple(s[1])) for s in statements] + if is_postgres(): + import psycopg + + with psycopg.connect(_pg_conninfo(), options="-c lock_timeout=5000") as conn: + for sql, params in items: + conn.execute(re.sub(r"\?", "%s", sql), params) + conn.commit() + return + import sqlite3 + + db = sqlite3.connect(settings.db_path, timeout=5) + try: + for sql, params in items: + db.execute(sql, params) + db.commit() + finally: + db.close() + + +def query(sql: str, *params: Any) -> list[tuple[Any, ...]]: + """在另一条连接上读:看别人这时提交了什么(不是调用方事务里的)。""" + if is_postgres(): + import psycopg + + with psycopg.connect(_pg_conninfo(), options="-c lock_timeout=5000") as conn: + return [tuple(r) for r in conn.execute(re.sub(r"\?", "%s", sql), params).fetchall()] + import sqlite3 + + db = sqlite3.connect(settings.db_path, timeout=5) + try: + return [tuple(r) for r in db.execute(sql, params).fetchall()] + finally: + db.close() + + +def refuse_inserts(table: str, condition: str, *, name: str) -> tuple[list[str], list[str]]: + """建一个触发器:往 table 插入满足 condition(用 NEW.列 写)的行时报错,报的是完整性错误(IntegrityError),同 SQLite + 的 RAISE(ABORT)。返回 (建的语句, 删的语句),各是一串语句,逐条在应用库的连接上执行。 + + PostgreSQL 要先建一个触发器函数(plpgsql)再挂上去,错误码给 23000(integrity_constraint_violation)。""" + if is_postgres(): + create = [f"CREATE FUNCTION {name}_fn() RETURNS trigger AS $$ BEGIN IF {condition} THEN " + f"RAISE EXCEPTION 'refused by test' USING ERRCODE = 'integrity_constraint_violation'; END IF; " + f"RETURN NEW; END $$ LANGUAGE plpgsql", + f"CREATE TRIGGER {name} BEFORE INSERT ON {table} FOR EACH ROW EXECUTE FUNCTION {name}_fn()"] + drop = [f"DROP TRIGGER IF EXISTS {name} ON {table}", f"DROP FUNCTION IF EXISTS {name}_fn()"] + return create, drop + return ([f"CREATE TRIGGER {name} BEFORE INSERT ON {table} WHEN {condition} " + f"BEGIN SELECT RAISE(ABORT, 'refused by test'); END"], [f"DROP TRIGGER IF EXISTS {name}"]) + + +def admin_conninfo() -> str: + """测试用的 PostgreSQL 服务器上能建库删库的连接串(就是这个 xdist 进程自己的库,建删别的库用)。""" + return _pg_conninfo() + + +@contextlib.contextmanager +def fresh_postgres_database(tag: str): + """同一台测试服务器上另建一个空库,交回它的地址(postgresql+asyncpg://…),用完删掉。只在 PostgreSQL 时能用。""" + import uuid + + import psycopg + from sqlalchemy.engine import make_url + + url = make_url(settings.db_url) + name = f"{url.database}_{tag}_{uuid.uuid4().hex[:8]}" + with psycopg.connect(_pg_conninfo(), autocommit=True) as conn: + conn.execute(f'CREATE DATABASE "{name}"') + try: + yield url.set(database=name).render_as_string(hide_password=False) + finally: + with psycopg.connect(_pg_conninfo(), autocommit=True) as conn: + conn.execute(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)') diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index c8f7d71d..d4be71bf 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -12,19 +12,74 @@ app.db.base 导入时就按 db_url 建好了。conftest 早于测试模块加载 from __future__ import annotations +import atexit import os import tempfile from pathlib import Path _TMP = Path(tempfile.mkdtemp(prefix="agentlab-tests-")) os.environ["AGENTLAB_DATA_DIR"] = str(_TMP) +# 下面这几项也都要在任何 app.* 导入之前设(两种库同一个顺序):PostgreSQL 分支要先导入 app.db.base 装连接池,settings 在那一刻 +# 就按当时的环境定下来了——以前这两项写在那之后,PostgreSQL 上 AGENTLAB_SCHEDULER 没生效(scheduler_on 成了 True) # 别把真实的 key 带进测试环境 os.environ.setdefault("AGENTLAB_SECRET_KEY", "test-only-not-a-real-key") # 定时任务的调度循环不在测试里起:走 lifespan 的测试不该到点替人发起运行。要测调度的直接调它 os.environ.setdefault("AGENTLAB_SCHEDULER", "0") + +def _postgres_database(admin_url: str) -> str: + """AGENTLAB_TEST_DATABASE_URL 给的是管理用的库:每个 xdist 进程在同一台服务器上建一个自己的空库(库名加进程号后缀, + 已有的同名库先删掉),进程退出时删掉。返回这个库的地址。scripts/test-postgres.sh 起临时容器、设好这个变量。 + 这里导入的 app.db.urls 只用 SQLAlchemy 的地址解析,不碰 settings(settings 要等环境变量都设好)。""" + import psycopg + from sqlalchemy.engine import make_url + + from app.db.urls import libpq_conninfo, normalize_database_url + + url = make_url(normalize_database_url(admin_url)) + name = f"{url.database}_{os.environ.get('PYTEST_XDIST_WORKER', 'main')}" + admin = libpq_conninfo(url.render_as_string(hide_password=False)) + + def run(sql: str) -> None: + with psycopg.connect(admin, autocommit=True) as conn: + conn.execute(sql) + + run(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)') + run(f'CREATE DATABASE "{name}"') + + def drop() -> None: + try: + run(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)') + except Exception: # noqa: BLE001 - 服务器先走了(容器已删)就算了 + pass + + atexit.register(drop) + return url.set(database=name).render_as_string(hide_password=False) + + +# 应用库:缺省数据目录里的 SQLite。设了 AGENTLAB_TEST_DATABASE_URL 就跑在 PostgreSQL 上。不管哪种,都盖掉环境和 .env +# 里可能有的 AGENTLAB_DATABASE_URL / AGENTLAB_CHECKPOINT_URL:测试绝不能连到真在用的库上 +_TEST_PG = os.environ.get("AGENTLAB_TEST_DATABASE_URL", "").strip() +os.environ["AGENTLAB_CHECKPOINT_URL"] = "" +os.environ["AGENTLAB_DATABASE_URL"] = _postgres_database(_TEST_PG) if _TEST_PG else "" +if _TEST_PG: + # 每个测试一个事件循环(pytest-asyncio 的缺省),asyncpg 的连接不能跨循环用:引擎先按不用连接池建, + # 再换成按事件循环分的池子(tests/pgpool.py),一个测试里的查库照样复用连接 + os.environ["AGENTLAB_DB_POOL_SIZE"] = "0" + + from app.db.base import engine as _app_engine # noqa: E402 + from tests.pgpool import install as _loop_local_pool # noqa: E402 + + _loop_local_pool(_app_engine) + import pytest # noqa: E402 +# 整个 JSON 值做比较、去重、分组、排序时编译就报错(tests/jsonguard.py):PostgreSQL 的 json 不支持,SQLite 的测试也拦下 +from app.db.base import engine as _guarded_engine # noqa: E402 +from tests.jsonguard import install as _json_guard # noqa: E402 + +_json_guard(_guarded_engine.sync_engine.dialect) + @pytest.fixture(scope="session", autouse=True) def _prepare_schema(): @@ -42,7 +97,18 @@ def _prepare_schema(): def pytest_report_header(config) -> str: - return f"测试数据目录:{_TMP}" + where = "PostgreSQL(AGENTLAB_TEST_DATABASE_URL)" if _TEST_PG else "SQLite" + return f"测试数据目录:{_TMP};应用库:{where}" + + +def pytest_collection_modifyitems(config, items) -> None: + """标了 sqlite_only 的测试只在应用库是 SQLite 时跑:它们断言的是 SQLite 自己的东西(数据目录里的库文件、PRAGMA)。""" + if not _TEST_PG: + return + skip = pytest.mark.skip(reason="只对 SQLite 应用库成立") + for item in items: + if "sqlite_only" in item.keywords: + item.add_marker(skip) @pytest.fixture(scope="session") diff --git a/backend/tests/fake_docker.py b/backend/tests/fake_docker.py new file mode 100644 index 00000000..9ff3cd24 --- /dev/null +++ b/backend/tests/fake_docker.py @@ -0,0 +1,368 @@ +"""给 deploy/deploy.sh 的测试用的假 docker(tests/test_deploy_script.py):不碰本机的 docker,状态放一个 JSON 文件里。 + +只认 deploy.sh 用到的那些子命令和 inspect 格式,认不出的直接报错退出(测试就会失败),免得假的悄悄放过脚本的新用法。 +每次调用记一行 JSON(argv)到 FAKE_DOCKER_LOG,测试按它核对先后顺序和参数。用完就删的容器(run --rm)里要做的事照做: +python -m app.ops 真跑(/data 换成挂进来的目录),chown 不做,mkdir、pg_dump 在目录里建文件,copy-to-postgres 按 +FAKE_COPY_FAIL 成功或失败。 + +几个开关(环境变量): + FAKE_COPY_FAIL=dry|real copy-to-postgres 的核对(--dry-run)或正式导入失败 + FAKE_PGDUMP_FAIL=1 pg_dump 失败 + FAKE_PSQL_FAIL=1 docker exec … psql 失败 + FAKE_UNHEALTHY_ENV=<变量名> 带这个环境变量(-e)起的容器健康检查不过 + FAKE_UNHEALTHY_IMAGE=<镜像> 用这个镜像起的容器健康检查不过 +""" + +from __future__ import annotations + +import json +import os +import platform +import sys +from pathlib import Path + +BACKEND = Path(__file__).resolve().parents[1] +# 测试里 import 这个模块只为 host_arch,不设这两个变量 +STATE = Path(os.environ.get("FAKE_DOCKER_STATE", "fake-docker-state.json")) +LOG = Path(os.environ.get("FAKE_DOCKER_LOG", "fake-docker-calls.jsonl")) + +VALUE_OPTS = { + "--name", "--label", "--restart", "--network", "-e", "--env", "-v", "--volume", "-p", "--publish", "--env-file", + "--entrypoint", "--user", "-u", "--health-cmd", "--health-interval", "--health-timeout", "--health-retries", + "--health-start-period", "--stop-timeout", "--log-opt", "--security-opt", "--device", "--group-add", "--tmpfs", +} +FLAG_OPTS = {"-d", "--detach", "--rm", "-i", "-t", "-it"} + + +def host_arch() -> str: + machine = platform.machine().lower() + return {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}.get(machine, machine) + + +def load() -> dict: + if STATE.exists(): + return json.loads(STATE.read_text()) + return {"images": {}, "containers": {}, "networks": [], "volumes": []} + + +def save(state: dict) -> None: + STATE.write_text(json.dumps(state, ensure_ascii=False, indent=1)) + + +def fail(message: str, code: int = 1) -> None: + print(message, file=sys.stderr) + sys.exit(code) + + +def unknown(argv: list[str]) -> None: + fail(f"fake docker: 认不出 {argv!r}", 99) + + +def image_info(state: dict, ref: str) -> dict | None: + return state["images"].get(ref) + + +def container_health(c: dict) -> str: + if c["state"] != "running": + return c.get("health", "") + bad_env = os.environ.get("FAKE_UNHEALTHY_ENV") + if bad_env and bad_env in c["env"]: + return "unhealthy" + if os.environ.get("FAKE_UNHEALTHY_IMAGE") and c["image"] == os.environ["FAKE_UNHEALTHY_IMAGE"]: + return "unhealthy" + return "healthy" + + +def fmt_container(c: dict, fmt: str) -> str: + if fmt == "{{.State.Status}}": + return c["state"] + if fmt == "{{.State.Running}}": + return "true" if c["state"] == "running" else "false" + if fmt == "{{.Config.Image}}": + return c["image"] + if ".Mounts" in fmt and '"/data"' in fmt: + return c["mounts"].get("/data", "") + if ".State.Health" in fmt: + return container_health(c) + if '"agentlab.role"' in fmt: + return c["labels"].get("agentlab.role", "") + if ".HostConfig.PortBindings" in fmt: + return "".join(p.rsplit(":", 2)[-2] for p in c["ports"]) + raise KeyError(fmt) + + +def fmt_image(img: dict, fmt: str) -> str: + if fmt == "{{.Architecture}}": + return img["arch"] + if fmt == "{{.Os}}/{{.Architecture}}": + return f"linux/{img['arch']}" + if fmt == "{{.Config.User}}": + return img.get("user", "") + if '"agentlab.microvm"' in fmt: + return img.get("microvm", "0") + raise KeyError(fmt) + + +def parse_run(args: list[str]) -> tuple[dict, list[str]]: + opts: dict = {"env": {}, "labels": {}, "mounts": {}, "ports": [], "flags": set(), "volumes": []} + i = 0 + while i < len(args): + a = args[i] + if a in FLAG_OPTS: + opts["flags"].add(a) + i += 1 + continue + if a.startswith("-") and "=" in a and a.split("=", 1)[0] in VALUE_OPTS: + a, value = a.split("=", 1) + i += 1 + elif a in VALUE_OPTS: + value = args[i + 1] + i += 2 + elif a.startswith("-"): + unknown(["run", *args]) + else: + break + if a in ("-e", "--env"): + key, _, val = value.partition("=") + opts["env"][key] = val + elif a == "--label": + key, _, val = value.partition("=") + opts["labels"][key] = val + elif a in ("-v", "--volume"): + src, dst = value.split(":")[:2] + opts["mounts"][dst] = src + if not src.startswith("/"): + opts["volumes"].append(src) + elif a in ("-p", "--publish"): + opts["ports"].append(value) + else: + opts[a] = value + return opts, args[i:] + + +def to_host(arg: str, data: str | None) -> str: + if data and (arg == "/data" or arg.startswith("/data/")): + return data + arg[len("/data"):] + return arg + + +def run_oneoff(state: dict, opts: dict, image: str, command: list[str]) -> int: + data = opts["mounts"].get("/data") + entry = opts.get("--entrypoint") + if entry == "python" and command[:2] == ["-m", "app.ops"]: + sys.path.insert(0, str(BACKEND)) + from app import ops + + rest = [to_host(a, data) for a in command[2:]] + if rest and rest[0] in ("postgres-secrets", "state", "backup-sqlite", "backup-dir") and "--data" not in rest: + rest[1:1] = ["--data", data or "/data"] + return ops.main(rest) + if entry == "python" and command[:3] == ["-m", "app.db", "copy-to-postgres"]: + which = "dry" if "--dry-run" in command else "real" + if os.environ.get("FAKE_COPY_FAIL") == which: + print("没有导入:fake 的失败", file=sys.stderr) + return 1 + print("核对通过") + return 0 + if entry == "chown": + return 0 + if entry == "mkdir": + for a in command: + if not a.startswith("-"): + Path(to_host(a, data)).mkdir(parents=True, exist_ok=True) + return 0 + if entry == "sh" and command[:1] == ["-c"] and 'pg_dump "$@"' in command[1]: + # sh -c '<脚本>' pg_dump <参数…>:照脚本里的 umask 建文件 + umask = 0o077 if "umask 077" in command[1] else 0o022 + if os.environ.get("FAKE_PGDUMP_FAIL"): + print("pg_dump: fake 的失败", file=sys.stderr) + return 1 + args = command[3:] + out = Path(to_host(args[args.index("-f") + 1], data)) + fd = os.open(out, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666 & ~umask) + with os.fdopen(fd, "wb") as fh: + fh.write(b"PGDMP fake") + return 0 + unknown(["run", "--rm", image, *command]) + return 99 + + +def main(argv: list[str]) -> int: + with LOG.open("a") as fh: + fh.write(json.dumps(argv, ensure_ascii=False) + "\n") + state = load() + if not argv: + unknown(argv) + cmd, args = argv[0], argv[1:] + + if cmd == "version": + print("27.0.0-fake") + return 0 + + if cmd == "image" and args[:1] == ["inspect"]: + fmt = None + rest = args[1:] + if rest[:1] == ["-f"]: + fmt, rest = rest[1], rest[2:] + img = image_info(state, rest[0]) + if img is None: + fail(f"Error: No such image: {rest[0]}") + if fmt: + try: + print(fmt_image(img, fmt)) + except KeyError: + unknown(argv) + else: + print("[{}]") + return 0 + + if cmd == "load": + tar = Path(args[args.index("-i") + 1]) + for line in tar.read_text().splitlines(): + ref, _, user = line.partition(" ") + if ref: + state["images"][ref] = {"arch": host_arch(), "user": user, "microvm": "0"} + print(f"Loaded image: {ref}") + save(state) + return 0 + + if cmd == "rmi": + for ref in args: + state["images"].pop(ref, None) + save(state) + return 0 + + if cmd == "container" and args[:1] == ["inspect"]: + return 0 if args[1] in state["containers"] else 1 + + if cmd == "inspect": + if args[:1] != ["-f"]: + unknown(argv) + fmt, name = args[1], args[2] + c = state["containers"].get(name) + if c is None: + fail(f"Error: No such object: {name}") + try: + print(fmt_container(c, fmt)) + except KeyError: + unknown(argv) + return 0 + + if cmd == "run": + opts, rest = parse_run(args) + image, command = rest[0], rest[1:] + if image_info(state, image) is None: + fail(f"Unable to find image '{image}' locally", 125) + network = opts.get("--network") + if network and network not in ("none", "host", "bridge") and network not in state["networks"]: + fail(f"docker: Error response from daemon: network {network} not found.", 125) + if "--rm" in opts["flags"]: + return run_oneoff(state, opts, image, command) + name = opts.get("--name") + if not name or "-d" not in opts["flags"]: + unknown(argv) + if name in state["containers"]: + fail(f'docker: Error response from daemon: Conflict. The container name "/{name}" is already in use.', 125) + for vol in opts["volumes"]: + if vol not in state["volumes"]: + state["volumes"].append(vol) + state["containers"][name] = { + "image": image, "state": "running", "env": opts["env"], "labels": opts["labels"], + "mounts": opts["mounts"], "ports": opts["ports"], "network": network or "bridge", + } + save(state) + print("f" * 64) + return 0 + + if cmd in ("start", "stop", "restart"): + names = [a for a in args if not a.startswith("-") and not a.isdigit()] + for name in names: + c = state["containers"].get(name) + if c is None: + fail(f"Error response from daemon: No such container: {name}") + c["state"] = "exited" if cmd == "stop" else "running" + save(state) + return 0 + + if cmd == "rm": + force = "-f" in args + for name in (a for a in args if not a.startswith("-")): + c = state["containers"].get(name) + if c is None: + fail(f"Error response from daemon: No such container: {name}") + if c["state"] == "running" and not force: + fail(f"Error response from daemon: cannot remove container {name}: container is running") + del state["containers"][name] + save(state) + return 0 + + if cmd == "rename": + old, new = args + if old not in state["containers"] or new in state["containers"]: + fail(f"Error response from daemon: cannot rename {old} to {new}") + state["containers"][new] = state["containers"].pop(old) + save(state) + return 0 + + if cmd == "exec": + name, command = args[0], args[1:] + c = state["containers"].get(name) + if c is None or c["state"] != "running": + fail(f"Error response from daemon: container {name} is not running") + if command[:1] == ["psql"]: + return 1 if os.environ.get("FAKE_PSQL_FAIL") else 0 + if command[:1] == ["python"]: + print("子进程(fake)") + return 0 + if command[:1] == ["pg_isready"]: + return 0 + unknown(argv) + + if cmd == "logs": + print("fake 的日志") + return 0 + + if cmd == "port": + c = state["containers"].get(args[0]) + if c is None: + return 1 + for p in c["ports"]: + host_part = p.rsplit(":", 1)[0] + print(host_part) + return 0 + + if cmd == "network": + sub, name = args[0], args[-1] + if sub == "inspect": + return 0 if name in state["networks"] else 1 + if sub == "create": + if name in state["networks"]: + fail(f"network with name {name} already exists") + state["networks"].append(name) + save(state) + return 0 + if sub == "rm": + if any(c["network"] == name for c in state["containers"].values()): + fail(f"error while removing network: network {name} has active endpoints") + state["networks"].remove(name) + save(state) + return 0 + unknown(argv) + + if cmd == "volume": + sub, name = args[0], args[-1] + if sub == "inspect": + return 0 if name in state["volumes"] else 1 + if sub == "create": + if name not in state["volumes"]: + state["volumes"].append(name) + save(state) + return 0 + unknown(argv) + + unknown(argv) + return 99 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/backend/tests/fixtures/compare_runs.py b/backend/tests/fixtures/compare_runs.py index 9e0d3944..455f28d5 100644 --- a/backend/tests/fixtures/compare_runs.py +++ b/backend/tests/fixtures/compare_runs.py @@ -187,7 +187,7 @@ class Fake: return artifact def side(self): - def sources(started): + def sources(started, card_node=""): outputs = {} for seq, etype, nid, data in self.events: if seq < started and etype == "node.finished" and data.get("artifact"): diff --git a/backend/tests/fixtures/dual_check_shop.py b/backend/tests/fixtures/dual_check_shop.py index 6f67f5b2..2b07ba76 100644 --- a/backend/tests/fixtures/dual_check_shop.py +++ b/backend/tests/fixtures/dual_check_shop.py @@ -19,7 +19,6 @@ from __future__ import annotations import json import re import sqlite3 -import uuid from pathlib import Path from langchain_core.messages import AIMessage, ToolMessage @@ -27,6 +26,7 @@ from langchain_core.messages import AIMessage, ToolMessage from app.data import introspect from app.db.base import SessionLocal from app.db.models import DataSource +from tests.fixtures.names import letters ROWS = [ (1, "华东", 101, 100.0, "paid", "2026-08-31", "2026-09-01", "online", 0), @@ -67,7 +67,7 @@ def build(path: Path, *, date_type: str = "DATE") -> Path: async def add_source(path: Path, *, options: dict | None = None) -> tuple[str, str]: """登记成只读的手工源,探查好表结构。返回 (数据源名, id)。""" - name = f"dc_shop_{uuid.uuid4().hex[:8]}" + name = f"dc_shop_{letters(8)}" async with SessionLocal() as session: row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True, options=dict(options or {})) @@ -135,7 +135,7 @@ class Blind: hook, blind.on_query = blind.on_query, None hook() call = {"name": f"db_query__{blind.source}", "args": {"sql": sqls[done]}, - "id": f"blind-{uuid.uuid4().hex[:6]}"} + "id": f"blind-{letters(6)}"} return AIMessage(content="查一下。", tool_calls=[call]) return AIMessage(content="查完了。") blind.writer_prompts.append(text) diff --git a/backend/tests/fixtures/names.py b/backend/tests/fixtures/names.py new file mode 100644 index 00000000..91910ca4 --- /dev/null +++ b/backend/tests/fixtures/names.py @@ -0,0 +1,19 @@ +"""测试里随机起的名字不带数字。 + +名字会拼进导出的文件、通知、任务说明这类文本,而不少断言写的是「文本里不许出现某个数」(不许出现 330、300、48……): +随机的十六进制里带着那几个数字的概率不大但不是零(8 位里出现「330」约千分之一点五),一撞上就无缘无故地失败。 +十六进制里的数字换成字母(同 test_run_data_versions 的写法),随机性来自 uuid4(不受 pytest-randomly 每个测试重设 +random 种子的影响,不会在不同测试里撞名)。 +""" + +from __future__ import annotations + +import uuid + +#: 十六进制里的数字换成字母:名字里不带数字,数字的断言才只看得见数据 +NO_DIGITS = str.maketrans("0123456789", "ghijklmnop") + + +def letters(n: int = 8) -> str: + """n 个随机小写字母(a–p),n 最多 32。""" + return uuid.uuid4().hex[:n].translate(NO_DIGITS) diff --git a/backend/tests/jsonguard.py b/backend/tests/jsonguard.py new file mode 100644 index 00000000..6dbb11e3 --- /dev/null +++ b/backend/tests/jsonguard.py @@ -0,0 +1,63 @@ +"""测试时挂在应用库编译器上的守卫:整个 JSON 值做比较、去重、分组、排序就报错(两种库都挂,跑全量时一直生效)。 + +PostgreSQL 的 json 没有相等和大小的运算符:`=`、`!=`、`IN`、`IS DISTINCT FROM`、`<` 之类,SELECT DISTINCT、GROUP BY、 +ORDER BY、UNION(去重的那几种)碰到 json 类型的值都报错;SQLite 照常执行。只在 PostgreSQL 上跑到才发现太晚:这里在编译 +那一刻就拦下,SQLite 的测试也拦得到。按表达式的类型认,不按写法认:跨行的 order_by、变量别名、 +`select(Run.origin).distinct()`、JSON 的下标(`Run.origin["k"] == 1`,取出来的还是 json)都算。按键筛选用 db/portable 的 +json_eq 等(类型是布尔、文本),不受影响。 +""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import JSON +from sqlalchemy.sql import operators +from sqlalchemy.sql.elements import UnaryExpression + + +class JsonMisuse(AssertionError): + """整个 JSON 值被拿去比较、去重、分组或排序。""" + + +_COMPARE = {operators.eq, operators.ne, operators.in_op, operators.not_in_op, operators.is_distinct_from, + operators.is_not_distinct_from, operators.lt, operators.le, operators.gt, operators.ge} + + +def _json(element: Any) -> bool: + while isinstance(element, UnaryExpression) and element.modifier is not None: + element = element.element + return isinstance(getattr(element, "type", None), JSON) + + +class JsonGuard: + """混进编译器类里用:class Guarded(JsonGuard, 原来的编译器)。""" + + def visit_binary(self, binary: Any, **kw: Any) -> str: + if binary.operator in _COMPARE and (_json(binary.left) or _json(binary.right)): + raise JsonMisuse(f"整个 JSON 值做了比较({binary.operator.__name__}):PostgreSQL 的 json 没有这个运算符," + "按键筛选用 db/portable.json_eq 等") + return super().visit_binary(binary, **kw) # type: ignore[misc] + + def visit_select(self, select: Any, **kw: Any) -> str: + if select._distinct and any(_json(c) for c in select.selected_columns): + raise JsonMisuse("SELECT DISTINCT 里有 JSON 列:PostgreSQL 的 json 没有相等运算符,去不了重") + if any(_json(c) for c in select._group_by_clauses): + raise JsonMisuse("GROUP BY 了 JSON 值:PostgreSQL 的 json 没有相等运算符") + if any(_json(c) for c in select._order_by_clauses): + raise JsonMisuse("ORDER BY 了 JSON 值:PostgreSQL 的 json 没有大小") + return super().visit_select(select, **kw) # type: ignore[misc] + + def visit_compound_select(self, cs: Any, **kw: Any) -> str: + keyword = str(getattr(cs.keyword, "value", cs.keyword)).upper() + if "ALL" not in keyword and any(_json(c) for c in cs.selected_columns): + raise JsonMisuse(f"{keyword or 'UNION'} 去重时碰到 JSON 列:PostgreSQL 的 json 没有相等运算符") + return super().visit_compound_select(cs, **kw) # type: ignore[misc] + + +def install(dialect: Any) -> None: + """挂到应用库引擎的方言实例上(PostgreSQL 时是 db/postgres 换过的编译器,SQLite 时是原来的)。""" + base = dialect.statement_compiler + if issubclass(base, JsonGuard): + return + dialect.statement_compiler = type(f"JsonGuarded{base.__name__}", (JsonGuard, base), {}) diff --git a/backend/tests/pgpool.py b/backend/tests/pgpool.py new file mode 100644 index 00000000..ae1b4a65 --- /dev/null +++ b/backend/tests/pgpool.py @@ -0,0 +1,123 @@ +"""应用库是 PostgreSQL 时测试用的连接池:每个事件循环一个池子,连接只给建它的那个循环用。 + +pytest-asyncio 每个测试一个事件循环,TestClient 在自己的线程里另起一个;asyncpg 的连接绑在建它的循环上,换个循环用 +就报「attached to a different loop」。不用连接池(每次查库新建连接)能跑,但一次运行几十次查库,每次多几毫秒, +合起来让计时类的测试(审批收尾不计时长、反查的时限)在满载时越界——生产上有连接池,没有这笔开销。 + +这里:连接按建它的循环归池子,取的时候只从当前循环的池子里拿,还回来放回它自己的池子;循环关掉之前(测试结束、 +TestClient 退出)把它池子里的连接在它自己的循环里关掉。只在 conftest 里、AGENTLAB_TEST_DATABASE_URL 时装上, +生产代码不用它。 +""" + +from __future__ import annotations + +import asyncio +import contextlib +import threading +import weakref +from typing import Any + +from sqlalchemy.pool import Pool + +_LOOP = "agentlab_loop" + + +class LoopLocalPool(Pool): + _is_asyncio = True + + def __init__(self, creator: Any, **kw: Any) -> None: + super().__init__(creator, **kw) + self._idle: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, list[Any]] = weakref.WeakKeyDictionary() + self._lock = threading.Lock() + + @classmethod + def replacing(cls, pool: Pool) -> LoopLocalPool: + """换掉引擎原来的池子:同一个建连接的函数、同一套连接事件(方言的 on_connect 挂在上面)。""" + return cls(pool._creator, recycle=pool._recycle, echo=pool.echo, logging_name=pool._orig_logging_name, + reset_on_return=pool._reset_on_return, pre_ping=False, _dispatch=pool.dispatch, + dialect=pool._dialect) + + def recreate(self) -> LoopLocalPool: + return LoopLocalPool.replacing(self) + + def _bucket(self, loop: asyncio.AbstractEventLoop) -> list[Any]: + with self._lock: + return self._idle.setdefault(loop, []) + + def _do_get(self) -> Any: + bucket = self._bucket(asyncio.get_running_loop()) + with self._lock: + if bucket: + return bucket.pop() + record = self._create_connection() + record.info[_LOOP] = weakref.ref(asyncio.get_running_loop()) + return record + + def _do_return_conn(self, record: Any) -> None: + ref = record.info.get(_LOOP) + loop = ref() if ref is not None else None + if loop is None or loop.is_closed(): + with contextlib.suppress(Exception): + record.close() + return + bucket = self._bucket(loop) + with self._lock: + bucket.append(record) + + def take(self, loop: asyncio.AbstractEventLoop) -> list[Any]: + """这个循环池子里空闲的连接,取出来(交给调用方在这个循环里关掉)。""" + with self._lock: + return self._idle.pop(loop, []) + + def dispose(self) -> None: + """只关当前循环的:别的循环的连接不能在这里关。没有正在跑的循环时什么都不做(各循环关掉时自己收拾)。""" + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + for record in self.take(loop): + with contextlib.suppress(Exception): + record.close() + + def status(self) -> str: + with self._lock: + idle = sum(len(v) for v in self._idle.values()) + return f"LoopLocalPool id:{id(self)} idle:{idle} loops:{len(self._idle)}" + + +def install(engine: Any) -> LoopLocalPool: + """把应用库引擎的池子换成 LoopLocalPool,并让每个事件循环关掉之前先关它自己的连接。""" + from sqlalchemy.util import greenlet_spawn + + pool = LoopLocalPool.replacing(engine.sync_engine.pool) + engine.sync_engine.pool = pool + + async def close_mine(records: list[Any]) -> None: + def run() -> None: + for record in records: + with contextlib.suppress(Exception): + record.close() + + await greenlet_spawn(run) + + base = asyncio.get_event_loop_policy() + + class Policy(type(base)): # type: ignore[misc] + def new_event_loop(self) -> asyncio.AbstractEventLoop: + loop = super().new_event_loop() + close = loop.close + + def closing() -> None: + current = engine.sync_engine.pool + if isinstance(current, LoopLocalPool) and not loop.is_closed() and not loop.is_running(): + records = current.take(loop) + if records: + with contextlib.suppress(Exception): + loop.run_until_complete(close_mine(records)) + close() + + loop.close = closing # type: ignore[method-assign] + return loop + + asyncio.set_event_loop_policy(Policy()) + return pool diff --git a/backend/tests/test_app_spec_tools.py b/backend/tests/test_app_spec_tools.py index d1234746..c67b7d8e 100644 --- a/backend/tests/test_app_spec_tools.py +++ b/backend/tests/test_app_spec_tools.py @@ -6,7 +6,6 @@ from __future__ import annotations import copy -import uuid import pytest @@ -21,6 +20,7 @@ from app.apps.spec import empty_spec, parse_spec from app.core.artifact_store import graph_hash from app.db.base import SessionLocal, utcnow from app.db.models import DataSource, Schedule, Workflow, WorkflowVersion +from tests.fixtures.names import letters def node(nid, ntype, label=None, **config): @@ -57,7 +57,7 @@ def approval(): async def make_workflow(graph, *, versions=((1, "governed"),), current=1, status="governed") -> tuple[str, dict]: async with SessionLocal() as session: - wf = Workflow(name=f"周报-{uuid.uuid4().hex[:6]}", graph=graph, status=status, published_version=current, + wf = Workflow(name=f"周报-{letters(6)}", graph=graph, status=status, published_version=current, version=max(v for v, _ in versions)) session.add(wf) await session.flush() @@ -281,7 +281,7 @@ async def test_process_pins(monkeypatch): async def test_on_demand_views_on_live_sources_need_a_ttl(tmp_path): - name = f"live_{uuid.uuid4().hex[:6]}" + name = f"live_{letters(6)}" async with SessionLocal() as session: session.add(DataSource(name=name, kind="sqlite", database=str(tmp_path / "x.db"), enabled=True)) await session.commit() diff --git a/backend/tests/test_atomic_writes.py b/backend/tests/test_atomic_writes.py new file mode 100644 index 00000000..8ac67581 --- /dev/null +++ b/backend/tests/test_atomic_writes.py @@ -0,0 +1,143 @@ +"""数据目录里的文件先写临时文件再改名(core/atomic):读的一方看不到写了一半的,只该生成一次的不会各生成一份。""" + +from __future__ import annotations + +import os +import stat +import threading + +import pytest + +from app.core import atomic +from app.db import models # noqa: F401 建表要先认得模型(单独跑这个文件时) + + +def _leftovers(folder) -> list[str]: + return [p.name for p in folder.iterdir() if atomic.TMP_MARK in p.name] + + +def test_write_replaces_whole_and_leaves_no_temp_file(tmp_path): + target = tmp_path / "a" / "b.json" + atomic.write_text(target, "第一版") + atomic.write_text(target, "第二版") + assert target.read_text(encoding="utf-8") == "第二版" + assert _leftovers(target.parent) == [] + + +def test_a_failed_write_keeps_the_old_file(tmp_path, monkeypatch): + target = tmp_path / "x.txt" + atomic.write_text(target, "旧的整份") + + def boom(src, dst): + raise OSError("磁盘满了") + + monkeypatch.setattr(os, "replace", boom) + with pytest.raises(OSError): + atomic.write_text(target, "新的") + assert target.read_text(encoding="utf-8") == "旧的整份" + assert _leftovers(tmp_path) == [] + + +def test_create_only_once_and_the_loser_reads_the_winner(tmp_path): + """几个线程同时第一次建同一个密钥文件:只有一个建成,文件内容就是那一个写的,没有临时文件留下。""" + target = tmp_path / ".secret" + results: list[tuple[str, bool]] = [] + barrier = threading.Barrier(8) + + def make(i: int) -> None: + barrier.wait() + results.append((f"key-{i}", atomic.create_text(target, f"key-{i}", mode=0o600))) + + threads = [threading.Thread(target=make, args=(i,)) for i in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + winners = [k for k, made in results if made] + assert len(winners) == 1 and target.read_text() == winners[0] + assert stat.S_IMODE(target.stat().st_mode) == 0o600 + assert _leftovers(tmp_path) == [] + + +def test_content_addressed_artifacts_are_written_atomically(tmp_path, monkeypatch): + """工件按内容寻址:同一份内容写两遍,文件只有一份、内容完整,不留临时文件。""" + import asyncio + + from app.core import artifact_store + from app.core.config import settings + + async def twice() -> tuple[str, str]: + a = await artifact_store.put_json({"区域": "虚构", "值": 1}, kind="test_atomic") + b = await artifact_store.put_json({"值": 1, "区域": "虚构"}, kind="test_atomic") + return a, b + + a, b = asyncio.run(twice()) + assert a == b and artifact_store.load(a) == {"区域": "虚构", "值": 1} + folder = settings.data_dir / "artifacts" / a[:2] + assert _leftovers(folder) == [] + + +async def test_file_write_tool_overwrites_atomically_and_appends_in_place(): + from types import SimpleNamespace + + from app.tools.builtin.files import file_write + + ctx = SimpleNamespace(sandbox_session=f"atomic-{os.getpid()}") + assert (await file_write(ctx, path="notes/a.txt", content="第一行\n"))["ok"] + from app.core.config import settings + + path = settings.session_dir(ctx.sandbox_session) / "notes" / "a.txt" + inode = path.stat().st_ino + out = await file_write(ctx, path="notes/a.txt", content="第二行\n", append=True) + assert path.read_text(encoding="utf-8") == "第一行\n第二行\n" and out["bytes"] == len("第一行\n第二行\n".encode()) + assert path.stat().st_ino == inode, "追加不换文件" + await file_write(ctx, path="notes/a.txt", content="重写\n") + assert path.read_text(encoding="utf-8") == "重写\n" + assert _leftovers(path.parent) == [] + + +def test_create_falls_back_when_the_filesystem_cannot_hard_link(tmp_path, monkeypatch): + """exFAT、一部分 SMB / FUSE 挂载不支持硬链接(EPERM、ENOTSUP、EXDEV):退回 O_EXCL 直接建,照样只建一次,不崩。""" + import errno + + def no_links(src, dst): + raise OSError(errno.EPERM, "Operation not permitted") + + monkeypatch.setattr(os, "link", no_links) + target = tmp_path / ".secret" + assert atomic.create_text(target, "第一把", mode=0o600) is True + assert atomic.create_text(target, "第二把", mode=0o600) is False + assert target.read_text() == "第一把" and stat.S_IMODE(target.stat().st_mode) == 0o600 + assert _leftovers(tmp_path) == [] + + +def test_replacing_keeps_the_permissions_of_the_old_file(tmp_path): + target = tmp_path / "run.sh" + target.write_text("echo 1") + target.chmod(0o750) + atomic.write_text(target, "echo 2") + assert target.read_text() == "echo 2" and stat.S_IMODE(target.stat().st_mode) == 0o750 + + +def test_appending_keeps_the_same_file(tmp_path): + """追加写不换文件:inode、权限都不变,原有内容不动(以前整份读出来重写、再改名)。""" + target = tmp_path / "log.txt" + target.write_text("一\n") + target.chmod(0o640) + before = target.stat().st_ino + atomic.append_bytes(target, "二\n".encode()) + assert target.read_text() == "一\n二\n" + assert target.stat().st_ino == before and stat.S_IMODE(target.stat().st_mode) == 0o640 + assert _leftovers(tmp_path) == [] + + +def test_the_data_dir_carries_a_marker_deploy_can_recognise(): + """数据目录里有 .agentlab-data:库在 PostgreSQL、密钥由环境变量给时,deploy.sh 也认得出这是 AgentLab 的数据目录。""" + from app.core.config import DATA_DIR_MARKER, settings + + settings.ensure_dirs() + marker = settings.data_dir / DATA_DIR_MARKER + assert marker.is_file() + text = marker.read_text(encoding="utf-8") + settings.ensure_dirs() + assert marker.read_text(encoding="utf-8") == text, "已有的不覆盖" diff --git a/backend/tests/test_background_drain.py b/backend/tests/test_background_drain.py new file mode 100644 index 00000000..4aab03af --- /dev/null +++ b/backend/tests/test_background_drain.py @@ -0,0 +1,81 @@ +"""core/background.drain:等后台任务做完,碰上别的事件循环上做完、还没摘掉的任务不空转。 + +以前 notify.service.drain、eval.cleanup.drain 写的是 `while _background: await gather(*_background)`:前一个测试的事件循环上 +刚做完的推送,done 回调(从集合里摘掉)排在那个已经不转的循环上、永远不跑;gather 拿到的全是做完了的、立刻返回,await 不让出 +事件循环,于是空转、整个测试进程卡死(PostgreSQL 全量里 test_personal_apps 收尾时出现过两次)。空转时事件循环也转不动, +asyncio.wait_for 的超时救不了,所以这里在另一个线程里跑、按线程等。 +""" + +from __future__ import annotations + +import asyncio +import threading +from typing import Any + +import pytest + +from app.core import background +from app.eval import cleanup +from app.notify import service + + +def _finished_on_a_dead_loop(tasks: set[asyncio.Future[Any]]) -> asyncio.Future[Any]: + """在另一个事件循环上做完、但 discard 回调没跑过的任务:照 spawn 的写法登记,结果设上(回调排进那个循环),循环不转就关。""" + other = asyncio.new_event_loop() + fut = other.create_future() + tasks.add(fut) + fut.add_done_callback(tasks.discard) + fut.set_result(None) + other.close() + return fut + + +def _returns_within(seconds: float, make: Any) -> bool: + done = threading.Event() + + def go() -> None: + asyncio.run(make()) + done.set() + + threading.Thread(target=go, daemon=True).start() + return done.wait(seconds) + + +@pytest.mark.parametrize("module", [service, cleanup], ids=["notify", "eval-cleanup"]) +def test_drain_does_not_spin_on_tasks_finished_on_another_loop(module): + stale = _finished_on_a_dead_loop(module._background) + try: + assert _returns_within(5, module.drain), "drain 卡住了(空转)" + assert stale not in module._background, "做完了的摘掉" + finally: + module._background.discard(stale) + + +def test_drain_waits_for_this_loops_tasks_and_skips_other_loops_pending_ones(): + tasks: set[asyncio.Future[Any]] = set() + other = asyncio.new_event_loop() + pending_elsewhere = other.create_future() # 别的循环上没做完的:等不到,不等 + tasks.add(pending_elsewhere) + seen: list[str] = [] + + async def scenario() -> None: + async def slow() -> None: + await asyncio.sleep(0.05) + seen.append("slow") + + task = asyncio.get_running_loop().create_task(slow()) + tasks.add(task) + task.add_done_callback(tasks.discard) + await background.drain(tasks) + seen.append("drained") + + try: + assert _returns_within(5, scenario) + assert seen == ["slow", "drained"], "本循环上的等它做完" + assert pending_elsewhere in tasks, "循环还开着的别人的任务留着" + other.close() + assert _returns_within(5, lambda: background.drain(tasks)) + assert tasks == set(), "循环关了的一并摘掉" + finally: + if not other.is_closed(): + other.close() diff --git a/backend/tests/test_bubblewrap_probe.py b/backend/tests/test_bubblewrap_probe.py index 81b1aaf8..55fa91d3 100644 --- a/backend/tests/test_bubblewrap_probe.py +++ b/backend/tests/test_bubblewrap_probe.py @@ -103,3 +103,58 @@ def test_auto_falls_back_to_local_and_says_why(monkeypatch: pytest.MonkeyPatch) # 节点上选 fast 档也不能点到一个起不来的 bubblewrap monkeypatch.setattr(mgr.sys, "platform", "linux") assert manager._resolve_named("fast") is None + + +# -------------------------------------------------------------------------- +# 环境变量:沙箱里的代码看不到后端进程的(模型的 API Key、带密码的 AGENTLAB_DATABASE_URL) +# -------------------------------------------------------------------------- + +_SECRETS = {"AGENTLAB_DATABASE_URL": "postgresql+asyncpg://u:not-a-real-password@h/db", + "ANTHROPIC_API_KEY": "sk-ant-not-real", "OPENAI_API_KEY": "sk-not-real"} + + +def _setenvs(argv: list[str]) -> dict[str, str]: + return {argv[i + 1]: argv[i + 2] for i, a in enumerate(argv) if a == "--setenv"} + + +def test_the_sandbox_starts_from_a_cleared_environment(monkeypatch, tmp_path): + from app.sandbox.base import SandboxLimits + + for key, value in _SECRETS.items(): + monkeypatch.setenv(key, value) + argv = bw.BubblewrapSandbox()._argv(tmp_path, SandboxLimits(), ["/usr/bin/python3", "/work/x.py"], + {"RUN_FLAG": "1", "HOME": "/work/custom"}) + # --clearenv 在所有 --setenv 前面:bwrap 按先后处理,放后面就把设好的也清掉了 + assert argv[1] == "--clearenv" and argv.index("--clearenv") < argv.index("--setenv") + env = _setenvs(argv) + assert env["RUN_FLAG"] == "1" and env["HOME"] == "/work/custom", "调用方给的照旧生效,能盖过缺省的" + assert env["PATH"] == "/usr/bin:/bin" + assert not any(k.startswith("AGENTLAB_") or k.endswith("_API_KEY") for k in env) + assert not any(v in " ".join(argv) for v in _SECRETS.values()) + + +async def test_bwrap_itself_gets_only_a_minimal_environment(monkeypatch, tmp_path): + for key, value in _SECRETS.items(): + monkeypatch.setenv(key, value) + seen: dict[str, Any] = {} + + class Done: + returncode = 0 + pid = 1 + + async def communicate(self): + return b"", b"" + + async def fake_exec(*argv, **kwargs): + seen["argv"], seen["env"] = list(argv), kwargs.get("env") + return Done() + + box = bw.BubblewrapSandbox() + monkeypatch.setattr(bw.BubblewrapSandbox, "available", staticmethod(lambda: True)) + monkeypatch.setattr(box, "_session_dir", lambda _sid: tmp_path) + monkeypatch.setattr(bw.asyncio, "create_subprocess_exec", fake_exec) + result = await box.run("print(1)", env={"RUN_FLAG": "1"}) + assert result.ok, result.error + assert seen["env"] is not None and set(seen["env"]) == {"PATH"}, "bwrap 进程本身不继承后端的环境变量" + assert not any(v in " ".join(seen["argv"]) for v in _SECRETS.values()) + assert _setenvs(seen["argv"])["RUN_FLAG"] == "1" diff --git a/backend/tests/test_challenge_masking.py b/backend/tests/test_challenge_masking.py index b1d6b89b..839098f4 100644 --- a/backend/tests/test_challenge_masking.py +++ b/backend/tests/test_challenge_masking.py @@ -23,6 +23,7 @@ from app.db.base import SessionLocal from app.db.models import DataSource from app.engine import masking from app.engine.nodes import challenge as challenge_mod +from tests.fixtures.names import letters from tests.test_challenge_node import Models, Play, engine_up, finish, graph, output_of # noqa: F401 - engine_up 是自动用上的夹具 #: 四笔订单,买家手机号设了遮罩;备注没设遮罩,其中一条写着换了写法的号码 @@ -42,7 +43,7 @@ async def masked_shop(tmp_path): db.executescript(SCRIPT) db.commit() db.close() - name = f"shop_{uuid.uuid4().hex[:8]}" + name = f"shop_{letters(8)}" async with SessionLocal() as session: row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True, options={"mask_columns": ["buyer_phone"]}) @@ -139,7 +140,7 @@ async def big_shop(tmp_path): db.executescript(BIG) db.commit() db.close() - name = f"shop_{uuid.uuid4().hex[:8]}" + name = f"shop_{letters(8)}" async with SessionLocal() as session: row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True, options={"mask_columns": ["amount", "buyer_name"]}) diff --git a/backend/tests/test_challenge_node.py b/backend/tests/test_challenge_node.py index 896b1dac..1b8768ea 100644 --- a/backend/tests/test_challenge_node.py +++ b/backend/tests/test_challenge_node.py @@ -1431,3 +1431,56 @@ def test_values_the_agent_found_are_masked_in_the_rationale_when_the_report_fill rationale=doc, usage={}, masked_values=seen.pooled()) masked = challenge_mod._masked_rationale(outcome, secrets, given=[]) assert masked.result["rationale"] == "号码 已遮罩 的客户在华南。" + + +# -------------------------------------------------------------------------- +# 子工作流里的反方审稿 +# -------------------------------------------------------------------------- + + +async def _in_subworkflow(spec: dict) -> dict: + """把 spec 发布成子工作流,父图:开始 → 子工作流 → 成果。""" + from tests import ledger_samples as S + + sub_id = await S.publish("周报子流程", graph=spec, level="draft") + return chain(node("start", "input"), node("sub", "subgraph", "周报子流程", workflow_id=sub_id, workflow_version=1), + node("top", "output", fields=[{"name": "周报", "value": "{{ nodes.sub.output.周报 }}"}])) + + +async def test_a_challenge_inside_a_subworkflow_names_the_report_by_its_run_id(monkeypatch, shop): + """子工作流里的报告在父运行里按全局 id(「sub/write」)记(report.checked、报告文档):审稿记录也按它记,证据面板点开 + 子图里那份报告的句子时认得出审它的那次审稿,不再因为记的是子图里的 id(write)对不上、什么都不显示。""" + from httpx import ASGITransport, AsyncClient + + from app.main import app + + Models(monkeypatch, shop) + row = await finish(await _in_subworkflow(graph(shop, contract=False)), run_class="exploratory") + assert row.status == "succeeded", row.error + out = await output_of(row.id, "sub/chal") + record = artifact_store.load(out["challenge_artifact"]) + [checked] = await events(row.id, "report.checked", "sub/write") + assert (record["report"], record["doc_artifact"]) == ("sub/write", checked.data["doc_artifact"]) + doc = artifact_store.load(checked.data["doc_artifact"]) + south = unit_of(doc, SOUTH) + seg = next(s for _, u in iter_units(doc) if u["id"] == south + for s in u.get("segments") or [] if s.get("kind") != "structural") + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + res = await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}?report=sub/write") + assert res.status_code == 200, res.text + challenge = res.json()["unit"].get("challenge") or {} + assert (challenge.get("verdict"), challenge.get("node_id")) == ("refuted", "sub/chal") + + +async def test_an_approval_inside_a_subworkflow_points_at_the_report_by_its_run_id(monkeypatch, shop): + """子工作流里「报告 → 反方审稿 → 人工审批」:审批卡上的报告按这次运行里的 id 记(「sub/write」),审批卡据此打开证据面板, + 反方审稿的概况照样带上。""" + Models(monkeypatch, shop) + row = await finish(await _in_subworkflow(gated(shop)), run_class="exploratory", + statuses=("interrupted", "failed", "succeeded")) + assert row.status == "interrupted", row.error + async with SessionLocal() as session: + approval = (await session.execute(select(Approval).where(Approval.run_id == row.id))).scalar_one() + [report] = approval.payload["reports"] + assert report["node_id"] == "sub/write" + assert report["challenge"]["counts"]["refuted"] == 1 diff --git a/backend/tests/test_chat_turns.py b/backend/tests/test_chat_turns.py index fff8502e..77104fe7 100644 --- a/backend/tests/test_chat_turns.py +++ b/backend/tests/test_chat_turns.py @@ -239,7 +239,8 @@ async def test_the_stream_replays_then_follows_and_reconnects_by_seq(client, mon first: list[dict[str, Any]] = [] stream = turn_log.stream(turn_id, 0, active=active, heartbeat=0.2) async for frame in stream: - if frame.startswith("data: "): + # 助手的心跳不带 seq、不落库(chat/log.beat):按 seq 补齐时不算它,同页面 + if frame.startswith("data: ") and "seq" in json.loads(frame[6:]): first.append(json.loads(frame[6:])) if any(e.get("type") == "stage" for e in first): break @@ -253,7 +254,9 @@ async def test_the_stream_replays_then_follows_and_reconnects_by_seq(client, mon if frame == turn_log.PING: pings += 1 continue - rest.append(json.loads(frame[6:])) + event = json.loads(frame[6:]) + if "seq" in event or event.get("type") == "stream.end": + rest.append(event) assert rest[-1]["type"] == "stream.end" and rest[-1]["status"] == "succeeded" got = [e["seq"] for e in first + rest[:-1]] assert got == list(range(1, len(got) + 1)), "补齐、不重复" @@ -330,8 +333,9 @@ async def test_stop_while_running_cancels_the_run(client, monkeypatch, source): # -------------------------------------------------------------------------- -async def _stranded(conversation_id: str, *, run: Run | None, stage: str, mode: str = "ask") -> str: - """造一轮「上次进程没了时还在跑」的:那一行 running、任务 running(不在这个进程里)。""" +async def _stranded(conversation_id: str, *, run: Run | None, stage: str, mode: str = "ask", + principal: dict[str, Any] | None = None) -> str: + """造一轮「上次进程没了时还在跑」的:那一行 running、任务 running(不在这个进程里)。principal 是发起时的身份快照。""" async with SessionLocal() as session: if run is not None: session.add(run) @@ -341,7 +345,7 @@ async def _stranded(conversation_id: str, *, run: Run | None, stage: str, mode: session.add(turn) await session.flush() session.add(ChatTurnJob(id=turn.id, conversation_id=conversation_id, status="running", stage=stage, mode=mode, - run_id=run.id if run else None, request={"started_ms": 0}, principal={})) + run_id=run.id if run else None, request={"started_ms": 0}, principal=principal or {})) await session.commit() return turn.id @@ -935,3 +939,233 @@ async def test_a_stop_that_lands_while_launching_still_cancels_the_run(client, m assert (await session.get(Run, job.run_id)).status == "cancelled", "这次运行跟着取消了,没有没人管的运行" row = await _row(turn_id) assert row.status == "error" and row.error == "已取消" and row.meta["runId"] == job.run_id + + +# -------------------------------------------------------------------------- +# 事件日志:心跳不落库、已结束的轮次按期清理 +# -------------------------------------------------------------------------- + + +async def test_heartbeats_go_out_on_the_stream_but_are_not_stored(client, monkeypatch, source): + """心跳(模型几秒没出声时助手发的一拍 heartbeat 操作):换了阶段的那一拍照落(刷新后过程里那一行阶段回放得出来),同一阶段 + 接着的是纯保活,只在实时流上发、不落库,重连时流自己补最近的一拍;别的事件(思考、操作、阶段……)照旧。以前每 3 秒落一条, + 一轮下来几百条心跳。""" + from app.api import copilot + + gate = asyncio.Event() + es.use_copilot(monkeypatch, Gated(source["name"], gate)) + monkeypatch.setattr(copilot, "_HEARTBEAT_SECONDS", 0.05) # use_copilot 把心跳调成了一小时一拍 + es.use_report(monkeypatch) + conversation_id = await _conversation(client) + turn_id = await _ask(client, conversation_id, datasource_ids=[source["id"]]) + await _stage(turn_id, "build") + + async def active() -> str | None: + return await chat_job.stream_status(turn_id) + + beats: list[dict[str, Any]] = [] + stream = turn_log.stream(turn_id, 0, active=active, heartbeat=0.2) + async for frame in stream: + if frame.startswith("data: "): + event = json.loads(frame[6:]) + if event.get("type") == "op" and event["data"].get("op") == "heartbeat": + beats.append(event) + if any("seq" not in b for b in beats): + break + await stream.aclose() + assert [b for b in beats if "seq" not in b], "同一阶段接着的心跳不占 seq:重连按 seq 补齐时不会多出来、也不会漏" + # 重连(从最后一条落库的事件接着读):流先补一拍最近的心跳,页面接着知道还在等模型 + stored = await _events(turn_id) + again = turn_log.stream(turn_id, stored[-1]["seq"], active=active, heartbeat=0.2) + first = await again.__anext__() + await again.aclose() + assert first.startswith("data: ") and json.loads(first[6:])["data"]["op"] == "heartbeat" + + gate.set() + await _done(turn_id) + events = await _events(turn_id) + stored = [e for e in events if e["type"] == "op" and e["data"].get("op") == "heartbeat"] + # 只落换了阶段的那一拍(规划、查看……各一拍),同一阶段接着的几十拍都不落 + phases = [e["data"]["phase"] for e in stored] + assert phases and phases[0] == "planning" and all(a != b for a, b in zip(phases, phases[1:])), phases + assert [e["seq"] for e in events] == list(range(1, len(events) + 1)) + assert turn_log.latest_beat(turn_id) is None, "这一轮结束了,进程里记的那一拍跟着清掉" + + +async def test_a_later_event_supersedes_the_last_heartbeat(): + """心跳只说「上一条事件之后还在等」:之后落了新事件,重连时不再补那一拍(不然页面的进度会退回去)。""" + async with SessionLocal() as session: + conversation = Conversation(kind="chat", title="虚构") + session.add(conversation) + await session.flush() + turn = ConversationTurn(conversation_id=conversation.id, question="q", status="running") + session.add(turn) + await session.flush() + session.add(ChatTurnJob(id=turn.id, conversation_id=conversation.id, status="running")) + await session.commit() + turn_id = turn.id + turn_log.beat(turn_id, {"op": "heartbeat", "phase": "planning", "elapsed_ms": 3000}) + assert turn_log.latest_beat(turn_id)["phase"] == "planning" + async with SessionLocal() as session: + await turn_log.append(session, turn_id, "stage", {"stage": "run"}) + assert turn_log.latest_beat(turn_id) is None + turn_log.forget(turn_id) + + +async def _finished_turn(*, days_ago: float, status: str = "succeeded", events: int = 3) -> str: + from datetime import timedelta + + from app.db.base import utcnow + + async with SessionLocal() as session: + conversation = Conversation(kind="chat", title="虚构的旧会话") + session.add(conversation) + await session.flush() + turn = ConversationTurn(conversation_id=conversation.id, question="虚构的问题", answer="虚构的回答", + status="done" if status == "succeeded" else "error", meta={"outcome": "done"}) + session.add(turn) + await session.flush() + when = utcnow() - timedelta(days=days_ago) + session.add(ChatTurnJob(id=turn.id, conversation_id=conversation.id, status=status, last_seq=events, + finished_at=None if status in ("queued", "running") else when)) + for seq in range(1, events + 1): + session.add(ChatTurnEvent(turn_id=turn.id, seq=seq, type="op", ts=0.0, data={"op": "thinking"})) + await session.commit() + return turn.id + + +async def test_finished_turns_lose_their_event_log_after_the_retention_period(): + """已结束超过保留期的轮次整轮删事件日志:页面重开已结束的一轮读的是 conversation_turns 那一行(答案、图、复核、用量都在 + 那里),不读事件日志;任务行留着,last_seq 接着发号,之后再在这一轮上重试、继续运行不撞号。还在跑的、刚结束的不动。""" + from app.db.base import utcnow + + old = await _finished_turn(days_ago=turn_log.RETAIN_DAYS + 1) + stuck = await _finished_turn(days_ago=turn_log.RETAIN_DAYS + 1, status="interrupted") + recent = await _finished_turn(days_ago=1) + live = await _finished_turn(days_ago=turn_log.RETAIN_DAYS + 1, status="running") + purged = await turn_log.purge(utcnow()) + assert set(purged) >= {old, stuck} and not {recent, live} & set(purged) + assert await _events(old) == [] and await _events(stuck) == [] + assert len(await _events(recent)) == 3 and len(await _events(live)) == 3 + async with SessionLocal() as session: + job = await session.get(ChatTurnJob, old) + row = await session.get(ConversationTurn, old) + assert job is not None and job.last_seq == 3, "任务行留着:seq 接着往下发" + assert row.answer == "虚构的回答", "那一行不动" + async with SessionLocal() as session: + assert await turn_log.add(session, old, "turn.attempt", {"mode": "retry"}) == 4 + await session.rollback() + + +async def test_the_cleanup_runs_in_the_background_tick(monkeypatch): + """清理挂在调度循环的那一拍里(scheduler/loop.process_due),一小时最多清一次。""" + from app.db.base import utcnow + from app.scheduler import loop + + old = await _finished_turn(days_ago=turn_log.RETAIN_DAYS + 2) + monkeypatch.setattr(turn_log, "_last_purge", 0.0) + await loop.process_due(utcnow()) + assert await _events(old) == [] + again = await _finished_turn(days_ago=turn_log.RETAIN_DAYS + 2) + await loop.process_due(utcnow()) + assert len(await _events(again)) == 3, "一小时内不再清第二次" + + +# -------------------------------------------------------------------------- +# 服务重启后接着收尾:按接手时的账号重新核对 +# -------------------------------------------------------------------------- + + +async def _snapshot_of(name: str) -> dict[str, Any]: + from app.db.models_identity import User + + async with SessionLocal() as session: + user = (await session.execute(select(User).where(User.username == name))).scalar_one() + return {"username": user.username, "role": user.role, "display_name": user.display_name, "user_id": user.id, + "groups": [], "verified": True, "mode": "local", "via": "session"} + + +async def _set_user(name: str, **values: Any) -> None: + from sqlalchemy import update as sa_update + + from app.db.models_identity import User + + async with SessionLocal() as session: + await session.execute(sa_update(User).where(User.username == name).values(**values)) + await session.commit() + + +def _finished_run(by: str) -> Run: + return Run(status="succeeded", output={"答案": "华东 300。"}, graph={"nodes": [], "edges": []}, run_class="exploratory", + origin_kind="chat", started_by=by) + + +async def test_resuming_after_a_restart_checks_the_account_again(people, monkeypatch, source): + """服务重启后接着收尾一轮(已发起运行的接着等、复核),按接手这一刻的账号重新核对,不再照发起时的身份快照接着做: + 发起之后账号被停用的,照实标中断、写清原因;那一行不再往下改答案。""" + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + mine = await _conversation(people["uma"], "uma 的对话") + turn_id = await _stranded(mine, run=_finished_run("uma"), stage="review", principal=await _snapshot_of("uma")) + await _set_user("uma", disabled=True) + await chat_job.startup(resume=True) + await chat_job.manager.wait(turn_id, timeout=30) + row = await _row(turn_id) + assert row.status == "error" and row.meta["cause"]["kind"] == "identity", row.meta + assert row.error == "发起这一轮的账号「uma」已停用,本轮没有接着收尾" and not row.answer + async with SessionLocal() as session: + assert (await session.get(ChatTurnJob, turn_id)).status == "interrupted" + assert (await _events(turn_id))[-1]["data"]["outcome"] == "error" + + +async def test_a_downgraded_account_without_chat_is_interrupted(people, monkeypatch, source): + """发起时是搭建者,之后被降成使用者、管理员又没给使用者开问数据:已经没有权限了,照实标中断。""" + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + theirs = await _conversation(people["bob"], "bob 的对话") + turn_id = await _stranded(theirs, run=_finished_run("bob"), stage="review", principal=await _snapshot_of("bob")) + await _set_user("bob", role="user") + assert (await people["admin"].put("/api/auth/settings", json={"user_can_chat": False})).status_code == 200 + await chat_job.startup(resume=True) + await chat_job.manager.wait(turn_id, timeout=30) + row = await _row(turn_id) + assert row.status == "error" and row.meta["cause"]["kind"] == "identity" + assert row.error == "发起这一轮的账号「bob」已经没有问数据的权限,本轮没有接着收尾" + + +async def test_a_downgraded_account_that_can_still_chat_finishes_as_who_they_are_now(people, monkeypatch, source): + """降了角色、但仍然用得了问数据(使用者开着问数据):照常收尾,按现在的身份(角色、组)接着做,不按快照里的搭建者。""" + es.use_copilot(monkeypatch, es.Copilot(source["name"])) + es.use_report(monkeypatch) + seen: list[Any] = [] + real = chat_job.ChatDriver.drive + + async def spy(self, *a, **k): + from app.api.deps import current_principal + + seen.append(current_principal()) + return await real(self, *a, **k) + + monkeypatch.setattr(chat_job.ChatDriver, "drive", spy) + theirs = await _conversation(people["bob"], "bob 的对话") + turn_id = await _stranded(theirs, run=_finished_run("bob"), stage="review", principal=await _snapshot_of("bob")) + await _set_user("bob", role="user") + await chat_job.startup(resume=True) + await chat_job.manager.wait(turn_id, timeout=30) + row = await _row(turn_id) + assert row.status == "done" and row.answer == "华东 300。" + assert seen and seen[-1].username == "bob" and seen[-1].role == "user" + + +async def test_the_cleanup_tick_without_a_time_uses_now(monkeypatch): + """调度循环那一拍不一定给 now(process_due(None)):清理按当前时间算,不报错。""" + from app.scheduler import loop + + old = await _finished_turn(days_ago=turn_log.RETAIN_DAYS + 2) + monkeypatch.setattr(turn_log, "_last_purge", 0.0) + assert old in await turn_log.purge_tick() + assert await _events(old) == [] + again = await _finished_turn(days_ago=turn_log.RETAIN_DAYS + 2) + monkeypatch.setattr(turn_log, "_last_purge", 0.0) + await loop.process_due(None) + assert await _events(again) == [] diff --git a/backend/tests/test_cluster.py b/backend/tests/test_cluster.py new file mode 100644 index 00000000..8b7aea5d --- /dev/null +++ b/backend/tests/test_cluster.py @@ -0,0 +1,871 @@ +"""多进程协调(app/cluster,docs/design/orchestration-round §5 W2-b)在一个进程里能验的那几样:别的进程用库里的行装出来 +(一个登记着、租约没过期的「同伴」,一个从没登记过的「已经没了的进程」)。两种库都跑;真起两个进程的端到端见 +test_multiprocess_e2e.py。""" +from __future__ import annotations + +import asyncio +import os +import subprocess +import sys +import time +from datetime import timedelta +from pathlib import Path + +import pytest +from sqlalchemy import delete, select + +from app.cluster import identity, leader as leader_mod, slots, state, workers +from app.core.config import settings as app_settings +from app.db.base import SessionLocal, new_id, utcnow +from app.db.models import Run, RunEvent +from app.db.models_cluster import ClusterLease, ClusterRequest, ClusterWorker, LaneTicket, RunSlot + +PEER = "peer-host:4242:feedbeef" +GHOST = "gone-host:999:deadbeef" + + +@pytest.fixture(autouse=True) +async def clean(): + async def wipe() -> None: + async with SessionLocal() as session: + for model in (RunSlot, LaneTicket, ClusterRequest, ClusterWorker): + await session.execute(delete(model)) + await session.execute(delete(ClusterLease).where(ClusterLease.name.like("test-%"))) + await session.commit() + state.forget() + slots.forget_policy() + + await wipe() + yield + await wipe() + + +async def _peer(owner: str = PEER, *, alive: bool = True) -> None: + now = utcnow() + async with SessionLocal() as session: + session.add(ClusterWorker(id=owner, host="peer-host", pid=4242, started_at=now, heartbeat_at=now, + lease_until=now + timedelta(seconds=60 if alive else -60))) + await session.commit() + state.forget() + + +def _policy(monkeypatch, total: int, reserve: int | None = None, **classes: tuple[int, int]) -> None: + stored: dict = {"max_total": total, "classes": {k: {"limit": lim, "priority": pri} for k, (lim, pri) in classes.items()}} + if reserve is not None: + stored["interactive_reserve"] = reserve + value = slots.values(stored) + + async def fixed(*, fresh: bool = False): # noqa: ARG001 + return value + + monkeypatch.setattr(slots, "policy", fixed) + + +async def test_liveness_counts_only_unexpired_peers_and_never_lists_itself(): + await workers.register() + await _peer(PEER, alive=True) + await _peer(GHOST, alive=False) + live = await workers.liveness() + assert live.others == frozenset({PEER}) + assert live.alive(PEER) and not live.alive(GHOST) and not live.alive(None) + assert not live.alive(identity.owner()), "属主是自己的只看本进程里是不是真的在跑" + assert live.alive(identity.owner(), local=True) and live.trusting().alive(identity.owner()) + + +async def test_slots_go_to_the_higher_priority_class_first(monkeypatch): + """全站 1 个名额:系统类的先占着、又一个系统类排着,后来的交互类先拿到空出来的名额;已经在跑的不打断。""" + _policy(monkeypatch, 1, interactive=(1, 40), system=(1, 10)) + order: list[str] = [] + release = {name: asyncio.Event() for name in ("e1", "e2", "chat")} + + async def run(name: str, origin: str) -> None: + async with slots.hold(f"run-{name}", origin): + order.append(name) + await release[name].wait() + + first = asyncio.create_task(run("e1", "eval")) + await asyncio.sleep(0.2) + second = asyncio.create_task(run("e2", "eval")) + await asyncio.sleep(0.2) + chat = asyncio.create_task(run("chat", "chat")) + await asyncio.sleep(0.2) + assert order == ["e1"], "名额满了,后来的两个都在等" + release["e1"].set() + await asyncio.wait_for(asyncio.shield(_until(lambda: len(order) == 2)), timeout=10) + assert order == ["e1", "chat"], "交互类的优先级高,先拿到名额" + release["chat"].set() + release["e2"].set() + await asyncio.wait_for(asyncio.gather(first, second, chat), timeout=10) + assert order == ["e1", "chat", "e2"] + + +async def test_a_class_at_its_cap_does_not_hold_up_the_others(monkeypatch): + """系统类的上限是 1:第二个系统类的等着,交互类的照样拿到名额(全站还有空)。""" + _policy(monkeypatch, 5, interactive=(5, 40), system=(1, 10)) + got: list[str] = [] + gate = asyncio.Event() + + async def run(name: str, origin: str) -> None: + async with slots.hold(f"run-{name}", origin): + got.append(name) + await gate.wait() + + tasks = [asyncio.create_task(run("e1", "eval")), asyncio.create_task(run("e2", "recall"))] + await asyncio.sleep(0.3) + tasks.append(asyncio.create_task(run("chat", "canvas"))) + await _until(lambda: "chat" in got) + assert len({"e1", "e2"} & set(got)) == 1, "系统类到了上限:另一个等着" + snapshot = await slots.snapshot() + assert snapshot["classes"]["system"] == {"running": 1, "waiting": 1}, snapshot + gate.set() + await asyncio.wait_for(asyncio.gather(*tasks), timeout=10) + + +def test_the_defaults_keep_a_reserve_for_interactive_and_lend_the_rest(): + pol = slots.defaults(20) + assert pol["interactive_reserve"] == 5 and pol["classes"]["interactive"]["limit"] == 20 + assert {pol["classes"][k]["limit"] for k in ("app", "schedule", "system")} == {15}, "别的类缺省借满到 全站 − 留底" + assert slots.defaults(2)["interactive_reserve"] == 1 and slots.defaults(1)["interactive_reserve"] == 0, \ + "至少留 1 个;全站只有 1 个名额时不留(别的类才跑得起来)" + assert slots.values({"max_total": 8})["classes"]["system"]["limit"] == 6, "缺省按存着的全站上限现算" + assert slots.problem({"max_total": 4, "interactive_reserve": 4}) is not None + + +async def _holders(names_origins: list[tuple[str, str]], got: list[str], gate: asyncio.Event) -> list[asyncio.Task]: + async def run(name: str, origin: str) -> None: + async with slots.hold(f"run-{name}", origin): + got.append(name) + await gate.wait() + + tasks = [] + for name, origin in names_origins: + tasks.append(asyncio.create_task(run(name, origin))) + await asyncio.sleep(0.05) + return tasks + + +async def test_idle_system_runs_borrow_all_but_the_interactive_reserve(monkeypatch): + """全站 4 个、留底 1 个:没人问数据时评测能同时跑 3 个(全站 − 留底),第 4 个等着。""" + _policy(monkeypatch, 4, reserve=1) + got: list[str] = [] + gate = asyncio.Event() + tasks = await _holders([(f"e{i}", "eval") for i in range(4)], got, gate) + await _until(lambda: len(got) == 3) + await asyncio.sleep(0.3) + assert len(got) == 3, "留底那一个不借给评测" + assert (await slots.snapshot())["classes"]["system"] == {"running": 3, "waiting": 1} + gate.set() + await asyncio.wait_for(asyncio.gather(*tasks), timeout=10) + + +async def test_interactive_takes_the_reserve_at_once_and_it_is_never_lent(monkeypatch): + """评测借满了(3 个在跑),应用类的再来也借不到留底(全站还空着 1 个);问数据一来马上拿到留底,不用等评测跑完。""" + _policy(monkeypatch, 4, reserve=1) + got: list[str] = [] + gate = asyncio.Event() + tasks = await _holders([("e1", "eval"), ("e2", "eval"), ("e3", "eval")], got, gate) + await _until(lambda: len(got) == 3) + tasks += await _holders([("app", "app_view")], got, gate) + await asyncio.sleep(0.3) + assert "app" not in got, "留底不借给应用类" + tasks += await _holders([("chat", "chat")], got, gate) + await _until(lambda: "chat" in got) + assert "app" not in got and set(got) == {"e1", "e2", "e3", "chat"} + gate.set() + await asyncio.wait_for(asyncio.gather(*tasks), timeout=10) + + +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3),没有别的进程的名额行") +async def test_slots_held_by_a_process_that_is_gone_are_taken_back(monkeypatch): + _policy(monkeypatch, 1, interactive=(1, 40)) + async with SessionLocal() as session: + session.add(RunSlot(run_id="ghost-run", klass="interactive", priority=40, owner=GHOST, granted=True)) + await session.commit() + async with slots.hold("mine", "chat"): + async with SessionLocal() as session: + rows = (await session.execute(select(RunSlot.run_id))).scalars().all() + assert rows == ["mine"], "没了的进程占着的名额收回了" + + +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3),没有别的进程的名额行") +async def test_slots_of_a_live_peer_are_left_alone(monkeypatch): + _policy(monkeypatch, 1, interactive=(1, 40)) + await _peer(PEER) + async with SessionLocal() as session: + session.add(RunSlot(run_id="peer-run", klass="interactive", priority=40, owner=PEER, granted=True)) + await session.commit() + waiting = asyncio.create_task(_hold_once("mine")) + await asyncio.sleep(0.5) + assert not waiting.done(), "活着的同伴占着唯一的名额:等着" + async with SessionLocal() as session: + await session.execute(delete(RunSlot).where(RunSlot.run_id == "peer-run")) + await session.commit() + slots.wake_local() + await asyncio.wait_for(waiting, timeout=10) + + +async def _hold_once(run_id: str) -> None: + async with slots.hold(run_id, "chat"): + pass + + +async def test_the_source_lane_is_shared_and_skips_tickets_of_a_gone_process(): + from app.core import job_runtime as runtime + + import app.data.catalog_jobs # noqa: F401 - 登记 draft、profile + + async with SessionLocal() as session: + session.add(LaneTicket(source_id="src-x", kind="profile", job_id="ghost-job", owner=GHOST, granted=True)) + session.add(LaneTicket(source_id="src-y", kind="profile", job_id="peer-job", owner=PEER, granted=True)) + await session.commit() + await _peer(PEER) + mine = await runtime.lanes.enter("src-x", "draft", "job-1") + assert mine.granted.is_set(), "没了的进程占着的位置让出来了" + other = await runtime.lanes.enter("src-y", "draft", "job-2") + assert not other.granted.is_set(), "活着的同伴在这个源上做着事:排在后面" + assert await runtime.lanes.ahead("src-y", "job-2") == ["profile"] + with pytest.raises(runtime.LaneBusy) as busy: + await runtime.lanes.try_enter("src-y", "import") + assert busy.value.holder.kind == "profile" + await mine.release() + await other.release() + + +async def test_the_source_lane_does_not_look_up_liveness_while_holding_its_lock(monkeypatch): + """复核:持着这个源的咨询锁时不另取连接查「谁还活着」(连接池用尽时会卡到取连接超时)——拿锁之前备好;拿锁之后才出现的 + 别的属主这一次当活着,不删它的行。""" + from app.core import job_runtime as runtime + + import app.data.catalog_jobs # noqa: F401 - 登记 draft、profile + + lanes = runtime.lanes + holding = {"now": False, "looked_inside": 0} + real_locked, real_rows, real_live_for = lanes._locked, lanes._rows, state.live_for + + async def locked(session, source_id): + await real_locked(session, source_id) + holding["now"] = True + + async def rows(session, source_id, dead_owners): + try: + return await real_rows(session, source_id, dead_owners) + finally: + holding["now"] = False + + async def live_for(owners): + if holding["now"]: + holding["looked_inside"] += 1 + return await real_live_for(owners) + + monkeypatch.setattr(lanes, "_locked", locked) + monkeypatch.setattr(lanes, "_rows", rows) + monkeypatch.setattr(state, "live_for", live_for) + async with SessionLocal() as session: + session.add(LaneTicket(source_id="src-lock", kind="profile", job_id="ghost-job", owner=GHOST, granted=True)) + await session.commit() + mine = await lanes.enter("src-lock", "draft", "job-lock") + assert mine.granted.is_set(), "没了的进程占着的位置照旧让出来(拿锁之前就查清楚了)" + assert holding["looked_inside"] == 0 + await mine.release() + + async with SessionLocal() as session: + session.add(LaneTicket(source_id="src-late", kind="profile", job_id="late-job", owner=GHOST, granted=True)) + await session.commit() + kept = await real_rows(session, "src-late", frozenset()) + await session.commit() + assert [r.owner for r in kept] == [GHOST], "拿锁之前那份里没有的属主这一次当活着" + async with SessionLocal() as session: + await session.execute(delete(LaneTicket).where(LaneTicket.source_id.in_(["src-lock", "src-late"]))) + await session.commit() + + +async def test_reaping_runs_only_touches_the_ones_whose_owner_is_gone(): + from app.engine.runner import run_manager + + await _peer(PEER) + async with SessionLocal() as session: + live_peer = Run(status="running", owner=PEER, graph={}, input={}) + dead = Run(status="running", owner=GHOST, graph={}, input={}) + queued = Run(status="queued", owner=GHOST, graph={}, input={}) + legacy = Run(status="running", owner=None, graph={}, input={}) + session.add_all([live_peer, dead, queued, legacy]) + await session.commit() + done = await run_manager.reap(await state.current(fresh=True)) + assert done >= 3 + async with SessionLocal() as session: + rows = {r.id: r for r in (await session.execute(select(Run).where( + Run.id.in_([live_peer.id, dead.id, queued.id, legacy.id])))).scalars()} + events = list((await session.execute(select(RunEvent.run_id, RunEvent.type, RunEvent.data).where( + RunEvent.run_id.in_([dead.id, queued.id])))).all()) + assert rows[live_peer.id].status == "running" and rows[live_peer.id].owner == PEER, "活着的同伴在跑的不碰" + assert rows[dead.id].status == "interrupted" and rows[dead.id].owner is None + assert rows[queued.id].status == "failed" + assert rows[legacy.id].status == "interrupted", "升级前没记属主的照「属主不在了」处理" + assert any(e.type == "log" and (e.data or {}).get("code") == "server_shutdown" for e in events) + assert any(e.type == "run.failed" for e in events) + + +async def test_stopping_or_noting_a_run_on_a_live_peer_goes_to_that_peer(): + from app.core.events import EventType + from app.engine.runner import run_manager + + await _peer(PEER) + async with SessionLocal() as session: + run = Run(status="running", owner=PEER, graph={}, input={}) + session.add(run) + await session.commit() + assert await run_manager.cancel(run.id, actor="张工") == "stopping" + await run_manager.note(run.id, EventType.LOG, level="info", message="别的进程记的一条") + async with SessionLocal() as session: + sent = list((await session.execute(select(ClusterRequest).where(ClusterRequest.target_id == run.id) + .order_by(ClusterRequest.created_at))).scalars()) + row = await session.get(Run, run.id) + assert [(r.owner, r.kind, r.target) for r in sent] == [(PEER, "stop", "run"), (PEER, "note", "run")] + assert sent[0].actor == "张工" and sent[1].payload["data"]["message"] == "别的进程记的一条" + assert row.status == "running", "停止交给属主去做,这里不改状态" + + +async def test_stopping_a_run_whose_owner_is_gone_takes_it_over_and_abandons_it(): + from app.engine.runner import run_manager + + async with SessionLocal() as session: + run = Run(status="running", owner=GHOST, graph={}, input={}) + session.add(run) + await session.commit() + assert await run_manager.cancel(run.id, actor="张工") == "cancelled" + async with SessionLocal() as session: + row = await session.get(Run, run.id) + assert row.status == "cancelled" and row.end_reason == "stopped" + + +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,名额在进程里(复核 R-b3)") +async def test_a_slot_is_released_under_the_identity_it_was_taken_with(monkeypatch): + """复核 R-b10:拿着名额的时候隔离换了身份,放名额照样删得到那一行(按要名额时的身份删),不用等旧身份过期才收回。""" + _policy(monkeypatch, 2, interactive=(2, 40)) + monkeypatch.setattr(identity, "OWNER", identity.OWNER) + async with slots.hold("rb10-run", "chat"): + identity.regenerate() + async with SessionLocal() as session: + left = list((await session.execute(select(RunSlot).where(RunSlot.run_id == "rb10-run"))).scalars()) + assert left == [] + + +async def test_registering_again_tidies_up_what_the_retired_identity_left(monkeypatch): + """复核 R-b10:隔离之后新身份登记上,旧身份的登记、名额、同源队的位置删掉,当班租约交出,发给它的请求转到新身份名下照办—— + 停止请求对停在断点上的那次运行照停止的规矩收成 cancelled。""" + from app.cluster import requests, version + from app.cluster import service as service_mod + from app.cluster.fence import now as fence_now + + monkeypatch.setattr(identity, "OWNER", f"{identity.host()}:{os.getpid()}:retired0") # 测试完换回原来的身份 + old = identity.owner() + now = utcnow() + async with SessionLocal() as session: + session.add(ClusterWorker(id=old, host="this-host", pid=1, started_at=now, heartbeat_at=now, + lease_until=now + timedelta(seconds=60), code_version=version.code_version(), + schema_version=version.schema_version())) + session.add(RunSlot(run_id="retired-slot", klass="interactive", priority=40, owner=old, granted=True, + requested_at=now)) + session.add(LaneTicket(source_id="src-retired", kind="profile", job_id="retired-job", owner=old, granted=True)) + session.add(ClusterLease(name="test-retired", owner=old, lease_until=now + timedelta(seconds=60), epoch=1)) + run = Run(status="interrupted", owner=old, graph={}, input={}) + session.add(run) + await session.flush() + session.add(ClusterRequest(owner=old, kind="stop", target="run", target_id=run.id, actor="张工", payload={})) + await session.commit() + svc = service_mod._Service() + svc.retired = [old] + identity.regenerate() + svc.pending_since = fence_now() + await svc.register_again() + new = identity.owner() + async with SessionLocal() as session: + ids = set((await session.execute(select(ClusterWorker.id))).scalars()) + slot_rows = list((await session.execute(select(RunSlot).where(RunSlot.owner == old))).scalars()) + lane_rows = list((await session.execute(select(LaneTicket).where(LaneTicket.owner == old))).scalars()) + lease = await session.get(ClusterLease, "test-retired") + sent = list((await session.execute(select(ClusterRequest.owner).where(ClusterRequest.target_id == run.id))).scalars()) + assert old not in ids and new in ids, "旧身份的登记删掉:别的进程马上按属主不在了接手" + assert slot_rows == [] and lane_rows == [] and lease.owner is None + assert sent == [new], "发给旧身份的请求转到新身份名下" + await requests.drain() + await requests.settle() + async with SessionLocal() as session: + row = await session.get(Run, run.id) + assert row.status == "cancelled" and row.end_reason == "stopped", "停止请求照停止的规矩落下了" + + +async def test_leadership_is_handed_over_when_the_lease_runs_out(): + me = leader_mod.Leadership("test-lead") + assert await me.claim() and me.held() + first = me.epoch + async with SessionLocal() as session: + row = await session.get(ClusterLease, "test-lead") + row.owner, row.lease_until = PEER, utcnow() + timedelta(seconds=60) + await session.commit() + assert not await me.claim() and not me.held(), "别的进程拿着没到期的租约:不当班" + async with SessionLocal() as session: + row = await session.get(ClusterLease, "test-lead") + row.lease_until = utcnow() - timedelta(seconds=1) + await session.commit() + assert await me.claim() and me.epoch == first + 1, "到期了接班,换一次当班加一" + await me.release() + assert (await leader_mod.current("test-lead"))["owner"] is None + + +def test_the_fence_judges_by_the_sleep_aware_clock_and_falls_back_to_the_wall(monkeypatch): + """复核 R-b7、R-b14:取得到计入休眠的钟时只看它——它走过两个租约(合盖、挂起)算续晚了,墙钟往前、往后跳一小时不算(只算 + 两只钟对不上,jumped);取不到计入休眠的钟时,monotonic 和墙钟里走得多的那只说了算(墙钟当后备)。""" + from app.cluster import fence + + clocks = {"mono": 1000.0, "wall": 5000.0} + monkeypatch.setattr(fence, "_clock", lambda: clocks["mono"]) + monkeypatch.setattr(fence, "_wall", lambda: clocks["wall"]) + monkeypatch.setattr(fence, "SLEEP_AWARE", True) + fence.activate() + try: + assert not fence.stale() and not fence.jumped() + for jump in (3600.0, -3600.0): # NTP 一步校时、手动改时间 + fence.renewed() + clocks["wall"] += jump + assert not fence.stale() and fence.jumped(), "只是墙钟跳了:不算续晚了" + fence.renewed() + clocks["mono"] += 2 * workers.lease_s() # 计入休眠的钟走过两个租约(合盖) + assert fence.stale() + fence.renewed() + clocks["mono"] += workers.lease_s() / 2 + clocks["wall"] += workers.lease_s() / 2 + assert not fence.stale() and not fence.jumped() + + monkeypatch.setattr(fence, "SLEEP_AWARE", False) # 取不到计入休眠的钟:monotonic 不计休眠,墙钟当后备 + fence.renewed() + clocks["wall"] += 2 * workers.lease_s() + assert fence.stale() + finally: + fence.deactivate() + + +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="SQLite 只有一个进程,租约用本机的钟") +async def test_leases_use_the_database_clock_not_the_local_wall_clock(monkeypatch): + """复核 R-b7:本机墙钟快了一个小时——别的进程照样算活着、它拿着的当班租约照样抢不走;按库的时钟自己的租约已经过期时,续租 + 不再续这个身份,而是隔离。""" + from app.cluster import fence + from app.cluster.service import service + from app.db.base import utcnow as real_utcnow + import app.db.base as db_base + + await _peer(PEER) + lead = leader_mod.Leadership("test-skew") + async with SessionLocal() as session: + session.add(ClusterLease(name="test-skew", owner=PEER, lease_until=real_utcnow() + timedelta(seconds=60), epoch=1)) + await session.commit() + skewed = lambda: real_utcnow() + timedelta(hours=1) # noqa: E731 + monkeypatch.setattr(workers, "utcnow", skewed) + monkeypatch.setattr(db_base, "utcnow", skewed) + assert PEER in (await workers.liveness()).others, "按库的时钟它的租约没过期" + assert not await lead.claim(), "当班租约按库的时钟判:抢不走" + async with SessionLocal() as session: + await session.execute(delete(ClusterWorker).where(ClusterWorker.id == PEER)) # 它没记版本:登记时会被版本守卫拦下 + await session.execute(delete(ClusterLease).where(ClusterLease.name == "test-skew")) + await session.commit() + + await workers.register() + fence.activate() + before = fence.generation() + try: + assert await service.renew(), "按库的时钟没过期:续上" + monkeypatch.setattr(fence, "SLEEP_AWARE", True) + for jump in (3600.0, -3600.0): # 单进程、墙钟往前往后跳一小时(复核 R-b14) + monkeypatch.setattr(fence, "_wall", lambda jump=jump: time.time() + jump) + assert not fence.stale() and fence.jumped() + fence.check() + assert await service.renew() and fence.generation() == before, "库的时钟说租约还在:续上、不隔离" + assert not fence.jumped(), "续上之后两只钟重新对齐" + monkeypatch.setattr(fence, "_wall", time.time) + async with SessionLocal() as session: + row = await session.get(ClusterWorker, identity.owner()) + row.lease_until = real_utcnow() - timedelta(seconds=1) # 库的时钟已经过了它的租约,本机的两只钟都没觉得 + await session.commit() + assert not await service.renew() and fence.generation() == before + 1, "过期了不续这个身份:隔离" + finally: + fence.deactivate() + if service._lease_conn is not None: + await service._lease_conn.close() + service._lease_conn = None + + +async def test_the_daily_cap_is_not_overrun_by_concurrent_charges(): + from app.engine.spend import Ledger + + book = Ledger(f"test_spend_{new_id()[:8]}") + other = Ledger(book.key) # 同一本账的另一个实例(另一个进程里就是这样) + got = await asyncio.gather(*(b.charge(0.1, limit=1.0, priced=True) for b in [book, other] * 10), + *(book.charge(0.1, limit=1.0, priced=True) for _ in range(5))) + assert sum(1 for d in got if d) == 10 + assert (await book.daily())["usd"] <= 1.0 + 1e-9 + + +@pytest.mark.sqlite_only +def test_sqlite_refuses_a_second_process_on_the_same_data_dir(): + from app.cluster import guard + + guard.acquire() + try: + backend = Path(__file__).resolve().parents[1] + env = {**os.environ, "PYTHONPATH": str(backend)} + out = subprocess.run([sys.executable, "-c", "from app.cluster import guard; guard.acquire()"], cwd=str(backend), + env=env, capture_output=True, text=True, timeout=60) + assert out.returncode != 0 and "只支持单进程" in out.stderr, out.stderr[-2000:] + many = subprocess.run([sys.executable, "-c", "from app.cluster import guard; guard.acquire()"], cwd=str(backend), + env={**env, "AGENTLAB_WORKERS": "2", "AGENTLAB_DATA_DIR": str(Path(env["AGENTLAB_DATA_DIR"]) / "other")}, + capture_output=True, text=True, timeout=60) + assert many.returncode != 0 and "AGENTLAB_WORKERS=2" in many.stderr, many.stderr[-2000:] + finally: + guard.release() + + +async def _until(check, timeout: float = 10.0) -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while not check(): + if loop.time() > deadline: + raise AssertionError("等超时了") + await asyncio.sleep(0.05) + + +async def test_a_daily_quota_is_taken_up_to_the_limit_by_concurrent_callers(): + """先占额度再发起(cluster/quota):10 个协程同时占、上限 5,正好 5 个占到;这一天第一次占按已有的记录起算;退回一份又能占。 + (同时占的个数不超过连接池能直接给的连接数:SQLite 的池子是各测试共用的,排队等连接会把它绑到这个测试的事件循环上。)""" + from app.cluster import quota + + name = f"test-quota-{new_id()[:8]}" + + async def seed(_session) -> int: + return 2 # 今天已经做过 2 次 + + got = await asyncio.gather(*(quota.take(name, 5, seed=seed) for _ in range(10))) + assert sum(1 for g in got if g) == 3 and await quota.used(name) == 5 + assert not await quota.take(name, 5) + await quota.give_back(name) + assert await quota.take(name, 5) and not await quota.take(name, 5) + + +async def test_a_quota_is_given_back_to_the_day_it_was_taken(monkeypatch): + """复核 R-b11:23:59:59 占、00:00:01 发起失败——退回前一天那一份,第二天的不动。""" + from app.cluster import quota + + name = f"test-quota-{new_id()[:8]}" + days = iter(["2026-10-18", "2026-10-19", "2026-10-19", "2026-10-19"]) + monkeypatch.setattr(quota, "today", lambda: next(days)) + day = await quota.take(name, 5) + assert day == "2026-10-18" + assert await quota.take(name, 5) == "2026-10-19" # 过了零点又占了一份 + await quota.give_back(name, day=day) # 前一天占的那一份发起失败 + assert await quota.used(name, day="2026-10-18") == 0 + assert await quota.used(name, day="2026-10-19") == 1, "第二天的那一份不动" + + +def test_a_daily_quota_holds_across_two_processes(): + """两个进程同时各占 15 次、上限 10:合计正好 10 次。""" + name = f"test-quota-{new_id()[:8]}" + backend = Path(__file__).resolve().parents[1] + script = ("import asyncio, sys\n" + "from app.cluster import quota\n" + "async def main():\n" + f" got = await asyncio.gather(*(quota.take({name!r}, 10) for _ in range(15)))\n" + " print(sum(1 for g in got if g))\n" + "asyncio.run(main())\n") + env = {**os.environ, "PYTHONPATH": str(backend), "AGENTLAB_DB_POOL_SIZE": "5"} + procs = [subprocess.Popen([sys.executable, "-c", script], cwd=str(backend), env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) for _ in range(2)] + outs = [p.communicate(timeout=120) for p in procs] + assert all(p.returncode == 0 for p in procs), [o[1][-2000:] for o in outs] + assert sum(int(o[0].strip().splitlines()[-1]) for o in outs) == 10 + + +# -------------------------------------------------------------------------- +# 不许新旧版本混跑(cluster/version.py) +# -------------------------------------------------------------------------- + +async def _peer_version(owner: str, code: str | None, *, lease_s: float = 60.0) -> None: + now = utcnow() + async with SessionLocal() as session: + session.add(ClusterWorker(id=owner, host="peer-host", pid=4242, started_at=now, heartbeat_at=now, + lease_until=now + timedelta(seconds=lease_s), code_version=code, + schema_version=None if code is None else _schema())) + await session.commit() + + +def _schema() -> str: + from app.cluster import version + + return version.schema_version() + + +async def _rows(): + async with SessionLocal() as session: + return list((await session.execute(select(ClusterWorker))).scalars()) + + +async def test_a_live_process_of_another_version_is_named_and_refused(): + from app.cluster import version + + await _peer_version(PEER, "0.0.9+old+aaaaaaaaaaaa") + with pytest.raises(version.VersionMismatch) as e: + version.check_rows(await _rows(), utcnow()) + assert "peer-host 上的进程 4242" in str(e.value) and "0.0.9+old+aaaaaaaaaaaa" in str(e.value) and "先停掉" in str(e.value) + + +async def test_the_same_version_joins_and_expired_or_unversioned_dead_ones_do_not_count(): + from app.cluster import version + + await _peer_version(PEER, version.code_version()) + await _peer_version(GHOST, "0.0.9+old+aaaaaaaaaaaa", lease_s=-5) + version.check_rows(await _rows(), utcnow()) + assert version.same("0.1.0+other-build+" + version.source_fingerprint(), version.schema_version()), \ + "比的是源码指纹:构建号写法不同、代码相同算同一版" + + +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="只在 PostgreSQL 上拦(SQLite 只有一个进程)") +async def test_preflight_refuses_a_beating_peer_and_waits_out_a_stale_one(): + from app.cluster import version + + await _peer_version(PEER, "0.0.9+old+aaaaaaaaaaaa") + + async def beat() -> None: + for _ in range(20): + await asyncio.sleep(0.1) + async with SessionLocal() as session: + row = await session.get(ClusterWorker, PEER) + row.heartbeat_at = utcnow() + await session.commit() + + beating = asyncio.create_task(beat()) + with pytest.raises(version.VersionMismatch): + await version.preflight(observe_s=0.5) + await beating + async with SessionLocal() as session: + await session.execute(delete(ClusterWorker).where(ClusterWorker.id == PEER)) + await session.commit() + # 强杀之后留下的旧版登记(心跳停着、租约还有 1 秒):等它过期就起 + await _peer_version(GHOST, "0.0.9+old+aaaaaaaaaaaa", lease_s=1.0) + loop = asyncio.get_running_loop() + t0 = loop.time() + await version.preflight(observe_s=0.3) + assert 0.5 < loop.time() - t0 < 5 + with pytest.raises(version.VersionMismatch): + await _peer_version(PEER, "0.0.9+old+aaaaaaaaaaaa") + await workers.register() + + +@pytest.mark.skipif(app_settings.db_dialect != "postgresql", reason="preflight 只在 PostgreSQL 上看") +async def test_preflight_names_every_live_process_of_another_version(): + """复核 R-b13:一个心跳在走、一个心跳停着——报错两个都列出来(只点名先看到心跳的那个,人停掉它再起还会被另一个拦下)。""" + from app.cluster import version + + await _peer_version(PEER, "0.0.9+old+aaaaaaaaaaaa") + await _peer_version(GHOST, "0.0.8+older+bbbbbbbbbbbb") + + async def beat() -> None: + for _ in range(15): + await asyncio.sleep(0.1) + async with SessionLocal() as session: + row = await session.get(ClusterWorker, PEER) + row.heartbeat_at = utcnow() + await session.commit() + + beating = asyncio.create_task(beat()) + with pytest.raises(version.VersionMismatch) as e: + await version.preflight(observe_s=0.5) + await beating + text = str(e.value) + assert "2 个版本不同" in text and "aaaaaaaaaaaa" in text and "bbbbbbbbbbbb" in text + assert "心跳在走" in text and "心跳停着" in text + + +def test_the_fingerprint_changes_with_the_dependency_lock(monkeypatch, tmp_path): + """复核 R-b13:依赖锁文件变了(升了依赖)源码指纹跟着变;锁文件取不到时换成几个关键包的版本。""" + from app.cluster import version + + lock = tmp_path / "requirements.lock" + monkeypatch.setattr(version, "LOCK_FILE", lock) + + def fingerprint() -> str: + version.source_fingerprint.cache_clear() + return version.source_fingerprint() + + try: + lock.write_text("sqlalchemy==2.0.40\n") + first = fingerprint() + assert fingerprint() == first + lock.write_text("sqlalchemy==2.0.41\n") + assert fingerprint() != first, "升了一个依赖:指纹不同" + lock.unlink() + assert version.dependency_marker().startswith(b"packages:") and b"sqlalchemy==" in version.dependency_marker() + assert fingerprint() not in (first,) + finally: + version.source_fingerprint.cache_clear() + + +async def test_a_known_seq_collision_is_not_rescanned_on_every_start(tmp_path, caplog): + """复核 R-b13:存量库里有撞了号的事件——第一次启动整表数一遍、记下结果、不建唯一索引;之后启动只核对记下的那几处,不再整表 + GROUP BY;撞号处理掉之后再数一遍、建索引、删掉记录。用一个单独的 SQLite 库(两种库的测试里都跑)。""" + import logging + + from sqlalchemy import event, inspect, text + + from app.db import base as db_base + from app.db.models import Setting + + target = db_base.make_engine(f"sqlite+aiosqlite:///{tmp_path / 'seq.db'}") + scans: list[str] = [] + event.listen(target.sync_engine, "before_cursor_execute", + lambda conn, cursor, statement, *a: scans.append(statement) if "GROUP BY run_id, seq" in statement else None) + try: + await db_base.create_schema(target) + from sqlalchemy.ext.asyncio import async_sessionmaker + + async with target.begin() as conn: + await conn.execute(text("DROP INDEX ux_run_events_run_seq")) + await conn.execute(text("CREATE INDEX ix_run_events_run_seq ON run_events (run_id, seq)")) + async with async_sessionmaker(target, expire_on_commit=False)() as session: + session.add(Run(id="r1", status="succeeded", graph={}, input={})) + await session.flush() + session.add_all([RunEvent(id=eid, run_id="r1", seq=seq, type="log", ts=0, data={}) + for eid, seq in (("e1", 1), ("e2", 1), ("e3", 2))]) + await session.commit() + + async def start() -> set[str]: + async with target.begin() as conn: + def run(sync_conn): + cols = {t: {c["name"] for c in inspect(sync_conn).get_columns(t)} for t in ("run_events", "settings")} + db_base._unique_run_event_seq(sync_conn, cols) + return {ix["name"] for ix in inspect(sync_conn).get_indexes("run_events")} + return await conn.run_sync(run) + + caplog.set_level(logging.INFO, logger="app.db.base") + assert "ux_run_events_run_seq" not in await start() and len(scans) == 1 + async with target.connect() as conn: + marker = (await conn.execute(text(f"SELECT value FROM settings WHERE key = '{db_base.SEQ_DUPLICATES_KEY}'"))).scalar() + assert '"r1"' in str(marker) and "1" in str(marker) + assert "ux_run_events_run_seq" not in await start() and len(scans) == 1, "第二次启动不再整表 GROUP BY" + assert any("还有撞了号的事件" in r.getMessage() for r in caplog.records) + async with target.begin() as conn: + await conn.execute(text("DELETE FROM run_events WHERE id = 'e2'")) + assert "ux_run_events_run_seq" in await start() and len(scans) == 2, "处理掉之后再数一遍、建上索引" + async with target.connect() as conn: + left = (await conn.execute(text(f"SELECT COUNT(*) FROM settings WHERE key = '{db_base.SEQ_DUPLICATES_KEY}'"))).scalar() + assert left == 0 + assert Setting.__tablename__ == "settings" + finally: + await target.dispose() + + +@pytest.mark.sqlite_only +async def test_an_idle_single_process_writes_nothing_for_coordination(monkeypatch): + """复核 R-b3:SQLite(只有一个进程)空闲时协调不写库——不续租约、不写当班、不周期接手;要名额、放名额也不落库。用另一条连接 + 读 PRAGMA data_version:别的连接提交过写事务它就会变。""" + import sqlite3 + + from app import cluster + from app.core.config import settings + + monkeypatch.setattr(settings, "worker_heartbeat_s", 0.05) + await cluster.start() + try: + await asyncio.sleep(0.2) + probe = sqlite3.connect(settings.db_path) + try: + before = probe.execute("PRAGMA data_version").fetchone()[0] + await asyncio.sleep(0.6) # 十来拍心跳 + async with slots.hold("idle-check", "chat"): # 一段运行要一个名额、用完放掉 + pass + after = probe.execute("PRAGMA data_version").fetchone()[0] + finally: + probe.close() + assert before == after, "空闲的单进程为协调写了库" + assert cluster.leading(), "只有本进程:直接当班" + finally: + await cluster.stop() + + +def test_the_worker_count_comes_from_the_backend_settings(): + """复核 R-b4:prod.sh、Docker 的入口按 python -m app.cluster workers 起几个进程(后端自己读环境变量和 .env);SQLite 而大于 1 时 + 照实报错、退出码 2。""" + backend = Path(__file__).resolve().parents[1] + base = {k: v for k, v in os.environ.items() if not k.startswith("AGENTLAB_")} + base.update(PYTHONPATH=str(backend), AGENTLAB_DATA_DIR=os.environ["AGENTLAB_DATA_DIR"], + AGENTLAB_SECRET_KEY="test-only-not-a-real-key") + + def ask(**env: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, "-m", "app.cluster", "workers"], cwd=str(backend), env={**base, **env}, + capture_output=True, text=True, timeout=60) + + sqlite = ask(AGENTLAB_DATABASE_URL="", AGENTLAB_WORKERS="3") + assert sqlite.returncode == 2 and "只支持单进程" in sqlite.stderr, sqlite.stderr[-1000:] + assert ask(AGENTLAB_DATABASE_URL="", AGENTLAB_WORKERS="1").stdout.strip() == "1" + pg = ask(AGENTLAB_DATABASE_URL="postgresql://u:p@db.internal:5432/agentlab", AGENTLAB_WORKERS="3") + assert pg.returncode == 0 and pg.stdout.strip() == "3", pg.stderr[-1000:] + + +async def test_the_relay_rereads_a_little_behind_so_a_late_commit_is_not_skipped(): + """复核 R-b5:号先取后提交,号大的先提交时游标会越过号小的那一条——转发器往回留一小段重读、按发布过的 seq 去重。""" + from app.cluster.relay import relay + from app.core.bus import bus + + async with SessionLocal() as session: + run = Run(status="running", owner=PEER, graph={}, input={}) + session.add(run) + await session.flush() + session.add_all([RunEvent(id=new_id(), run_id=run.id, seq=s, type="log", ts=0, data={"n": s}) for s in (1, 3)]) + await session.commit() + got: list[int] = [] + with bus.attach(run.id) as sub, relay.follow_run(run.id, 0): + async def collect() -> None: + async for event in sub.stream(): + got.append(event.seq) + + reader = asyncio.create_task(collect()) + await relay.pump(run.id) + async with SessionLocal() as session: # 号小的那一条后提交 + session.add(RunEvent(id=new_id(), run_id=run.id, seq=2, type="log", ts=0, data={"n": 2})) + await session.commit() + await relay.pump(run.id) + await relay.pump(run.id) + await asyncio.sleep(0.1) + reader.cancel() + assert sorted(got) == [1, 2, 3] and len(got) == 3, got + + +async def test_cache_invalidations_committed_out_of_order_are_all_applied(monkeypatch): + """复核 R-b5:缓存失效的通知 id 大的先提交,id 小的后到也照样处理(往回留一小段重读、按 id 去重)。""" + from app.cluster import caches + from app.db.models_cluster import CacheEvent + + seen: list[str | None] = [] + caches.on("test-cache", lambda key: seen.append(key)) + monkeypatch.setattr(caches, "_enabled", lambda: True) + await caches.start() + base = caches._last_seen or 0 + async with SessionLocal() as session: + session.add(CacheEvent(id=base + 5, name="test-cache", key="late-big", owner=PEER)) + await session.commit() + assert await caches.poll() == 1 + async with SessionLocal() as session: + session.add(CacheEvent(id=base + 3, name="test-cache", key="early-small", owner=PEER)) + await session.commit() + assert await caches.poll() == 1 and await caches.poll() == 0 + assert seen == ["late-big", "early-small"] + + +def test_a_long_merged_delta_is_split_not_cut(): + """复核 R-b6:几条正文增量合成一条之后超过载荷上限,按字切成几条(不切坏 UTF-8),拼回来一字不少。""" + import json + + from app.cluster.relay import PAYLOAD_MAX, _chunks + + items = [{"type": "llm.token", "node_id": "w", "seq": i, "data": {"delta": "数据" * 500 + "😀"}} for i in range(1, 9)] + chunks = list(_chunks(items)) + assert all(len(json.dumps(c, ensure_ascii=False).encode()) <= PAYLOAD_MAX for c in chunks) + text = "".join(e["data"]["delta"] for c in chunks for e in c) + assert text == ("数据" * 500 + "😀") * 8 diff --git a/backend/tests/test_cluster_fence.py b/backend/tests/test_cluster_fence.py new file mode 100644 index 00000000..f3146d5f --- /dev/null +++ b/backend/tests/test_cluster_fence.py @@ -0,0 +1,320 @@ +"""自我隔离(cluster/fence.py,复核 R-b1):本进程卡住了、续不上租约,别的进程已经把它在跑的运行接手了——它缓过来之后不再写: +不写事件(不和接手的一边撞号)、不写断点、不写终态(不把接手的结果盖掉)。在一个进程里装出来:把「上次续上租约」拨到一个租约 +时长以前,再把运行改成接手之后的样子。两种库都跑。""" +from __future__ import annotations + +import asyncio +import contextlib +import os + +import pytest +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError + +from app.cluster import fence +from app.cluster.workers import lease_s +from app.db.base import SessionLocal, new_id +from app.db.models import Run, RunEvent +from app.engine import compiler +from app.engine.runner import run_manager +from app.engine.schema import NodeType + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + fence.activate() + yield + fence.deactivate() + await run_manager.shutdown() + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +GRAPH = {"nodes": [node("start", "input", fields=[{"name": "question"}]), + node("slow", "transform", mode="template", template="慢的这步"), + node("next", "transform", mode="template", template="后面这步"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.next }}"}])], + "edges": [{"source": "start", "target": "slow"}, {"source": "slow", "target": "next"}, + {"source": "next", "target": "out"}]} + + +@pytest.fixture +def slow(monkeypatch): + """slow 那一步:第一次跑时卡住,等测试放行(好在它卡着的时候装出「续不上租约、被接手」);之后立刻返回。""" + calls = {"n": 0} + gate = asyncio.Event() + entered = asyncio.Event() + original = compiler.RUNNERS[NodeType.TRANSFORM] + + async def runner(state, ctx): + if ctx.node.id == "slow": + calls["n"] += 1 + if calls["n"] == 1: + entered.set() + await gate.wait() + return await original(state, ctx) + + monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, runner) + return {"calls": calls, "gate": gate, "entered": entered} + + +async def _taken_over(run_id: str) -> None: + """别的进程看它的租约过期了、接手:标中断、属主清空(同 run_manager.reap 做的)。""" + from sqlalchemy import update + + async with SessionLocal() as session: + await session.execute(update(Run).where(Run.id == run_id).values( + status="interrupted", owner=None, error="服务重启,运行已挂起,可从断点继续运行")) + await session.commit() + + +async def _events(run_id: str) -> list[RunEvent]: + async with SessionLocal() as session: + return list((await session.execute(select(RunEvent).where(RunEvent.run_id == run_id) + .order_by(RunEvent.seq))).scalars()) + + +async def _no_repeated_seq(run_id: str) -> None: + async with SessionLocal() as session: + rows = (await session.execute(select(RunEvent.seq, func.count()).where(RunEvent.run_id == run_id) + .group_by(RunEvent.seq).having(func.count() > 1))).all() + assert rows == [], f"撞号:{rows}" + + +async def _checkpointed_before_slow(run_id: str) -> None: + """等 slow 前面那一步的断点落了库(LangGraph 存断点和下一步并行:慢一点的库上 slow 已经开始、断点还在写)。""" + config = {"configurable": {"thread_id": run_id}} + for _ in range(300): + tup = await run_manager.checkpointer.aget_tuple(config) + if tup is not None and int((tup.metadata or {}).get("step", -1)) >= 1: + return + await asyncio.sleep(0.02) + raise AssertionError("slow 前面那一步的断点一直没落库") + + +async def _stall_and_take_over(slow) -> str: + run = await run_manager.start(graph=GRAPH, input_payload={"question": "q"}) + await asyncio.wait_for(slow["entered"].wait(), timeout=15) + await _checkpointed_before_slow(run.id) + before = fence.generation() + fence.renewed(ago=lease_s() + 1) # 卡了一个多租约时长 + await _taken_over(run.id) + slow["gate"].set() # 缓过来,手上的执行接着往下走 + for _ in range(200): + if not run_manager.is_active(run.id): + break + await asyncio.sleep(0.05) + assert not run_manager.is_active(run.id), "执行停了" + assert fence.generation() == before + 1, "发现续晚了:隔离了一次" + return run.id + + +async def test_a_stalled_executor_does_not_overwrite_the_takeover(slow, caplog): + import gc + import logging + + caplog.set_level(logging.ERROR, logger="asyncio") + run_id = await _stall_and_take_over(slow) + await asyncio.sleep(0.1) + gc.collect() + await asyncio.sleep(0) + noisy = [r.getMessage() for r in caplog.records if r.name == "asyncio"] + assert noisy == [], f"隔离时 asyncio 不该记回调异常、没人取的任务异常(复核 R-b8):{noisy}" + async with SessionLocal() as session: + row = await session.get(Run, run_id) + assert row.status == "interrupted" and row.owner is None, "接手之后的样子没被盖掉" + events = await _events(run_id) + assert not [e for e in events if e.type in ("run.finished", "run.failed", "run.cancelled")] + assert not [e for e in events if e.node_id == "next"], "缓过来之后一条都没写" + await _no_repeated_seq(run_id) + + +async def test_a_run_continued_elsewhere_finishes_once_and_the_stalled_side_stays_quiet(slow): + run_id = await _stall_and_take_over(slow) + fence.renewed() # 换了身份重新登记之后,这一世代照常 + await run_manager.continue_failed(run_id) # 接手的那一边接着跑 + for _ in range(300): + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed") and row.manifest_hash: + break + await asyncio.sleep(0.05) + assert row.status == "succeeded", row.error + events = await _events(run_id) + assert len([e for e in events if e.type == "run.finished"]) == 1 + assert len([e for e in events if e.type == "node.started" and e.node_id == "next"]) == 1, "后面那一步只跑了一次" + await _no_repeated_seq(run_id) + + +@pytest.mark.parametrize("jump", [3600.0, -3600.0]) +async def test_a_wall_clock_jump_does_not_fence_a_running_run(slow, monkeypatch, jump): + """复核 R-b14:执行到一半本机墙钟往前 / 往后跳一小时(NTP 一步校时、手动改时间)——计入休眠的钟没问题,不隔离,运行照常跑完。""" + import time as _time + + monkeypatch.setattr(fence, "SLEEP_AWARE", True) + run = await run_manager.start(graph=GRAPH, input_payload={"question": "q"}) + await asyncio.wait_for(slow["entered"].wait(), timeout=15) + before = fence.generation() + monkeypatch.setattr(fence, "_wall", lambda: _time.time() + jump) + slow["gate"].set() + for _ in range(300): + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed", "interrupted") and not run_manager.is_active(run.id): + break + await asyncio.sleep(0.05) + assert row.status == "succeeded", row.error + assert fence.generation() == before, "没有隔离" + + +async def test_a_run_that_cannot_start_after_fencing_is_failed_with_the_real_reason(monkeypatch): + """复核 R-b15:隔离之后新身份一直没登记上,新发起的运行等不到(fence.ready)——这一行照实收成失败(写明是隔离后没能重新 + 登记,不是「服务重启时还在排队」),run.failed 记下来,执行任务也从 run_manager 里摘掉。""" + from app.cluster import workers + from app.engine.runner import NOT_REREGISTERED_QUEUED + + monkeypatch.setattr(workers, "lease_s", lambda: 0.3) + fence.trip("测试") # 隔离了,新身份还没登记上 + run = await run_manager.start(graph=GRAPH, input_payload={"question": "q"}) + for _ in range(100): + if run.id not in run_manager._tasks: + break + await asyncio.sleep(0.05) + assert run.id not in run_manager._tasks, "执行任务摘掉了" + async with SessionLocal() as session: + row = await session.get(Run, run.id) + assert row.status == "failed" and row.error == NOT_REREGISTERED_QUEUED and row.owner is None + events = await _events(run.id) + assert [e.type for e in events if e.type == "run.failed"] == ["run.failed"] + assert events[-1].data["error"] == NOT_REREGISTERED_QUEUED + fence.renewed() + + +async def test_the_final_write_checks_the_owner(): + async with SessionLocal() as session: + run = Run(status="running", owner="peer-host:1:abcd", graph={}, input={}) + session.add(run) + await session.commit() + await run_manager._finalize(run.id, "succeeded", None, {}, 10, owner="mine:1:ffff") + async with SessionLocal() as session: + row = await session.get(Run, run.id) + assert row.status == "running" and row.finished_at is None, "属主已经不是自己:不写终态" + + +async def test_a_repeated_seq_is_refused_by_the_database(): + async with SessionLocal() as session: + run = Run(status="running", graph={}, input={}) + session.add(run) + await session.flush() + session.add_all([RunEvent(id=new_id(), run_id=run.id, seq=1, type="log", ts=0, data={}), + RunEvent(id=new_id(), run_id=run.id, seq=1, type="log", ts=0, data={})]) + with pytest.raises(IntegrityError): + await session.commit() + + +# ---------------- 复核 R-b9:谁会被打停、谁不会 ---------------- + + +async def test_only_executions_bound_to_a_generation_are_stopped_by_check(): + """续晚之后第一个问 check() 的不一定是运行本身:没绑定世代的调用者(HTTP 处理、调度那一拍)只隔离、不抛;绑定了的抛。 + 隔离之后新身份还没登记上,新开始的执行在 ready() 上等,等不到抛 Fenced;登记上了就放行。""" + gate = asyncio.Event() + seen: list[str] = [] + + async def bound_writer() -> None: + fence.bind() + await gate.wait() + try: + fence.check() + seen.append("wrote") + except fence.Fenced: + seen.append("fenced") + + task = asyncio.create_task(bound_writer()) + await asyncio.sleep(0) + before = fence.generation() + fence.renewed(ago=lease_s() + 1) + fence.check() # 没绑定世代:只隔离 + assert fence.generation() == before + 1 and fence.pending() + gate.set() + await task + assert seen == ["fenced"], "绑定在隔离之前的那一段执行:不许再写" + + with pytest.raises(fence.Fenced): + await fence.ready(timeout=0.1) # 新身份还没登记上:新开始的执行等不到就不开始 + waiting = asyncio.create_task(fence.ready(timeout=5)) + await asyncio.sleep(0.1) + assert not waiting.done() + fence.renewed() # 登记上了 + await asyncio.wait_for(waiting, timeout=2) + fence.bind() + fence.check() # 新世代照常 + + +async def test_the_scheduler_loop_survives_a_fenced_tick(monkeypatch): + from app.scheduler import loop as sloop + + calls = {"n": 0} + + async def fire_due(now=None): + calls["n"] += 1 + if calls["n"] == 1: + raise fence.Fenced() + return [] + + monkeypatch.setattr(sloop, "fire_due", fire_due) + await sloop.runner.start(interval=0.02) + try: + for _ in range(200): + if calls["n"] >= 3: + break + await asyncio.sleep(0.02) + assert calls["n"] >= 3 and sloop.runner.running(), "被 Fenced 打到的那一拍之后照常往下" + finally: + await sloop.runner.stop() + + +async def test_the_lease_task_keeps_going_when_registering_again_fails(monkeypatch): + """隔离之后重新登记:库一抖、版本守卫拦下都不让续租的任务退出,退避着重试;登记上了照常续租。""" + from app.cluster import identity, service as service_mod, workers + from app.cluster.version import VersionMismatch + + monkeypatch.setattr(identity, "OWNER", f"{identity.host()}:{os.getpid()}:retired1") # 测试完换回原来的身份 + monkeypatch.setattr(workers, "heartbeat_s", lambda: 0.01) + monkeypatch.setattr(workers, "lease_s", lambda: 5.0) + svc = service_mod._Service() + tries = {"register": 0, "renew": 0} + + async def register(*, exclusive: bool = False) -> None: # noqa: ARG001 + tries["register"] += 1 + if tries["register"] == 1: + raise VersionMismatch("库里还有版本不同的进程活着") + if tries["register"] == 2: + raise OSError("库一时连不上") + + async def renew() -> bool: + tries["renew"] += 1 + return True + + monkeypatch.setattr(workers, "register", register) + monkeypatch.setattr(svc, "renew", renew) + old = identity.owner() + fence.trip("测试") + svc._on_fenced() + assert identity.owner() != old and svc.pending_since is not None + task = asyncio.create_task(svc._lease_loop()) + try: + for _ in range(300): + if tries["renew"] >= 2: + break + await asyncio.sleep(0.01) + assert not task.done(), "续租的任务还在" + assert tries["register"] == 3 and svc.pending_since is None and not fence.pending() + assert tries["renew"] >= 2, "登记上之后照常续租" + finally: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task diff --git a/backend/tests/test_compare_api.py b/backend/tests/test_compare_api.py index 541f87b7..41fa1b8f 100644 --- a/backend/tests/test_compare_api.py +++ b/backend/tests/test_compare_api.py @@ -476,3 +476,79 @@ async def test_real_runs_split_data_and_parameter_changes(client, engine_up, ord body = await get(client, f"/api/runs/{third}/compare/base") assert (body["base_run_id"], body["reason"]) == (second, "same_workflow") json.dumps(body, allow_nan=False) + + +# -------------------------------------------------------------------------- +# 子工作流里的口径卡:还原输入按卡所在那一层找产出 +# -------------------------------------------------------------------------- + + +def sub_card_graph(): + """子工作流:取数 → 原样解析 → 口径卡(东区金额读单元格;区域数读解析出来的整份结果的行,要从解析节点当时的产出还原)。""" + return chain( + node("s", "input", "开始"), + node("fetch", "tool", "区域取数", tool=f"db_query__{E2E_SOURCE}", args={"sql": E2E_SQL.replace( + "{{ input.start }}", "2026-08-31").replace("{{ input.end }}", "2026-09-06")}), + node("parse", "transform", "原样解析", mode="json", template="{{ nodes.fetch }}", assign_to="rows"), + node("card", "metrics", "子口径卡", caliber="周报口径", caliber_version="v1", metrics=[ + {"id": "east", "name": "东区金额", "unit": "元", "decimals": 1, "format": "thousands", + "expression": "cell(nodes.fetch, 0, 'gmv')"}, + {"id": "regions", "name": "区域数", "unit": "个", "decimals": 0, "format": "plain", + "expression": "len(nodes.parse.rows)"}]), + node("o", "output", "出口", fields=[{"name": "指标", "value": "{{ nodes.card.text }}"}]), + ) + + +async def test_cards_inside_a_subworkflow_restore_inputs_from_their_own_layer(client, engine_up, orders): + """子工作流里的卡(这次运行里的 id「sub/card」)取值链写的是子图里的 id(nodes.parse):对比还原容器输入时要找「sub/parse」 + 的产出,不是父图里的 parse。父图里另有同名的 fetch、parse(查的是另一条 SQL),按层找才不会取错。""" + from tests import ledger_samples as S + + path, source_id = orders + sub_id = await S.publish("区域子流程", graph=sub_card_graph(), level="draft") + workflow = await new_workflow("子流程周报") + + def graph(): + return chain( + node("start", "input", "开始"), + # 父图里同名的取数、解析节点:一行,和子图里那份三行的结果不一样 + node("fetch", "tool", "父图取数", tool=f"db_query__{E2E_SOURCE}", + args={"sql": "SELECT COUNT(*) AS n FROM orders"}), + node("parse", "transform", "父图解析", mode="json", template="{{ nodes.fetch }}"), + node("sub", "subgraph", "区域子流程", workflow_id=sub_id, workflow_version=1), + node("out", "output", "成果", fields=[{"name": "指标", "value": "{{ nodes.sub.output.指标 }}"}]), + ) + + async def finished() -> str: + run = await run_manager.start(graph=graph(), input_payload={}, workflow_id=workflow, workflow_name="子流程周报") + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + assert row.status == "succeeded", row.error + return run.id + raise AssertionError(f"运行 {run.id} 没有结束") + + first = await finished() + db = sqlite3.connect(path) + db.execute("UPDATE orders SET amount = 1500.5 WHERE day = '2026-09-01'") + db.commit() + db.close() + await engines.invalidate(source_id) + second = await finished() + + body = await get(client, f"/api/runs/{second}/compare?base={first}") + metrics = {m["metric_id"] if "metric_id" in m else m["id"]: m for m in body["metrics"]} + east, regions = metrics["east"], metrics["regions"] + assert (east["classification"], east["base"], east["target"]) == ("data_only", 1200.5, 1500.5) + # 区域数:从子图里那次取数的产出还原(三行),两次一样 + assert (regions["base"], regions["target"]) == (3, 3) + assert regions["classification"] == "unchanged", regions + problems = [i.get("problem") for i in regions.get("inputs") or [] if i.get("problem")] + assert not problems, problems + # 区域数的那份结果顺着取值链认到子图里那次取数(两次之间改了数,快照不同),不是父图里同名的 fetch(两次一样) + [cell] = east["inputs"] + [rows] = regions["inputs"] + assert (rows["base_artifact"], rows["target_artifact"]) == (cell["base_artifact"], cell["target_artifact"]) + assert rows["base_artifact"] != rows["target_artifact"] diff --git a/backend/tests/test_copilot_reject_exit_guide.py b/backend/tests/test_copilot_reject_exit_guide.py new file mode 100644 index 00000000..e9b9cee7 --- /dev/null +++ b/backend/tests/test_copilot_reject_exit_guide.py @@ -0,0 +1,84 @@ +"""助手的节点手册要教「驳回」出口(docs/design/orchestration-round/00-总方案.md §3)。 + +手册里没写的出口,助手就不会接:驳回之后要通知申请人、退回修改、改走另一条路的,画出来的图只会在驳回时整轮结束。 +这里不只查关键词,还拿校验认的那一份(会被驳回的节点类型、出口的 sourceHandle、发布检查的「走不到结束」)来对, +再照手册的写法搭一张图,校验、发布检查都过。 +""" +from __future__ import annotations + +import re + +from app.api.copilot import NODE_REFERENCE +from app.engine.schema import GATE_TYPES, REJECT_HANDLE, GraphSpec, validate_graph + + +def _reject_entry() -> str: + start = NODE_REFERENCE.index("「驳回」出口") + return NODE_REFERENCE[start:NODE_REFERENCE.index("\n- 一个节点连出多条普通边", start)] + + +def test_the_guide_names_every_node_that_can_be_rejected_and_the_handle(): + entry = _reject_entry() + assert f'sourceHandle="{REJECT_HANDLE}"' in entry + # 能接驳回出口的节点类型(schema.GATE_TYPES)一个不漏,按 type 字段写 + assert all(f"{t}" in entry for t in GATE_TYPES), [t for t in GATE_TYPES if str(t) not in entry] + # 六种关卡都点到:审批、补充输入、编辑稿件、动作确认、代码 / 工具审批、双盲裁决 + for kind in ("审批", "补充输入", "编辑稿件", "动作确认", "代码执行审批", "工具调用审批", "裁决"): + assert kind in entry, kind + + +def test_the_guide_says_when_to_connect_it_and_what_happens_without_it(): + entry = _reject_entry() + # 什么时候接:驳回后要通知、退回修改、走另一条路 + assert "通知" in entry and "退回修改" in entry and "另一条路" in entry + # 不接时:运行结束、记被驳回(不算失败);审批模式没设驳回即终止的老规矩照实说 + assert "记为被驳回" in entry and "不算失败" in entry + assert "stop_on_reject" in entry + # 驳回那条路必须走到成果节点或结束(发布检查 reject_exit.no_end),绕回去改完再问一次也可以 + assert "成果节点" in entry and "不再往下连" in entry and "发布检查" in entry + # 驳回不是故障:失败重试、出错时继续不管它 + assert "失败重试" in entry + # 驳回之后下游取得到谁、备注 + assert "{{ nodes.关卡id.note }}" in entry + + +def test_the_guide_does_not_leak_internal_names(): + entry = _reject_entry() + for internal in ("reject_exit", "NodeRejected", "end_reason", "cancelled", "__decision__", "GATE_TYPES"): + assert internal not in entry, internal + + +def test_the_old_one_line_rule_is_replaced_not_duplicated(): + # 以前只有一句「human 节点(approve 模式):sourceHandle 用 approved 和 rejected」:驳回出口说一处,不留两种说法 + assert len(re.findall(r'sourceHandle="rejected"', NODE_REFERENCE)) == 1 + + +def _node(nid: str, ntype: str, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def test_a_graph_drawn_the_way_the_guide_says_passes_the_checks(): + """照手册写:审批批准走 approved,驳回接 sourceHandle="rejected" 到「通知申请人」再到一个成果节点;工具的确认被驳回 + 退回上游修改(连回起草,改完再问一次)。画布校验、各级发布检查都不报驳回出口的问题。""" + from app.engine.governance import lint_for_publish + + graph = { + "nodes": [_node("start", "input"), _node("draft", "llm", prompt="起草"), + _node("gate", "human", mode="approve", title="可以吗"), + _node("send", "tool", tool="calculator", args={"expression": "1+1"}, approval="always"), + _node("tell", "llm", prompt="通知申请人:{{ nodes.gate.note }}"), + _node("done", "output", fields=[{"name": "结果", "value": "{{ nodes.send.text }}"}]), + _node("back", "output", fields=[{"name": "结果", "value": "{{ nodes.tell.text }}"}])], + "edges": [{"source": "start", "target": "draft"}, {"source": "draft", "target": "gate"}, + {"source": "gate", "target": "send", "sourceHandle": "approved"}, + {"source": "gate", "target": "tell", "sourceHandle": "rejected"}, + {"source": "tell", "target": "back"}, + {"source": "send", "target": "done"}, + {"source": "send", "target": "draft", "sourceHandle": "rejected"}], + } + spec = GraphSpec.model_validate(graph) + codes = {i.code for i in validate_graph(spec).issues if i.level == "error"} + assert not {c for c in codes if c and "reject" in c}, codes + for level in ("draft", "published", "governed"): + issues = lint_for_publish(spec, level=level).issues + assert not [i for i in issues if (i.code or "").startswith("reject_exit")], level diff --git a/backend/tests/test_copy_to_postgres.py b/backend/tests/test_copy_to_postgres.py new file mode 100644 index 00000000..5bc37b31 --- /dev/null +++ b/backend/tests/test_copy_to_postgres.py @@ -0,0 +1,448 @@ +"""python -m app.db copy-to-postgres:SQLite 数据目录导入 PostgreSQL(db/to_postgres.py)。 + +端到端那几项要 PostgreSQL(scripts/test-postgres.sh):另起一个进程在 SQLite 数据目录里跑一次停在审批上的运行(带事件、 +审批、工件引用),导入一个新建的空库,再另起一个进程连这个 PostgreSQL、同一个数据目录,把审批批了,运行接着跑完。 +不连库的几项(来源认不认、服务停没停、参数)两种库都跑。全是虚构数据。 +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from sqlalchemy import text + +from app.core.config import settings +from app.db import to_postgres +from app.db.to_postgres import CopyRefused, copy_to_postgres + +PG = bool(os.environ.get("AGENTLAB_TEST_DATABASE_URL", "").strip()) +needs_pg = pytest.mark.skipif(not PG, reason="要 AGENTLAB_TEST_DATABASE_URL(scripts/test-postgres.sh)") +BACKEND = Path(__file__).resolve().parents[1] +#: 不会去连的地址:导入前的检查先于连目标库 +NOWHERE = "postgresql+asyncpg://nobody@127.0.0.1:1/none" + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +GRAPH = { + "nodes": [node("start", "input", fields=[{"name": "question"}]), + node("prep", "transform", mode="template", template="第一段"), + node("gate", "human", mode="approve", title="看一眼"), + node("after", "transform", mode="template", template="第二段"), + node("out", "output", fields=[{"name": "结果", "value": "{{ nodes.after }}"}])], + "edges": [{"source": a, "target": b} for a, b in + (("start", "prep"), ("prep", "gate"), ("gate", "after"), ("after", "out"))], +} + +MAKE_SOURCE = r''' +import asyncio, json, sys + +async def main(): + from sqlalchemy import text + from app.core import artifact_store + from app.db.base import SessionLocal, engine, init_db + from app.db.models import DataSource, Run + from app.db.models_sources import SourceChange + from app.engine.runner import run_manager + + await init_db() + await run_manager.setup() + run = await run_manager.start(graph=json.loads(sys.argv[1]), input_payload={"question": "虚构的问题"}) + status = None + for _ in range(400): + async with SessionLocal() as s: + status = (await s.get(Run, run.id)).status + if status in ("interrupted", "failed", "succeeded"): + break + await asyncio.sleep(0.05) + await run_manager.wait_idle(run.id) + artifact = await artifact_store.put_json({"区域": "虚构", "数": 1}, kind="test_copy", run_id=run.id) + async with SessionLocal() as s: + source = DataSource(name="虚构的源", kind="sqlite", database="/tmp/虚构的源.db") + s.add(source) + await s.flush() + s.add(SourceChange(source_id=source.id, kind="column_added", fingerprint="f" * 64, table_name="订单")) + dropped = SourceChange(source_id=source.id, kind="column_removed", fingerprint="d" * 64, table_name="订单") + s.add(dropped) + await s.flush() + # 删掉最后一条:SQLite 的 AUTOINCREMENT 计数器停在 2,表里的 MAX 是 1。导过去之后新的变化要从 3 编起 + await s.delete(dropped) + long = Run(workflow_name="长" * 300, last_seq=5_000_000_000, origin={"b": 2, "a": 1}) + sql_null = Run(workflow_name="SQL 的空", origin={"x": 1}) + json_null = Run(workflow_name="JSON 的空", origin=None) + s.add_all([long, sql_null, json_null]) + await s.commit() + await s.execute(text("UPDATE runs SET origin = NULL WHERE id = :i"), {"i": sql_null.id}) + await s.commit() + out = {"long": long.id, "sql_null": sql_null.id, "json_null": json_null.id, "source": source.id} + async with SessionLocal() as s: + last_seq = (await s.get(Run, run.id)).last_seq + await run_manager.shutdown() + await engine.dispose() + print(json.dumps({"run": run.id, "status": status, "last_seq": last_seq, "artifact": artifact, **out})) + +asyncio.run(main()) +''' + +RESUME = r''' +import asyncio, json, sys + +async def main(): + from sqlalchemy import select + from app.core import artifact_store + from app.db.base import SessionLocal, engine, init_db + from app.db.models import Approval, Run, RunEvent + from app.db.models_sources import SourceChange + from app.engine.runner import run_manager + + run_id, source_id, artifact = sys.argv[1], sys.argv[2], sys.argv[3] + await init_db() + await run_manager.setup() + await run_manager.resume(run_id, {"approved": True}) + status = None + for _ in range(400): + async with SessionLocal() as s: + status = (await s.get(Run, run_id)).status + if status in ("failed", "succeeded"): + break + await asyncio.sleep(0.05) + await run_manager.wait_idle(run_id) + async with SessionLocal() as s: + seqs = list((await s.execute(select(RunEvent.seq).where(RunEvent.run_id == run_id) + .order_by(RunEvent.seq))).scalars()) + approvals = list((await s.execute(select(Approval.status).where(Approval.run_id == run_id))).scalars()) + row = await s.get(Run, run_id) + change = SourceChange(source_id=source_id, kind="column_removed", fingerprint="e" * 64, table_name="订单") + s.add(change) + await s.commit() + out = {"status": status, "error": row.error, "output": row.output, "seqs": seqs, "approvals": approvals, + "new_change_seq": change.seq, "artifact": artifact_store.load(artifact)} + await run_manager.shutdown() + await engine.dispose() + print(json.dumps(out, ensure_ascii=False)) + +asyncio.run(main()) +''' + + +def _run(script: str, *args: str, data_dir: Path, database_url: str = "") -> dict: + env = {**os.environ, "AGENTLAB_DATA_DIR": str(data_dir), "AGENTLAB_DATABASE_URL": database_url, + "AGENTLAB_CHECKPOINT_URL": "", "AGENTLAB_SCHEDULER": "0", "PYTHONPATH": str(BACKEND)} + env.pop("AGENTLAB_DB_POOL_SIZE", None) + done = subprocess.run([sys.executable, "-c", script, *args], env=env, capture_output=True, text=True, + timeout=180, cwd=str(data_dir)) + assert done.returncode == 0, done.stderr[-4000:] + return json.loads(done.stdout.strip().splitlines()[-1]) + + +def _sql(url: str, sql: str, *params) -> list[tuple]: + import psycopg + + from app.db.urls import libpq_conninfo + + with psycopg.connect(libpq_conninfo(url)) as conn: + return [tuple(r) for r in conn.execute(sql, params).fetchall()] + + +# -------------------------------------------------------------------------- +# 不连库:来源、服务停没停、参数 +# -------------------------------------------------------------------------- + + +def test_the_source_must_be_an_sqlite_app_database(tmp_path): + with pytest.raises(CopyRefused, match="找不到应用库"): + to_postgres.resolve_source(tmp_path) + (tmp_path / "agentlab.db").write_bytes(b"not a database at all") + with pytest.raises(CopyRefused, match="不是 SQLite 库文件"): + to_postgres.resolve_source(tmp_path) + + +async def test_a_source_still_open_is_refused_before_anything_is_written(tmp_path): + import sqlite3 + + sqlite3.connect(tmp_path / "agentlab.db").execute("CREATE TABLE t (x)").connection.close() + (tmp_path / "agentlab.db-wal").write_bytes(b"") + with pytest.raises(CopyRefused, match="先停服务"): + await copy_to_postgres(tmp_path, NOWHERE, progress=lambda _: None) + + +def test_the_command_needs_a_target(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(settings, "database_url", "") + with pytest.raises(SystemExit) as exited: + to_postgres.main([str(tmp_path)]) + assert exited.value.code == 2 and "没有目标地址" in capsys.readouterr().err + assert to_postgres.main([str(tmp_path), "mysql://x@h/db"]) == 2 + assert "只认 PostgreSQL" in capsys.readouterr().err + + +def test_an_unreachable_target_is_reported_in_one_line(tmp_path, capsys): + """目标库连不上(库的容器没起来、被停了):一句话说清楚、退出码 1,不甩一屏调用栈(deploy.sh migrate-to-postgres 照它回退)。""" + import sqlite3 + + sqlite3.connect(tmp_path / "agentlab.db").execute("CREATE TABLE t (x)").connection.close() + for extra, what in (([], "没有导入完"), (["--dry-run"], "没有核对完")): + assert to_postgres.main([str(tmp_path), "postgresql+asyncpg://u:p@127.0.0.1:1/agentlab", *extra]) == 1 + err = capsys.readouterr().err + assert err.startswith(what) and "Traceback" not in err and len(err.strip().splitlines()) == 1 + + +def test_the_dispatcher_knows_its_commands(capsys): + from app.db.__main__ import main + + assert main([]) == 2 and "copy-to-postgres" in capsys.readouterr().out + assert main(["drop-everything"]) == 2 and "认不出的命令" in capsys.readouterr().err + + +def test_pending_writes_keep_their_order_and_special_channels_apart(): + from langgraph.checkpoint.base import WRITES_IDX_MAP + + special = next(iter(WRITES_IDX_MAP)) + groups = to_postgres._split_writes([("t1", special, 1), ("t1", "a", 2), ("t2", "b", 3), ("t1", "c", 4)]) + assert groups == {"t1": ([(special, 1)], [("a", 2), ("c", 4)]), "t2": ([], [("b", 3)])} + + +def test_values_sqlite_kept_but_postgresql_refuses_are_fixed_and_counted(): + from sqlalchemy import Column, Integer, MetaData, String, Table + + from app.db.base import PortableJSON + + table = Table("t", MetaData(), Column("n", Integer), Column("s", String(5)), Column("j", PortableJSON)) + fixed: dict[str, int] = {} + fix = to_postgres._Fixer(fixed) + assert fix.value(table.c.n, 3.0) == 3 and fix.value(table.c.n, " 12 ") == 12 and fix.value(table.c.n, 7) == 7 + assert fix.value(table.c.s, 42) == "42" and fix.value(table.c.s, "a\x00b") == "ab" + assert fix.value(table.c.j, '{"b": 1, "a": NaN}') == '{"b": 1, "a": null}' + assert fix.value(table.c.j, '{"b": 1, "a": 2}') == '{"b": 1, "a": 2}', "平常的 JSON 原文照搬,键的先后不变" + assert fix.value(table.c.j, None) is None + assert set(fixed) == {"整数列里的浮点写法", "整数列里的数字字符串", "文本列里的非文本值", "文本里的 NUL 字符", + "JSON 里的 NaN / Infinity / NUL"} + + +# -------------------------------------------------------------------------- +# 端到端:SQLite 上停在审批的运行,导入 PostgreSQL 后接着跑完 +# -------------------------------------------------------------------------- + + +@pytest.fixture +def source_dir(tmp_path): + """另起一个进程,在 SQLite 数据目录里留下一次停在审批上的运行和几行只看值的数据。""" + data = tmp_path / "data" + data.mkdir() + made = _run(MAKE_SOURCE, json.dumps(GRAPH), data_dir=data) + assert made["status"] == "interrupted" + assert not Path(f"{data}/agentlab.db-wal").exists(), "来源进程正常退出后不该留着 -wal" + return data, made + + +@needs_pg +async def test_a_waiting_run_is_imported_and_finishes_on_postgresql(source_dir): + from tests.appdb import fresh_postgres_database + + data, made = source_dir + lines: list[str] = [] + with fresh_postgres_database("copy") as target: + report = await copy_to_postgres(data, target, progress=lines.append) + assert report.ok, "\n".join(to_postgres.render(report)) + by_table = {t.name: t for t in report.tables} + assert by_table["runs"].source == by_table["runs"].present_after == 4 + assert by_table["run_events"].source > 0 and by_table["approvals"].present_after == 1 + assert by_table["artifacts"].present_after >= 1 and by_table["source_changes"].present_after == 1 + checkpoints, writes = report.checkpoints + assert checkpoints > 0 and report.checkpoints_target == (checkpoints, writes) + assert report.threads == 1 and report.checkpoints_written == checkpoints + + # 值:超长的名字、过 21 亿的整数照存;SQL 的 NULL 和 JSON 的 null 分开;JSON 原文(键的先后)不变 + [(name, seq)] = _sql(target, "SELECT workflow_name, last_seq FROM runs WHERE id = %s", made["long"]) + assert name == "长" * 300 and seq == 5_000_000_000 + assert _sql(target, "SELECT CAST(origin AS TEXT) FROM runs WHERE id = %s", made["long"]) == \ + [('{"b": 2, "a": 1}',)] + assert _sql(target, "SELECT origin IS NULL FROM runs WHERE id = %s", made["sql_null"]) == [(True,)] + assert _sql(target, "SELECT CAST(origin AS TEXT) FROM runs WHERE id = %s", made["json_null"]) == [("null",)] + + # 再导一遍:什么都不重复,目标上已有的不动 + again = await copy_to_postgres(data, target, progress=lambda _: None) + assert again.ok and again.checkpoints_written == 0 + assert all(t.present_before == t.source for t in again.tables) + assert {t.name: t.target_after for t in again.tables} == {t.name: t.target_after for t in report.tables} + + # PostgreSQL 上、同一个数据目录:批了审批,运行从断点接着跑完 + done = _run(RESUME, made["run"], made["source"], made["artifact"], data_dir=data, database_url=target) + assert done["status"] == "succeeded", done["error"] + assert done["output"]["结果"] == "第二段" + assert done["approvals"] == ["resolved"], "导入的那条审批就是被批掉的那条,没有另起一条" + seqs = done["seqs"] + assert seqs == sorted(set(seqs)) and seqs[-1] > made["last_seq"], "接着导入前的序号往下编" + assert done["artifact"] == {"区域": "虚构", "数": 1} + assert done["new_change_seq"] == 3, "序列接在 SQLite 的 AUTOINCREMENT 计数器后面,删过的号不复用(召回按它当游标)" + + +@needs_pg +async def test_dry_run_counts_and_writes_nothing(source_dir): + from tests.appdb import fresh_postgres_database + + data, _ = source_dir + with fresh_postgres_database("dry") as target: + report = await copy_to_postgres(data, target, dry_run=True, progress=lambda _: None) + assert report.ok and report.checkpoints[0] > 0 + runs = next(t for t in report.tables if t.name == "runs") + assert runs.source == 4 and runs.present_before == runs.present_after == 0 + assert _sql(target, "SELECT count(*) FROM information_schema.tables WHERE table_name IN " + "('runs', 'checkpoints')") == [(0,)] + assert "只核对,什么都没写" in to_postgres.render(report)[-1] + + +@needs_pg +async def test_a_target_with_the_service_still_connected_is_refused(source_dir): + import psycopg + + from app.db.urls import libpq_conninfo + from tests.appdb import fresh_postgres_database + + data, _ = source_dir + with fresh_postgres_database("busy") as target: + with psycopg.connect(libpq_conninfo(target), application_name="agentlab"): + with pytest.raises(CopyRefused, match="服务的连接"): + await copy_to_postgres(data, target, progress=lambda _: None) + assert _sql(target, "SELECT count(*) FROM information_schema.tables WHERE table_name = 'runs'") == [(0,)] + + +# -------------------------------------------------------------------------- +# 老库:补列时没带缺省值的 NOT NULL 列、SQLite 当时没强制外键留下的孤儿行 +# -------------------------------------------------------------------------- + + +def _legacy_source(folder: Path) -> Path: + """照模型建一个 SQLite 应用库,再做旧:定时任务的 trigger 列删掉、按第一版迁移(不带缺省值)补回来,存量行是 NULL; + 关着外键写几行孤儿行(对话轮次指向已删的运行、事件指向已删的运行、轮次指向已删的对话)。""" + from datetime import datetime, timezone + + from sqlalchemy import create_engine, insert + + from app.db import models # noqa: F401 + from app.db.base import Base + + folder.mkdir(parents=True, exist_ok=True) + path = folder / "agentlab.db" + engine = create_engine(f"sqlite:///{path}") + tables = Base.metadata.tables + try: + Base.metadata.create_all(engine) + with engine.begin() as conn: + conn.execute(insert(tables["schedules"]).values(id="s1", workflow_id="w1", cron="0 9 * * *", + timezone="Asia/Shanghai")) + conn.execute(insert(tables["schedule_fires"]).values( + id="f1", schedule_id="s1", fire_at=datetime(2026, 1, 1, tzinfo=timezone.utc))) + conn.execute(insert(tables["conversations"]).values(id="c1", title="虚构的对话")) + conn.execute(insert(tables["conversation_turns"]).values(id="t1", conversation_id="c1", seq=0, + run_id="run-gone")) + conn.execute(insert(tables["conversation_turns"]).values(id="t2", conversation_id="conv-gone", seq=0)) + conn.execute(insert(tables["run_events"]).values(id="e1", run_id="run-gone", seq=1, type="log", ts=0.0, + data={})) + with engine.begin() as conn: + conn.execute(text("ALTER TABLE schedules DROP COLUMN \"trigger\"")) + conn.execute(text("ALTER TABLE schedules ADD COLUMN \"trigger\" JSON")) + finally: + engine.dispose() + return path + + +def test_old_sqlite_databases_get_null_not_null_columns_filled(tmp_path): + """SQLite 这边:启动时的迁移把老版本补列留下的 NULL 填成模型的缺省值,两种库存的一样。""" + from sqlalchemy import create_engine + + from app.db.base import _migrate + + path = _legacy_source(tmp_path / "old") + engine = create_engine(f"sqlite:///{path}") + try: + with engine.begin() as conn: + assert conn.execute(text("SELECT \"trigger\" FROM schedules")).scalar_one() is None + _migrate(conn) + assert conn.execute(text("SELECT \"trigger\" FROM schedules")).scalar_one() == "{}" + finally: + engine.dispose() + + +def test_orphan_rows_are_nulled_or_left_out_on_the_snapshot(tmp_path): + path = _legacy_source(tmp_path / "old") + fixed: dict[str, int] = {} + orphans = to_postgres.settle_orphans(path, fixed=fixed) + assert fixed == {"外键指向已不存在的行,置空": 1} + assert orphans == {"conversation_turns": 1, "run_events": 1} + import sqlite3 + + db = sqlite3.connect(path) + try: + assert db.execute("SELECT id, run_id FROM conversation_turns").fetchall() == [("t1", None)] + assert db.execute("SELECT count(*) FROM run_events").fetchone() == (0,) + finally: + db.close() + + +@needs_pg +async def test_an_old_database_with_nulls_and_orphans_imports_cleanly(tmp_path): + """真实开发库上核对不过的那两种:定时任务 trigger 为 NULL(NOT NULL 列),对话轮次指向已删的运行。导入照样核对通过, + 孤儿行照实列出、不算失败,重跑也一样。""" + from tests.appdb import fresh_postgres_database + + folder = tmp_path / "old" + _legacy_source(folder) + with fresh_postgres_database("legacy") as target: + report = await copy_to_postgres(folder, target, progress=lambda _: None) + lines = to_postgres.render(report) + assert report.ok, "\n".join(lines) + assert report.orphans == {"conversation_turns": 1, "run_events": 1} + assert report.fixed.get("外键指向已不存在的行,置空") == 1 + assert any("孤儿行" in line and "run_events 1 行" in line for line in lines) + assert _sql(target, "SELECT CAST(\"trigger\" AS TEXT) FROM schedules") == [("{}",)] + assert _sql(target, "SELECT count(*) FROM schedule_fires") == [(1,)] + assert _sql(target, "SELECT run_id FROM conversation_turns") == [(None,)] + again = await copy_to_postgres(folder, target, progress=lambda _: None) + assert again.ok and again.orphans == report.orphans + + +def test_null_not_null_values_fall_back_to_the_model_default(): + from app.db import models # noqa: F401 表登记进元数据(这个进程里之前没有测试导入过模型时 schedules 还不在) + from app.db.base import Base + + schedules = Base.metadata.tables["schedules"] + fixed: dict[str, int] = {} + fix = to_postgres._Fixer(fixed) + assert fix.value(schedules.c["trigger"], None) == "{}" # JSON 列按原文绑定 + assert fix.value(schedules.c["trigger_kind"], None) == "cron" + assert fix.value(schedules.c["target_id"], None) is None # 可空的照旧是 NULL + assert fix.value(schedules.c["cron"], None) is None # 没有缺省值的:照实交给库去拒 + assert fixed == {"NOT NULL 列的空值填成缺省值": 2} + + +# -------------------------------------------------------------------------- +# 停服务之后数据目录里不留 -wal(导入按它认服务停没停) +# -------------------------------------------------------------------------- + +START_STOP = r''' +from fastapi.testclient import TestClient +from app.main import app + +with TestClient(app) as client: + assert client.get("/api/health").status_code == 200 +''' + + +def test_stopping_the_service_leaves_no_wal_behind(tmp_path): + data = tmp_path / "data" + data.mkdir() + env = {**os.environ, "AGENTLAB_DATA_DIR": str(data), "AGENTLAB_DATABASE_URL": "", "AGENTLAB_CHECKPOINT_URL": "", + "AGENTLAB_SCHEDULER": "0", "PYTHONPATH": str(BACKEND)} + done = subprocess.run([sys.executable, "-c", START_STOP], env=env, capture_output=True, text=True, timeout=180, + cwd=str(data)) + assert done.returncode == 0, done.stderr[-4000:] + assert (data / "agentlab.db").is_file() and (data / "checkpoints.db").is_file() + assert sorted(p.name for p in data.iterdir() if p.name.endswith(("-wal", "-shm"))) == [] + assert to_postgres.source_in_use(data / "agentlab.db", data / "checkpoints.db") is None diff --git a/backend/tests/test_datasources_upload.py b/backend/tests/test_datasources_upload.py index 51ca5fb7..30060844 100644 --- a/backend/tests/test_datasources_upload.py +++ b/backend/tests/test_datasources_upload.py @@ -556,9 +556,16 @@ async def test_startup_locks_legacy_sources_registered_under_other_spellings(cli def _app_db() -> str: - """测试进程实际连着的元数据库(conftest 建的那个)。data_dir 被夹具换过,和 settings.db_path 不是同一个。""" + """测试进程实际连着的元数据库(conftest 建的那个)。data_dir 被夹具换过,和 settings.db_path 不是同一个。 + + 应用库是 PostgreSQL 时没有连着的库文件:拿数据目录里的 agentlab.db 顶上(导入 PostgreSQL 之前留下的那一份, + 同一批元数据,照样不许登记),先建出这个文件,硬链接、大小写这些变体才有东西可指。""" from app.db import base as db_base + if db_base.engine.dialect.name != "sqlite": + settings.db_path.parent.mkdir(parents=True, exist_ok=True) + settings.db_path.touch() + return str(settings.db_path) return str(db_base.engine.url.database) diff --git a/backend/tests/test_db_direct.py b/backend/tests/test_db_direct.py new file mode 100644 index 00000000..13c52dd0 --- /dev/null +++ b/backend/tests/test_db_direct.py @@ -0,0 +1,133 @@ +"""直连地址和 PgBouncer 事务模式(AGENTLAB_DATABASE_DIRECT_URL、AGENTLAB_DB_PGBOUNCER):地址怎么解析、哪些连接走直连、事务模式下 +关掉了哪些东西。只建引擎、不连库,两种库都跑。""" +from __future__ import annotations + +import pytest +from sqlalchemy.pool import NullPool + +from app.core.config import Settings, settings + +APP = "postgres://agentlab:secret@pgbouncer.internal:6432/agentlab" +DIRECT = "postgresql://agentlab:secret@db.internal:5432/agentlab" + + +def test_the_direct_url_is_normalized_and_used_for_session_bound_connections(): + s = Settings(database_url=APP, database_direct_url=DIRECT, _env_file=None) + assert s.database_url.startswith("postgresql+asyncpg://") and "pgbouncer.internal:6432" in s.database_url + assert s.database_direct_url.startswith("postgresql+asyncpg://") and "db.internal:5432" in s.database_direct_url + assert s.session_db_url == s.database_direct_url + assert "db.internal" in s.checkpoint_conninfo and s.checkpoint_conninfo.startswith("postgresql://"), "断点库缺省走直连" + plain = Settings(database_url=APP, _env_file=None) + assert plain.session_db_url == plain.database_url and "pgbouncer.internal" in plain.checkpoint_conninfo, "不设就全用同一个" + with pytest.raises(ValueError, match="AGENTLAB_DATABASE_DIRECT_URL"): + Settings(database_url=APP, database_direct_url="mysql://x@y/z", _env_file=None) + + +def test_transaction_mode_turns_off_client_pooling_and_statement_caches(): + from app.db.base import pg_engine_kwargs + + kw = pg_engine_kwargs(pgbouncer=True) + assert kw["poolclass"] is NullPool + args = kw["connect_args"] + assert args["statement_cache_size"] == 0 and args["prepared_statement_cache_size"] == 0 + names = {args["prepared_statement_name_func"]() for _ in range(3)} + assert len(names) == 3, "预备语句的名字每次都不一样,不和别的客户端撞名" + direct = pg_engine_kwargs(pgbouncer=False, pool_size=5) + assert direct["pool_size"] == 5 and "statement_cache_size" not in direct["connect_args"] + + +def test_listen_and_cluster_locks_go_direct(monkeypatch): + from app.cluster.signal import listen_dsn + from app.db import base + + monkeypatch.setattr(settings, "database_url", Settings(database_url=APP, _env_file=None).database_url) + monkeypatch.setattr(settings, "database_direct_url", Settings(database_url=APP, database_direct_url=DIRECT, + _env_file=None).database_direct_url) + monkeypatch.setattr(base, "_session_engine", None) + assert "db.internal:5432" in listen_dsn() and listen_dsn().startswith("postgresql://") + made = base.session_engine() + try: + assert made is not base.engine and made.url.host == "db.internal" and made.url.port == 5432 + assert base.session_engine() is made, "直连那个池子只建一个" + finally: + monkeypatch.setattr(base, "_session_engine", None) + made.sync_engine.dispose() + monkeypatch.setattr(settings, "database_direct_url", "") + assert base.session_engine() is base.engine and "pgbouncer.internal" in listen_dsn() + + +def test_transaction_mode_without_a_direct_url_refuses_to_start(monkeypatch): + from app.db import base + + monkeypatch.setattr(settings, "database_url", Settings(database_url=APP, _env_file=None).database_url) + monkeypatch.setattr(settings, "db_pgbouncer", True) + monkeypatch.setattr(settings, "database_direct_url", "") + with pytest.raises(RuntimeError, match="AGENTLAB_DATABASE_DIRECT_URL"): + base.check_pgbouncer() + monkeypatch.setattr(settings, "database_direct_url", Settings(database_url=APP, database_direct_url=DIRECT, + _env_file=None).database_direct_url) + base.check_pgbouncer() + + +def test_transaction_mode_with_a_direct_url_that_is_the_same_endpoint_refuses_to_start(monkeypatch): + """复核 R-b12:直连地址和应用库地址规范化之后连的是同一个主机端口(还是那个 PgBouncer)——同样拒绝,说清楚。""" + from app.db import base + + monkeypatch.setattr(settings, "database_url", Settings(database_url=APP, _env_file=None).database_url) + monkeypatch.setattr(settings, "db_pgbouncer", True) + for same in (APP, "postgresql+asyncpg://other:pw@PGBOUNCER.internal:6432/agentlab?ssl=require"): + monkeypatch.setattr(settings, "database_direct_url", + Settings(database_url=APP, database_direct_url=same, _env_file=None).database_direct_url) + with pytest.raises(RuntimeError, match="指着同一个地址"): + base.check_pgbouncer() + monkeypatch.setattr(settings, "database_direct_url", Settings( + database_url=APP, database_direct_url="postgresql://agentlab:secret@pgbouncer.internal:5432/agentlab", + _env_file=None).database_direct_url) + base.check_pgbouncer() # 同一台机器上的另一个端口(直连 PostgreSQL 或会话模式):放行 + + +def test_copying_checkpoints_goes_through_the_direct_url(monkeypatch, tmp_path): + """复核 R-b12:copy-to-postgres 搬断点缺省走直连地址(设了的话);单独给了断点库地址就用它。""" + from app.db import to_postgres + + assert "db.internal:5432" in to_postgres.checkpoint_conninfo(APP, direct_url=DIRECT) + assert "pgbouncer.internal:6432" in to_postgres.checkpoint_conninfo(APP) + assert "ckpt.internal" in to_postgres.checkpoint_conninfo(APP, checkpoint_url="postgresql://u:p@ckpt.internal/c", + direct_url=DIRECT) + seen: dict = {} + + async def fake_copy(source, target, **kw): + seen.update(kw, target=target) + raise to_postgres.CopyRefused("测试:到这里为止") + + monkeypatch.setattr(to_postgres, "copy_to_postgres", fake_copy) + monkeypatch.setattr(settings, "database_url", Settings(database_url=APP, _env_file=None).database_url) + monkeypatch.setattr(settings, "database_direct_url", + Settings(database_url=APP, database_direct_url=DIRECT, _env_file=None).database_direct_url) + monkeypatch.setattr(settings, "checkpoint_url", "") + assert to_postgres.main([str(tmp_path)]) == 2 + assert seen["direct_url"] == settings.database_direct_url, "目标就是应用库:断点走它的直连地址" + assert to_postgres.main([str(tmp_path), "postgresql://u:p@elsewhere.internal:5432/x"]) == 2 + assert seen["direct_url"] is None, "目标是别的库:不套用应用库的直连地址" + + +def test_loopback_spellings_count_as_the_same_endpoint(monkeypatch): + """R-b12 的尾巴:localhost、127.0.0.1、::1 是同一台机器——同一个端口时照样拒绝;IP 按地址比,解析不成 IP 的按字面比。""" + from app.db import base + + def norm(url: str) -> str: + return Settings(database_url=url, _env_file=None).database_url + + monkeypatch.setattr(settings, "db_pgbouncer", True) + for app_url, direct in (("postgresql://u:p@localhost:6432/db", "postgresql://u:p@127.0.0.1:6432/db"), + ("postgresql://u:p@127.0.0.1:6432/db", "postgresql://u:p@[::1]:6432/db"), + ("postgresql://u:p@10.0.0.5:6432/db", "postgresql://u:p@[::ffff:10.0.0.5]:6432/db")): + monkeypatch.setattr(settings, "database_url", norm(app_url)) + monkeypatch.setattr(settings, "database_direct_url", norm(direct)) + with pytest.raises(RuntimeError, match="指着同一个地址"): + base.check_pgbouncer() + monkeypatch.setattr(settings, "database_url", norm("postgresql://u:p@localhost:6432/db")) + monkeypatch.setattr(settings, "database_direct_url", norm("postgresql://u:p@127.0.0.1:5432/db")) + base.check_pgbouncer() # 本机另一个端口:放行 + monkeypatch.setattr(settings, "database_direct_url", norm("postgresql://u:p@db.internal:6432/db")) + base.check_pgbouncer() # 名字不同、不查 DNS:按字面算不同 diff --git a/backend/tests/test_db_portable.py b/backend/tests/test_db_portable.py new file mode 100644 index 00000000..1d0e134e --- /dev/null +++ b/backend/tests/test_db_portable.py @@ -0,0 +1,529 @@ +"""应用库在 SQLite、PostgreSQL 上行为一致(db/portable.py、db/postgres.py、db/base.py 的迁移)。 + +连着库的那几项在两种库上各跑一遍(scripts/test-postgres.sh 跑 PostgreSQL),断言同一套结果;只看编译结果的不连库; +PostgreSQL 老库迁移另建一个库(只在 AGENTLAB_TEST_DATABASE_URL 时跑)。全是虚构数据。 +""" + +from __future__ import annotations + +import os +import re +import uuid + +import pytest +from sqlalchemy import func, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.schema import CreateTable + +from app.db import base as db_base +from app.db.base import Base, SessionLocal +from app.db.models import Run, TableRecipe +from app.db.portable import json_eq, json_flag, json_is_string, json_text, unique_violation + +PG = bool(os.environ.get("AGENTLAB_TEST_DATABASE_URL", "").strip()) + + +def _pg_dialect(): + from sqlalchemy.dialects.postgresql.asyncpg import dialect + + from app.db import postgres + + made = dialect() + postgres.install(made) + return made + + +# -------------------------------------------------------------------------- +# 守着的约定 +# -------------------------------------------------------------------------- + + +def test_every_migrated_column_is_in_the_models(): + """PostgreSQL 上补列的类型取模型里这一列的(db/base._column_ddl):迁移表里的每一列模型里都得有。""" + missing = [(t, c) for t, c, _ in db_base._COLUMN_MIGRATIONS + if t not in Base.metadata.tables or c not in Base.metadata.tables[t].c] + assert missing == [] + + +def test_every_not_null_migrated_column_gets_a_value_for_old_rows(): + """迁移补的列模型里是 NOT NULL 的,存量行不能留 NULL(PostgreSQL 的表照模型带 NOT NULL,导入时整行插不进):要么补列时 + 带 DEFAULT 回填,要么登记在 _NOT_NULL_BACKFILLS 里每次启动按模型的缺省值填。""" + bad = [] + for table, column, ddl in db_base._COLUMN_MIGRATIONS: + col = Base.metadata.tables[table].c[column] + if col.nullable: + continue + if not re.search(r"\bDEFAULT\b", ddl, re.I) and (table, column) not in db_base._NOT_NULL_BACKFILLS: + bad.append((table, column)) + assert bad == [] + for table, column in db_base._NOT_NULL_BACKFILLS: + assert db_base.model_default(Base.metadata.tables[table].c[column])[0], (table, column) + + +def test_checkpoint_tables_do_not_clash_with_app_tables(): + """断点和应用表放在同一个库、同一个 schema:两边的表名不能撞。名字从两个 saver 自己的建表语句里取。""" + import inspect as pyinspect + + from langgraph.checkpoint.postgres.base import BasePostgresSaver + from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver + + created = set(re.findall(r"CREATE TABLE IF NOT EXISTS (\w+)", " ".join(BasePostgresSaver.MIGRATIONS))) + created |= set(re.findall(r"CREATE TABLE IF NOT EXISTS (\w+)", pyinspect.getsource(AsyncSqliteSaver.setup))) + assert {"checkpoints", "checkpoint_blobs", "checkpoint_writes", "checkpoint_migrations"} <= created + assert created & set(Base.metadata.tables) == set() + + +def test_the_compile_guard_catches_whole_json_values_however_they_are_written(): + """整个 JSON 值做比较、去重、分组、排序:编译时就报错(tests/jsonguard.py,conftest 挂在应用库引擎上,跑全量时一直生效)。 + 按表达式的类型认,逐行正则抓不到的写法也抓得到;按键筛选的 json_eq 等、IS NULL、UNION ALL 不受影响。""" + from sqlalchemy import union, union_all + + from tests.jsonguard import JsonMisuse + + dialect = db_base.engine.sync_engine.dialect + alias = Run.origin + caught = [ + select(Run.origin).distinct(), + select(Run).distinct(), + select(Run.id).order_by( + Run.created_at, + Run.origin, + ), + select(Run.id).order_by(alias.desc()), + select(Run.id).where(alias == {"a": 1}), + select(Run.id).where(Run.input.in_([{"q": 1}])), + select(Run.id).where(Run.origin["k"] == 1), + select(func.count()).select_from(Run).group_by(Run.options), + union(select(Run.origin), select(Run.input)), + ] + for statement in caught: + with pytest.raises(JsonMisuse): + statement.compile(dialect=dialect) + fine = [ + select(Run.origin).where(json_eq(Run.origin, "k", 1), Run.origin.is_(None), + Run.origin != None), # noqa: E711 - 写成 IS NOT NULL,json 上也能用 + select(Run.id).where(json_text(Run.origin, "k") == "x").order_by(json_text(Run.origin, "k")).distinct(), + union_all(select(Run.origin), select(Run.input)), + ] + for statement in fine: + statement.compile(dialect=dialect) + + +def test_postgres_columns_are_unbounded_varchar_bigint_and_json(): + """应用库上 String(n) 建成不限长的 VARCHAR、文本列按字节比(COLLATE "C",同 SQLite)、Integer 是 BIGINT(自增主键 + BIGSERIAL)、JSON 是 json(不是 JSONB:JSONB 会重排对象里的键)、时间是不带时区的 TIMESTAMP。SQLite 上的建表语句不变。""" + from sqlalchemy.dialects.sqlite.aiosqlite import dialect as sqlite_dialect + + runs = Base.metadata.tables["runs"] + pg = CreateTable(runs).compile(dialect=_pg_dialect()).string + assert 'id VARCHAR COLLATE "C" NOT NULL' in pg and "VARCHAR(" not in pg + assert 'error TEXT COLLATE "C"' in pg + assert "last_seq BIGINT" in pg and "INTEGER" not in pg + assert "input JSON NOT NULL" in pg and "JSONB" not in pg + assert "created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL" in pg + changes = CreateTable(Base.metadata.tables["source_changes"]).compile(dialect=_pg_dialect()).string + assert "seq BIGSERIAL NOT NULL" in changes + lite = CreateTable(runs).compile(dialect=sqlite_dialect()).string + assert "id VARCHAR(32) NOT NULL" in lite and "last_seq INTEGER" in lite and "input JSON NOT NULL" in lite + assert "COLLATE" not in lite + + +def test_postgres_casts_keep_their_length(): + """CAST(x AS VARCHAR(n)) 是截断的意思,照旧写长度;只有建表时的列不限长。""" + from sqlalchemy import String, cast + + sql = str(select(cast(Run.workflow_name, String(4))).compile(dialect=_pg_dialect())) + assert "CAST(runs.workflow_name AS VARCHAR(4))" in sql + + +def test_postgres_orders_nulls_like_sqlite(): + """SQLite 升序 NULL 在前、降序 NULL 在后:PostgreSQL 上可空列照这个补写,不可空的列、写明了的不动。""" + from app.db.models import Task + + sql = str(select(Run.id, func.row_number().over(order_by=Run.started_at).label("n")) + .order_by(Run.started_at, Run.finished_at.desc(), Run.created_at.desc(), + Task.due_at.desc().nulls_first(), Task.remind_at.asc().nulls_last()) + .compile(dialect=_pg_dialect())) + assert "OVER (ORDER BY runs.started_at ASC NULLS FIRST)" in sql + order = sql.split("ORDER BY", 2)[2] + assert "runs.started_at ASC NULLS FIRST" in order + assert "runs.finished_at DESC NULLS LAST" in order + assert "runs.created_at DESC," in order # 不可空:不补 + assert "tasks.due_at DESC NULLS FIRST" in order and "NULLS FIRST NULLS" not in order + assert "tasks.remind_at ASC NULLS LAST" in order + + +def test_postgres_integer_binds_are_bigint(): + """和 Integer 列比较、写进 Integer 列的参数按 BIGINT 转换,超过 21 亿的值不在半路溢出。""" + sql = str(select(Run.id).where(Run.last_seq > 5).compile(dialect=_pg_dialect())) + assert "runs.last_seq > $1::BIGINT" in sql + + +def test_database_urls_are_normalized_and_never_logged_with_the_password(): + from app.db.urls import describe, libpq_conninfo, normalize_checkpoint_url, normalize_database_url + + assert normalize_database_url("postgres://u:p%40w@h:5433/db") == "postgresql+asyncpg://u:p%40w@h:5433/db" + assert normalize_checkpoint_url("postgresql+asyncpg://u:pw@h/db") == "postgresql://u:pw@h/db" + # asyncpg 的 ssl= 换成 libpq 的 sslmode=,只给 asyncpg 看的参数不带给 psycopg + assert libpq_conninfo("postgresql+asyncpg://u:pw@h/db?ssl=require&prepared_statement_cache_size=0") \ + == "postgresql://u:pw@h/db?sslmode=require" + # 应用库地址写 libpq 的 sslmode=:换成 asyncpg 的 ssl=(以前原样交给 asyncpg,连的时候 TypeError);断点库换回 sslmode= + app_url = normalize_database_url("postgresql://u:pw@h/db?sslmode=verify-full") + assert app_url == "postgresql+asyncpg://u:pw@h/db?ssl=verify-full" + assert libpq_conninfo(app_url) == "postgresql://u:pw@h/db?sslmode=verify-full" + for bad, says in (("postgresql://u@h/db?sslrootcert=/etc/ca.pem", "PGSSLROOTCERT"), + ("postgresql://u@h/db?sslmode=strict", "sslmode=strict"), + ("postgresql://u@h/db?ssl=require&sslmode=require", "只写一个"), + ("postgresql://u@h/db?application_name=x", "application_name")): + with pytest.raises(ValueError, match=says): + normalize_database_url(bad) + assert "pw" not in describe("postgresql+asyncpg://u:pw@h:5433/db") + for bad in ("mysql://u@h/db", "sqlite:///x.db", "postgresql://h", "这不是地址"): + with pytest.raises(ValueError): + normalize_database_url(bad) + from app.core.config import Settings + + with pytest.raises(ValueError, match="AGENTLAB_DATABASE_URL"): + Settings(database_url="mysql://u@h/db") + assert Settings(database_url=" ").db_dialect == "sqlite" + + +def test_a_postgres_url_without_the_driver_installed_says_what_to_install(monkeypatch): + import importlib.util + + real = importlib.util.find_spec + monkeypatch.setattr(importlib.util, "find_spec", lambda name, *a, **k: None if name == "asyncpg" else real(name, *a, **k)) + with pytest.raises(RuntimeError, match=r"backend\[postgres\]"): + db_base.make_engine("postgresql+asyncpg://u:pw@127.0.0.1:1/db") + + +def test_the_startup_line_names_the_database_without_the_password(): + from sqlalchemy.engine import make_url + + line = db_base.describe_database() + password = make_url(db_base.settings.db_url).password + assert ("PostgreSQL" if PG else "SQLite") in line + assert not password or password not in line + + +# -------------------------------------------------------------------------- +# 连着库:两种库同一套结果 +# -------------------------------------------------------------------------- + + +async def _runs(origins: list[dict | None]) -> tuple[str, list[str]]: + tag = uuid.uuid4().hex[:12] + ids = [] + async with SessionLocal() as session: + for i, origin in enumerate(origins): + row = Run(workflow_name=f"便携-{tag}-{i}", origin=origin, input={"question": "虚构的问题"} if i % 2 else + {"question": 7}) + session.add(row) + await session.flush() + ids.append(row.id) + await session.commit() + return tag, ids + + +async def _match(tag: str, *where) -> list[int]: + async with SessionLocal() as session: + names = (await session.execute(select(Run.workflow_name).where( + Run.workflow_name.like(f"便携-{tag}-%"), *where))).scalars().all() + return sorted(int(n.rsplit("-", 1)[1]) for n in names) + + +async def test_json_helpers_match_the_same_rows_on_both_databases(): + tag, _ = await _runs([ + {"v": 3, "s": "3", "flag": True, "k": "甲"}, # 0 + {"v": "3", "s": 3, "flag": 1, "k": None}, # 1 + {"v": 4, "flag": 0, "k": "乙", "uses": ["r-1", "r-22"]}, # 2 + {"flag": False}, # 3 + {}, # 4 + None, # 5(JSON 的 null) + {"v": True, "n": 1.0, "b": False}, # 6 + {"v": 1, "n": 1, "b": 0}, # 7 + ]) + # 按 JSON 的类型比:数字 3 不等于字符串 "3";true 不等于 1、false 不等于 0(SQLite 的 json_extract 单用时分不开); + # 数 1 等于 1.0 + assert await _match(tag, json_eq(Run.origin, "v", 3)) == [0] + assert await _match(tag, json_eq(Run.origin, "v", "3")) == [1] + assert await _match(tag, json_eq(Run.origin, "s", "3")) == [0] + assert await _match(tag, json_eq(Run.origin, "v", 1)) == [7] + assert await _match(tag, json_eq(Run.origin, "v", True)) == [6] + assert await _match(tag, json_eq(Run.origin, "b", False)) == [6] + assert await _match(tag, json_eq(Run.origin, "b", 0)) == [7] + assert await _match(tag, json_eq(Run.origin, "n", 1)) == [6, 7] + assert await _match(tag, json_eq(Run.origin, "n", 1.0)) == [6, 7] + with pytest.raises(TypeError): + json_eq(Run.origin, "v", {"a": 1}) + # None:没有这个键、或者值是 null + assert await _match(tag, json_eq(Run.origin, "k", None)) == [1, 3, 4, 5, 6, 7] + # 标记位:true 和非零的数算真 + assert await _match(tag, json_flag(Run.origin, "flag")) == [0, 1] + from sqlalchemy import not_ + + assert await _match(tag, not_(json_flag(Run.origin, "flag"))) == [2, 3, 4, 5, 6, 7] + # 文本:字符串是它本身,数、true / false 是 JSON 写法(两种库都是) + assert await _match(tag, json_text(Run.origin, "k") == "乙") == [2] + assert await _match(tag, func.coalesce(json_text(Run.origin, "k"), "").in_(["甲", ""])) == [0, 1, 3, 4, 5, 6, 7] + assert await _match(tag, json_text(Run.origin, "v") == "3") == [0, 1] + assert await _match(tag, json_text(Run.origin, "v") == "1") == [7] + assert await _match(tag, json_text(Run.origin, "v") == "true") == [6] + assert await _match(tag, json_text(Run.origin, "n") == "1.0") == [6] + assert await _match(tag, json_text(Run.origin, "uses").contains("r-22", autoescape=True)) == [2] + # 是不是字符串:input.question 偶数行是数字、奇数行是字符串 + assert await _match(tag, json_is_string(Run.input, "question")) == [1, 3, 5, 7] + + +async def test_json_columns_keep_the_order_of_keys(): + """JSON 列存原文:对象里键的先后读回来不变(JSONB 会重排,应用里按键的先后出列名、出段落的地方就乱了)。""" + keys = ["单号", "金额", "电话", "b", "a", "aa"] + async with SessionLocal() as session: + row = Run(workflow_name="键序", origin={k: i for i, k in enumerate(keys)}) + session.add(row) + await session.commit() + rid = row.id + async with SessionLocal() as session: + got = (await session.execute(select(Run.origin).where(Run.id == rid))).scalar_one() + assert list(got) == keys + + +async def test_json_null_and_sql_null_stay_apart(): + """JSON 列写 None 存的是 JSON 的 null(不是 SQL 的 NULL),两种库一样:按 IS NULL 筛不到它。""" + async with SessionLocal() as session: + row = Run(workflow_name="空值", origin=None) + session.add(row) + await session.commit() + rid = row.id + async with SessionLocal() as session: + is_sql_null = (await session.execute(select(Run.origin.is_(None)).where(Run.id == rid))).scalar_one() + raw = (await session.execute(text("SELECT CAST(origin AS TEXT) FROM runs WHERE id = :i"), {"i": rid})).scalar_one() + assert is_sql_null is False and raw == "null" + + +async def test_long_strings_and_big_integers_are_kept(): + """String(n) 超长、Integer 过 21 亿:SQLite 照存,PostgreSQL 上也照存(列不限长、是 BIGINT)。""" + long_name = "长" * 500 # workflow_name 是 String(200) + big = 5_000_000_000 + async with SessionLocal() as session: + row = Run(workflow_name=long_name, last_seq=big) + session.add(row) + await session.commit() + rid = row.id + async with SessionLocal() as session: + got = (await session.execute(select(Run.workflow_name, Run.last_seq).where( + Run.id == rid, Run.last_seq > big - 1))).one() + assert tuple(got) == (long_name, big) + + +async def test_non_finite_floats_in_json_are_written(): + """JSON 里的 NaN:SQLite 照存;PostgreSQL 的 json 不收,写成 null(同浏览器的 JSON.stringify),不让整条写入失败。""" + async with SessionLocal() as session: + row = Run(workflow_name="非有限", origin={"x": float("nan"), "y": 1.5, "s": "a\x00b"}) + session.add(row) + await session.commit() + rid = row.id + async with SessionLocal() as session: + got = (await session.execute(select(Run.origin).where(Run.id == rid))).scalar_one() + assert got["y"] == 1.5 + if db_base.engine.dialect.name == "postgresql": + assert got["x"] is None and got["s"] == "ab" + else: + assert got["x"] != got["x"] and got["s"] == "a\x00b" # NaN、NUL 原样 + + +async def test_unique_violation_names_the_table_and_columns(): + sid = uuid.uuid4().hex + async with SessionLocal() as session: + session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256="c" * 64)) + await session.commit() + session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256="c" * 64)) + with pytest.raises(IntegrityError) as caught: + await session.commit() + assert unique_violation(caught.value, "table_recipes", "source_id", "seq") + assert not unique_violation(caught.value, "table_recipes", "source_id") + assert not unique_violation(caught.value, "data_sources", "source_id", "seq") + + +async def test_order_by_puts_nulls_where_sqlite_does(): + tag = uuid.uuid4().hex[:12] + from datetime import datetime, timezone + + async with SessionLocal() as session: + for i, started in enumerate([datetime(2026, 1, 2, tzinfo=timezone.utc), None, + datetime(2026, 1, 1, tzinfo=timezone.utc)]): + session.add(Run(workflow_name=f"排序-{tag}-{i}", started_at=started)) + await session.commit() + async with SessionLocal() as session: + where = Run.workflow_name.like(f"排序-{tag}-%") + up = (await session.execute(select(Run.workflow_name).where(where).order_by(Run.started_at))).scalars().all() + down = (await session.execute(select(Run.workflow_name).where(where) + .order_by(Run.started_at.desc()))).scalars().all() + assert [n[-1] for n in up] == ["1", "2", "0"] # 升序:NULL 在前 + assert [n[-1] for n in down] == ["0", "2", "1"] # 降序:NULL 在后 + + +async def test_text_sorts_and_compares_byte_by_byte_like_sqlite(): + """大写在小写前、标点按码位:两种库排出来一样(PostgreSQL 上不看建库时的区域设置)。""" + tag = uuid.uuid4().hex[:12] + names = ["b", "B", "a-c", "ab", "_z", "Z", "中"] + async with SessionLocal() as session: + for name in names: + session.add(Run(workflow_name=f"字节-{tag}-{name}")) + await session.commit() + async with SessionLocal() as session: + got = (await session.execute(select(Run.workflow_name).where(Run.workflow_name.like(f"字节-{tag}-%")) + .order_by(Run.workflow_name))).scalars().all() + lower = (await session.execute(select(func.count()).select_from(Run).where( + Run.workflow_name.like(f"字节-{tag}-%"), Run.workflow_name < f"字节-{tag}-a"))).scalar_one() + assert [g.removeprefix(f"字节-{tag}-") for g in got] == sorted(names) + assert lower == sum(1 for n in names if n < "a") + + +async def test_nul_characters_in_text_columns(): + """文本列里的 NUL:SQLite 照存;PostgreSQL 的 text / varchar 不收,写入时去掉(同 JSON 列),不让整条写入失败。""" + async with SessionLocal() as session: + row = Run(workflow_name="带\x00NUL\x00的名字", error="错\x00误") + session.add(row) + await session.commit() + rid = row.id + await session.execute(text("UPDATE runs SET end_reason = :r WHERE id = :i"), {"r": "x\x00y", "i": rid}) + await session.commit() + async with SessionLocal() as session: + got = (await session.execute(select(Run.workflow_name, Run.error, Run.end_reason).where(Run.id == rid))).one() + if db_base.engine.dialect.name == "postgresql": + assert tuple(got) == ("带NUL的名字", "错误", "xy") + else: + assert tuple(got) == ("带\x00NUL\x00的名字", "错\x00误", "x\x00y") + + +async def test_case_insensitive_search_folds_like_the_database_does(): + """contains_ci:ASCII 两种库都不分大小写;非 ASCII(É/é、Ж/ж、全角A/a)按库自己会不会折——SQLite 看构建,PostgreSQL 看 + 建库的区域设置(列本身是 COLLATE "C",列上的 ILIKE 只认 ASCII,所以模式带 COLLATE "default")。% 和 _ 按字面。""" + from app.db.portable import contains_ci + + tag = uuid.uuid4().hex[:12] + async with SessionLocal() as session: + session.add(Run(workflow_name=f"大小写-{tag}-Éclair Жук Ab Report_100%")) + await session.commit() + pg = db_base.engine.dialect.name == "postgresql" + folds = (await session.execute(text( + "SELECT lower('ÉЖA' COLLATE \"default\") = 'éжa'" if pg else "SELECT lower('ÉЖA') = 'éжa'"))).scalar() + + async def hit(needle: str) -> bool: + return bool((await session.execute(select(func.count()).select_from(Run).where( + Run.workflow_name.like(f"大小写-{tag}-%"), contains_ci(Run.workflow_name, needle)))).scalar()) + + assert await hit("report") and await hit("REPORT_100%") + assert not await hit("report-100") and not await hit("100%%") + assert await hit("éclair") == bool(folds) and await hit("жук") == bool(folds) and await hit("ab") == bool(folds) + assert await hit("Éclair") and await hit("Жук") + + +async def test_timestamps_read_back_as_aware_utc(): + from datetime import datetime, timedelta, timezone + + local = datetime(2026, 3, 4, 9, 30, tzinfo=timezone(timedelta(hours=8))) + async with SessionLocal() as session: + row = Run(workflow_name="时区", started_at=local) + session.add(row) + await session.commit() + rid = row.id + async with SessionLocal() as session: + got = (await session.execute(select(Run.started_at).where(Run.id == rid, Run.started_at == local))).scalar_one() + assert got == local and got.tzinfo == timezone.utc and got.hour == 1 + + +# -------------------------------------------------------------------------- +# PostgreSQL 上的老库迁移 +# -------------------------------------------------------------------------- + + +@pytest.fixture +def scratch_pg(): + """同一台服务器上另建一个空库,给迁移测试造「老版本建的库」。只在 PostgreSQL 时有。""" + if not PG: + pytest.skip("要 AGENTLAB_TEST_DATABASE_URL(scripts/test-postgres.sh)") + import psycopg + from sqlalchemy import create_engine + from sqlalchemy.engine import make_url + + from app.db.urls import libpq_conninfo + + admin = make_url(db_base.settings.db_url) + name = f"{admin.database}_mig_{uuid.uuid4().hex[:8]}" + conninfo = libpq_conninfo(admin.render_as_string(hide_password=False)) + with psycopg.connect(conninfo, autocommit=True) as conn: + conn.execute(f'CREATE DATABASE "{name}"') + url = admin.set(drivername="postgresql+psycopg", database=name) + engine = create_engine(url) + try: + yield engine + finally: + engine.dispose() + with psycopg.connect(conninfo, autocommit=True) as conn: + conn.execute(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)') + + +def _columns(conn, table: str) -> dict[str, tuple[str, str | None]]: + rows = conn.execute(text("SELECT column_name, data_type, column_default FROM information_schema.columns " + "WHERE table_name = :t"), {"t": table}).all() + return {r[0]: (r[1], r[2]) for r in rows} + + +def test_old_postgres_databases_get_the_missing_columns_and_indexes(scratch_pg): + """老版本建的 PostgreSQL 库(少几列、少索引、有并发留下的重复行):补上的列类型按模型的、缺省值回填,布尔的缺省是 + false 不是 0,索引补上,唯一索引建之前先去重;再跑一遍什么都不做。""" + with scratch_pg.begin() as conn: + conn.execute(text("CREATE TABLE runs (id VARCHAR PRIMARY KEY, workflow_id VARCHAR, version BIGINT, " + "status VARCHAR, last_seq BIGINT)")) + conn.execute(text("INSERT INTO runs (id, status, last_seq) VALUES ('r1', 'succeeded', 9)")) + conn.execute(text("CREATE TABLE schedules (id VARCHAR PRIMARY KEY, name VARCHAR)")) + conn.execute(text("INSERT INTO schedules (id, name) VALUES ('s1', '虚构的定时')")) + conn.execute(text("CREATE TABLE conversations (id VARCHAR PRIMARY KEY, kind VARCHAR, archived BOOLEAN, " + "workflow_id VARCHAR, last_active_at TIMESTAMP)")) + for cid, at in (("c1", "2026-01-01"), ("c2", "2026-01-02")): + conn.execute(text("INSERT INTO conversations (id, kind, archived, last_active_at) " + "VALUES (:i, 'app', false, :at)"), {"i": cid, "at": at}) + db_base._migrate(conn) + db_base._migrate(conn) + runs = _columns(conn, "runs") + assert runs["options"][0] == "json" and runs["options"][1].startswith("'{}'") + assert runs["schedule_id"][0] == "character varying" and runs["manifest_seq"][0] == "bigint" + assert runs["started_by"][0] == "character varying" + assert conn.execute(text("SELECT CAST(options AS TEXT), run_class FROM runs WHERE id = 'r1'")).one() \ + == ("{}", "exploratory") + schedules = _columns(conn, "schedules") + assert schedules["trigger_kind"][1].startswith("'cron'") + assert conn.execute(text("SELECT trigger_kind FROM schedules")).scalar_one() == "cron" + indexes = {r[0] for r in conn.execute(text("SELECT indexname FROM pg_indexes WHERE tablename = 'runs'"))} + assert {"ix_runs_schedule_id", "ix_runs_wf_ver_status"} <= indexes + # 同一个人在同一个应用里两段没归档的助手对话:created_by 是迁移补上的列(NULL 不受约束),唯一索引照样建上 + convs = {r[0] for r in conn.execute(text("SELECT indexname FROM pg_indexes WHERE tablename = 'conversations'"))} + assert "ux_conversations_app_owner" in convs + + +def test_old_postgres_duplicates_are_settled_before_the_unique_index(scratch_pg): + with scratch_pg.begin() as conn: + conn.execute(text("CREATE TABLE conversations (id VARCHAR PRIMARY KEY, kind VARCHAR, archived BOOLEAN, " + "app_id VARCHAR, created_by VARCHAR, last_active_at TIMESTAMP)")) + for cid, at in (("c1", "2026-01-01"), ("c2", "2026-01-03"), ("c3", "2026-01-02")): + conn.execute(text("INSERT INTO conversations VALUES (:i, 'app', false, 'a1', '甲', :at)"), + {"i": cid, "at": at}) + db_base._migrate(conn) + kept = conn.execute(text("SELECT id FROM conversations WHERE archived = false")).scalars().all() + assert kept == ["c2"], "留最近活跃的那一段,别的归档" + with pytest.raises(Exception, match="ux_conversations_app_owner"): + with conn.begin_nested(): + conn.execute(text("INSERT INTO conversations VALUES ('c4', 'app', false, 'a1', '甲', '2026-02-01')")) + + +async def test_new_postgres_databases_have_the_partial_unique_indexes(): + """新库靠 create_all:部分唯一索引在 PostgreSQL 上也是部分索引(postgresql_where),条件里的布尔写成 false。""" + if not PG: + pytest.skip("要 AGENTLAB_TEST_DATABASE_URL(scripts/test-postgres.sh)") + async with db_base.engine.connect() as conn: + rows = await conn.execute(text("SELECT indexname, indexdef FROM pg_indexes WHERE indexname LIKE 'ux_%'")) + got = {r[0]: r[1] for r in rows} + assert "WHERE (((kind)::text = 'app'::text) AND (archived = false))" in got["ux_conversations_app_owner"] + assert "WHERE ((status)::text = ANY" in got["ux_catalog_draft_jobs_active"] diff --git a/backend/tests/test_db_startup.py b/backend/tests/test_db_startup.py new file mode 100644 index 00000000..1ff06ffb --- /dev/null +++ b/backend/tests/test_db_startup.py @@ -0,0 +1,161 @@ +"""启动时等库(db/base.wait_for_database):和 PostgreSQL 容器一起起、库还没就绪时按退避重试、照实打日志,等够了再报错; +密码不对、库不存在这种等也没用的立刻报错;SQLite 不等。""" + +from __future__ import annotations + +import logging + +import pytest + +from app.db.base import make_engine, wait_for_database + + +class _Dialect: + def __init__(self, name: str) -> None: + self.name = name + + +class _Conn: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def execute(self, _statement): + return None + + +class _Engine: + """connect() 依次抛 errors 里的错,用完了就连上。""" + + def __init__(self, errors: list[BaseException], dialect: str = "postgresql") -> None: + self.errors = list(errors) + self.dialect = _Dialect(dialect) + self.attempts = 0 + + def connect(self): + self.attempts += 1 + if self.errors: + raise self.errors.pop(0) + return _Conn() + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + self.slept: list[float] = [] + + def __call__(self) -> float: + return self.now + + async def sleep(self, seconds: float) -> None: + self.slept.append(seconds) + self.now += seconds + + +class _SqlState(Exception): + def __init__(self, sqlstate: str) -> None: + super().__init__(f"sqlstate {sqlstate}") + self.sqlstate = sqlstate + + +async def test_retries_with_backoff_until_the_database_answers(caplog): + clock = _Clock() + engine = _Engine([ConnectionRefusedError("拒绝连接"), OSError("名字解析不了"), _SqlState("57P03")]) + with caplog.at_level(logging.INFO, logger="app.db.base"): + await wait_for_database(engine, timeout=60, sleep=clock.sleep, clock=clock) + assert engine.attempts == 4 and clock.slept == [1.0, 2.0, 4.0] + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings) == 3 and all("应用库还没就绪" in w for w in warnings) + assert "拒绝连接" in warnings[0] and "最多 60 秒" in warnings[0] + assert any("应用库连上了(第 4 次" in r.getMessage() for r in caplog.records) + + +async def test_gives_up_after_the_budget_and_raises_the_last_error(caplog): + clock = _Clock() + engine = _Engine([ConnectionRefusedError(f"第 {i} 次") for i in range(100)]) + with caplog.at_level(logging.INFO, logger="app.db.base"), pytest.raises(ConnectionRefusedError, match="第 5 次"): + await wait_for_database(engine, timeout=20, sleep=clock.sleep, clock=clock) + # 退避 1、2、4、5、5:等到第 17 秒,再等 5 秒就过了 20 秒,不等了,抛第 6 次(下标 5)的错 + assert clock.slept == [1.0, 2.0, 4.0, 5.0, 5.0] + assert any("不再等了" in r.getMessage() for r in caplog.records if r.levelno == logging.ERROR) + + +@pytest.mark.parametrize("sqlstate", ["28P01", "28000", "3D000", "42501"]) +async def test_errors_waiting_cannot_fix_are_raised_at_once(sqlstate): + clock = _Clock() + engine = _Engine([_SqlState(sqlstate)]) + with pytest.raises(_SqlState): + await wait_for_database(engine, timeout=60, sleep=clock.sleep, clock=clock) + assert engine.attempts == 1 and clock.slept == [] + + +async def test_sqlstate_on_the_wrapped_cause_counts(): + clock = _Clock() + wrapper = RuntimeError("SQLAlchemy 包了一层") + wrapper.__cause__ = _SqlState("28P01") + with pytest.raises(RuntimeError): + await wait_for_database(_Engine([wrapper]), timeout=60, sleep=clock.sleep, clock=clock) + assert clock.slept == [] + + +async def test_sqlite_does_not_wait(): + engine = _Engine([ConnectionRefusedError("不该连")], dialect="sqlite") + await wait_for_database(engine, timeout=60) + assert engine.attempts == 0 + + +async def test_a_real_refused_connection_is_treated_as_not_ready_yet(): + """真的 asyncpg:端口上没有服务时(库容器还没起来)算「还没就绪」,按退避重试,到时间才抛。""" + pytest.importorskip("asyncpg") + clock = _Clock() + engine = make_engine("postgresql+asyncpg://agentlab:x@127.0.0.1:1/agentlab") + try: + with pytest.raises(OSError): + await wait_for_database(engine, timeout=4, sleep=clock.sleep, clock=clock) + finally: + await engine.dispose() + assert clock.slept == [1.0, 2.0] + + +async def test_a_missing_database_on_a_real_server_is_raised_at_once(): + """真的 PostgreSQL(AGENTLAB_TEST_DATABASE_URL):库不存在(3D000)立刻报错,不等。""" + from app.core.config import settings + + if settings.db_dialect != "postgresql": + pytest.skip("要 PostgreSQL") + base, _, _ = settings.db_url.rpartition("/") + clock = _Clock() + engine = make_engine(f"{base}/agentlab_no_such_database") + try: + with pytest.raises(Exception) as caught: + await wait_for_database(engine, timeout=60, sleep=clock.sleep, clock=clock) + finally: + await engine.dispose() + assert clock.slept == [], caught.value + assert "agentlab_no_such_database" in str(caught.value) + + +async def test_startup_waits_for_the_database_before_the_version_check(monkeypatch): + """lifespan 里第一次连库的是版本检查(cluster/version.preflight):等库要在它前面,库的容器晚一步起来时服务不至于直接崩。""" + from app.cluster import guard, version + from app.db import base + from app.main import app, lifespan + + calls: list[str] = [] + + async def waited(*_a, **_k) -> None: + calls.append("wait") + + async def checked(*_a, **_k) -> None: + calls.append("preflight") + raise RuntimeError("到这里就够了") + + monkeypatch.setattr(guard, "acquire", lambda: None) + monkeypatch.setattr(base, "wait_for_database", waited) + monkeypatch.setattr(version, "preflight", checked) + with pytest.raises(RuntimeError, match="到这里就够了"): + async with lifespan(app): + pass + assert calls == ["wait", "preflight"] diff --git a/backend/tests/test_deploy_script.py b/backend/tests/test_deploy_script.py new file mode 100644 index 00000000..87a36497 --- /dev/null +++ b/backend/tests/test_deploy_script.py @@ -0,0 +1,708 @@ +"""deploy/deploy.sh:内置 PostgreSQL 的起停顺序、生成的 docker 参数、模式互斥、升级前备份、migrate-to-postgres 的回退。 + +docker 换成 tests/fake_docker.py(PATH 最前面),不碰本机的 docker;脚本照打包时那样填好版本和架构,放进临时目录跑。 +真起容器的端到端见 deploy/README.md「用 PostgreSQL」末尾记的做法(不在单测里跑)。 +""" + +from __future__ import annotations + +import json +import os +import shutil +import sqlite3 +import stat +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.fake_docker import host_arch + +REPO = Path(__file__).resolve().parents[2] +FAKE = Path(__file__).resolve().parent / "fake_docker.py" +VERSION = "t1" +APP = "agl-t" +PG = f"{APP}-postgres" +NET = f"{APP}-net" +VOL = f"{APP}-pgdata" +PG_IMAGE = "postgres:16.15" + +pytestmark = pytest.mark.skipif(shutil.which("bash") is None, reason="要 bash") + + +class Deploy: + def __init__(self, root: Path, *, env_lines: list[str] | None = None) -> None: + self.root = root + self.pkg = root / "pkg" + self.pkg.mkdir() + text = (REPO / "deploy" / "deploy.sh").read_text(encoding="utf-8") + text = text.replace("__AGENTLAB_VERSION__", VERSION).replace("__AGENTLAB_ARCH__", host_arch()) + (self.pkg / "deploy.sh").write_text(text, encoding="utf-8") + self.write_env(env_lines or []) + (self.pkg / "image.tar").write_text(f"agentlab:{VERSION} 10001:10001\n") + (self.pkg / "postgres-image.tar").write_text(f"{PG_IMAGE} \n") + bin_dir = root / "bin" + bin_dir.mkdir() + docker = bin_dir / "docker" + docker.write_text(f'#!/bin/sh\nexec "{sys.executable}" -S "{FAKE}" "$@"\n') + docker.chmod(0o755) + self.state_file = root / "docker-state.json" + self.log_file = root / "docker-calls.jsonl" + self.env = { + "PATH": f"{bin_dir}:/usr/bin:/bin:/usr/sbin:/sbin", "HOME": str(root), "LANG": "C.UTF-8", + "FAKE_DOCKER_STATE": str(self.state_file), "FAKE_DOCKER_LOG": str(self.log_file), + } + + @property + def data(self) -> Path: + return (self.pkg / "data").resolve() + + def write_env(self, extra: list[str]) -> None: + lines = ["HOST_PORT=18765", "BIND_ADDR=127.0.0.1", "DATA_DIR=./data", f"CONTAINER_NAME={APP}", + "ENABLE_MICROVM=0", "ENABLE_BWRAP=0", "AGENTLAB_SECRET_KEY=", *extra] + (self.pkg / "agentlab.env").write_text("\n".join(lines) + "\n") + + def __call__(self, *args: str, ok: bool = True, **env: str) -> subprocess.CompletedProcess: + self.log_file.write_text("") + out = subprocess.run(["bash", str(self.pkg / "deploy.sh"), *args], capture_output=True, text=True, + env={**self.env, **env}, timeout=120) + if ok: + assert out.returncode == 0, out.stdout + out.stderr + return out + + def calls(self) -> list[list[str]]: + return [json.loads(line) for line in self.log_file.read_text().splitlines() if line.strip()] + + def state(self) -> dict: + return json.loads(self.state_file.read_text()) if self.state_file.exists() else {"containers": {}} + + def containers(self) -> dict: + return self.state()["containers"] + + def mode(self) -> str: + path = self.data / ".postgres" / "state" + if not path.exists(): + return "sqlite" + return next(line.split("=", 1)[1] for line in path.read_text().splitlines() if line.startswith("mode=")) + + +def index_of(calls: list[list[str]], pred) -> int: + for i, argv in enumerate(calls): + if pred(argv): + return i + raise AssertionError("没找到这次调用:\n" + "\n".join(" ".join(c) for c in calls)) + + +def runs_named(name: str): + return lambda argv: argv[:1] == ["run"] and "--name" in argv and argv[argv.index("--name") + 1] == name + + +def is_cmd(cmd: str, name: str): + return lambda argv: argv[:1] == [cmd] and argv[-1] == name + + +def oneoff(entry: str, *words: str): + return lambda argv: (argv[:1] == ["run"] and "--rm" in argv and "--entrypoint" in argv + and argv[argv.index("--entrypoint") + 1] == entry and all(w in argv for w in words)) + + +def make_sqlite_data(data: Path) -> None: + data.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(data / "agentlab.db") + db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)") + db.execute("INSERT INTO t (v) VALUES ('一')") + db.commit() + db.close() + (data / ".secret_key").write_text("x") + + +@pytest.fixture +def deploy(tmp_path): + return Deploy(tmp_path) + + +# -------------------------------------------------------------------------- +# 参数 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("args", [["bogus"], ["start", "--postgres"], ["upgrade", "--postgres"], ["status", "--postgres"], + ["restart", "--all"], + ["install", "--purge-image"], ["migrate-to-postgres", "-f"], ["status", "--tail", "3"]]) +def test_unknown_arguments_are_refused_before_doing_anything(deploy, args): + out = deploy(*args, ok=False) + assert out.returncode == 2 + assert "认不出" in out.stderr + assert deploy.calls() == [], "认参数时一次 docker 都没调" + + +def test_help_lists_the_postgres_commands(deploy): + out = deploy("--help") + for word in ("install [--postgres]", "migrate-to-postgres", "restart [--postgres]", "logs [-f] [--tail N] [--postgres]", + "DEPLOY_<名字>"): + assert word in out.stdout + + +# -------------------------------------------------------------------------- +# install --postgres +# -------------------------------------------------------------------------- + + +def test_install_postgres_starts_the_database_first_and_wires_the_app_without_the_password(deploy): + out = deploy("install", "--postgres") + calls = deploy.calls() + net = index_of(calls, lambda a: a[:2] == ["network", "create"] and a[-1] == NET) + pg = index_of(calls, runs_named(PG)) + pg_ready = index_of(calls, lambda a: a[:1] == ["inspect"] and a[-1] == PG and ".State.Health" in a[2]) + app = index_of(calls, runs_named(APP)) + assert net < pg < pg_ready < app, "先建网络、起库、等库就绪,再起服务" + + pg_args = calls[pg] + assert "-p" not in pg_args and not any(a.startswith("--publish") for a in pg_args), "库的端口不对外" + assert pg_args[pg_args.index("--network") + 1] == NET + assert f"{VOL}:/var/lib/postgresql/data" in pg_args + assert "POSTGRES_PASSWORD_FILE=/run/agentlab/password" in pg_args + assert any(a.endswith("/.postgres/password:/run/agentlab/password:ro") for a in pg_args) + assert "pg_isready -h 127.0.0.1 -U agentlab -d agentlab" in pg_args + assert pg_args[-1] == PG_IMAGE + + app_args = calls[app] + assert app_args[app_args.index("--network") + 1] == NET + assert f"AGENTLAB_DATABASE_URL=postgresql+asyncpg://agentlab@{PG}:5432/agentlab" in app_args + assert "PGPASSFILE=/data/.postgres/pgpass" in app_args + assert "127.0.0.1:18765:8000" in app_args + + secret = (deploy.data / ".postgres" / "password").read_text().strip() + assert len(secret) >= 32 + everything = json.dumps(calls, ensure_ascii=False) + out.stdout + out.stderr + assert secret not in everything, "密码不进任何 docker 参数和输出" + for name, mode in (("password", 0o600), ("pgpass", 0o600), ("state", 0o644)): + assert stat.S_IMODE((deploy.data / ".postgres" / name).stat().st_mode) == mode, name + assert stat.S_IMODE((deploy.data / ".postgres").stat().st_mode) == 0o711 + pgpass = (deploy.data / ".postgres" / "pgpass").read_text().strip() + assert pgpass.startswith(f"{PG}:5432:agentlab:agentlab:") + assert deploy.mode() == "builtin" + assert "内置 PostgreSQL" in out.stdout + assert deploy.containers()[PG]["labels"]["agentlab.role"] == "postgres" + + +def test_once_builtin_later_installs_keep_using_it_without_the_flag(deploy): + deploy("install", "--postgres") + secret = (deploy.data / ".postgres" / "password").read_text() + deploy("install") + calls = deploy.calls() + assert not any(runs_named(PG)(a) for a in calls), "库的容器已经在跑,不重建" + app_args = calls[index_of(calls, runs_named(APP))] + assert f"AGENTLAB_DATABASE_URL=postgresql+asyncpg://agentlab@{PG}:5432/agentlab" in app_args + assert (deploy.data / ".postgres" / "password").read_text() == secret, "密码沿用" + + +def test_default_install_is_sqlite_and_touches_no_postgres(deploy): + deploy("install") + calls = deploy.calls() + app_args = calls[index_of(calls, runs_named(APP))] + assert not any("AGENTLAB_DATABASE_URL" in a for a in app_args) and "--network" not in app_args + assert not any(a[:2] == ["network", "create"] for a in calls) + assert PG not in deploy.containers() + assert not (deploy.data / ".postgres").exists() + + +def test_builtin_and_external_are_mutually_exclusive(tmp_path): + deploy = Deploy(tmp_path, env_lines=["AGENTLAB_DATABASE_URL=postgresql+asyncpg://u:p@db.example:5432/agentlab"]) + out = deploy("install", "--postgres", ok=False) + assert out.returncode == 1 and "两种库只能选一种" in out.stderr and "AGENTLAB_DATABASE_URL" in out.stderr + assert APP not in deploy.containers() and PG not in deploy.containers() + out = deploy("migrate-to-postgres", ok=False) + assert "外接 PostgreSQL" in out.stderr + # 不加 --postgres:外接的照常用,不起内置的 + deploy("install") + assert PG not in deploy.containers() + assert "外接 PostgreSQL" in deploy("status").stdout + + +def test_external_url_on_a_builtin_data_dir_is_refused(deploy): + deploy("install", "--postgres") + deploy.write_env(["AGENTLAB_DATABASE_URL=postgresql+asyncpg://u:p@db.example:5432/agentlab"]) + out = deploy("install", ok=False) + assert "两种库只能选一种" in out.stderr + # 停服务这类不改配置的照样能做,按数据目录里记的管两个容器 + deploy("stop") + assert deploy.containers()[PG]["state"] == "exited" + + +def test_install_postgres_on_a_sqlite_data_dir_points_to_migrate(deploy): + make_sqlite_data(deploy.pkg / "data") + out = deploy("install", "--postgres", ok=False) + assert "migrate-to-postgres" in out.stderr + assert deploy.containers() == {} and deploy.mode() == "sqlite" + + +def test_agentlab_settings_reach_the_container_through_the_env_file(tmp_path): + """agentlab.env 里的 AGENTLAB_*(多进程、连接池……)经 --env-file 原样进应用容器;内置库的地址、密码文件另用 -e 给。""" + deploy = Deploy(tmp_path, env_lines=["AGENTLAB_WORKERS=3", "AGENTLAB_DB_POOL_SIZE=5", "AGENTLAB_WORKER_LEASE_S=40"]) + deploy("install", "--postgres") + calls = deploy.calls() + app_args = calls[index_of(calls, runs_named(APP))] + env_file = Path(app_args[app_args.index("--env-file") + 1]) + assert env_file == deploy.pkg / "agentlab.env" + lines = env_file.read_text().splitlines() + assert {"AGENTLAB_WORKERS=3", "AGENTLAB_DB_POOL_SIZE=5", "AGENTLAB_WORKER_LEASE_S=40"} <= set(lines) + + +@pytest.mark.parametrize("line", ["AGENTLAB_DATABASE_URL=postgresql+asyncpg://u:p@db.example:5432/agentlab", + "AGENTLAB_CHECKPOINT_URL=postgresql://u:p@db.example:5432/ckpt"]) +def test_any_external_database_setting_conflicts_with_builtin(tmp_path, line): + deploy = Deploy(tmp_path, env_lines=[line]) + out = deploy("install", "--postgres", ok=False) + assert "两种库只能选一种" in out.stderr and line.split("=")[0] in out.stderr + assert deploy.containers() == {} + + +def test_direct_url_needs_the_external_url(tmp_path): + deploy = Deploy(tmp_path, env_lines=["AGENTLAB_DATABASE_DIRECT_URL=postgresql+asyncpg://u:p@db.example:5432/agentlab"]) + out = deploy("install", ok=False) + assert "AGENTLAB_DATABASE_DIRECT_URL" in out.stderr and "要和 AGENTLAB_DATABASE_URL 一起设" in out.stderr + assert deploy.containers() == {} + out = deploy("install", "--postgres", ok=False) + assert "两种库只能选一种" in out.stderr, "和内置库一起:照互斥报" + + +def test_pgbouncer_switch_is_refused_for_the_builtin_database(tmp_path): + deploy = Deploy(tmp_path, env_lines=["AGENTLAB_DB_PGBOUNCER=1"]) + out = deploy("install", "--postgres", ok=False) + assert "内置 PostgreSQL 是直连" in out.stderr and deploy.containers() == {} + make_sqlite_data(deploy.pkg / "data") + out = deploy("migrate-to-postgres", ok=False) + assert "内置 PostgreSQL 是直连" in out.stderr + assert not any(a[:1] == ["stop"] for a in deploy.calls()), "切换之前就拦下,服务没停" + # 外接的库配 PgBouncer:照常 + deploy.write_env(["AGENTLAB_DB_PGBOUNCER=1", "AGENTLAB_DATABASE_URL=postgresql+asyncpg://u:p@bouncer.example:6432/agentlab", + "AGENTLAB_DATABASE_DIRECT_URL=postgresql+asyncpg://u:p@db.example:5432/agentlab"]) + deploy("install") + assert "外接 PostgreSQL" in deploy("status").stdout + + +def test_more_than_one_worker_needs_postgres(tmp_path): + deploy = Deploy(tmp_path, env_lines=["AGENTLAB_WORKERS=4"]) + out = deploy("install", ok=False) + assert "SQLite 只支持单进程" in out.stderr and "migrate-to-postgres" in out.stderr + assert deploy.containers() == {} + deploy("install", "--postgres") + assert deploy.containers()[APP]["state"] == "running", "内置 PostgreSQL 时可以多进程" + + +def test_install_postgres_refuses_a_leftover_volume(deploy): + deploy.state_file.write_text(json.dumps({"images": {}, "containers": {}, "networks": [], "volumes": [VOL]})) + out = deploy("install", "--postgres", ok=False) + assert VOL in out.stderr and "docker volume rm" in out.stderr + assert PG not in deploy.containers() + + +def test_deploy_overrides_take_names_from_the_environment(deploy): + env = {"DEPLOY_CONTAINER_NAME": "agl-x", "DEPLOY_POSTGRES_CONTAINER": "agl-x-db", "DEPLOY_DOCKER_NETWORK": "agl-x-n", + "DEPLOY_POSTGRES_VOLUME": "agl-x-v", "DEPLOY_HOST_PORT": "18766", "DEPLOY_POSTGRES_IMAGE": "postgres:16"} + (deploy.pkg / "postgres-image.tar").write_text("postgres:16 \n") + deploy("install", "--postgres", **env) + calls = deploy.calls() + pg_args = calls[index_of(calls, runs_named("agl-x-db"))] + assert pg_args[-1] == "postgres:16" and "agl-x-v:/var/lib/postgresql/data" in pg_args + app_args = calls[index_of(calls, runs_named("agl-x"))] + assert "127.0.0.1:18766:8000" in app_args and app_args[app_args.index("--network") + 1] == "agl-x-n" + assert set(deploy.containers()) == {"agl-x", "agl-x-db"} + + +@pytest.mark.parametrize("name,value", [("DEPLOY_POSTGRES_CONTAINER", APP), ("DEPLOY_DOCKER_NETWORK", "bad name"), + ("DEPLOY_BACKUP_KEEP", "0"), ("DEPLOY_POSTGRES_IMAGE", "a b")]) +def test_bad_overrides_are_refused(deploy, name, value): + out = deploy("install", "--postgres", ok=False, **{name: value}) + assert out.returncode == 1 and "错误:" in out.stderr + assert deploy.containers() == {} + + +# -------------------------------------------------------------------------- +# start / stop / restart / status / logs / uninstall +# -------------------------------------------------------------------------- + + +def test_start_stop_restart_order_the_two_containers(deploy): + deploy("install", "--postgres") + deploy("stop") + calls = deploy.calls() + assert index_of(calls, is_cmd("stop", APP)) < index_of(calls, is_cmd("stop", PG)), "先停服务再停库" + assert {c["state"] for c in deploy.containers().values()} == {"exited"} + + deploy("start") + calls = deploy.calls() + pg_start = index_of(calls, is_cmd("start", PG)) + pg_ready = index_of(calls, lambda a: a[:1] == ["inspect"] and a[-1] == PG and ".State.Health" in a[2]) + assert pg_start < pg_ready < index_of(calls, is_cmd("start", APP)), "先起库、等就绪,再起服务" + + deploy("restart", "--postgres") + calls = deploy.calls() + order = [index_of(calls, is_cmd("stop", APP)), index_of(calls, is_cmd("stop", PG)), + index_of(calls, is_cmd("start", PG)), index_of(calls, is_cmd("start", APP))] + assert order == sorted(order), "--postgres:先停服务、再停库,先起库、再起服务" + assert {c["state"] for c in deploy.containers().values()} == {"running"} + + +def test_restart_by_default_leaves_the_database_alone(deploy): + deploy("install", "--postgres") + out = deploy("restart") + calls = deploy.calls() + assert ["restart", APP] in calls + assert not any(a[:1] in (["stop"], ["start"], ["restart"]) and a[-1] == PG for a in calls), "库不跟着重启" + assert "已重启" in out.stdout + # 库停着时先把库起来,服务才连得上 + state = deploy.state() + state["containers"][PG]["state"] = "exited" + deploy.state_file.write_text(json.dumps(state)) + deploy("restart") + calls = deploy.calls() + assert index_of(calls, is_cmd("start", PG)) < index_of(calls, lambda a: a == ["restart", APP]) + assert not any(is_cmd("stop", PG)(a) for a in calls) + assert deploy.containers()[PG]["state"] == "running" + + +def test_restart_postgres_needs_the_builtin_database(deploy): + deploy("install") + out = deploy("restart", "--postgres", ok=False) + assert "没用内置 PostgreSQL" in out.stderr + assert not any(a[:1] in (["stop"], ["restart"]) for a in deploy.calls()), "什么都没动" + + +def test_status_and_logs_show_the_database(deploy): + deploy("install", "--postgres") + out = deploy("status").stdout + assert "内置 PostgreSQL" in out and "pg_isready:healthy" in out and "端口不对外" in out + deploy("logs", "--postgres", "--tail", "5") + assert deploy.calls()[-1] == ["logs", "--tail", "5", PG] + deploy("logs") + assert deploy.calls()[-1] == ["logs", "--tail", "200", APP] + + +def test_logs_postgres_without_builtin_says_so(deploy): + deploy("install") + out = deploy("logs", "--postgres", ok=False) + assert "没用内置 PostgreSQL" in out.stderr + + +def test_uninstall_removes_both_containers_and_the_network_but_keeps_the_volume(deploy): + deploy("install", "--postgres") + out = deploy("uninstall") + state = deploy.state() + assert state["containers"] == {} and NET not in state["networks"] + assert VOL in state["volumes"], "数据卷留着" + assert f"docker volume rm {VOL}" in out.stdout + assert PG_IMAGE in state["images"], "PostgreSQL 的镜像不删" + assert (deploy.data / ".postgres" / "password").exists() + # 卸载后同一个数据目录再装:照旧用那个数据卷和密码 + deploy("install") + assert deploy.containers()[PG]["mounts"]["/var/lib/postgresql/data"] == VOL + + +def test_uninstall_refuses_a_foreign_container_with_the_postgres_name(deploy): + deploy("install") + state = deploy.state() + state["images"]["redis:7"] = {"arch": host_arch(), "user": ""} + state["containers"][PG] = {"image": "redis:7", "state": "running", "env": {}, "labels": {}, "mounts": {}, + "ports": [], "network": "bridge"} + deploy.state_file.write_text(json.dumps(state)) + out = deploy("uninstall", ok=False) + assert "不是 deploy.sh 起的内置 PostgreSQL" in out.stderr + assert PG in deploy.containers() + + +# -------------------------------------------------------------------------- +# upgrade:先备份 +# -------------------------------------------------------------------------- + + +def test_upgrade_builtin_dumps_after_stopping_the_app_and_keeps_the_last_n(deploy): + deploy("install", "--postgres") + folder = deploy.data / "backups" / "postgres" + folder.mkdir(parents=True) + for i in range(4): + (folder / f"agentlab-2026010{i}T000000Z.dump").write_bytes(b"old") + deploy("upgrade", DEPLOY_BACKUP_KEEP="3") + calls = deploy.calls() + stop = index_of(calls, is_cmd("stop", APP)) + dump = index_of(calls, oneoff("sh", "pg_dump")) + new_app = index_of(calls, runs_named(APP)) + assert stop < dump < new_app, "停了旧服务再备份,备份完才起新的" + dump_args = calls[dump] + assert dump_args[dump_args.index("--network") + 1] == NET and "PGPASSFILE=/data/.postgres/pgpass" in dump_args + assert dump_args[dump_args.index("-c") + 1].startswith("umask 077 && "), "备份文件从建出来就是 0600" + assert dump_args[dump_args.index("--user") + 1] == "10001:10001", "属主照数据目录:容器里的用户" + kept = sorted(p.name for p in folder.iterdir()) + assert len(kept) == 3 and kept[-1].startswith("agentlab-") and kept[-1] > "agentlab-20260103", kept + assert (folder / kept[-1]).read_bytes() == b"PGDMP fake" + assert stat.S_IMODE((folder / kept[-1]).stat().st_mode) == 0o600 + for level in (deploy.data / "backups", folder): + assert stat.S_IMODE(level.stat().st_mode) == 0o700, level + + +def test_upgrade_sqlite_backs_up_the_database_files(deploy): + deploy("install") + make_sqlite_data(deploy.pkg / "data") + deploy("upgrade") + calls = deploy.calls() + assert index_of(calls, is_cmd("stop", APP)) < index_of(calls, oneoff("python", "backup-sqlite")) \ + < index_of(calls, runs_named(APP)) + (stamp,) = list((deploy.data / "backups" / "sqlite").iterdir()) + copy = sqlite3.connect(stamp / "agentlab.db") + assert copy.execute("SELECT v FROM t").fetchall() == [("一",)] + copy.close() + for level in (deploy.data / "backups", deploy.data / "backups" / "sqlite", stamp): + assert stat.S_IMODE(level.stat().st_mode) == 0o700, level + assert stat.S_IMODE((stamp / "agentlab.db").stat().st_mode) == 0o600 + + +def test_failed_backup_aborts_the_upgrade_and_restarts_the_old_service(deploy): + deploy("install", "--postgres") + before = deploy.containers()[APP] + out = deploy("upgrade", ok=False, FAKE_PGDUMP_FAIL="1") + assert "备份没做成,没有升级" in out.stderr + after = deploy.containers()[APP] + assert after["state"] == "running" and after == before + assert not any(runs_named(APP)(a) for a in deploy.calls()) + + +def test_upgrade_with_a_new_postgres_image_recreates_the_database_container(deploy): + deploy("install", "--postgres") + (deploy.pkg / "postgres-image.tar").write_text("postgres:16.99 \n") + deploy("upgrade", DEPLOY_POSTGRES_IMAGE="postgres:16.99") + calls = deploy.calls() + assert index_of(calls, is_cmd("stop", APP)) < index_of(calls, runs_named(PG)) + pg = deploy.containers()[PG] + assert pg["image"] == "postgres:16.99" and pg["mounts"]["/var/lib/postgresql/data"] == VOL + + +def test_a_postgres_image_that_cannot_start_is_rolled_back(deploy): + deploy("install", "--postgres") + (deploy.pkg / "postgres-image.tar").write_text("postgres:17.1 \n") + out = deploy("upgrade", ok=False, DEPLOY_POSTGRES_IMAGE="postgres:17.1", FAKE_UNHEALTHY_IMAGE="postgres:17.1") + assert "已换回原来的 postgres:16.15" in out.stderr + containers = deploy.containers() + assert containers[PG]["image"] == PG_IMAGE and containers[PG]["state"] == "running" + assert containers[APP]["state"] == "running", "原来在跑的服务起回来了" + + +# -------------------------------------------------------------------------- +# migrate-to-postgres +# -------------------------------------------------------------------------- + + +def _sqlite_deployment(deploy: Deploy) -> None: + make_sqlite_data(deploy.pkg / "data") + deploy("install") + assert deploy.mode() == "sqlite" + + +def test_migrate_runs_the_steps_in_order_and_switches(deploy): + _sqlite_deployment(deploy) + out = deploy("migrate-to-postgres") + calls = deploy.calls() + steps = [ + index_of(calls, is_cmd("stop", APP)), + index_of(calls, oneoff("python", "backup-sqlite")), + index_of(calls, runs_named(PG)), + index_of(calls, lambda a: a[:2] == ["exec", PG] and "psql" in a), + index_of(calls, oneoff("python", "copy-to-postgres", "--dry-run")), + index_of(calls, lambda a: oneoff("python", "copy-to-postgres")(a) and "--dry-run" not in a), + index_of(calls, oneoff("python", "state", "builtin")), + index_of(calls, runs_named(APP)), + ] + assert steps == sorted(steps), steps + copy = calls[steps[5]] + assert f"postgresql+asyncpg://agentlab@{PG}:5432/agentlab" in copy and "--assume-stopped" in copy + assert "PGPASSFILE=/data/.postgres/pgpass" in copy + assert deploy.mode() == "builtin" + app = deploy.containers()[APP] + assert app["env"]["AGENTLAB_DATABASE_URL"].endswith(f"@{PG}:5432/agentlab") and app["state"] == "running" + assert f"{APP}-previous" not in deploy.containers() + assert (deploy.data / "agentlab.db").exists(), "SQLite 的库文件留着" + assert list((deploy.data / "backups" / "sqlite").iterdir()) + assert "已切到内置 PostgreSQL" in out.stdout + # 再跑一次:什么都不做 + again = deploy("migrate-to-postgres") + assert "已经切到内置 PostgreSQL" in again.stdout + assert not any(a[:1] in (["stop"], ["run"]) for a in deploy.calls() if "--rm" not in a) + + +@pytest.mark.parametrize("fail", ["dry", "real"]) +def test_a_failed_copy_leaves_the_sqlite_service_as_it_was_and_rerunning_works(deploy, fail): + _sqlite_deployment(deploy) + before = deploy.containers()[APP] + out = deploy("migrate-to-postgres", ok=False, FAKE_COPY_FAIL=fail) + assert "没有切换,仍用 SQLite" in out.stderr + containers = deploy.containers() + assert containers[APP] == before, "服务原样起回来,还是 SQLite 的那个容器" + assert containers[PG]["state"] == "exited", "内置库停掉" + assert deploy.mode() == "migrating" + assert "上次 migrate-to-postgres 没切成" in deploy("status").stdout + # 重新运行:数据卷认得出是自己上次留下的,清空再导 + deploy("migrate-to-postgres") + calls = deploy.calls() + assert index_of(calls, lambda a: a[:2] == ["exec", PG] and "psql" in a) \ + < index_of(calls, oneoff("python", "copy-to-postgres", "--dry-run")) + psql = calls[index_of(calls, lambda a: a[:2] == ["exec", PG] and "psql" in a)] + assert any("DROP DATABASE IF EXISTS" in a for a in psql) + assert deploy.mode() == "builtin" + + +def test_a_stopped_service_stays_stopped_after_a_failed_migration(deploy): + _sqlite_deployment(deploy) + deploy("stop") + deploy("migrate-to-postgres", ok=False, FAKE_COPY_FAIL="real") + assert deploy.containers()[APP]["state"] == "exited" + + +def test_an_unhealthy_new_service_is_swapped_back_to_the_sqlite_one(deploy): + _sqlite_deployment(deploy) + before = deploy.containers()[APP] + out = deploy("migrate-to-postgres", ok=False, FAKE_UNHEALTHY_ENV="AGENTLAB_DATABASE_URL") + assert "没有切换,仍用 SQLite" in out.stderr + containers = deploy.containers() + assert containers[APP] == before and f"{APP}-previous" not in containers + assert containers[PG]["state"] == "exited" + assert deploy.mode() == "migrating", "记成 builtin 的改回来了" + + +def test_migrate_without_an_app_container_removes_the_failed_one(deploy): + make_sqlite_data(deploy.pkg / "data") + deploy("migrate-to-postgres", ok=False, FAKE_UNHEALTHY_ENV="AGENTLAB_DATABASE_URL") + assert APP not in deploy.containers(), "按内置库起的、没通过健康检查的容器不留着" + assert deploy.mode() == "migrating" + + +def test_migrate_refuses_a_volume_it_did_not_create(deploy): + _sqlite_deployment(deploy) + state = deploy.state() + state["volumes"].append(VOL) + deploy.state_file.write_text(json.dumps(state)) + out = deploy("migrate-to-postgres", ok=False) + assert VOL in out.stderr and "没有动它" in out.stderr + calls = deploy.calls() + assert not any(a[:1] == ["stop"] or a[:1] == ["exec"] for a in calls), "服务没停、库没碰" + assert deploy.mode() == "sqlite" + + +def test_migrate_needs_a_sqlite_database(deploy): + out = deploy("migrate-to-postgres", ok=False) + assert "install --postgres" in out.stderr + + +def test_package_script_takes_the_postgres_image_from_deploy_sh(): + """包里带的 PostgreSQL 镜像和 deploy.sh 要的是同一个标签:标签只在 deploy.sh 里定一处。""" + deploy_sh = (REPO / "deploy" / "deploy.sh").read_text(encoding="utf-8") + assert f'POSTGRES_IMAGE="{PG_IMAGE}"' in deploy_sh + package = (REPO / "scripts" / "package-docker.sh").read_text(encoding="utf-8") + assert "postgres:16" not in package, "打包脚本里别再写一份标签" + out = subprocess.run(["sed", "-n", r's/^POSTGRES_IMAGE="\([^"]*\)"$/\1/p', str(REPO / "deploy" / "deploy.sh")], + capture_output=True, text=True, check=True) + assert out.stdout.strip() == PG_IMAGE + + +def test_env_template_lists_the_postgres_settings_commented_out_with_the_defaults(): + """agentlab.env 模板末尾的「内置 PostgreSQL(可选)」:几项都注释着(不改任何人的部署),缺省值和 deploy.sh 里的一致。""" + template = (REPO / "deploy" / "agentlab.env").read_text(encoding="utf-8") + start = template.index("# ---- 内置 PostgreSQL(可选)----") + tail = template[start:] + assert template.rstrip().endswith(tail.rstrip()), "追加在文件末尾" + section = tail.split("\n# ---- 外接 PostgreSQL 与多进程(可选)----")[0] + commented = dict(line[2:].split("=", 1) for line in section.splitlines() + if line.startswith("# ") and "=" in line and line[2:3].isupper()) + assert commented == {"POSTGRES_IMAGE": PG_IMAGE, "POSTGRES_CONTAINER": "agentlab-postgres", + "DOCKER_NETWORK": "agentlab-net", "POSTGRES_VOLUME": "agentlab-pgdata", "BACKUP_KEEP": "5"} + assert not any(line.split("=", 1)[0] in commented for line in template.splitlines() if not line.startswith("#")), \ + "都是注释" + + +def test_env_template_ends_with_the_external_postgres_and_multiprocess_settings(): + """模板最后一段「外接 PostgreSQL 与多进程(可选)」:要的几项都在、都注释着,缺省值和代码里的一致。""" + from app.core.config import Settings + + template = (REPO / "deploy" / "agentlab.env").read_text(encoding="utf-8") + section = template[template.index("# ---- 外接 PostgreSQL 与多进程(可选)----"):] + commented = dict(line[2:].split("=", 1) for line in section.splitlines() + if line.startswith("# AGENTLAB_") and "=" in line) + assert set(commented) == { + "AGENTLAB_DATABASE_URL", "AGENTLAB_CHECKPOINT_URL", "AGENTLAB_DB_PGBOUNCER", "AGENTLAB_DATABASE_DIRECT_URL", + "AGENTLAB_WORKERS", "AGENTLAB_WORKER_LEASE_S", "AGENTLAB_WORKER_HEARTBEAT_S", "AGENTLAB_DB_POOL_SIZE", + "AGENTLAB_DB_MAX_OVERFLOW", "AGENTLAB_DB_POOL_TIMEOUT", "AGENTLAB_DB_POOL_RECYCLE", "AGENTLAB_DB_STARTUP_TIMEOUT"} + assert "172.17.0.1" in commented["AGENTLAB_DATABASE_URL"] and "172.17.0.0/16" in section + fields = Settings.model_fields + for key, shown in commented.items(): + default = fields[key.removeprefix("AGENTLAB_").lower()].default + if isinstance(default, bool): + assert shown == str(int(default)), key + elif isinstance(default, (int, float)): + assert float(shown) == float(default), key + assert not any(line.startswith("AGENTLAB_") for line in section.splitlines()), "都是注释" + + +def test_settings_written_in_agentlab_env_take_effect(tmp_path): + deploy = Deploy(tmp_path, env_lines=["POSTGRES_CONTAINER=agl-db", "DOCKER_NETWORK=agl-n", "POSTGRES_VOLUME=agl-v", + "BACKUP_KEEP=1"]) + deploy("install", "--postgres") + assert set(deploy.containers()) == {APP, "agl-db"} + assert deploy.containers()["agl-db"]["mounts"]["/var/lib/postgresql/data"] == "agl-v" + assert deploy.containers()[APP]["network"] == "agl-n" + + +def test_env_example_lists_every_database_pool_and_multiprocess_setting(): + """core/config.py 里应用库、连接池、PgBouncer、多进程、调度这几类配置,每一项都在 .env.example 里有一行(注释掉的示例); + dev.sh --postgres 读的 AGENTLAB_DEV_PG_* 也在。以后加了配置忘了写,这里会报出来。""" + import re + + from app.core.config import Settings + + prefixes = ("database_", "checkpoint_", "db_", "worker", "scheduler") + names = {f"AGENTLAB_{name.upper()}" for name in Settings.model_fields if name.startswith(prefixes)} + assert {"AGENTLAB_DATABASE_URL", "AGENTLAB_DATABASE_DIRECT_URL", "AGENTLAB_DB_PGBOUNCER", "AGENTLAB_WORKERS", + "AGENTLAB_WORKER_LEASE_S", "AGENTLAB_DB_STARTUP_TIMEOUT", "AGENTLAB_SCHEDULER_TICK_S"} <= names + dev = set(re.findall(r"dev_pg_setting (AGENTLAB_DEV_PG_[A-Z_]+)", (REPO / "scripts" / "dev.sh").read_text())) + assert len(dev) == 4 + example = (REPO / ".env.example").read_text(encoding="utf-8") + listed = set(re.findall(r"^# (AGENTLAB_[A-Z_]+)=", example, re.M)) + assert names | dev <= listed, f".env.example 里缺:{sorted((names | dev) - listed)}" + + +def test_dev_script_reads_its_postgres_settings_from_dotenv(tmp_path): + """dev.sh --postgres 的几项:环境变量优先,没设时读 .env / backend/.env(后一个优先),和后端的配置同一个规矩。 + 在一份拷出来的脚本上跑,假的 conda、pnpm、docker:docker run 记下参数就失败退出,不真起容器、不起前后端。""" + root = tmp_path / "repo" + (root / "scripts").mkdir(parents=True) + (root / "frontend" / "node_modules").mkdir(parents=True) + shutil.copy(REPO / "scripts" / "dev.sh", root / "scripts" / "dev.sh") + (root / "backend").symlink_to(REPO / "backend") + (root / ".env").write_text("AGENTLAB_DEV_PG_CONTAINER=from-root-env\nAGENTLAB_DEV_PG_PORT=55491\n" + "AGENTLAB_DEV_PG_VOLUME='quoted-vol' \n") + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + log = tmp_path / "docker.log" + (bin_dir / "conda").write_text(f'#!/bin/sh\ncase "$1" in env) echo "agentlab /x" ;; run) echo "{sys.executable}" ;; esac\n') + (bin_dir / "pnpm").write_text("#!/bin/sh\nexit 0\n") + (bin_dir / "docker").write_text(f'#!/bin/sh\necho "$*" >> "{log}"\ncase "$1" in version) echo 27 ;; ' + 'container) exit 1 ;; run) exit 1 ;; esac\nexit 0\n') + for f in bin_dir.iterdir(): + f.chmod(0o755) + env = {"PATH": f"{bin_dir}:/usr/bin:/bin:/usr/sbin:/sbin", "HOME": str(tmp_path), "AGENTLAB_DATA_DIR": str(tmp_path / "data"), + "AGENTLAB_DEV_PG_IMAGE": "postgres:from-env", "AGENTLAB_PORT": "18799", "LANG": "C.UTF-8"} + out = subprocess.run(["bash", str(root / "scripts" / "dev.sh"), "--postgres"], capture_output=True, text=True, env=env, + timeout=60) + assert out.returncode != 0, "假的 docker run 失败,脚本停在这里" + [run] = [line for line in log.read_text().splitlines() if line.startswith("run ")] + assert "--name from-root-env" in run and "127.0.0.1:55491:5432" in run, ".env 里的生效" + assert "quoted-vol:/var/lib/postgresql/data" in run, "引号和行尾空白去掉" + assert run.endswith("postgres:from-env"), "环境变量优先" + (root / "backend").unlink() + + +def test_dev_script_refuses_unknown_arguments_and_documents_postgres(): + env = {**os.environ, "PATH": "/usr/bin:/bin"} + out = subprocess.run(["bash", str(REPO / "scripts" / "dev.sh"), "--bogus"], capture_output=True, text=True, env=env) + assert out.returncode == 2 and "认不出的参数" in out.stdout + out = subprocess.run(["bash", str(REPO / "scripts" / "dev.sh"), "--help"], capture_output=True, text=True, env=env) + assert out.returncode == 0 and "--postgres" in out.stdout and "AGENTLAB_DEV_PG_PORT" in out.stdout diff --git a/backend/tests/test_dual_check_node.py b/backend/tests/test_dual_check_node.py index 01b207b7..f7f2d1f0 100644 --- a/backend/tests/test_dual_check_node.py +++ b/backend/tests/test_dual_check_node.py @@ -376,8 +376,10 @@ async def test_disagreement_pauses_with_the_agreed_payload_and_card(monkeypatch, approval = await pending(run.id) assert approval.mode == "input" and approval.node_id == "dual" payload = approval.payload - # 约定的六个键,另加 on_reject(实际生效的处置):审核人在卡上要看得出没采用原流程会不会让运行失败 - assert set(payload) == {"kind", "node_id", "mode", "title", "card_artifact", "summary", "on_reject"} + # 约定的六个键,另加 on_reject(实际生效的处置):审核人在卡上要看得出没采用原流程会不会让运行失败;reject_route + # (驳回之后去哪,engine/replay.reject_route):没接驳回出口,驳回后运行就此结束 + assert set(payload) == {"kind", "node_id", "mode", "title", "card_artifact", "summary", "on_reject", "reject_route"} + assert payload["reject_route"] == {"ends": "run"} assert payload["kind"] == "dual_check" and payload["node_id"] == "dual" and payload["mode"] == "input" assert payload["on_reject"] == "degrade" [item] = payload["summary"] diff --git a/backend/tests/test_eval_runner.py b/backend/tests/test_eval_runner.py index a9685e62..6aaaa484 100644 --- a/backend/tests/test_eval_runner.py +++ b/backend/tests/test_eval_runner.py @@ -289,6 +289,87 @@ async def test_a_case_over_its_time_limit_stops_its_run(monkeypatch): assert item.status == "ok" and item.path == "build_failed" and "每题时限" in item.error +async def test_time_queued_for_a_run_slot_does_not_count_against_the_case_limit(monkeypatch): + """复核:满载时作答运行排队等名额的那一段不算每题时限(排了 2 秒、时限 1 秒,拿到名额后很快答完:不记超时)。""" + from app.cluster import slots + + stored = slots.values({"max_total": 2, "interactive_reserve": 1}) + + async def fixed(*, fresh: bool = False): # noqa: ARG001 + return stored + + monkeypatch.setattr(slots, "policy", fixed) + slots.forget_policy() + gate = asyncio.Event() + + async def blocker() -> None: + async with slots.hold("eval-slot-blocker", "eval"): + await gate.wait() + + async def drive(self): + self.turn.current_run = "eval-answer-in-queue" + await self._phase("build") + async with slots.hold(self.turn.current_run, "eval"): + await asyncio.sleep(0.05) + self.turn.path = "asked" + self.turn.usage["assistant"] = {"by_model": {}, "input_tokens": 0, "output_tokens": 0, "calls": 1, + "cost_usd": 0.0, "unpriced_calls": 0} + return self.turn + + monkeypatch.setattr(answer_mod.Driver, "drive", drive) + original = runner.EvalExecution._prepare + + async def prepare(self): + units = await original(self) + self.budget["per_case_timeout_s"] = 1.0 + return units + + monkeypatch.setattr(runner.EvalExecution, "_prepare", prepare) + held = asyncio.create_task(blocker()) + try: + while not (await slots.snapshot())["classes"]["system"]["running"]: + await asyncio.sleep(0.02) + suite_id = await es.make_suite("none", cases=ask_cases(1)) + [run] = await start(suite_id) + while not await slots.waiting("eval-answer-in-queue"): + await asyncio.sleep(0.02) + await asyncio.sleep(2.0) + gate.set() + await es.wait_eval(run.id) + finally: + gate.set() + await held + [item] = await results(run.id) + assert item.status == "ok" and item.path == "asked" and not item.error, item.error + + +async def test_within_does_not_count_the_paused_stretches(): + paused = {"now": True} + + async def is_paused() -> bool: + return paused["now"] + + async def work() -> str: + await asyncio.sleep(0.5) + paused["now"] = False + await asyncio.sleep(0.05) + return "done" + + assert await runner.within(work(), 0.2, is_paused, tick=0.02) == "done", "暂停的那 0.5 秒不算" + + async def never(): + await asyncio.sleep(10) + + async def awake() -> bool: + return False + + loop = asyncio.get_running_loop() + t0 = loop.time() + with pytest.raises(asyncio.TimeoutError): + await runner.within(never(), 0.2, awake, tick=0.02) + assert loop.time() - t0 < 1.0 + + async def test_a_run_stopped_at_an_approval_is_cancelled_not_approved(client, monkeypatch, tmp_path): from app.eval.answer import APPROVAL_REASON diff --git a/backend/tests/test_eval_schedule.py b/backend/tests/test_eval_schedule.py index 1bf3c7b3..a999e294 100644 --- a/backend/tests/test_eval_schedule.py +++ b/backend/tests/test_eval_schedule.py @@ -12,6 +12,7 @@ from __future__ import annotations import asyncio import json +import re import uuid from datetime import timedelta, timezone @@ -218,8 +219,10 @@ async def test_a_regression_is_pushed_without_numbers(client, monkeypatch, recei assert message["event"] == "eval.regressed" and message["title"] == "定时评测「虚构周回归」有退步" text = message["text"] assert "通过:0 / 1 题" in text and "退步:1 题" in text + # 消息里的编号(32 位十六进制)和发送时间是随机的、可能正好带着「310」:去掉再查,查的是写出来的文字 + readable = re.sub(r"[0-9a-f]{32}", "", json.dumps({**message, "sent_at": ""}, ensure_ascii=False)) for secret in ("330", "310", "gmv", "orders", "paid_date"): - assert secret not in json.dumps(message, ensure_ascii=False), f"通知里不该有 {secret}" + assert secret not in readable, f"通知里不该有 {secret}" from app.db.models import NotifyDelivery async with SessionLocal() as session: diff --git a/backend/tests/test_eval_snapshots.py b/backend/tests/test_eval_snapshots.py index a6702b06..955cfcb1 100644 --- a/backend/tests/test_eval_snapshots.py +++ b/backend/tests/test_eval_snapshots.py @@ -120,7 +120,7 @@ async def test_shadow_drafting_writes_only_the_copy(source, monkeypatch): assert snap.status in ("queued", "running") # 不占数据源的队:同一个源照样能马上起草、剖析 await asyncio.sleep(0) - assert job_runtime.lanes.holders(source["id"]) == [] + assert await job_runtime.lanes.holders(source["id"]) == [] done = await wait_done(snap.id) assert done.status == "succeeded", (done.error, done.progress) # 线上目录一项不动 diff --git a/backend/tests/test_evals_api.py b/backend/tests/test_evals_api.py index c4efa1f5..910938e7 100644 --- a/backend/tests/test_evals_api.py +++ b/backend/tests/test_evals_api.py @@ -7,7 +7,6 @@ from __future__ import annotations import json import sqlite3 -import uuid from datetime import datetime, timedelta from pathlib import Path from urllib.parse import quote @@ -27,6 +26,7 @@ from app.main import app from tests.fixtures import dual_check_shop as shop from tests.fixtures.acceptance import retail from tests.fixtures.catalog.fake_model import FakeModel +from tests.fixtures.names import letters from tests.fixtures.sources import drop_source SUITE_FILE = Path(__file__).parent / "fixtures" / "eval" / "retail_suite.json" @@ -57,7 +57,7 @@ async def source(tmp_path): async def new_suite(client, **body) -> dict: - res = await client.post("/api/evals/suites", json={"name": f"虚构题集_{uuid.uuid4().hex[:6]}", **body}, headers=ACTOR) + res = await client.post("/api/evals/suites", json={"name": f"虚构题集_{letters(6)}", **body}, headers=ACTOR) assert res.status_code == 201, res.text return res.json() @@ -264,7 +264,7 @@ def _weekly(source_name: str, *, refunds: bool = False) -> dict: async def test_caliber_cases_list_and_dispose_upgrades(client, source): async with SessionLocal() as session: - wf = Workflow(name=f"周销售_{uuid.uuid4().hex[:6]}", graph=_weekly(source["name"]), status="published", + wf = Workflow(name=f"周销售_{letters(6)}", graph=_weekly(source["name"]), status="published", published_version=1, version=1) session.add(wf) await session.flush() @@ -329,7 +329,7 @@ async def test_caliber_cases_list_and_dispose_upgrades(client, source): async def test_caliber_cases_pin_only_published_versions(client, source): async with SessionLocal() as session: - wf = Workflow(name=f"草稿口径_{uuid.uuid4().hex[:6]}", graph=_weekly(source["name"]), status="draft", version=1) + wf = Workflow(name=f"草稿口径_{letters(6)}", graph=_weekly(source["name"]), status="draft", version=1) session.add(wf) await session.flush() session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=_weekly(source["name"]))) diff --git a/backend/tests/test_evidence_judge_api.py b/backend/tests/test_evidence_judge_api.py index 923aec55..25cea0dd 100644 --- a/backend/tests/test_evidence_judge_api.py +++ b/backend/tests/test_evidence_judge_api.py @@ -446,7 +446,7 @@ async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_even """核对过状态、还没写入的那一下,接着跑的条件更新(runner._claim)先提交了:追加判定的条件更新落空, 照样不记——核对和写入是同一行上的条件更新,不是先读后写。""" from app.core.bus import bus - from app.core.config import settings + from tests import appdb Models(monkeypatch, verdicts={"新客": "contradicted"}) row = await finish(weekly(claims="judge")) @@ -456,12 +456,7 @@ async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_even def claimed_in_between(run_id, value): # _append_judged 读完运行、分配序号之前正好调用它:在这里用另一条连接把运行占成 running if run_id == row.id: - db = sqlite3.connect(settings.db_path, timeout=5) - try: - db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (row.id,)) - db.commit() - finally: - db.close() + appdb.execute(("UPDATE runs SET status = 'running' WHERE id = ?", (row.id,))) set_seq(run_id, value) monkeypatch.setattr(bus, "set_seq", claimed_in_between) diff --git a/backend/tests/test_introspect_jobs.py b/backend/tests/test_introspect_jobs.py index feb03bda..62734cf7 100644 --- a/backend/tests/test_introspect_jobs.py +++ b/backend/tests/test_introspect_jobs.py @@ -211,7 +211,7 @@ async def test_stop_keeps_what_was_read_and_restart_marks_interrupted(client, bi async def test_introspect_queues_behind_profile_and_blocks_sync_writes(client, small, monkeypatch): sid, _ = small - held = runtime.lanes.enter(sid, "profile", "someone-else") + held = await runtime.lanes.enter(sid, "profile", "someone-else") try: job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] await asyncio.sleep(0.1) @@ -220,7 +220,7 @@ async def test_introspect_queues_behind_profile_and_blocks_sync_writes(client, s r = await client.post(f"/api/datasources/{sid}/introspect") assert r.status_code == 409 and r.json()["code"] == "introspect_busy" finally: - held.release() + await held.release() await introspect_jobs.manager.wait(job["id"], timeout=30) assert (await client.get(f"/api/datasources/{sid}/introspect-jobs/{job['id']}")).json()["job"]["status"] \ == "succeeded" @@ -237,7 +237,8 @@ async def test_introspect_queues_behind_profile_and_blocks_sync_writes(client, s job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] for _ in range(100): await asyncio.sleep(0.02) - if runtime.lanes.holders(sid) and runtime.lanes.holders(sid)[0].kind == "introspect": + holders = await runtime.lanes.holders(sid) + if holders and holders[0].kind == "introspect": break r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": ["alpha"]}) assert r.status_code == 409 and r.json()["code"] == "introspect_busy" and "探查" in r.json()["detail"] @@ -436,14 +437,14 @@ async def test_draft_and_profile_jobs_say_they_wait_for_introspect(small): try: for kind, job, waiting in (("profile", profile, catalog_profile_jobs.waiting_for), ("draft", draft, catalog_jobs.waiting_for)): - held = runtime.lanes.enter(sid, "introspect", "an-introspect-job") - mine = runtime.lanes.enter(sid, kind, job.id) + held = await runtime.lanes.enter(sid, "introspect", "an-introspect-job") + mine = await runtime.lanes.enter(sid, kind, job.id) try: async with SessionLocal() as session: assert await waiting(session, job) == "introspect", kind finally: - held.release() - mine.release() + await held.release() + await mine.release() finally: async with SessionLocal() as session: for model, job_id in ((CatalogProfileJob, profile.id), (CatalogDraftJob, draft.id)): @@ -644,7 +645,7 @@ async def test_tables_added_while_the_job_is_queued_stay_in_scope(client, small) got = await so.add_to_scope(sid, ["eta"], trail=so.Trail(via=so.VIA_LOOK, actor="tester")) assert got.added == ["eta"] await _scope(client, sid, mode="rules", include=["alpha"], removed=["eta"]) - held = runtime.lanes.enter(sid, "profile", "someone-else") + held = await runtime.lanes.enter(sid, "profile", "someone-else") try: job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] assert job["status"] == "queued" and job["total"] == 1 @@ -655,7 +656,7 @@ async def test_tables_added_while_the_job_is_queued_stay_in_scope(client, small) # 任务还没开始:范围还是上一次的(eta 还在),新加的两张已经进了表结构 assert set((await _cache(sid))["tables"]) == {"alpha", "beta", "gamma", "eta"} finally: - held.release() + await held.release() await introspect_jobs.manager.wait(job["id"], timeout=30) assert (await client.get(f"/api/datasources/{sid}/introspect-jobs/{job['id']}")).json()["job"]["status"] \ == "succeeded" diff --git a/backend/tests/test_job_runtime.py b/backend/tests/test_job_runtime.py index ffda8c90..742e3fe2 100644 --- a/backend/tests/test_job_runtime.py +++ b/backend/tests/test_job_runtime.py @@ -32,7 +32,7 @@ def test_an_unregistered_kind_is_refused(): async def go() -> None: with pytest.raises(ValueError, match="没有登记"): - lanes.enter("src", "nonsense-kind") + await lanes.enter("src", "nonsense-kind") asyncio.run(go()) @@ -52,16 +52,16 @@ def test_a_new_kind_queues_behind_the_existing_ones_on_the_same_source(): lanes = job_runtime.SourceLanes() async def go() -> None: - first = lanes.enter("src-a", "draft", "job-1") - second = lanes.enter("src-a", "test-introspect", "job-2") - other = lanes.enter("src-b", "test-introspect", "job-3") + first = await lanes.enter("src-a", "draft", "job-1") + second = await lanes.enter("src-a", "test-introspect", "job-2") + other = await lanes.enter("src-b", "test-introspect", "job-3") assert first.granted.is_set() and not second.granted.is_set() assert other.granted.is_set(), "不同源并行" - assert lanes.ahead("src-a", "job-2") == ["draft"] - first.release() + assert await lanes.ahead("src-a", "job-2") == ["draft"] + await first.release() await asyncio.wait_for(second.wait(), timeout=1) - second.release() - other.release() + await second.release() + await other.release() asyncio.run(go()) diff --git a/backend/tests/test_kb_progress.py b/backend/tests/test_kb_progress.py index 3d8055a1..f4ab5c37 100644 --- a/backend/tests/test_kb_progress.py +++ b/backend/tests/test_kb_progress.py @@ -178,7 +178,9 @@ async def test_a_second_reindex_while_one_is_running_is_refused(client, monkeypa async def slow(session, collection=None, *, on_progress=None): await gate.wait() - return await real(session, collection, on_progress=on_progress) + # 放行之后只重建一个空集合:这里测的是「同时只跑一个」,不是重建本身。按全部集合重建要把同一个 xdist 进程里 + # 别的测试留下的几百段都重算一遍,满载时超过 _wait_done 的 10 秒,后面两条测试跟着撞上「正在重建」 + return await real(session, "reindex_gate_empty", on_progress=on_progress) monkeypatch.setattr(kb, "reindex", slow) try: @@ -211,7 +213,7 @@ async def test_two_reindexes_started_at_the_same_moment_do_not_both_run(client, async def slow(session, collection=None, *, on_progress=None): started.append(collection) await gate.wait() - return await real(session, collection, on_progress=on_progress) + return await real(session, "reindex_gate_empty", on_progress=on_progress) # 理由同上一条 monkeypatch.setattr(kb, "reindex", slow) try: diff --git a/backend/tests/test_ledger_issue.py b/backend/tests/test_ledger_issue.py index 19803753..00c7f4df 100644 --- a/backend/tests/test_ledger_issue.py +++ b/backend/tests/test_ledger_issue.py @@ -297,6 +297,28 @@ async def test_users_issue_only_versions_pinned_by_apps_they_can_use(monkeypatch assert created and formal.started_by == "carol" and formal.origin["granted_by_app"] == board_id +async def test_different_answers_issued_at_once_stop_exactly_at_the_daily_limit(monkeypatch): + """几份不同的答案同时出具(不再一把全站锁一个接一个):先在库里占今天的额度再发起,正好到上限为止,多出来的回 429。""" + wf = await S.publish(level="published") + rows = [await equivalent_run(monkeypatch) for _ in range(3)] + from app.cluster import quota + + async with SessionLocal() as session: + issued = await ledger_issue.issued_today(session) + # 今天的额度在库里记着(同一个测试进程里前面的测试也出具过);还没记过的按今天已出具的起算 + already = await quota.used(ledger_issue.QUOTA) + already = issued if already is None else already + await set_ledger(issue_daily_limit=max(already, issued) + 2) + results = await asyncio.gather(*(ledger_issue.issue(r.id, wf, 1, principal=ADMIN) for r in rows), + return_exceptions=True) + made = [r for r in results if not isinstance(r, BaseException)] + refused = [r for r in results if isinstance(r, ledger_issue.IssueRefused)] + assert len(made) == 2 and all(created for _, created in made), results + assert len(refused) == 1 and refused[0].status_code == 429 and refused[0].code == ledger_issue.DAILY_LIMIT + async with SessionLocal() as session: + assert await ledger_issue.issued_today(session) == issued + 2 + + async def test_concurrent_requests_issue_once(monkeypatch): wf = await S.publish(level="published") row = await equivalent_run(monkeypatch) diff --git a/backend/tests/test_ledger_projection.py b/backend/tests/test_ledger_projection.py index 9fa3a257..6ab642fd 100644 --- a/backend/tests/test_ledger_projection.py +++ b/backend/tests/test_ledger_projection.py @@ -131,8 +131,9 @@ def test_without_a_projection_every_pair_is_exactly_the_old_result(): ] for sql, columns, cited in cases: row = compare(sql, columns, cited) - # 逐格比(algo 4)是另加的 cells、one_row 两个键:整条查询这一层的结论和候选照旧一字不差 - assert {k: v for k, v in row.items() if k not in ("cells", "one_row")} == compare(sql, columns, None), sql + # 逐格比(algo 4)是另加的 cells 键(one_row、ordered 两边都记:行认不认得住,指标标签要看):整条查询这一层的 + # 结论和候选照旧一字不差 + assert {k: v for k, v in row.items() if k != "cells"} == compare(sql, columns, None), sql assert ("cells" in row) == (len(cited) >= 2 and row["status"] == "related"), sql diff --git a/backend/tests/test_ledger_recall.py b/backend/tests/test_ledger_recall.py index 7e7232c7..5610451a 100644 --- a/backend/tests/test_ledger_recall.py +++ b/backend/tests/test_ledger_recall.py @@ -913,3 +913,131 @@ async def test_restart_resumes_the_auto_batch_and_settles_finished_runs(monkeypa await recall.resume() async with SessionLocal() as session: assert len(await recall.recall_runs(session, wf, 2)) == 2, "再重启一次:都算过了,不再发起" + + +async def test_a_probe_that_cannot_assemble_its_inputs_says_which_are_missing(engine_up, caplog): + """表结构变了、口径凑不齐参数(这一版没有正式运行、也没有等价的问数回答,必填的入口字段又没有缺省值):不试算,照实记一条 + 「缺哪几个参数」——日志里有,变化记录的影响里也有(变化页照它写),不再一声不响地跳过。之后这条变化再被探查到一次,记下的 + 不丢。""" + import logging + + from app.data import source_changes as sc + from app.db.models import DataSource + from app.db.models_sources import SourceChange + + wf = await S.publish() + async with SessionLocal() as session: + source = (await session.execute(select(DataSource).where(DataSource.name == S.CURRENT["source"]))).scalar_one() + change, _ = await sc.record(session, source, sc.NewChange(kind="column_removed", table="orders", column="amount", + severity="action", key="v3"), + sc.Detected(job_kind="introspect")) + await session.commit() + out = sc.change_out(change) + with caplog.at_level(logging.INFO, logger="app.ledger.recall"): + await recall.process_change(out) + async with SessionLocal() as session: + probes = [r for r in (await session.execute(select(Run).where(Run.origin_kind == "recall"))).scalars() + if (r.origin or {}).get("change") == out["id"]] + row = (await session.execute(select(SourceChange).where(SourceChange.id == out["id"]))).scalar_one() + assert probes == [], "凑不齐参数:不试算" + [note] = row.affects["probes"] + assert (note["workflow_id"], note["version"], note["outcome"]) == (wf, 1, "skipped") + assert note["missing"] == ["start", "end"] and "start" in note["reason"], note + assert any("start、end" in r.getMessage() for r in caplog.records), [r.getMessage() for r in caplog.records] + # 变化接口(变化页)照样带着 + assert sc.change_out(row)["affects"]["probes"] == [note] + # 同一条变化再被探查到一次:影响按新的一次刷新,试算的记录不丢 + async with SessionLocal() as session: + source = await session.get(DataSource, row.source_id) + await sc.record(session, source, sc.NewChange(kind="column_removed", table="orders", column="amount", + severity="action", key="v3"), sc.Detected(job_kind="introspect")) + await session.commit() + again = (await session.execute(select(SourceChange).where(SourceChange.id == out["id"]))).scalar_one() + assert again.affects["probes"] == [note] + + +async def test_probe_notes_and_change_refreshes_do_not_overwrite_each_other(engine_up): + """同一条变化上几处同时写 affects(几版口径各记一条没试算、探查又刷新了一次影响):先占住那一行再读改写,谁也不盖掉谁。""" + import asyncio + from types import SimpleNamespace + + from app.data import source_changes as sc + from app.db.models import DataSource + from app.db.models_sources import SourceChange + + async with SessionLocal() as session: + source = (await session.execute(select(DataSource).where(DataSource.name == S.CURRENT["source"]))).scalar_one() + change, _ = await sc.record(session, source, sc.NewChange(kind="column_removed", table="orders", column="amount", + severity="action", key="v4", + affects={"catalog": [], "templates": [], + "rules_touched": False}), + sc.Detected(job_kind="introspect")) + await session.commit() + out = sc.change_out(change) + source_id = source.id + + async def refresh() -> None: + async with SessionLocal() as session: + src = await session.get(DataSource, source_id) + await sc.record(session, src, sc.NewChange(kind="column_removed", table="orders", column="amount", + severity="action", key="v4", + affects={"catalog": [], "templates": [], "rules_touched": True}), + sc.Detected(job_kind="introspect")) + await session.commit() + + calibers = [SimpleNamespace(workflow_id=f"wf-{i}", version=1) for i in range(6)] + await asyncio.gather(*(recall._probe_skipped(out, c, None, {}, "defaults", "缺参数") for c in calibers), refresh()) + async with SessionLocal() as session: + row = (await session.execute(select(SourceChange).where(SourceChange.id == out["id"]))).scalar_one() + assert sorted(p["workflow_id"] for p in row.affects["probes"]) == [f"wf-{i}" for i in range(6)], row.affects + assert row.affects["rules_touched"] is True, "探查刷新的那一份也在" + + +# -------------------------------------------------------------------------- +# 行序不定的指标(召回比不出变没变):不当「没变」 +# -------------------------------------------------------------------------- + + +async def test_all_metrics_with_unstable_row_order_cannot_be_checked_and_open_a_task(monkeypatch): + """全部指标都行序不定:结论写「核对不了(行序不定)」,开召回待办请人核对,不在原答案上记「已核对,不影响」。""" + wf = await S.publish() + a1 = await answer(monkeypatch) + graph = S.caliber_graph() + graph["nodes"][3]["data"]["config"]["instructions"] = "写周销售(新的写法)" + await publish_v2(wf, graph=graph) + monkeypatch.setattr(recall, "_rows_stable", lambda pinned, metric: False) + [done] = await run_recall(wf) + assert done.status == "succeeded", done.error + [settled] = [d for d in await events(done.id, EventType.LEDGER_RECALLED) if d.get("role") == "recall"] + assert settled["outcome"] == "row_order" and settled["changed"] is None + assert settled["row_order_metrics"] == ["销售额"] and settled["metrics"][0]["row_order"] == "unstable" + [task] = await recall_tasks(done.id) + assert task.title == "口径「周销售」升到 v2,答案里的数行序不定、没能核对,请人工核对" + assert await events(a1.id, EventType.LEDGER_RECALLED) == [], "核对不了的不在原答案上记「不影响」" + async with SessionLocal() as session: + view = await recall.batches(session, wf, 2) + state, _ = await recall.outcome_of(session, done) + assert state == "row_order" and view["groups"][0]["state"] == "row_order" and view["counts"]["row_order"] == 1 + + +async def test_some_metrics_with_unstable_row_order_are_listed_and_the_rest_decide(monkeypatch): + """部分行序不定:已判定的照常判(数没变就在原答案上记不影响),行序不定的那几项在结论和原答案的记账里照实列出。""" + two = S.caliber_graph() + card = two["nodes"][2]["data"]["config"] + card["metrics"] = [*card["metrics"], {**card["metrics"][0], "id": "gmv2", "name": "销售额(另算)"}] + wf = await S.publish(graph=two) + a1 = await answer(monkeypatch) + v2 = S.caliber_graph() + v2["nodes"][2]["data"]["config"]["metrics"] = card["metrics"] + v2["nodes"][3]["data"]["config"]["instructions"] = "写周销售(新的写法)" + await publish_v2(wf, graph=v2) + monkeypatch.setattr(recall, "_rows_stable", lambda pinned, metric: metric.get("id") != "gmv2") + [done] = await run_recall(wf) + assert done.status == "succeeded", done.error + [settled] = [d for d in await events(done.id, EventType.LEDGER_RECALLED) if d.get("role") == "recall"] + assert settled["outcome"] == "unchanged" and settled["changed"] is False + assert settled["row_order_metrics"] == ["销售额(另算)"] + assert [m.get("changed") for m in settled["metrics"]] == [False, None] + [booked] = await events(a1.id, EventType.LEDGER_RECALLED) + assert booked["changed"] is False and booked["row_order_metrics"] == ["销售额(另算)"] + assert await recall_tasks(done.id) == [] diff --git a/backend/tests/test_ledger_reconcile.py b/backend/tests/test_ledger_reconcile.py index 68beded9..432ad856 100644 --- a/backend/tests/test_ledger_reconcile.py +++ b/backend/tests/test_ledger_reconcile.py @@ -263,7 +263,9 @@ async def test_the_reconcile_view_gives_the_panel_what_it_draws(client, monkeypa body = {"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED, "confirm": True} res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json=body) assert res.status_code == 409 and res.json()["code"] == ledger_reconcile.NEEDS_PAIRS - assert res.json()["options"] == {"columns": ["gmv"], "metrics": [{"id": "gmv", "name": "销售额", "unit": "元"}]} + # suggested:逐格证明了的预选(这条查询只引用了一列,没有逐格比,空) + assert res.json()["options"] == {"columns": ["gmv"], "metrics": [{"id": "gmv", "name": "销售额", "unit": "元"}], + "suggested": []} done = await reconcile(client, row.id, {**body, "pairs": [{"ours": "gmv", "metric": "gmv"}]}) [listed] = (await ledger_of(client, row.id))["reconciled"] assert listed["run_id"] == done.id and listed["card"] == "reconcile" @@ -280,3 +282,18 @@ async def test_multi_row_reconcile_groups_carry_the_row_labels(client, monkeypat assert [g["label"] for g in listed["groups"]] == ["区域=east · gmv", "区域=west · gmv"] assert listed["groups"][1] == {"label": "区域=west · gmv", "ours": "ours_1_0", "caliber": "cal_1_0", "diff": "diff_1_0", "rel": "rel_1_0"} + + +async def test_a_related_query_prefills_the_pairing_from_the_cells_that_were_proven(client, monkeypatch): + """要人选对照时,按逐格比过的结论预选:这次的 gmv 那一列单独比是和口径取数的 gmv 等价的(逐格证明了),口径卡上读这一列的 + 指标就是它最可能的对照,options.suggested 给出来,界面预选上、人可以改;单数那一列没有证明,不替人猜。""" + wf = await S.publish(graph=S.caliber_graph(expression="cell(nodes.fetch, 0, 'gmv') * 1")) + sql = ("SELECT SUM(amount) AS gmv, COUNT(*) AS 单数 FROM orders WHERE status = 'paid' " + "AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08'") + row = await S.explore(monkeypatch, sql, text="本周销售额 [[v:Q1.r0.gmv]],共 [[v:Q1.r0.单数]] 单。") + [action] = [a for a in (await ledger_of(client, row.id))["actions"] if a["kind"] == "reconcile"] + body = {"query": action["query"], "workflow_id": wf, "version": 1, "inputs": CHECKED, "confirm": True} + res = await client.post(f"/api/ledger/answers/{row.id}/reconcile", json=body) + assert res.status_code == 409 and res.json()["code"] == ledger_reconcile.NEEDS_PAIRS, res.text + options = res.json()["options"] + assert options["suggested"] == [{"ours": "gmv", "metric": "gmv"}], options diff --git a/backend/tests/test_ledger_row_order.py b/backend/tests/test_ledger_row_order.py new file mode 100644 index 00000000..17bad01d --- /dev/null +++ b/backend/tests/test_ledger_row_order.py @@ -0,0 +1,119 @@ +"""按行号取多行查询里的一格,重跑、核对时行序不一定一样(next-round 验收遗留): + +- 认法(data/sql_semantics.row_stable):最外层带 ORDER BY 的按位置认;一定只出一行的第 0 行认得住;别的(多行、又没有 + ORDER BY)行序不定——库可以按任何顺序交回这些行; +- 口径账本的指标标签:答案的一格和口径卡的一格按「同一行」认来自口径,只在行认得住时成立;行序不定的照实标「相关」、 + 写明行序不定,不再标成来自口径; +- 口径召回:两版都按行号取一格、查询又行序不定的指标,照实记「行序不定」(changed 为空),不按位置硬比; +- 对账:多行结果按维度对齐之后,每一组对账指标先核对那一行的维度是不是它(重跑时缺了一行、多了一行,后面的行会错位), + 对不上的记空值,不拿别的维度的数冒充。 +数据是虚构店铺库。 +""" +from __future__ import annotations + +import pytest + +from app.data.sql_semantics import ordered, row_stable +from app.engine.runner import run_manager +from app.ledger import match as ledger_match +from app.ledger import recall as ledger_recall +from app.ledger import reconcile as ledger_reconcile +from tests import ledger_samples as S + +UNORDERED = ("SELECT region AS 区域, SUM(amount) AS gmv FROM orders WHERE status = 'paid' " + "AND paid_at >= '{{ input.start }}' AND paid_at < '{{ input.end }}' GROUP BY region") +UNORDERED_ALL = ("SELECT region AS 区域, SUM(amount) AS gmv FROM orders WHERE status = 'paid' " + "AND paid_at >= '2026-09-01' AND paid_at < '2026-09-08' GROUP BY region") + + +def test_which_rows_survive_a_rerun(): + assert ordered("SELECT a, b FROM t ORDER BY a") and not ordered("SELECT a, b FROM t") + assert row_stable("SELECT a, SUM(b) FROM t GROUP BY a ORDER BY a", 3) + assert row_stable("SELECT SUM(b) FROM t", 0) and not row_stable("SELECT SUM(b) FROM t", 1) + assert not row_stable("SELECT a, SUM(b) FROM t GROUP BY a", 0) + assert not row_stable("这不是 SQL", 0) + + +@pytest.fixture +async def shop(tmp_path): + await run_manager.setup() + await S.fresh_source(tmp_path / "shop.db") + yield + await ledger_match.drain() + await run_manager.shutdown() + + +async def _label(monkeypatch, caliber_sql: str, answer_sql: str, row: int) -> dict: + from tests.test_ledger_match import matched, only_label + + await S.publish(graph=S.caliber_graph(caliber_sql, expression=f"cell(nodes.fetch, {row}, 'gmv')")) + run = await S.explore(monkeypatch, answer_sql, text="这一区销售额 [[m:gmv]]。", + card_expression=f"cell(nodes.q, {row}, 'gmv')") + recorded, _ = await matched(run.id) + return only_label(recorded.content) + + +async def test_an_unordered_multi_row_cell_is_not_claimed_to_come_from_the_caliber(shop, monkeypatch): + label = await _label(monkeypatch, UNORDERED, UNORDERED_ALL, 1) + assert label["kind"] == "related" and label["reason"] == ledger_match.ROW_ORDER_UNSTABLE, label + + +async def test_with_order_by_the_same_row_still_comes_from_the_caliber(shop, monkeypatch): + label = await _label(monkeypatch, UNORDERED + " ORDER BY region", UNORDERED_ALL + " ORDER BY region", 1) + assert label["kind"] == "caliber", label + + +def _pinned(sql: str, expression: str, version: int): + from app.ledger.derive import Pinned + + graph = S.caliber_graph(sql, expression=expression) + card = next(n for n in graph["nodes"] if n["id"] == "card") + fetch = next(n for n in graph["nodes"] if n["id"] == "fetch") + return Pinned(workflow_id="wf", workflow_name="周销售", version=version, node_id="card", level="governed", + published=True, retired=False, card=card, fetches=(fetch,), reads=(("fetch", S.tool(), ("gmv",)),), + entry_fields=(), defaults={}) + + +def test_recall_marks_unordered_row_comparisons_instead_of_comparing_by_position(): + pair = ledger_recall.Pair(old=_pinned(UNORDERED, "cell(nodes.fetch, 1, 'gmv')", 1), + new=_pinned(UNORDERED + " AND 1 = 1", "cell(nodes.fetch, 1, 'gmv')", 2)) + _, meta = ledger_recall.recall_card([pair]) + assert meta[0]["row_order"] == "unstable" + values = {"old_0": {"value": 1.0, "rendered": "1"}, "new_0": {"value": 2.0, "rendered": "2"}, + "diff_0": {"value": 1.0, "rendered": "1"}} + [item] = ledger_recall.compare(meta, values) + assert item["changed"] is None and item["row_order"] == "unstable" + # 带 ORDER BY 的照常比 + pair = ledger_recall.Pair(old=_pinned(UNORDERED + " ORDER BY region", "cell(nodes.fetch, 1, 'gmv')", 1), + new=_pinned(UNORDERED + " ORDER BY region", "cell(nodes.fetch, 1, 'gmv')", 2)) + _, meta = ledger_recall.recall_card([pair]) + assert "row_order" not in meta[0] + [item] = ledger_recall.compare(meta, values) + assert item["changed"] is True + + +def test_reconcile_checks_each_aligned_row_is_the_one_it_stands_for(): + plan = ledger_reconcile.Plan(pairs=[ledger_reconcile.Pair(ours="gmv", column="gmv")], rows=[0, 1], + keys=[("区域", "区域")], key_values=[("east",), ("west",)]) + nodes = ledger_reconcile.tail_nodes(plan, card_id="card", fetch_id="fetch") + card = next(n for n in nodes if n["id"] == "reconcile") + metrics = {m["id"]: m["expression"] for m in card["data"]["config"]["metrics"]} + from app.engine.expressions import eval_expression + + def run(rows): + table = {"columns": ["区域", "ours_0", "cal_0"], "rows": rows} + return {mid: _eval(eval_expression, expr, table) for mid, expr in metrics.items() if mid.startswith("ours_")} + + # 重跑时东区那一行没了:西区挪到了第 0 行。以前东区那一组拿的是西区的数 + got = run([["west", 58.0, 20.0]]) + assert got["ours_0_0"] is None, "东区对不上:记空值,不拿西区的数冒充" + assert run([["east", 163.0, 150.5], ["west", 58.0, 20.0]]) == {"ours_0_0": 163.0, "ours_1_0": 58.0} + + +def _eval(fn, expression, table): + from app.engine.expressions import CellError, ExpressionError + + try: + return fn(expression, {"nodes": {"align": table}}) + except (CellError, ExpressionError, IndexError, KeyError, TypeError): + return None diff --git a/backend/tests/test_multiprocess_e2e.py b/backend/tests/test_multiprocess_e2e.py new file mode 100644 index 00000000..7181b10e --- /dev/null +++ b/backend/tests/test_multiprocess_e2e.py @@ -0,0 +1,622 @@ +"""多进程端到端(docs/design/orchestration-round/00-总方案.md §5 W2-b 的验收):PostgreSQL 上起两个工作进程(uvicorn,A、B), +指着同一个库、同一个数据目录,用演示模型(Mock)跑: + +1. 两个进程都登记了、只有一个当调度班; +2. 运行由接到请求的那个进程认领;另一个进程上的 WebSocket 收得到实时事件(含不落库的流式增量);同一个审批决定同时发给两个 + 进程只执行一次;两边的事件流都按 seq 递增、不重不漏,封存核对得上; +3. 在 A 上跑的运行,从 B 停止:照本进程的停止办(run.cancelled 带上是谁停的); +4. 定时任务到点只触发一次(两个进程都起着调度循环,只有当班的做); +5. 评测和问数据并发时优先级生效:全站只给 1 个名额(没有留底),评测的作答运行(系统类,由本测试进程作为第三个登记的工作进程 + 以评测的来源发起——工作进程里发起评测要真的去调模型建图)排着队时,后来的问数据运行(交互类,经 A 的接口发起)先拿到名额; + 全站 2 个、给交互留底 1 个:评测借满了别的名额、还有题排着,问数据一来马上拿到留底,不等评测跑完; +6. 每日花费上限跨进程不越过:两个进程同时往同一本账上记,记下的合计不超过上限; +7. 强杀 B:租约过期后,B 上在跑的运行被照实标中断(可从断点继续),停在审批上的运行照常能在 A 上批、接着跑完;B 当着调度班的 + 话 A 接班; +8. 卡住但没死(复核 R-b1):A 在跑一次运行时被 SIGSTOP,B 等它的租约过期两倍时长后接手、标中断;A 缓过来(SIGCONT)之后发现续晚了, + 停手、不写终态、不写事件,换一个身份重新登记——不管 B 有没有在这期间把那次运行接着跑完,都不双写、不盖掉终态、序号不重。 + +缺省不跑:要 AGENTLAB_MULTIPROCESS_E2E=1 和 PostgreSQL 测试库(scripts/test-multiprocess.sh 起临时容器、设好这两样)。 +""" +from __future__ import annotations + +import asyncio +import json +import os +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +pytestmark = pytest.mark.skipif( + os.environ.get("AGENTLAB_MULTIPROCESS_E2E") != "1" or not os.environ.get("AGENTLAB_TEST_DATABASE_URL"), + reason="多进程端到端缺省不跑:用 scripts/test-multiprocess.sh(要本机 docker 的 postgres:16)") + +BACKEND = Path(__file__).resolve().parents[1] +#: 工作进程的租约和心跳:调小,强杀之后几秒就能看到接手 +HEARTBEAT_S = 0.5 +LEASE_S = 2.0 + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _node(nid: str, ntype: str, **config: Any) -> dict[str, Any]: + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def _chain(*nodes: dict[str, Any]) -> dict[str, Any]: + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def _sleep_code(seconds: float) -> dict[str, Any]: + return _node("wait", "code", language="python", code=f"import time\ntime.sleep({seconds})\nprint('ok')") + + +def slow_graph(seconds: float) -> dict[str, Any]: + return _chain(_node("start", "input", fields=[{"name": "question"}]), _sleep_code(seconds), + _node("out", "output", fields=[{"name": "r", "value": "{{ nodes.wait.text }}"}])) + + +APPROVAL_GRAPH = _chain( + _node("start", "input", fields=[{"name": "question"}]), + _sleep_code(1.0), + _node("gate", "human", mode="approve", title="看一眼再接着"), + _node("write", "llm", prompt="用三句话写一段说明"), + _node("out", "output", fields=[{"name": "结果", "value": "{{ nodes.write.text }}"}]), +) + + +class Worker: + def __init__(self, name: str, env: dict[str, str], log_dir: Path) -> None: + self.name = name + self.port = _free_port() + self.base = f"http://127.0.0.1:{self.port}" + self.log = log_dir / f"worker-{name}.log" + self.proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(self.port)], + cwd=str(BACKEND), env=env, stdout=open(self.log, "wb"), stderr=subprocess.STDOUT) + self.owner: str | None = None + + def kill(self) -> None: + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGKILL) + self.proc.wait(timeout=10) + + def pause(self) -> None: + self.proc.send_signal(signal.SIGSTOP) + + def resume(self) -> None: + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGCONT) + + def stop(self) -> None: + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGCONT) + self.proc.send_signal(signal.SIGTERM) + try: + self.proc.wait(timeout=15) + except subprocess.TimeoutExpired: + self.kill() + + def tail(self) -> str: + try: + return self.log.read_text(encoding="utf-8", errors="replace")[-12000:] + except OSError: + return "" + + +def _worker_env() -> dict[str, str]: + env = {k: v for k, v in os.environ.items() + if k not in ("ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY", + "PYTEST_XDIST_WORKER", "PYTEST_CURRENT_TEST")} + env.update({ + "PYTHONPATH": str(BACKEND), + # 应用库、数据目录:和本测试进程同一个(conftest 已经设好) + "AGENTLAB_DATABASE_URL": os.environ["AGENTLAB_DATABASE_URL"], + "AGENTLAB_DATA_DIR": os.environ["AGENTLAB_DATA_DIR"], + "AGENTLAB_DB_POOL_SIZE": "5", + "AGENTLAB_SCHEDULER": "1", + "AGENTLAB_SCHEDULER_TICK_S": "0.5", + "AGENTLAB_WORKER_HEARTBEAT_S": str(HEARTBEAT_S), + "AGENTLAB_WORKER_LEASE_S": str(LEASE_S), + "AGENTLAB_SANDBOX_BACKEND": "local", + "AGENTLAB_AUTH_MODE": "off", + }) + return env + + +@pytest.fixture(scope="module") +def workers(tmp_path_factory): + import httpx + + log_dir = tmp_path_factory.mktemp("multiprocess") + env = _worker_env() + started = [Worker("A", env, log_dir), Worker("B", env, log_dir)] + try: + deadline = time.monotonic() + 90 + for w in started: + while True: + if w.proc.poll() is not None: + raise AssertionError(f"工作进程 {w.name} 没起来:\n{w.tail()}") + try: + if httpx.get(f"{w.base}/api/health", timeout=2).status_code == 200: + break + except httpx.HTTPError: + pass + if time.monotonic() > deadline: + raise AssertionError(f"工作进程 {w.name} 90 秒没起来:\n{w.tail()}") + time.sleep(0.3) + w.owner = httpx.get(f"{w.base}/api/system", timeout=10).json()["cluster"]["me"] + yield {w.name: w for w in started} + finally: + for w in started: + w.stop() + for w in started: + print(f"---- 工作进程 {w.name} 的日志(末尾)----\n{w.tail()}") + + +async def _client(): + import httpx + + return httpx.AsyncClient(timeout=30) + + +async def _until(check, timeout: float = 20.0, what: str = "") -> Any: + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = await check() + if last: + return last + await asyncio.sleep(0.1) + raise AssertionError(f"等超时了:{what}(最后一次:{last!r})") + + +async def _run(client, worker: Worker, run_id: str) -> dict[str, Any]: + return (await client.get(f"{worker.base}/api/runs/{run_id}")).json() + + +async def _db(sql: str, *args: Any) -> list[Any]: + from sqlalchemy import text + + from app.db.base import SessionLocal + + async with SessionLocal() as session: + return list((await session.execute(text(sql), _params(args))).all()) + + +def _params(args: tuple[Any, ...]) -> dict[str, Any]: + return {f"p{i}": v for i, v in enumerate(args)} + + +class Stream: + """一条 WebSocket(/api/runs/{id}/stream)收到的全部消息。""" + + def __init__(self, worker: Worker, run_id: str) -> None: + self.url = f"ws://127.0.0.1:{worker.port}/api/runs/{run_id}/stream?after=0" + self.messages: list[dict[str, Any]] = [] + self.task: asyncio.Task[None] | None = None + + async def start(self) -> Stream: + import websockets + + ready = asyncio.Event() + + async def pump() -> None: + async with websockets.connect(self.url) as ws: + ready.set() + async for raw in ws: + msg = json.loads(raw) + self.messages.append(msg) + if msg.get("type") == "stream.end": + return + + self.task = asyncio.create_task(pump()) + await asyncio.wait_for(ready.wait(), timeout=10) + return self + + def types(self) -> list[str]: + return [m.get("type") for m in self.messages] + + def seqs(self) -> list[int]: + """落库事件的 seq(不落库的流式增量另算)。""" + return [int(m["seq"]) for m in self.messages if m.get("seq") and m.get("type") not in ("llm.token", + "llm.thinking.delta")] + + async def ended(self, timeout: float = 30.0) -> None: + assert self.task is not None + await asyncio.wait_for(self.task, timeout=timeout) + + +# -------------------------------------------------------------------------- + + +async def test_1_two_workers_one_leader(workers): + a, b = workers["A"], workers["B"] + async with await _client() as client: + async def both() -> Any: + info = (await client.get(f"{a.base}/api/system")).json()["cluster"] + alive = {w["id"] for w in info["workers"] if w["alive"]} + return info if {a.owner, b.owner} <= alive else None + + info = await _until(both, what="两个进程都登记") + assert info["database"] == "postgresql" + assert info["leader"]["owner"] in (a.owner, b.owner) + other = (await client.get(f"{b.base}/api/system")).json()["cluster"] + assert other["leader"]["owner"] == info["leader"]["owner"], "两边看到的当班是同一个" + + +async def test_2_live_events_across_workers_and_one_decision(workers): + a, b = workers["A"], workers["B"] + async with await _client() as client: + made = (await client.post(f"{a.base}/api/runs", json={"graph": APPROVAL_GRAPH, "input": {"question": "q"}, + "origin": "canvas"})).json() + run_id = made["id"] + # 两边各接一条流(趁它还在跑:停在审批上之后才接的流回放完就结束,要等恢复之后重新接) + on_b = await Stream(b, run_id).start() + on_a = await Stream(a, run_id).start() + owner = await _until(lambda: _owner_while_active(run_id), what="运行被认领") + assert owner == a.owner, "运行由接到请求的进程认领" + + async def paused() -> Any: + return (await _run(client, b, run_id))["status"] == "interrupted" + + await _until(paused, what="停在审批上") + await _until(lambda: _has(on_b, "run.interrupted"), what="B 上的流收到 run.interrupted") + # 同一个决定同时发给两个进程:只执行一次 + decided = await asyncio.gather( + client.post(f"{a.base}/api/runs/{run_id}/resume", json={"response": {"approved": True}}), + client.post(f"{b.base}/api/runs/{run_id}/resume", json={"response": {"approved": True}})) + assert sorted(r.status_code for r in decided)[0] == 200, [r.text for r in decided] + + async def finished() -> Any: + return (await _run(client, a, run_id))["status"] in ("succeeded", "failed") + + await _until(finished, timeout=60, what="运行跑完") + assert (await _run(client, a, run_id))["status"] == "succeeded" + await on_a.ended() + await on_b.ended() + for stream in (on_a, on_b): + assert "run.finished" in stream.types(), stream.types() + seqs = stream.seqs() + assert seqs == sorted(set(seqs)), f"事件流有重复或乱序:{seqs}" + started = [m for m in on_a.messages if m.get("type") == "node.started" and m.get("node_id") == "write"] + assert len(started) == 1, "审批之后的那一步只执行了一次" + # 执行后半段的进程不是谁、另一个进程上的流照样收到了流式增量(不落库,只转给在看的进程) + assert any(m.get("type") == "llm.token" for m in on_a.messages) and \ + any(m.get("type") == "llm.token" for m in on_b.messages), "两边都收到了正文增量" + rows = await _db("SELECT seq, count(*) FROM run_events WHERE run_id = :p0 GROUP BY seq HAVING count(*) > 1", run_id) + assert rows == [], f"库里的事件撞了号:{rows}" + verified = (await client.get(f"{b.base}/api/runs/{run_id}/verify")).json() + assert verified["ok"] is True, verified + + +async def _owner_while_active(run_id: str) -> str | None: + rows = await _db("SELECT owner, status FROM runs WHERE id = :p0", run_id) + if rows and rows[0][1] in ("running", "queued") and rows[0][0]: + return rows[0][0] + return None + + +async def _has(stream: Stream, kind: str) -> bool: + return kind in stream.types() + + +async def test_3_stop_from_the_other_worker(workers): + a, b = workers["A"], workers["B"] + async with await _client() as client: + run_id = (await client.post(f"{a.base}/api/runs", json={"graph": slow_graph(20), "input": {"question": "q"}, + "origin": "canvas"})).json()["id"] + await _until(lambda: _owner_while_active(run_id), what="运行被认领") + t0 = time.monotonic() + stop = await client.post(f"{b.base}/api/runs/{run_id}/cancel", headers={"X-Actor": "e2e-stopper"}) + assert stop.status_code == 200 and stop.json()["status"] == "stopping", stop.text + + async def cancelled() -> Any: + row = await _run(client, b, run_id) + return row if row["status"] == "cancelled" else None + + row = await _until(cancelled, timeout=15, what="停止生效") + assert row["end_reason"] == "stopped" + assert time.monotonic() - t0 < 10, "不是等它跑完才停的" + events = (await client.get(f"{a.base}/api/runs/{run_id}/events")).json() + stopped = [e for e in events if e["type"] == "run.cancelled"] + assert stopped and stopped[-1]["data"].get("actor") == "e2e-stopper", stopped + + +async def test_4_a_schedule_fires_once(workers): + a = workers["A"] + async with await _client() as client: + wf = (await client.post(f"{a.base}/api/workflows", json={"name": "定时验收", "graph": slow_graph(0.1)})).json() + made = await client.post(f"{a.base}/api/schedules", json={ + "name": "每分钟一次", "workflow_id": wf["id"], "cron": "* * * * *", "timezone": "Asia/Shanghai", "run_class": "exploratory", + "input_template": {"question": "定时"}}) + assert made.status_code == 201, made.text + schedule_id = made.json()["id"] + # 拨到「刚过点」:两个进程的调度循环都在转,只有当班的那个做 + from sqlalchemy import text + + from app.db.base import SessionLocal, utcnow + + due = utcnow().replace(microsecond=0) + async with SessionLocal() as session: + await session.execute(text("UPDATE schedules SET next_run_at = :t WHERE id = :i"), {"t": due.replace(tzinfo=None), + "i": schedule_id}) + await session.commit() + + async def fired() -> Any: + rows = await _db("SELECT id FROM schedule_fires WHERE schedule_id = :p0", schedule_id) + return rows or None + + await _until(fired, timeout=15, what="定时触发") + await asyncio.sleep(3) # 再等几拍:不会再发一次 + fires = await _db("SELECT kind, status, run_id FROM schedule_fires WHERE schedule_id = :p0", schedule_id) + runs = await _db("SELECT id FROM runs WHERE schedule_id = :p0", schedule_id) + assert len(fires) == 1 and len(runs) == 1, (fires, runs) + await client.patch(f"{a.base}/api/schedules/{schedule_id}", json={"enabled": False}) + + +async def test_5_chat_goes_before_queued_evaluation_runs(workers, monkeypatch): + """全站 1 个名额:评测的作答运行(系统类)排着两个,问数据的运行(交互类)后来,先拿到名额。""" + from app import cluster + from app.core.config import settings + from app.cluster import slots + from app.cluster.relay import relay + from app.db.base import SessionLocal + from app.db.models import Run + from app.engine.runner import run_manager + + a = workers["A"] + async with await _client() as client: + put = await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": { + "max_total": 1, "classes": {"interactive": {"limit": 1, "priority": 40}, "system": {"limit": 1, "priority": 10}}}}}) + assert put.status_code == 200, put.text + slots.forget_policy() + monkeypatch.setattr(settings, "sandbox_backend", "local") + await cluster.start() # 本测试进程登记成第三个工作进程,替评测发起作答运行 + await relay.start() + await run_manager.setup() + try: + evals = [await run_manager.start(graph=slow_graph(1.5), input_payload={"question": f"评测 {i}"}, + origin_kind="eval", origin={"kind": "eval"}) for i in range(3)] + + async def first_running() -> Any: + async with SessionLocal() as session: + return (await session.get(Run, evals[0].id)).status == "running" + + await _until(first_running, what="第一题的作答运行拿到名额") + await asyncio.sleep(0.3) + chat_id = (await client.post(f"{a.base}/api/runs", json={ + "graph": slow_graph(0.2), "input": {"question": "问数据"}, "origin": "chat"})).json()["id"] + + async def all_done() -> Any: + async with SessionLocal() as session: + rows = [await session.get(Run, rid, populate_existing=True) for rid in [*[e.id for e in evals], chat_id]] + return rows if all(r.status in ("succeeded", "failed") for r in rows) else None + + rows = await _until(all_done, timeout=60, what="都跑完") + assert all(r.status == "succeeded" for r in rows), [(r.id, r.status, r.error) for r in rows] + e1, e2, e3, chat = rows + assert e2.created_at < chat.created_at and e3.created_at < chat.created_at, "评测的两题先来、排着队" + assert e1.started_at < chat.started_at < e2.started_at < e3.started_at, \ + [(r.origin_kind, r.created_at, r.started_at) for r in rows] + finally: + await run_manager.shutdown() + await relay.stop() + await cluster.stop() + await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": slots.defaults()}}) + + +async def test_5b_the_interactive_reserve_is_there_when_evaluations_fill_the_rest(workers, monkeypatch): + """全站 2 个名额、留底 1 个:评测(系统类)只借得到 1 个,第二题排着;问数据一来马上拿到留底,评测的第一题还在跑。""" + from app import cluster + from app.cluster import slots + from app.cluster.relay import relay + from app.core.config import settings + from app.db.base import SessionLocal + from app.db.models import Run + from app.engine.runner import run_manager + + a = workers["A"] + async with await _client() as client: + put = await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": { + "max_total": 2, "interactive_reserve": 1, "classes": {}}}}) + assert put.status_code == 200, put.text + got = put.json()["concurrency"] + assert got["max_total"] - got["interactive_reserve"] == 1, "别的类最多借得到 全站 − 留底 = 1 个" + slots.forget_policy() + monkeypatch.setattr(settings, "sandbox_backend", "local") + await cluster.start() + await relay.start() + await run_manager.setup() + try: + evals = [await run_manager.start(graph=slow_graph(3), input_payload={"question": f"评测 {i}"}, + origin_kind="eval", origin={"kind": "eval"}) for i in range(2)] + + async def first_running() -> Any: + async with SessionLocal() as session: + return (await session.get(Run, evals[0].id)).status == "running" + + await _until(first_running, what="第一题的作答运行拿到名额") + chat_id = (await client.post(f"{a.base}/api/runs", json={ + "graph": slow_graph(0.2), "input": {"question": "问数据"}, "origin": "chat"})).json()["id"] + + async def all_done() -> Any: + async with SessionLocal() as session: + rows = [await session.get(Run, rid, populate_existing=True) for rid in [*[e.id for e in evals], chat_id]] + return rows if all(r.status in ("succeeded", "failed") for r in rows) else None + + e1, e2, chat = await _until(all_done, timeout=60, what="都跑完") + assert chat.started_at < e1.finished_at, "问数据拿到的是留底:评测的第一题还没跑完它就开始了" + assert e2.started_at >= e1.finished_at, "评测的第二题借不到留底,等第一题跑完" + finally: + await run_manager.shutdown() + await relay.stop() + await cluster.stop() + await client.put(f"{a.base}/api/settings", json={"values": {"concurrency": slots.defaults()}}) + + +def test_6_the_daily_cap_holds_across_processes(workers): + """两个进程同时往评测那本账上各记 20 次、每次 0.1 美元,上限 1 美元:合计记下的正好 10 次、不超过上限。""" + script = ( + "import asyncio, json\n" + "from app.engine.spend import Ledger\n" + "async def main():\n" + " book = Ledger('e2e_spend')\n" + " got = await asyncio.gather(*(book.charge(0.1, limit=1.0, priced=True) for _ in range(20)))\n" + " print(json.dumps(sum(1 for d in got if d)))\n" + "asyncio.run(main())\n") + env = {**_worker_env(), "AGENTLAB_SCHEDULER": "0"} + procs = [subprocess.Popen([sys.executable, "-c", script], cwd=str(BACKEND), env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) for _ in range(2)] + outs = [p.communicate(timeout=120) for p in procs] + assert all(p.returncode == 0 for p in procs), [o[1].decode()[-2000:] for o in outs] + charged = sum(int(o[0].decode().strip().splitlines()[-1]) for o in outs) + assert charged == 10, f"两个进程合计记下 {charged} 次(上限只容得下 10 次)" + total = asyncio.run(_spent()) + assert total <= 1.0 + 1e-9, total + + +async def _spent() -> float: + from app.engine.spend import Ledger + + return float((await Ledger("e2e_spend").daily())["usd"]) + + +async def _stall_a_during_a_run(client, a: Worker, b: Worker, seconds: float) -> str: + """A 上发起一次运行,跑起来之后 SIGSTOP A,等 B 接手(标中断)。交回运行 id(A 还停着,调用方负责 SIGCONT)。""" + run_id = (await client.post(f"{a.base}/api/runs", json={"graph": slow_graph(seconds), "input": {"question": "q"}, + "origin": "canvas"})).json()["id"] + await _until(lambda: _owner_while_active(run_id), what="A 认领了运行") + await asyncio.sleep(0.5) + a.pause() + + async def taken_over() -> Any: + row = await _run(client, b, run_id) + return row if row["status"] == "interrupted" else None + + await _until(taken_over, timeout=LEASE_S * 6 + 10, what="B 接手了卡住的 A 上的运行") + return run_id + + +async def _a_reidentified(client, a: Worker) -> None: + """A 缓过来之后换了身份重新登记:fixture 里记的 owner 跟着换。""" + old = a.owner + + async def fresh() -> Any: + try: + me = (await client.get(f"{a.base}/api/system")).json()["cluster"]["me"] + except Exception: # noqa: BLE001 + return None + return me if me != old else None + + a.owner = await _until(fresh, timeout=15, what="A 换身份重新登记") + + +async def _settled_events(client, worker: Worker, run_id: str) -> list[dict[str, Any]]: + return (await client.get(f"{worker.base}/api/runs/{run_id}/events")).json() + + +def _seqs_unique(events: list[dict[str, Any]]) -> None: + seqs = [e["seq"] for e in events] + assert len(seqs) == len(set(seqs)), f"撞号:{sorted(seqs)}" + + +async def test_6b_a_stalled_worker_does_not_write_after_it_was_taken_over(workers): + a, b = workers["A"], workers["B"] + async with await _client() as client: + try: + run_id = await _stall_a_during_a_run(client, a, b, 6) + finally: + a.resume() + await _a_reidentified(client, a) + await asyncio.sleep(max(4.0, 3 * HEARTBEAT_S)) # A 手上那一步本来再过几秒就跑完:给它机会(它不该写) + row = await _run(client, b, run_id) + assert row["status"] == "interrupted", f"接手之后的状态被盖掉了:{row['status']}" + events = await _settled_events(client, b, run_id) + assert not [e for e in events if e["type"] in ("run.finished", "run.failed", "run.cancelled")] + assert any(e["type"] == "log" and e["data"].get("code") == "server_shutdown" for e in events) + _seqs_unique(events) + + +async def test_6c_a_run_continued_by_the_other_worker_is_not_written_twice(workers): + a, b = workers["A"], workers["B"] + async with await _client() as client: + try: + run_id = await _stall_a_during_a_run(client, a, b, 3) + cont = await client.post(f"{b.base}/api/runs/{run_id}/continue", json={}) + assert cont.status_code == 200, cont.text + finally: + a.resume() + await _a_reidentified(client, a) + + async def done() -> Any: + row = await _run(client, b, run_id) + return row if row["status"] in ("succeeded", "failed") and row.get("manifest_hash") else None + + row = await _until(done, timeout=60, what="B 接着跑完") + await asyncio.sleep(2.0) + assert row["status"] == "succeeded", row + assert (await _run(client, b, run_id))["status"] == "succeeded", "A 缓过来之后没有把终态改掉" + events = await _settled_events(client, b, run_id) + assert len([e for e in events if e["type"] == "run.finished"]) == 1 + _seqs_unique(events) + verified = (await client.get(f"{b.base}/api/runs/{run_id}/verify")).json() + assert verified["ok"] is True, verified + + +async def test_7_kill_a_worker(workers): + a, b = workers["A"], workers["B"] + async with await _client() as client: + leader_before = (await client.get(f"{a.base}/api/system")).json()["cluster"]["leader"]["owner"] + # B 上:一次在跑(会被照实标中断),一次停在审批上(不属于任何进程,B 没了照样能在 A 上批) + busy = (await client.post(f"{b.base}/api/runs", json={"graph": slow_graph(30), "input": {"question": "q"}, + "origin": "canvas"})).json()["id"] + paused = (await client.post(f"{b.base}/api/runs", json={"graph": APPROVAL_GRAPH, "input": {"question": "q"}, + "origin": "canvas"})).json()["id"] + await _until(lambda: _owner_while_active(busy), what="B 认领了运行") + + async def waiting() -> Any: + return (await _run(client, a, paused))["status"] == "interrupted" + + await _until(waiting, what="停在审批上") + b.kill() + t0 = time.monotonic() + + async def taken_over() -> Any: + row = await _run(client, a, busy) + return row if row["status"] == "interrupted" else None + + row = await _until(taken_over, timeout=LEASE_S * 6 + 10, what="租约过期后被接手") + assert "可从断点继续运行" in (row["error"] or "") + events = (await client.get(f"{a.base}/api/runs/{busy}/events")).json() + assert any(e["type"] == "log" and e["data"].get("code") == "server_shutdown" for e in events) + print(f"强杀 B 之后 {time.monotonic() - t0:.1f} 秒被接手") + info = (await client.get(f"{a.base}/api/system")).json()["cluster"] + assert b.owner not in {w["id"] for w in info["workers"] if w["alive"]} + if leader_before == b.owner: + async def a_leads() -> Any: + got = (await client.get(f"{a.base}/api/system")).json()["cluster"]["leader"] + return got["owner"] == a.owner + + await _until(a_leads, timeout=LEASE_S * 6, what="A 接班") + # 停在审批上的那一次:在 A 上批,接着跑完 + ok = await client.post(f"{a.base}/api/runs/{paused}/resume", json={"response": {"approved": True}}) + assert ok.status_code == 200, ok.text + + async def done() -> Any: + return (await _run(client, a, paused))["status"] == "succeeded" + + await _until(done, timeout=60, what="批了之后在 A 上跑完") + # 被标中断的那一次:在 A 上能接着跑(这里接上就停掉,不等它睡完) + cont = await client.post(f"{a.base}/api/runs/{busy}/continue", json={}) + assert cont.status_code == 200, cont.text + await client.post(f"{a.base}/api/runs/{busy}/cancel") diff --git a/backend/tests/test_notify_channels.py b/backend/tests/test_notify_channels.py index b5ffd120..b25379c6 100644 --- a/backend/tests/test_notify_channels.py +++ b/backend/tests/test_notify_channels.py @@ -8,7 +8,6 @@ from __future__ import annotations import asyncio import json -import uuid import httpx import pytest @@ -19,6 +18,7 @@ from app.db.base import SessionLocal from app.db.models import App, AppVersion, NotifyChannel, NotifyDelivery, Run, Setting from app.engine.runner import run_manager from app.notify import core, service +from tests.fixtures.names import letters URL = "https://hooks.example.com/agentlab/inbox-7f3a9c2e1b" @@ -141,7 +141,7 @@ async def test_task_opened_is_pushed_to_the_app_subscription_without_numbers(rec async with SessionLocal() as session: channel = NotifyChannel(name="财务群", format="generic", url=core_encrypt(URL), events=[]) session.add(channel) - app = App(name=f"报销-{uuid.uuid4().hex[:6]}", published_version=1) + app = App(name=f"报销-{letters(6)}", published_version=1) session.add(app) await session.flush() session.add(AppVersion(app_id=app.id, version=1, spec_hash="h" * 64, bindings={}, diff --git a/backend/tests/test_notify_run.py b/backend/tests/test_notify_run.py index a57e32ff..0ccf2967 100644 --- a/backend/tests/test_notify_run.py +++ b/backend/tests/test_notify_run.py @@ -10,7 +10,6 @@ from __future__ import annotations import asyncio import json -import uuid import httpx import pytest @@ -21,6 +20,7 @@ from app.db.base import SessionLocal from app.db.models import App, AppVersion, NotifyChannel, NotifyDelivery, Run, User from app.engine.runner import run_manager from app.notify import core, service +from tests.fixtures.names import letters URL = "https://hooks.example.com/agentlab/run-3b8e1f" @@ -68,9 +68,9 @@ def flow(**human): async def _app(events: list[str]) -> tuple[str, str]: async with SessionLocal() as session: - channel = NotifyChannel(name=f"群-{uuid.uuid4().hex[:6]}", format="generic", url=encrypt(URL), events=[]) + channel = NotifyChannel(name=f"群-{letters(6)}", format="generic", url=encrypt(URL), events=[]) session.add(channel) - app = App(name=f"报销-{uuid.uuid4().hex[:6]}", published_version=1) + app = App(name=f"报销-{letters(6)}", published_version=1) session.add(app) await session.flush() session.add(AppVersion(app_id=app.id, version=1, spec_hash="h" * 64, bindings={}, @@ -180,7 +180,7 @@ async def _finished_instance(app_id: str, *, started_by: str, verified: bool) -> async def test_the_initiator_s_personal_channel_receives_my_instance_finished(received): """B1:个人渠道订阅了 run.finished,发起人自己发起的实例结束了收得到;应用自己没订阅也一样。""" app_id, _ = await _app([]) - name = f"uma{uuid.uuid4().hex[:6]}" + name = f"uma{letters(6)}" mine = await _personal(name, ["run.finished"]) run_id = await _finished_instance(app_id, started_by=name, verified=True) assert await service.notify_run(run_id, "run.finished") == 1 @@ -192,7 +192,7 @@ async def test_the_initiator_s_personal_channel_receives_my_instance_finished(re async def test_a_self_declared_initiator_does_not_reach_a_personal_channel(received): """关登录时署名是自己填的,谁都能填别人的名字:不按它推给那个人的个人渠道。""" app_id, _ = await _app([]) - name = f"vic{uuid.uuid4().hex[:6]}" + name = f"vic{letters(6)}" mine = await _personal(name, ["run.finished"]) run_id = await _finished_instance(app_id, started_by=name, verified=False) assert await service.notify_run(run_id, "run.finished") == 0 diff --git a/backend/tests/test_ops.py b/backend/tests/test_ops.py new file mode 100644 index 00000000..92c27604 --- /dev/null +++ b/backend/tests/test_ops.py @@ -0,0 +1,122 @@ +"""app/ops.py:deploy.sh、dev.sh 在容器里调的几件小事(密码文件、状态、SQLite 备份、删旧备份)。""" + +from __future__ import annotations + +import sqlite3 +import stat + +import pytest + +from app import ops + + +def _mode(path) -> int: + return stat.S_IMODE(path.stat().st_mode) + + +def test_secrets_are_generated_once_with_tight_permissions(tmp_path, capsys): + assert ops.main(["postgres-secrets", "--data", str(tmp_path), "--host", "agl-postgres"]) == 0 + assert capsys.readouterr().out.strip() == "生成了新密码" + folder = tmp_path / ".postgres" + password = (folder / "password").read_text().strip() + assert len(password) >= 32 + assert _mode(folder / "password") == 0o600 and _mode(folder / "pgpass") == 0o600 + assert _mode(folder) == 0o711, "别人列不出目录,但按名字读得到 state" + assert (folder / "pgpass").read_text() == f"agl-postgres:5432:agentlab:agentlab:{password}\n" + + assert ops.main(["postgres-secrets", "--data", str(tmp_path), "--host", "other", "--port", "55432"]) == 0 + out = capsys.readouterr().out + assert out.strip() == "沿用已有的密码" and password not in out, "密码不打印" + assert (folder / "password").read_text().strip() == password + assert (folder / "pgpass").read_text().startswith("other:55432:agentlab:agentlab:"), "pgpass 按现在的主机名重写" + + +def test_pgpass_escapes_colons_and_backslashes(tmp_path): + folder = tmp_path / ".postgres-dev" + folder.mkdir() + (folder / "password").write_text("a:b\\c\n") + assert ops.postgres_secrets(tmp_path, folder=".postgres-dev", host="127.0.0.1", port=55432, database="agentlab", + user="agentlab") is False + assert (folder / "pgpass").read_text() == "127.0.0.1:55432:agentlab:agentlab:a\\:b\\\\c\n" + + +def test_an_empty_password_file_is_refused(tmp_path): + (tmp_path / ".postgres").mkdir() + (tmp_path / ".postgres" / "password").write_text("\n") + with pytest.raises(SystemExit, match="是空的"): + ops.postgres_secrets(tmp_path, folder=".postgres", host="h", port=5432, database="d", user="u") + + +def test_state_defaults_to_sqlite_and_records_switches(tmp_path, capsys): + assert ops.read_state(tmp_path) == "sqlite" + for mode in ("migrating", "builtin", "sqlite"): + assert ops.main(["state", "--data", str(tmp_path), "--set", mode]) == 0 + assert capsys.readouterr().out.strip() == mode + assert ops.read_state(tmp_path) == mode + path = tmp_path / ".postgres" / "state" + assert _mode(path) == 0o644, "宿主机上的 deploy.sh 直接读它" + assert "since=" in path.read_text() + path.write_text("mode=whatever\n") + assert ops.read_state(tmp_path) == "sqlite", "认不出的当没切过" + with pytest.raises(SystemExit): + ops.main(["state", "--data", str(tmp_path), "--set", "external"]) + + +def test_prune_keeps_the_newest_by_name(tmp_path): + for name in ("agentlab-20260101T000000Z.dump", "agentlab-20260103T000000Z.dump", "agentlab-20260102T000000Z.dump"): + (tmp_path / name).write_bytes(b"x") + (tmp_path / "20260101T000000Z").mkdir() + (tmp_path / "20260101T000000Z" / "agentlab.db").write_bytes(b"x") + (tmp_path / ".hidden").write_bytes(b"x") + gone = ops.prune(tmp_path, 2) + assert sorted(p.name for p in gone) == ["20260101T000000Z", "agentlab-20260101T000000Z.dump"] + assert sorted(p.name for p in tmp_path.iterdir()) == [".hidden", "agentlab-20260102T000000Z.dump", + "agentlab-20260103T000000Z.dump"] + assert ops.prune(tmp_path / "missing", 2) == [] + + +def test_backup_sqlite_copies_a_consistent_snapshot_including_the_wal(tmp_path, capsys): + db = sqlite3.connect(tmp_path / "agentlab.db") + db.execute("PRAGMA journal_mode=WAL") + db.execute("CREATE TABLE t (v TEXT)") + db.execute("INSERT INTO t VALUES ('还在 WAL 里')") + db.commit() + # 不关连接、不做检查点:新写的那行还只在 -wal 里 + assert (tmp_path / "agentlab.db-wal").exists() + sqlite3.connect(tmp_path / "checkpoints.db").close() + try: + assert ops.main(["backup-sqlite", "--data", str(tmp_path), "--keep", "2"]) == 0 + finally: + db.close() + target = tmp_path / "backups" / "sqlite" + (stamp,) = list(target.iterdir()) + assert capsys.readouterr().out.strip() == str(stamp) + copy = sqlite3.connect(stamp / "agentlab.db") + assert copy.execute("SELECT v FROM t").fetchall() == [("还在 WAL 里",)] + copy.close() + assert (stamp / "checkpoints.db").exists() + # 备份里是整个库:目录只有属主进得去,文件只有属主读得到 + for level in (tmp_path / "backups", target, stamp): + assert _mode(level) == 0o700, level + assert _mode(stamp / "agentlab.db") == 0o600 and _mode(stamp / "checkpoints.db") == 0o600 + assert sorted(p.name for p in stamp.iterdir()) == ["agentlab.db", "checkpoints.db"], "没留下 -journal 之类的" + # 只留最近 keep 份 + for name in ("20200101T000000Z", "20200102T000000Z"): + (target / name).mkdir() + ops.backup_sqlite(tmp_path, keep=2) + assert len(list(target.iterdir())) == 2 and not (target / "20200101T000000Z").exists() + + +def test_backup_dir_creates_and_tightens_every_level(tmp_path, capsys): + (tmp_path / "backups").mkdir(mode=0o755) + (tmp_path / "backups").chmod(0o755) + assert ops.main(["backup-dir", "--data", str(tmp_path), "postgres"]) == 0 + assert capsys.readouterr().out.strip() == str(tmp_path / "backups" / "postgres") + assert _mode(tmp_path / "backups") == 0o700, "已有的收紧" + assert _mode(tmp_path / "backups" / "postgres") == 0o700 + + +def test_backup_sqlite_without_databases_does_nothing(tmp_path, capsys): + assert ops.main(["backup-sqlite", "--data", str(tmp_path)]) == 0 + assert "没有 SQLite 库" in capsys.readouterr().out + assert not (tmp_path / "backups").exists() diff --git a/backend/tests/test_p3_api_errors.py b/backend/tests/test_p3_api_errors.py index 6fb872de..1b3466ae 100644 --- a/backend/tests/test_p3_api_errors.py +++ b/backend/tests/test_p3_api_errors.py @@ -136,6 +136,7 @@ def held_guard(store): table_versions._release_guard() +@pytest.mark.sqlite_only # 文件锁守卫只在 SQLite(单进程)时有:PostgreSQL 可以多进程,版本存储靠跨进程的锁(app/cluster/mutex) async def test_read_only_process_refuses_writes_with_503_and_keeps_reads(client, held_guard): name = unique() base = await first_import(client, name, seed=505) @@ -170,6 +171,7 @@ async def test_read_only_process_refuses_writes_with_503_and_keeps_reads(client, assert await table_versions.expire_stagings(session) == [] +@pytest.mark.sqlite_only # 文件锁守卫只在 SQLite(单进程)时有:PostgreSQL 可以多进程,版本存储靠跨进程的锁(app/cluster/mutex) async def test_read_only_process_refuses_activate_and_commit_keeps_the_trial(client, held_guard): """启用旧版本 503;提交 503 时不消耗试运行库(守卫拿回来之后照常提交)。""" name = unique() @@ -196,6 +198,7 @@ async def test_read_only_process_refuses_activate_and_commit_keeps_the_trial(cli assert resp.json()["parts"] == 3 +@pytest.mark.sqlite_only # 文件锁守卫只在 SQLite(单进程)时有:PostgreSQL 可以多进程,版本存储靠跨进程的锁(app/cluster/mutex) async def test_read_only_process_refuses_version_and_edit_writes_before_doing_any_work(client, held_guard, store, monkeypatch): """只读进程:移除一期(要物化)、作废接受(累积模式,要物化)、edits/apply、edits/undo、POST /{id}/redraft 一律 503, @@ -253,6 +256,7 @@ async def test_read_only_process_refuses_version_and_edit_writes_before_doing_an assert len((await db_get(ImportStaging, applied["id"])).edits) == 1, "撤销没有生效" +@pytest.mark.sqlite_only # 文件锁守卫只在 SQLite(单进程)时有:PostgreSQL 可以多进程,版本存储靠跨进程的锁(app/cluster/mutex) async def test_read_only_process_still_deletes_a_manual_source(client, held_guard): """手工登记的源没有导入记录、配方、暂存区和隔离文件,不碰版本存储:只读进程里照常删除(9.1 只要求上传源 503)。""" sid = "manual-" + unique("m") diff --git a/backend/tests/test_process_actions.py b/backend/tests/test_process_actions.py index c093f98d..2110b39c 100644 --- a/backend/tests/test_process_actions.py +++ b/backend/tests/test_process_actions.py @@ -96,7 +96,9 @@ async def _events(run_id: str, kind: str) -> list[RunEvent]: async def _confirm(run_id: str) -> Task: - """等确认卡出来,按卡上的内容指纹确认。""" + """等运行停到确认上、确认卡出来,按卡上的内容指纹确认。待办和审批同一个事务开出来,那一刻运行可能还没收成 interrupted、 + 还在跑——这时去确认会被拒(「运行正在运行,没有待处理的审批」):先等运行停下来,再取待办。""" + await _wait(run_id, ("interrupted",)) task = await _task(run_id, "action_confirm") card = (await tasks.detail(task.id, ANN))["payload"] out = await tasks.respond(task.id, ANN, decision="approve", payload_hash=card["payload_hash"]) @@ -166,6 +168,11 @@ async def test_unknown_result_with_lookup_is_reconciled_automatically_and_sent_o await _confirm(run_id) held = await _wait(run_id, ("interrupted", "failed", "succeeded")) assert held.status == "interrupted" and held.manifest_hash is None + # 运行先挂起、runner 再安排核实(actions.on_unknown):等排上查询(记下 action.unknown)再看记录 + for _ in range(100): + if await _events(run_id, "action.unknown"): + break + await asyncio.sleep(0.05) [rec] = await _records(run_id) assert rec.state == "unknown" and rec.next_lookup_at is not None assert len(await _events(run_id, "action.unknown")) == 1 diff --git a/backend/tests/test_recipe_acceptance_p3.py b/backend/tests/test_recipe_acceptance_p3.py index a502564c..84ad61f0 100644 --- a/backend/tests/test_recipe_acceptance_p3.py +++ b/backend/tests/test_recipe_acceptance_p3.py @@ -1683,6 +1683,7 @@ sys.stdin.read() """ +@pytest.mark.sqlite_only # 文件锁守卫只在 SQLite(单进程)时有:PostgreSQL 可以多进程,版本存储靠跨进程的锁(app/cluster/mutex) async def test_l5_another_process_holding_the_store_guard(client, store): """L5:子进程先占住 .store.lock → 本进程 acquire_store_guard() 返回 False;本进程调 startup:事先放好的临时文件和孤儿 构建库一个都没删;/upload、stage、trial、commit、activate 返回 503 store_unavailable,GET 接口照常;过期处理不抛; diff --git a/backend/tests/test_recipe_imports.py b/backend/tests/test_recipe_imports.py index 74b54535..2f54d653 100644 --- a/backend/tests/test_recipe_imports.py +++ b/backend/tests/test_recipe_imports.py @@ -42,6 +42,7 @@ from app.data.xlsx_scan import Bounds, SheetScan, UnsupportedTable, WorkbookScan from app.db.base import SessionLocal from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableImport, TableRecipe from app.main import app +from tests.fixtures.names import letters FLOW_RECIPE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text("utf-8")) FLOW_FULL = Recipe.model_validate(FLOW_RECIPE).model_dump(mode="json") @@ -294,7 +295,7 @@ async def client(): def unique(prefix: str = "wp5c") -> str: - return f"{prefix}_{uuid.uuid4().hex[:10]}" + return f"{prefix}_{letters(10)}" def raw_bytes() -> bytes: diff --git a/backend/tests/test_reject_exit_wording.py b/backend/tests/test_reject_exit_wording.py new file mode 100644 index 00000000..5f072208 --- /dev/null +++ b/backend/tests/test_reject_exit_wording.py @@ -0,0 +1,197 @@ +"""驳回出口之后,待办、通知、定时任务的 Webhook 照实说「被驳回,转到 X」(docs/design/orchestration-round/00-总方案.md §3)。 + +实例页已经这么说(test_reject_exit.test_the_instance_says_rejected_and_where_it_went)。这里对的是另外几处: +- 待办:中断内容带着驳回之后去哪(reject_route):接了驳回出口的写转到哪几步,没接的写运行就此结束,审批模式没接、没设 + 驳回即终止的写这一路到此为止。待办卡的提示、驳回后的提示、处理过的结果照它说,不再一律写「转入驳回分支」「本次运行终止」; +- 子工作流里的关卡走了驳回出口:父运行的终态事件照实带上(「子工作流 › 把关」被驳回,转到「子工作流 › 退回」),实例页、 + 通知说这一句,不写内部 id、不写成「已完成」; +- 定时任务的 Webhook:途经驳回、照驳回那条路成功结束的,标题和状态写途经驳回,另起一行写谁被驳回、转到哪一步。 +图和数据都是虚构的。 +""" +from __future__ import annotations + +import asyncio + +import pytest +from sqlalchemy import select + +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent, Task +from app.engine.runner import run_manager +from tests import ledger_samples as S + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +START = node("start", "input", fields=[{"name": "q"}]) +DONE = node("done", "output", "办结", fields=[{"name": "结果", "value": "通过了"}]) +BACK = node("back", "output", "退回申请人", fields=[{"name": "结果", "value": "退回:{{ nodes.gate.note }}"}]) + + +def gated(gate: dict, *, exit_to: dict | None = BACK) -> dict: + normal = "approved" if gate["data"]["config"].get("mode") == "approve" else "out" + nodes = [START, gate, DONE] + ([exit_to] if exit_to else []) + edges = [{"source": "start", "target": "gate"}, {"source": "gate", "target": "done", "sourceHandle": normal}] + if exit_to: + edges.append({"source": "gate", "target": exit_to["id"], "sourceHandle": "rejected"}) + return {"nodes": nodes, "edges": edges} + + +async def _wait(run_id: str, statuses: tuple[str, ...], timeout: float = 20.0) -> Run: + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row is not None and row.status in statuses and not run_manager.is_active(run_id): + return row + await asyncio.sleep(0.05) + raise AssertionError(f"运行 {run_id} 没有进入 {statuses}") + + +async def _parked(graph: dict) -> tuple[Run, dict]: + """发起、停在关卡上,交回运行和那条待办的中断内容。""" + run = await run_manager.start(graph=graph, input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + async with SessionLocal() as session: + [task] = list((await session.execute(select(Task).where(Task.run_id == run.id))).scalars()) + from app.db.models import Approval + + approval = await session.get(Approval, task.approval_id) + return run, dict(approval.payload or {}) + + +APPROVE = node("gate", "human", "把关", mode="approve", title="可以吗") +INPUT = node("gate", "human", "补充", mode="input", title="补一下") +TOOL = node("gate", "tool", "算一下", tool="calculator", args={"expression": "1+1"}, approval="always") + + +@pytest.mark.parametrize("gate", [APPROVE, INPUT, TOOL], ids=["approve", "input", "tool"]) +async def test_the_task_says_where_a_rejection_goes(gate): + _, payload = await _parked(gated(gate)) + assert payload["reject_route"] == {"to": [{"node_id": "back", "label": "退回申请人"}]} + + +@pytest.mark.parametrize("gate", [INPUT, TOOL], ids=["input", "tool"]) +async def test_without_the_exit_the_task_says_the_run_ends(gate): + _, payload = await _parked(gated(gate, exit_to=None)) + assert payload["reject_route"] == {"ends": "run"} + + +async def test_an_approval_without_the_exit_says_this_path_stops_here(): + _, payload = await _parked(gated(APPROVE, exit_to=None)) + assert payload["reject_route"] == {"ends": "path"} + stop = node("gate", "human", "把关", mode="approve", title="可以吗", stop_on_reject=True) + _, payload = await _parked(gated(stop)) + assert payload["reject_route"] == {"ends": "run"}, "驳回即终止优先:驳回出口那条线走不到" + + +async def test_an_agent_tool_approval_has_no_route(): + """agent 自己调工具的审批不是关卡:驳回了 agent 照常往下想,中断内容不带 reject_route。""" + from app.engine.replay import reject_route + from app.engine.schema import GraphSpec + + spec = GraphSpec.model_validate(gated(node("gate", "agent", "助手", prompt="x"))) + + class Ctx: + class run: # noqa: N801 - 只要 spec 和 global_id + pass + + ctx = Ctx() + ctx.node = spec.node_map()["gate"] + ctx.run.spec = spec + assert reject_route(ctx) is None + + +# -------------------------------------------------------------------------- +# 子工作流里的关卡走了驳回出口 +# -------------------------------------------------------------------------- + + +async def _sub_rejected() -> Run: + inner = {"nodes": [node("s", "input"), node("gate", "human", "把关", mode="approve", title="可以吗"), + node("ok", "output", "通过", fields=[{"name": "r", "value": "ok"}]), + node("back", "output", "退回", fields=[{"name": "r", "value": "退回:{{ nodes.gate.note }}"}])], + "edges": [{"source": "s", "target": "gate"}, + {"source": "gate", "target": "ok", "sourceHandle": "approved"}, + {"source": "gate", "target": "back", "sourceHandle": "rejected"}]} + sub_id = await S.publish("复核子流程", graph=inner, level="draft") + graph = S.chain(node("start", "input"), node("sub", "subgraph", "复核子流程", workflow_id=sub_id, workflow_version=1), + node("out", "output", "成果", fields=[{"name": "r", "value": "{{ nodes.sub.output.r }}"}])) + run = await run_manager.start(graph=graph, input_payload={}) + await _wait(run.id, ("interrupted",)) + await run_manager.wait_idle(run.id) + await run_manager.resume(run.id, {"approved": False, "note": "不对"}, actor="张工") + return await _wait(run.id, ("cancelled", "failed", "succeeded")) + + +async def test_a_reject_exit_inside_a_subworkflow_is_named_in_the_parent(): + from app.apps.resolve import _trails, instance_state + + final = await _sub_rejected() + assert final.status == "succeeded", final.error + assert final.output == {"r": "退回:不对"} + async with SessionLocal() as session: + [ended] = list((await session.execute(select(RunEvent).where( + RunEvent.run_id == final.id, RunEvent.type == "run.finished"))).scalars()) + run = await session.get(Run, final.id) + state = instance_state(run, (await _trails(session, [run]))[run.id]) + [via] = ended.data["rejected_via"] + assert (via["node_id"], via["label"]) == ("sub/gate", "复核子流程 › 把关") + assert via["to"] == [{"node_id": "sub/back", "label": "复核子流程 › 退回"}] + # 实例页、通知(notify_run 取的就是这一句):照实说被驳回、转到哪,不写「sub/gate」,不画成已完成 + assert state == {"code": "cancelled", "text": "「复核子流程 › 把关」被驳回,转到「复核子流程 › 退回」"} + + +async def test_the_inner_task_names_the_route_with_the_subworkflow(): + inner = {"nodes": [node("s", "input"), node("gate", "human", "补充", mode="input", title="补一下"), + node("back", "output", "退回", fields=[{"name": "r", "value": "x"}]), + node("ok", "output", "通过", fields=[{"name": "r", "value": "y"}])], + "edges": [{"source": "s", "target": "gate"}, {"source": "gate", "target": "ok"}, + {"source": "gate", "target": "back", "sourceHandle": "rejected"}]} + sub_id = await S.publish("补充子流程", graph=inner, level="draft") + graph = S.chain(node("start", "input"), node("sub", "subgraph", "补充子流程", workflow_id=sub_id, workflow_version=1), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.sub.output.r }}"}])) + _, payload = await _parked(graph) + assert payload["reject_route"] == {"to": [{"node_id": "sub/back", "label": "补充子流程 › 退回"}]} + + +# -------------------------------------------------------------------------- +# 定时任务的 Webhook +# -------------------------------------------------------------------------- + + +def test_the_webhook_says_a_run_went_through_a_rejection(): + from app.scheduler.webhook import Notice, message + + via = [{"node_id": "gate", "label": "把关", "kind": "approve", "decision": "reject", "actor": "张工", "note": "不对", + "to": [{"node_id": "back", "label": "退回申请人"}]}] + title, lines, _ = message(Notice(event="finished", schedule_id="s1", schedule_name="周报", timezone="UTC", + status="succeeded", rejected_via=via)) + assert "途经驳回" in title + said = dict(lines) + assert said["状态"] == "已完成(途经驳回)" + assert said["驳回"] == "「把关」被驳回,转到「退回申请人」" + # 没途经驳回的照旧 + title, lines, _ = message(Notice(event="finished", schedule_id="s1", schedule_name="周报", timezone="UTC", + status="succeeded")) + assert title == "定时任务「周报」运行完成" and dict(lines)["状态"] == "已完成" and "驳回" not in dict(lines) + + +async def test_the_webhook_notice_reads_the_route_from_the_end_event(): + from app.scheduler.webhook import _rejected_via_of + + final = await _sub_rejected() + async with SessionLocal() as session: + run = await session.get(Run, final.id) + via = await _rejected_via_of(session, run) + assert [v["label"] for v in via] == ["复核子流程 › 把关"] diff --git a/backend/tests/test_review_subgraph.py b/backend/tests/test_review_subgraph.py new file mode 100644 index 00000000..6ea79e4a --- /dev/null +++ b/backend/tests/test_review_subgraph.py @@ -0,0 +1,105 @@ +"""复核规则(engine/review.scan)对子工作流里失败的那一步只报一条:子图里那一步一条 node.failed、子工作流节点又一条,原话 +相同,以前合成两条「执行失败」,问数据页的复核提示里同一件事写两遍。保留更具体的那一步,名字写清在哪个子工作流里 +(「子工作流 › 那一步」,不写内部 id)。图和数据都是虚构的。 +""" +from __future__ import annotations + +import asyncio + +import pytest +from sqlalchemy import select + +from app.core.events import EventType +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent +from app.engine import review as rv +from app.engine.runner import run_manager +from tests import ledger_samples as S + +OK = {"结果": "x"} + + +def ev(etype, **data): + return {"type": str(etype), "node_id": data.pop("node_id", ""), "data": data} + + +def test_a_failure_inside_a_subworkflow_is_reported_once_with_where_it_happened(): + signals = rv.scan([ + ev(EventType.NODE_STARTED, node_id="sub", label="复核子流程"), + ev(EventType.NODE_STARTED, node_id="sub/calc", label="算一下"), + ev(EventType.NODE_FAILED, node_id="sub/calc", error="表达式写错了"), + ev(EventType.NODE_FAILED, node_id="sub", error="表达式写错了"), + ], OK) + [signal] = [s for s in signals if s.kind == "node_failed"] + assert signal.detail == "节点「复核子流程 › 算一下」执行失败:表达式写错了" + assert "sub/" not in signal.detail and signal.severity == rv.BROKEN + + +def test_nested_two_levels_and_different_errors_stay_apart(): + signals = rv.scan([ + ev(EventType.NODE_STARTED, node_id="a", label="外层"), + ev(EventType.NODE_STARTED, node_id="a/b", label="内层"), + ev(EventType.NODE_STARTED, node_id="a/b/c", label="取数"), + ev(EventType.NODE_FAILED, node_id="a/b/c", error="连不上"), + ev(EventType.NODE_FAILED, node_id="a/b", error="连不上"), + ev(EventType.NODE_FAILED, node_id="a", error="连不上"), + # 父图里另一个节点原话不同的失败照报 + ev(EventType.NODE_STARTED, node_id="other", label="另一步"), + ev(EventType.NODE_FAILED, node_id="other", error="超时"), + ], OK) + assert [s.detail for s in signals if s.kind == "node_failed"] == [ + "节点「外层 › 内层 › 取数」执行失败:连不上", "节点「另一步」执行失败:超时"] + + +def test_a_rejection_inside_a_subworkflow_is_reported_once_too(): + signals = rv.scan([ + ev(EventType.NODE_STARTED, node_id="sub", label="复核子流程"), + ev(EventType.NODE_STARTED, node_id="sub/gate", label="把关"), + ev(EventType.NODE_FAILED, node_id="sub/gate", error="人工驳回:不对", reason="rejected"), + ev(EventType.NODE_FAILED, node_id="sub", error="人工驳回:不对", reason="rejected"), + ], OK) + assert [s.detail for s in signals if s.kind == "node_rejected"] == ["节点「复核子流程 › 把关」被驳回:人工驳回:不对"] + + +def test_a_subworkflow_failing_on_its_own_still_reports(): + """子工作流节点自己失败(没选工作流、版本不存在):子图里没有哪一步失败,照报这一条。""" + signals = rv.scan([ + ev(EventType.NODE_STARTED, node_id="sub", label="复核子流程"), + ev(EventType.NODE_FAILED, node_id="sub", error="「子工作流」节点尚未选择要嵌套的工作流"), + ], OK) + assert [s.detail for s in signals if s.kind == "node_failed"] == [ + "节点「复核子流程」执行失败:「子工作流」节点尚未选择要嵌套的工作流"] + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +async def test_a_real_run_failing_inside_a_subworkflow_gives_one_signal(engine_up): + inner = S.chain(node("s", "input"), node("calc", "transform", "算一下", mode="expression", expression="1 / 0"), + node("o", "output", fields=[{"name": "r", "value": "{{ nodes.calc.text }}"}])) + sub_id = await S.publish("复核子流程", graph=inner, level="draft") + graph = S.chain(node("start", "input"), node("sub", "subgraph", "复核子流程", workflow_id=sub_id, workflow_version=1), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.sub.output.r }}"}])) + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("failed", "succeeded") and not run_manager.is_active(run.id): + break + assert row.status == "failed" + async with SessionLocal() as session: + rows = list((await session.execute(select(RunEvent).where(RunEvent.run_id == run.id) + .order_by(RunEvent.seq))).scalars()) + events = [{"type": r.type, "node_id": r.node_id, "data": r.data} for r in rows] + failed = [s for s in rv.scan(events, {}) if s.kind == "node_failed"] + assert len(failed) == 1, [s.detail for s in failed] + assert failed[0].detail.startswith("节点「复核子流程 › 算一下」执行失败:") diff --git a/backend/tests/test_run_append.py b/backend/tests/test_run_append.py index e73d5f8d..7c5103e2 100644 --- a/backend/tests/test_run_append.py +++ b/backend/tests/test_run_append.py @@ -16,7 +16,6 @@ from __future__ import annotations import asyncio import json -import sqlite3 from collections import Counter import pytest @@ -25,7 +24,6 @@ from pydantic import BaseModel from sqlalchemy import select from app.core.bus import bus -from app.core.config import settings from app.core.events import EventType from app.db.base import SessionLocal from app.db.models import Approval, Run, RunEvent @@ -34,6 +32,7 @@ from app.engine.context import NodeError from app.engine.runner import RunManager, run_manager, verify_manifest from app.engine.schema import NodeType from app.tools.registry import register +from tests import appdb #: Agent 场景里工具真正执行的次数。注册表是进程级的,工具只注册一次,计数按测试清零 CALLS: list[str] = [] @@ -162,12 +161,7 @@ async def _pending(run_id: str) -> list[Approval]: def _sql(statement: str, *params) -> None: """另开一条连接改库:模拟别处(恢复、继续运行、另一个写入者)抢在中间提交。""" - db = sqlite3.connect(settings.db_path, timeout=5) - try: - db.execute(statement, params) - db.commit() - finally: - db.close() + appdb.execute((statement, params)) # -------------------------------------------------------------------------- @@ -226,12 +220,7 @@ async def test_a_resume_that_wins_the_row_leaves_nothing_behind(monkeypatch): def resumed_in_between(run_id, value): if run_id == paused.id: - db = sqlite3.connect(settings.db_path, timeout=5) - try: - db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (paused.id,)) - db.commit() - finally: - db.close() + appdb.execute(("UPDATE runs SET status = 'running' WHERE id = ?", (paused.id,))) set_seq(run_id, value) monkeypatch.setattr(bus, "set_seq", resumed_in_between) @@ -252,14 +241,9 @@ async def test_events_written_elsewhere_in_between_win(monkeypatch): def written_in_between(run_id, value): if run_id == paused.id: - db = sqlite3.connect(settings.db_path, timeout=5) - try: - db.execute("INSERT INTO run_events (id, run_id, seq, type, node_id, ts, data) " - "VALUES ('elsewhere-1', ?, ?, 'log', NULL, 0, '{}')", (paused.id, taken)) - db.execute("UPDATE runs SET last_seq = ? WHERE id = ?", (taken, paused.id)) - db.commit() - finally: - db.close() + appdb.execute(("INSERT INTO run_events (id, run_id, seq, type, node_id, ts, data) " + "VALUES ('elsewhere-1', ?, ?, 'log', NULL, 0, '{}')", (paused.id, taken)), + ("UPDATE runs SET last_seq = ? WHERE id = ?", (taken, paused.id))) set_seq(run_id, value) monkeypatch.setattr(bus, "set_seq", written_in_between) @@ -352,12 +336,8 @@ async def test_published_only_once_stored_and_never_when_not_written(monkeypatch async def stored_when_published(event): if event.type == EventType.APPROVAL_INQUIRY: - db = sqlite3.connect(settings.db_path, timeout=5) - try: - (count,) = db.execute("SELECT count(*) FROM run_events WHERE run_id = ? AND seq = ?", - (event.run_id, event.seq)).fetchone() - finally: - db.close() + [(count,)] = appdb.query("SELECT count(*) FROM run_events WHERE run_id = ? AND seq = ?", + event.run_id, event.seq) published.append((event.seq, count)) await publish(event) @@ -430,12 +410,7 @@ async def test_a_sealed_run_that_changes_in_between_gets_nothing(monkeypatch, ch def changed_in_between(run_id, value): if run_id == final.id: - db = sqlite3.connect(settings.db_path, timeout=5) - try: - db.execute(change, (final.id,)) - db.commit() - finally: - db.close() + appdb.execute((change, (final.id,))) set_seq(run_id, value) monkeypatch.setattr(bus, "set_seq", changed_in_between) @@ -647,10 +622,10 @@ def _judged_args(run: Run) -> tuple: async def test_a_judge_interleaved_with_a_post_seal_append_survives_a_restart(flaky, monkeypatch): - """按需裁判记判定(_append_judged)的条件更新不带 last_seq 条件:它取了号、还没写,封存后追加抢先提交了, - 它再提交时把 last_seq 写回自己那个更小的号。同一进程里序号计数只增不减,不会撞号;进程重启后计数从 0 起, - 各处又按 last_seq 接号——裁判自己也是。所以启动时按库里实际的最后一条改正 last_seq:之后再裁判、继续运行, - 都接在实际的最后一条后面。""" + """按需裁判记判定(_append_judged)和封存后追加交错:裁判取了号、还没写,封存后追加抢先提交了。以前裁判的条件更新不带 + last_seq 条件,照样提交、把 last_seq 写回自己那个更小的号(同一进程里计数只增不减,不撞号;多进程时两个进程各自的计数 + 会撞号)。现在同 append_events:last_seq 也要还是读到的那个,后到的裁判这一次不记(判定照样交回给点的人),last_seq + 停在追加的那一条上。启动时照旧按库里实际的最后一条改正 last_seq:之后再裁判、继续运行,都接在实际的最后一条后面。""" from app.api import evidence from app.engine.run_append import append_events @@ -687,9 +662,9 @@ async def test_a_judge_interleaved_with_a_post_seal_append_survives_a_restart(fl await asyncio.sleep(0.01) written = await append_events(first.id, [inquiry("这次为什么失败")], require="sealed") held.set() - assert await judging == {"seq": first.last_seq + 1} + assert await judging is None, "追加先提交了:后到的裁判不记,免得两条撞号" assert written is not None and written[0].seq == first.last_seq + 2 - assert (await _row(first.id)).last_seq == first.last_seq + 1, "裁判后提交,last_seq 被写回了小的号" + assert (await _row(first.id)).last_seq == written[0].seq monkeypatch.setattr(evidence, "SessionLocal", opened) # 进程重启:内存里的序号计数没了,启动时 setup 照常执行一遍。只改落后的那条,别的运行原样不动 diff --git a/backend/tests/test_run_launch.py b/backend/tests/test_run_launch.py index baf0935c..5331592b 100644 --- a/backend/tests/test_run_launch.py +++ b/backend/tests/test_run_launch.py @@ -322,15 +322,16 @@ async def test_launch_events_are_written_with_the_run_or_not_at_all(): from app.db.base import engine from app.engine.launch import launch_run + from tests.appdb import refuse_inserts lib = await upstream_with_two_versions() wf, _ = await workflow(weekly(lib, upgrade_policy="dual"), published=1, versions=(1,)) before = await _run_count() # 口径升版处置那条写不进去。走异步引擎建删触发器:同步连接等写锁时会卡住事件循环,而持锁的正是这个循环里的写入 + create, drop = refuse_inserts("run_events", "NEW.type = 'caliber.upgrade'", name="refuse_upgrade_note") async with engine.begin() as conn: - await conn.execute(text( - "CREATE TRIGGER refuse_upgrade_note BEFORE INSERT ON run_events WHEN NEW.type = 'caliber.upgrade' " - "BEGIN SELECT RAISE(ABORT, 'refused by test'); END")) + for sql in create: + await conn.execute(text(sql)) try: async with SessionLocal() as session: with pytest.raises(IntegrityError): @@ -338,7 +339,8 @@ async def test_launch_events_are_written_with_the_run_or_not_at_all(): assert await _run_count() == before, "没写进治理事件,却留下了一条运行" finally: async with engine.begin() as conn: - await conn.execute(text("DROP TRIGGER IF EXISTS refuse_upgrade_note")) + for sql in drop: + await conn.execute(text(sql)) async def test_an_exploratory_launch_takes_the_saved_graph_and_the_run_defaults(): diff --git a/backend/tests/test_run_options.py b/backend/tests/test_run_options.py index ef456a27..f4e7fa53 100644 --- a/backend/tests/test_run_options.py +++ b/backend/tests/test_run_options.py @@ -112,9 +112,11 @@ def test_a_missing_table_is_left_alone(): async def test_new_databases_index_schedule_id(): + from sqlalchemy import inspect + async with app_engine.connect() as conn: - names = {row[1] for row in (await conn.execute(text("PRAGMA index_list(runs)"))).all()} - assert "ix_runs_schedule_id" in names + indexes = await conn.run_sync(lambda sync: inspect(sync).get_indexes("runs")) + assert {ix["name"]: ix["column_names"] for ix in indexes}.get("ix_runs_schedule_id") == ["schedule_id"] async def test_a_migrated_row_reads_back_empty_options(): diff --git a/backend/tests/test_run_races.py b/backend/tests/test_run_races.py index 6e89d179..a7d10951 100644 --- a/backend/tests/test_run_races.py +++ b/backend/tests/test_run_races.py @@ -254,11 +254,10 @@ async def test_abandoning_while_the_pause_is_being_recorded(slow, monkeypatch): async def test_an_approval_closed_just_before_it_is_marked_answered(slow, monkeypatch): """放弃恰好提交在「读到这条审批还是 pending」和「把它记成已回复」之间。""" - import sqlite3 - from sqlalchemy import event from app.db.base import engine + from tests import appdb run = await run_manager.start(graph=_graph(gate=True), input_payload={"q": "x"}) await _until(lambda: _is(run.id, "interrupted")) @@ -268,11 +267,8 @@ async def test_an_approval_closed_just_before_it_is_marked_answered(slow, monkey def abandon_first(conn, cursor, statement, params, context, executemany): if armed["on"] and statement.lstrip().upper().startswith("UPDATE APPROVALS"): armed["on"] = False - other = sqlite3.connect(engine.url.database, timeout=5) - other.execute("UPDATE approvals SET status='cancelled' WHERE run_id=?", (run.id,)) - other.execute("UPDATE runs SET status='cancelled' WHERE id=?", (run.id,)) - other.commit() - other.close() + appdb.execute(("UPDATE approvals SET status='cancelled' WHERE run_id=?", (run.id,)), + ("UPDATE runs SET status='cancelled' WHERE id=?", (run.id,))) event.listen(engine.sync_engine, "before_cursor_execute", abandon_first) try: diff --git a/backend/tests/test_schedule_events_api.py b/backend/tests/test_schedule_events_api.py index e07d7561..6158c206 100644 --- a/backend/tests/test_schedule_events_api.py +++ b/backend/tests/test_schedule_events_api.py @@ -356,15 +356,25 @@ async def test_trigger_options(client): table = await upload_table("门店销售") flat = await upload_table("门店名单", mode="replace") wf, wf_name = await workflow("日报") - draft, _ = await workflow("草稿", published=False) + draft, draft_name = await workflow("草稿", published=False) r = await client.get("/api/schedules/trigger-options") assert r.status_code == 200 out = r.json() events = {e["event"]: e for e in out["events"]} assert list(events) == ["table.version", "workflow.finished", "source.changed"] - tables = {o["id"]: o for o in events["table.version"]["objects"]} + + # 对象按名字排、一页 100 个:同一个库里别的测试建的工作流、数据源多了,这里建的可能排到第一页外面。按名字(q)查要的那几个 + async def objects(event: str, name: str) -> dict[str, dict]: + got = await client.get("/api/schedules/trigger-options", params={"event": event, "q": name}) + assert got.status_code == 200 + [entry] = got.json()["events"] + return {o["id"]: o for o in entry["objects"]} + + async with SessionLocal() as session: + table_name, flat_name = [(await session.get(DataSource, i)).name for i in (table, flat)] + tables = {**await objects("table.version", table_name), **await objects("table.version", flat_name)} assert tables[table]["ok"] is True and tables[flat]["ok"] is False and "按期累积" in tables[flat]["reason"] - flows = {o["id"]: o for o in events["workflow.finished"]["objects"]} + flows = {**await objects("workflow.finished", wf_name), **await objects("workflow.finished", draft_name)} assert flows[wf] == {"id": wf, "name": wf_name, "ok": True, "reason": None} assert flows[draft]["ok"] is False and "发布" in flows[draft]["reason"] assert "hook" in [o["kind"] for o in out["origins"]] and out["max_depth"] == 5 diff --git a/backend/tests/test_schedule_events_scale.py b/backend/tests/test_schedule_events_scale.py new file mode 100644 index 00000000..40bd429d --- /dev/null +++ b/backend/tests/test_schedule_events_scale.py @@ -0,0 +1,214 @@ +"""事件触发的规模(scheduler/outbox.py、scheduler/triggers.py、api/schedules.py): + +- 定时任务列表里每个事件触发任务「有几条在等」一次查齐(outbox.pending_for_many:一条 SQL 按任务分组算待派发数),不再 + 每行各查一次; +- 每一拍派发只读监听着这几种事件、这几个对象的任务(按触发条件里的事件种类、对象筛,用 SQLAlchemy 的通用 JSON 访问), + 不再把全部事件触发任务读出来;保存时查环只顺着「上游工作流运行成功结束」那几条边往下找; +- 选对象(trigger-options)按事件、关键字查、分页:返回 more 说还有没有,当前选中的那一个总在里面; +- 去抖窗口最早那一条因为任务改了条件被标 dropped 时,「什么时候合成触发」按剩下的最早一条算;一个窗口的结束时刻和这个任务 + 已经有的另一次触发撞在同一刻(别的期次、早一拍发的)时往后挪,那几条事件照常发起,不并进已有的触发。 +工作流、表格、数据源都是虚构的。 +""" +from __future__ import annotations + +from contextlib import contextmanager +from datetime import timedelta + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import event as sa_event +from sqlalchemy import select + +from app.db.base import SessionLocal, engine, utcnow +from app.db.models import Schedule, ScheduleFire, Workflow +from app.db.models_triggers import TriggerDelivery +from app.scheduler import outbox +from tests.test_schedule_events import ( # noqa: F401 - engine_up 是本模块也要的 autouse 夹具 + deliveries, engine_up, event_schedule, fires, put_event, settle, workflow, +) + + +@contextmanager +def counted(): + """数这一段里发了几条 SQL。""" + seen: list[str] = [] + + def hook(conn, cursor, statement, *_a): # noqa: ANN001 + seen.append(statement) + + target = engine.sync_engine + sa_event.listen(target, "before_cursor_execute", hook) + try: + yield seen + finally: + sa_event.remove(target, "before_cursor_execute", hook) + + +@pytest.fixture +async def client(): + from app.main import app + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def _many(n: int) -> tuple[str, list[str]]: + upstream = await workflow("门店日报") + downstream = await workflow("区域汇总") + ids = [await event_schedule(downstream, {"event": "workflow.finished", "workflow_id": upstream, "debounce_s": 600}, + name=f"汇总{i}") for i in range(n)] + return upstream, ids + + +async def test_waiting_counts_for_many_schedules_in_one_go(): + upstream, ids = await _many(3) + t0 = utcnow().replace(microsecond=0) + for i in range(2): + await put_event("workflow.finished", upstream, {"workflow_id": upstream, "run_id": f"r{i}"}, t0 + timedelta(seconds=i)) + await outbox.route(t0 + timedelta(seconds=5)) + await put_event("workflow.finished", upstream, {"workflow_id": upstream, "run_id": "r9"}, t0 + timedelta(seconds=9)) + async with SessionLocal() as session: + rows = list((await session.execute(select(Schedule).where(Schedule.id.in_(ids)))).scalars()) + one = {r.id: await outbox.pending_for(session, r) for r in rows} + with counted() as sql: + many = await outbox.pending_for_many(session, rows) + assert many == one + assert all(v == {"events": 1, "waiting": 2, "due_at": (t0 + timedelta(seconds=600)).isoformat()} for v in many.values()) + assert len(sql) <= 3, sql + + +async def test_the_schedule_list_does_not_query_once_per_row(client): + await _many(2) + with counted() as few: + assert (await client.get("/api/schedules")).status_code == 200 + await _many(5) + with counted() as more: + assert (await client.get("/api/schedules")).status_code == 200 + assert len(more) <= len(few), (len(few), len(more)) + + +async def test_a_tick_reads_only_the_schedules_listening_to_the_pending_events(): + upstream, ids = await _many(2) + other = await workflow("别的日报") + elsewhere = await event_schedule(await workflow("别的汇总"), {"event": "workflow.finished", "workflow_id": other}) + await put_event("workflow.finished", upstream, {"workflow_id": upstream, "run_id": "r1"}, utcnow()) + async with SessionLocal() as session: + from app.db.models_triggers import TriggerEvent + + pending = list((await session.execute(select(TriggerEvent).where(TriggerEvent.status == "pending"))).scalars()) + listeners = await outbox.listeners(session, pending) + assert sorted(sid for sid, _ in listeners) == sorted(ids) and elsewhere not in {sid for sid, _ in listeners} + + +async def test_trigger_options_search_and_page(client): + names = [f"检索样例{c}-{i}" for c in "甲乙" for i in range(3)] + async with SessionLocal() as session: + rows = [Workflow(name=n, graph={"nodes": [], "edges": []}) for n in names] + session.add_all(rows) + await session.commit() + chosen = next(r.id for r in rows if r.name == "检索样例乙-2") + r = await client.get("/api/schedules/trigger-options", params={"event": "workflow.finished", "q": "检索样例甲", + "limit": 2}) + assert r.status_code == 200, r.text + [entry] = r.json()["events"] + assert entry["event"] == "workflow.finished" and entry["more"] is True + assert [o["name"] for o in entry["objects"]] == ["检索样例甲-0", "检索样例甲-1"] + r = await client.get("/api/schedules/trigger-options", params={"event": "workflow.finished", "q": "检索样例甲", + "limit": 2, "selected": chosen}) + [entry] = r.json()["events"] + assert chosen in [o["id"] for o in entry["objects"]], "当前选中的那一个总在里面(不被分页挤掉、不显示成已不存在)" + # 不带参数:照旧给全部种类,每种最多一页 + body = (await client.get("/api/schedules/trigger-options")).json() + assert {e["event"] for e in body["events"]} == {"table.version", "workflow.finished", "source.changed"} + assert all("more" in e for e in body["events"]) + + +async def test_a_dropped_earliest_event_no_longer_sets_the_window_end(): + """窗口最早那一条因为任务改了条件被标 dropped:什么时候合成触发按剩下的最早一条算(列表上的 due_at、真发起的 fire_at 一致)。""" + upstream = await workflow("门店日报") + downstream = await workflow("区域汇总") + sid = await event_schedule(downstream, {"event": "workflow.finished", "workflow_id": upstream, "debounce_s": 60}) + t0 = utcnow().replace(microsecond=0) + await put_event("workflow.finished", upstream, {"workflow_id": upstream, "run_id": "r1", "origin_kind": "launcher"}, t0) + e2 = await put_event("workflow.finished", upstream, {"workflow_id": upstream, "run_id": "r2", + "origin_kind": "app_entry"}, t0 + timedelta(seconds=30)) + await outbox.route(t0 + timedelta(seconds=31)) + async with SessionLocal() as session: + await session.execute(Schedule.__table__.update().where(Schedule.id == sid).values( + trigger={"event": "workflow.finished", "workflow_id": upstream, "debounce_s": 60, "origin_kind": "app_entry"})) + await session.commit() + state = await outbox.pending_for(session, await session.get(Schedule, sid)) + assert state["due_at"] == (t0 + timedelta(seconds=90)).isoformat(), state + assert await outbox.dispatch_due(t0 + timedelta(seconds=61)) == [] + [fire] = await outbox.dispatch_due(t0 + timedelta(seconds=91)) + assert fire.fire_at == t0 + timedelta(seconds=90) and fire.inputs["event"]["event_ids"] == [e2] + await settle(fire.run_id) + + +async def test_a_window_ending_where_another_fire_already_is_still_fires(): + """一个窗口的结束时刻和这个任务已有的另一次触发撞在同一刻(别的期次早一拍发了):往后挪,这几条照常发起, + 不并进已有的那次触发、不被它吞掉。""" + source = "src-虚构" + downstream = await workflow("月报") + sid = await event_schedule(downstream, {"event": "table.version", "source_id": source, "debounce_s": 60}) + t0 = utcnow().replace(microsecond=0) + aug = await put_event("table.version", source, {"period": "2026-08", "table": "日报"}, t0) + [first] = await outbox.dispatch_due(t0 + timedelta(seconds=61)) + assert first.fire_at == t0 + timedelta(seconds=60) and first.inputs["event"]["event_ids"] == [aug] + await settle(first.run_id) + # 另一期的事件发生在同一刻,晚到了一拍(比如补写的):窗口结束同样是 t0+60 + sep = await put_event("table.version", source, {"period": "2026-09", "table": "日报"}, t0) + [second] = await outbox.dispatch_due(t0 + timedelta(seconds=62)) + assert second.id != first.id and second.inputs["event"]["event_ids"] == [sep] + assert second.fire_at > first.fire_at + states = {d.event_id: (d.status, d.fire_id) for d in await deliveries(sid)} + assert states[sep] == ("fired", second.id) + await settle(second.run_id) + async with SessionLocal() as session: + assert len(list((await session.execute(select(ScheduleFire).where(ScheduleFire.schedule_id == sid))).scalars())) == 2 + assert len(await fires(sid)) == 2 + assert [d.status for d in await deliveries(sid)] == ["fired", "fired"] + assert TriggerDelivery is not None + + +async def test_trigger_option_search_ignores_case_and_takes_wildcards_literally(client): + """按名字查不分大小写(PostgreSQL 上 LIKE 分 ASCII 大小写、SQLite 不分:一律 ilike),% 和 _ 按字面匹配、不当通配符。""" + async with SessionLocal() as session: + session.add_all([Workflow(name=n, graph={"nodes": [], "edges": []}) + for n in ("Weekly_Report 样例", "WeeklyXReport 样例", "满 100% 样例", "满 1000 样例")]) + await session.commit() + + async def names(q: str) -> list[str]: + r = await client.get("/api/schedules/trigger-options", params={"event": "workflow.finished", "q": q}) + assert r.status_code == 200, r.text + return [o["name"] for o in r.json()["events"][0]["objects"]] + + assert await names("weekly_report") == ["Weekly_Report 样例"], "大小写不分;_ 不匹配任意一个字" + assert await names("100%") == ["满 100% 样例"], "% 不匹配任意几个字" + + +async def test_a_backlog_on_one_schedule_does_not_crowd_out_the_others(monkeypatch): + """积压多的任务不把别的任务挤掉:等着的事件、还没派发的事件按任务(按对象)各读各的上限,不是合起来一个上限。""" + monkeypatch.setattr(outbox, "DRAIN_BATCH", 3) + monkeypatch.setattr(outbox, "ROUTE_BATCH", 3) + busy_up, quiet_up = await workflow("积压的上游"), await workflow("安静的上游") + downstream = await workflow("区域汇总") + busy = await event_schedule(downstream, {"event": "workflow.finished", "workflow_id": busy_up, "debounce_s": 600}, + name="积压的") + quiet = await event_schedule(downstream, {"event": "workflow.finished", "workflow_id": quiet_up, "debounce_s": 600}, + name="安静的") + t0 = utcnow().replace(microsecond=0) + for i in range(5): + await put_event("workflow.finished", busy_up, {"workflow_id": busy_up, "run_id": f"b{i}"}, t0 + timedelta(seconds=i)) + await put_event("workflow.finished", quiet_up, {"workflow_id": quiet_up, "run_id": "q0"}, t0 + timedelta(seconds=10)) + for _ in range(4): # 一拍只派 ROUTE_BATCH 条:多派几拍,六条都派给任务 + await outbox.route(t0 + timedelta(seconds=20)) + for i in range(5): + await put_event("workflow.finished", busy_up, {"workflow_id": busy_up, "run_id": f"c{i}"}, t0 + timedelta(seconds=30 + i)) + await put_event("workflow.finished", quiet_up, {"workflow_id": quiet_up, "run_id": "q1"}, t0 + timedelta(seconds=40)) + async with SessionLocal() as session: + rows = list((await session.execute(select(Schedule).where(Schedule.id.in_([busy, quiet])))).scalars()) + state = await outbox.pending_for_many(session, rows) + assert state[quiet]["due_at"] == (t0 + timedelta(seconds=610)).isoformat(), state + assert state[quiet]["events"] == 1 and state[busy]["events"] == 3, state + assert state[busy]["due_at"] == (t0 + timedelta(seconds=600)).isoformat() and state[busy]["waiting"] == 5 diff --git a/backend/tests/test_source_upkeep.py b/backend/tests/test_source_upkeep.py index 0aaab038..ef51710f 100644 --- a/backend/tests/test_source_upkeep.py +++ b/backend/tests/test_source_upkeep.py @@ -243,7 +243,8 @@ async def test_background_jobs_follow_the_lane_order(client, shop): await session.commit() ids = {"introspect": intro.id, "draft": draft.id, "profile": profile.id} # 队里是 起草(在做)→ 剖析(排着);探查刚建、还没进队:排在队里的后面 - tickets = [runtime.lanes.enter(sid, "draft", ids["draft"]), runtime.lanes.enter(sid, "profile", ids["profile"])] + tickets = [await runtime.lanes.enter(sid, "draft", ids["draft"]), + await runtime.lanes.enter(sid, "profile", ids["profile"])] try: items = (await client.get(f"/api/datasources/{sid}/background-jobs")).json()["items"] assert [i["kind"] for i in items] == ["draft", "profile", "introspect"], items @@ -253,7 +254,7 @@ async def test_background_jobs_follow_the_lane_order(client, shop): assert {"total", "done", "status"} <= set(items[0]) finally: for t in tickets: - t.release() + await t.release() async with SessionLocal() as session: for model, key in ((IntrospectJob, "introspect"), (CatalogDraftJob, "draft"), (CatalogProfileJob, "profile")): await session.execute(delete(model).where(model.id == ids[key])) diff --git a/backend/tests/test_subgraph_repeat.py b/backend/tests/test_subgraph_repeat.py new file mode 100644 index 00000000..31e30365 --- /dev/null +++ b/backend/tests/test_subgraph_repeat.py @@ -0,0 +1,280 @@ +"""同一个子工作流节点在一次运行里执行多次(循环体里、退回修改绕回来):子图里的节点每次执行都要分得开。 + +以前子图每次都从空状态起步,里面的节点每次都记成「第 1 次执行」: +- 台账里同一个节点(「sub/fetch」)几条查询都记 exec 1,分不出是第几次; +- 写外部系统的幂等键按 (运行, 节点 id, 第几次执行) 算,子图里的节点 id 还是子图里的(pay,不带前缀):循环第二轮的 + 写入算出同一个键,找到第一轮生效的记录,交回第一轮的回执、这一轮不发送——外部系统少收一笔,界面上只有一句「已经生效」; + 父图里同名的节点、两个子工作流节点套同一个工作流时也撞同一个键。 +现在:子图里的节点接着父运行里同一个子工作流节点之前几次数(exec 1、2…),幂等键按这次运行里的节点 id(「sub/pay」)算; +升级前停下的运行(子图里的写入按子图里的 id 记的),恢复时照旧认得出已经生效的那一笔,不重发。 +图、能力、外部系统都是虚构的。 +""" +from __future__ import annotations + +import asyncio + +import pytest +from sqlalchemy import select + +from app.db.base import SessionLocal +from app.db.models import ActionRecord, Run, RunEvent +from app.engine.runner import run_manager +from app.process import actions +from tests import ledger_samples as S +from tests.fakes.fake_erp import PAY, FakeErp, endpoint_row, install + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def edge(source, target, handle=None): + return {"source": source, "target": target, **({"sourceHandle": handle} if handle else {})} + + +async def _finish(graph: dict, **kwargs) -> Run: + run = await run_manager.start(graph=graph, input_payload={}, **kwargs) + for _ in range(600): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed", "interrupted") and not run_manager.is_active(run.id): + return row + raise AssertionError(f"运行 {run.id} 没跑完:{row.status}") + + +async def _events(run_id: str, kind: str, node_id: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == kind) + if node_id: + q = q.where(RunEvent.node_id == node_id) + return list((await session.execute(q.order_by(RunEvent.seq))).scalars()) + + +def looped(sub_id: str, *, before: list[dict] | None = None) -> dict: + """开始 → (before…) → 循环两轮,每轮跑一次子工作流 → 成果。""" + nodes = [node("start", "input"), *(before or []), + node("each", "loop", "逐项", mode="foreach", items="[1, 2]", item_var="it", max_iterations=5), + node("sub", "subgraph", "子流程", workflow_id=sub_id, workflow_version=1, input={"n": "{{ vars.it }}"}), + node("out", "output", fields=[{"name": "r", "value": "完成"}])] + chain = ["start", *[b["id"] for b in before or []], "each"] + edges = [edge(a, b) for a, b in zip(chain, chain[1:])] + edges += [edge("each", "sub", "body"), edge("sub", "each"), edge("each", "out", "done")] + return {"nodes": nodes, "edges": edges} + + +async def test_inner_steps_count_on_across_invocations(tmp_path): + await S.fresh_source(tmp_path / "shop.db") + inner = S.chain(node("s", "input"), node("fetch", "tool", tool=S.tool(), args={"sql": "SELECT COUNT(*) AS n FROM orders"}), + node("o", "output", fields=[{"name": "n", "value": "{{ nodes.fetch.text }}"}])) + sub_id = await S.publish("取数子流程", graph=inner, level="draft") + row = await _finish(looped(sub_id)) + assert row.status == "succeeded", row.error + finished = await _events(row.id, "node.finished", "sub/fetch") + execs = [e["exec"] for f in finished for e in f.data.get("evidence") or [] if e.get("kind") == "query"] + assert execs == [1, 2], "第二轮是子图里那一步的第 2 次执行,不再又记成第 1 次" + + +@pytest.fixture +def erp() -> FakeErp: + return FakeErp() + + +async def _unattended(monkeypatch, erp) -> None: + """付款能力设成可以无人值守写入、节点不要确认:两轮都直接发,不停下等人(和确认无关,只看幂等键分不分得开)。""" + caps = install(monkeypatch, erp, await endpoint_row(test=True), lookup=False) + caps[PAY].unattended_write = True + + +def pay(nid: str = "pay") -> dict: + return node(nid, "tool", "付款", tool=PAY, approval="never", args={"amount": "{{ input.n }}", "card": "6222020200112233"}) + + +async def test_a_write_inside_a_loop_is_sent_on_every_round(monkeypatch, erp): + await _unattended(monkeypatch, erp) + inner = S.chain(node("s", "input", fields=[{"name": "n"}]), pay(), + node("o", "output", fields=[{"name": "单号", "value": "{{ nodes.pay.receipt.id }}"}])) + sub_id = await S.publish("付款子流程", graph=inner, level="draft") + row = await _finish(looped(sub_id)) + assert row.status == "succeeded", row.error + assert len(erp.writes("test")) == 2, "第二轮的写入照发,不交回第一轮的回执" + async with SessionLocal() as session: + records = list((await session.execute(select(ActionRecord).where(ActionRecord.run_id == row.id) + .order_by(ActionRecord.created_at))).scalars()) + assert [(r.node_id, r.exec_no) for r in records] == [("sub/pay", 1), ("sub/pay", 2)] + assert len({r.action_key for r in records}) == 2 + assert not [e for e in await _events(row.id, "log") if e.data.get("code") == "action_already_applied"] + + +async def test_a_namesake_in_the_parent_does_not_share_the_key(monkeypatch, erp): + """父图里也有一个 pay:子图里的 pay 不再和它撞同一个幂等键。""" + await _unattended(monkeypatch, erp) + inner = S.chain(node("s", "input", fields=[{"name": "n"}]), pay(), node("o", "output", fields=[{"name": "r", "value": "x"}])) + sub_id = await S.publish("付款子流程", graph=inner, level="draft") + graph = S.chain(node("start", "input", fields=[{"name": "n"}]), pay(), + node("sub", "subgraph", "子流程", workflow_id=sub_id, workflow_version=1, input={"n": "7"}), + node("out", "output", fields=[{"name": "r", "value": "完成"}])) + run = await run_manager.start(graph=graph, input_payload={"n": 5}) + for _ in range(600): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and not run_manager.is_active(run.id): + break + assert row.status == "succeeded", row.error + assert len(erp.writes("test")) == 2 + async with SessionLocal() as session: + ids = sorted(r.node_id for r in (await session.execute( + select(ActionRecord).where(ActionRecord.run_id == row.id))).scalars()) + assert ids == ["pay", "sub/pay"] + + +async def test_a_write_recorded_before_the_upgrade_is_still_found(monkeypatch, erp): + """升级前子图里的写入按子图里的 id(pay)记幂等键、记录上的 node_id 也是它;事件一向按这次运行里的 id(sub/pay)记。 + 写入节点本身重新执行时(结果未知核实之后接着跑):事件说是这个节点的、内容指纹也是这一次的,照旧按那个键交回回执、不重发; + 事件说是父图同名节点的、或者内容不一样(跨升级的循环,第二轮按老算法也是第 1 次执行)的不认。""" + from app.core.events import EventType + from app.db.models import RunEvent + + caps = install(monkeypatch, erp, await endpoint_row(test=True), lookup=False) + args = {"amount": "5", "card": "6222020200112233"} + digest = actions.payload_hash(PAY, actions.typed_args(caps[PAY], args)) + + async def old_record(writer: str, payload: str) -> tuple[str, str]: + async with SessionLocal() as session: + run = Run(graph={"nodes": [node("sub", "subgraph")], "edges": []}, status="running", run_class="exploratory") + session.add(run) + await session.flush() + legacy = actions.action_key(run.id, "pay", 1) + rec = ActionRecord(action_key=legacy, attempt=1, idem_key=f"{legacy}:1", run_id=run.id, node_id="pay", + exec_no=1, capability=PAY, environment="test", payload_hash=payload, state="confirmed", + receipt={"summary": {"id": "PAY-OLD"}}) + session.add(rec) + await session.flush() + session.add(RunEvent(run_id=run.id, seq=1, type=str(EventType.ACTION_PREPARED), node_id=writer, ts=0.0, + data={"action_id": rec.id, "capability": PAY, "payload_hash": payload})) + await session.commit() + return run.id, legacy + + run_id, legacy = await old_record("sub/pay", digest) + found = await actions.prepare(run_id=run_id, node_id="sub/pay", exec_no=1, capability=caps[PAY], args=args, + legacy_node_id="pay") + assert found.key == legacy and found.receipt is not None + fresh = await actions.prepare(run_id=run_id, node_id="sub/pay", exec_no=1, capability=caps[PAY], args=args) + assert fresh.key != legacy and fresh.receipt is None + # 父图里同名的 pay 按自己的键(和旧键一样)找到这一笔:事件说是 sub/pay 的,不是自己的,换一个只属于自己的键 + mine = await actions.prepare(run_id=run_id, node_id="pay", exec_no=1, capability=caps[PAY], args=args) + assert mine.key != legacy and mine.receipt is None + # 内容不一样(这一次是 7):不当成这一次已经生效 + other = await actions.prepare(run_id=run_id, node_id="sub/pay", exec_no=1, capability=caps[PAY], + args={**args, "amount": "7"}, legacy_node_id="pay") + assert other.key != legacy and other.receipt is None + # 那一笔是父图 pay 写的(事件说 pay):子图里的 pay 不认 + run_id, legacy = await old_record("pay", digest) + theirs = await actions.prepare(run_id=run_id, node_id="sub/pay", exec_no=1, capability=caps[PAY], args=args, + legacy_node_id="pay") + assert theirs.key != legacy and theirs.receipt is None + own = await actions.prepare(run_id=run_id, node_id="pay", exec_no=1, capability=caps[PAY], args=args) + assert own.key == legacy and own.receipt is not None + + +# -------------------------------------------------------------------------- +# 跨升级:升级前的运行按老算法记的键,升级后接着跑 +# -------------------------------------------------------------------------- + + +def _old_code(monkeypatch): + """装成升级前的代码:子工作流里的写入按子图里的 id 算幂等键、不认旧键,子图跑过的步骤不交回父图(exec 每次从 1 数)。 + 交回一个函数,调了就换回现在的代码(同一个进程里接力跑,等于升级后重启接着跑;能力登记的替身要重新装)。""" + from app.engine.nodes import multi, tools + + monkeypatch.setattr(tools, "_action_ids", lambda ctx: (ctx.node.id, None)) + monkeypatch.setattr(multi, "_forwarded_trail", lambda ran, prefix: []) + return monkeypatch.undo + + +def _amounts(erp) -> list[str]: + return [str((w.get("body") or {}).get("amount")) for w in erp.writes("test")] + + +async def _until_status(run_id: str, statuses: tuple[str, ...]) -> Run: + for _ in range(600): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in statuses and not run_manager.is_active(run_id): + return row + raise AssertionError(f"运行 {run_id} 停在 {row.status}") + + +async def test_an_unknown_write_inside_a_subworkflow_with_a_namesake_in_the_parent_survives_the_upgrade(monkeypatch, erp): + """升级前:子工作流里的 pay(1)发出去、结果未知、运行挂起,键按子图里的 id(pay)记——和父图里同名的 pay 一样。升级后 + 核实为已生效、接着跑:子图里的 pay 认出自己那一笔(事件记着是「sub/pay」写的),不再发;父图的 pay(99)按自己的键找到 + 的是子图那一笔、不是自己的,换用自己的键照常发。1 和 99 各只发一次。""" + from app.db.models import Task + from app.process import tasks + from tests.test_process_actions import ANN + + await _unattended(monkeypatch, erp) + inner = S.chain(node("s", "input", fields=[{"name": "n"}]), pay(), + node("o", "output", fields=[{"name": "r", "value": "x"}])) + sub_id = await S.publish("付款子流程", graph=inner, level="draft") + graph = S.chain(node("start", "input", fields=[{"name": "n"}]), + node("sub", "subgraph", "子流程", workflow_id=sub_id, workflow_version=1, input={"n": "1"}), + pay(), node("out", "output", fields=[{"name": "r", "value": "完成"}])) + upgrade = _old_code(monkeypatch) + erp.mode = "timeout_after_write" + run = await run_manager.start(graph=graph, input_payload={"n": 99}) + parked = await _until_status(run.id, ("interrupted", "failed", "succeeded")) + assert parked.status == "interrupted", parked.error + assert _amounts(erp) == ["1"] + upgrade() + await _unattended(monkeypatch, erp) # undo 连能力登记的替身一起撤了:重新装上 + erp.mode = "ok" + async with SessionLocal() as session: + task = (await session.execute(select(Task).where(Task.run_id == run.id, Task.kind == "reconcile", + Task.status == "pending"))).scalars().first() + assert task is not None + await tasks.respond(task.id, ANN, decision="confirmed", values={"external_id": "PAY-1"}) + final = await _until_status(run.id, ("succeeded", "failed")) + assert final.status == "succeeded", final.error + assert sorted(_amounts(erp)) == ["1", "99"], _amounts(erp) + + +async def test_a_loop_across_the_upgrade_sends_the_second_round(monkeypatch, erp): + """升级前跑完第一轮(子图里 pay 发了 1)、停在父图的审批上;升级后批准,第二轮按老算法同样是第 1 次执行,找到第一轮那一笔: + 内容指纹不一样(这一轮是 2),不当成已经生效,照常发;也不记「已经生效」的提示。""" + await _unattended(monkeypatch, erp) + inner = S.chain(node("s", "input", fields=[{"name": "n"}]), pay(), + node("o", "output", fields=[{"name": "r", "value": "x"}])) + sub_id = await S.publish("付款子流程", graph=inner, level="draft") + nodes = [node("start", "input"), + node("each", "loop", "逐项", mode="foreach", items="[1, 2]", item_var="it", max_iterations=5), + node("sub", "subgraph", "子流程", workflow_id=sub_id, workflow_version=1, input={"n": "{{ vars.it }}"}), + node("gate", "human", "把关", mode="approve", title="下一轮?"), + node("out", "output", fields=[{"name": "r", "value": "完成"}])] + edges = [edge("start", "each"), edge("each", "sub", "body"), edge("sub", "gate"), edge("gate", "each", "approved"), + edge("each", "out", "done")] + upgrade = _old_code(monkeypatch) + run = await run_manager.start(graph={"nodes": nodes, "edges": edges}, input_payload={}) + parked = await _until_status(run.id, ("interrupted", "failed", "succeeded")) + assert parked.status == "interrupted", parked.error + assert _amounts(erp) == ["1"] + upgrade() + await _unattended(monkeypatch, erp) + await run_manager.resume(run.id, {"approved": True}) + parked = await _until_status(run.id, ("interrupted", "failed", "succeeded")) + assert parked.status == "interrupted", parked.error + assert _amounts(erp) == ["1", "2"], "第二轮照发,不交回第一轮的回执" + assert not [e for e in await _events(run.id, "log") if e.data.get("code") == "action_already_applied"] + await run_manager.resume(run.id, {"approved": True}) + final = await _until_status(run.id, ("succeeded", "failed")) + assert final.status == "succeeded", final.error + assert _amounts(erp) == ["1", "2"] diff --git a/backend/tests/test_table_imports_api.py b/backend/tests/test_table_imports_api.py index 6146d18d..761cfdbe 100644 --- a/backend/tests/test_table_imports_api.py +++ b/backend/tests/test_table_imports_api.py @@ -11,7 +11,6 @@ from __future__ import annotations import datetime as dt import json -import uuid from pathlib import Path from typing import Any @@ -26,6 +25,7 @@ from app.data.engine import engines, run_query from app.db.base import SessionLocal from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableImport, TableRecipe from app.main import app +from tests.fixtures.names import letters from tests.fixtures.xlsx.flow import FLOW_EXPECT, flow_workbook from tests.test_recipe_imports import ( # noqa: F401 - fake 是夹具,导入即可用 FLOW_FULL, api_stage, fake, raw_bytes, recipe_source, staged, @@ -52,7 +52,7 @@ async def client(): def unique(prefix: str = "wp5c") -> str: - return f"{prefix}_{uuid.uuid4().hex[:10]}" + return f"{prefix}_{letters(10)}" async def stage(client, name: str, raw: bytes, filename: str, **form: str): diff --git a/backend/tests/test_table_versions_p2.py b/backend/tests/test_table_versions_p2.py index bacd95e6..f90bcae8 100644 --- a/backend/tests/test_table_versions_p2.py +++ b/backend/tests/test_table_versions_p2.py @@ -23,6 +23,7 @@ from typing import Any import pytest from sqlalchemy import create_engine, select, text +from sqlalchemy.exc import IntegrityError from app.core.config import settings from app.data import raw_store, recipe_parsers, recipe_types, table_versions @@ -32,6 +33,7 @@ from app.db.base import SessionLocal, utcnow from app.db.models import ( DataSource, ImportStaging, SourceSnapshot, TableBuild, TableImport, TableRecipe, ) +from app.db.portable import unique_violation @pytest.fixture(autouse=True) @@ -268,8 +270,10 @@ async def test_recipe_seq_is_unique_per_source(): session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256=RECIPE_SHA)) await session.commit() session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256=RECIPE_SHA)) - with pytest.raises(Exception, match="UNIQUE"): + with pytest.raises(IntegrityError) as caught: await session.commit() + # 撞的正是 (source_id, seq) 这一条唯一约束(两种库的报错写法不同,按表和列认) + assert unique_violation(caught.value, "table_recipes", "source_id", "seq") # -------------------------------------------------------------------------- diff --git a/backend/tests/test_table_versions_p3.py b/backend/tests/test_table_versions_p3.py index 13494004..444e1166 100644 --- a/backend/tests/test_table_versions_p3.py +++ b/backend/tests/test_table_versions_p3.py @@ -1162,6 +1162,7 @@ async def test_guard_is_idempotent_in_one_process(store, guard_reset): await table_versions.startup(session) +@pytest.mark.sqlite_only # 文件锁守卫只在 SQLite(单进程)时有:PostgreSQL 可以多进程,版本存储靠跨进程的锁(app/cluster/mutex) async def test_second_process_skips_startup_cleanup_and_refuses_writes(store, guard_reset, monkeypatch): """L5:子进程先占住守卫,本进程启动时一个文件都不删、不回收,写入一律 StoreUnavailable;子进程退出后恢复。""" name = unique() @@ -1243,6 +1244,7 @@ async def test_second_process_skips_startup_cleanup_and_refuses_writes(store, gu assert not half.exists() and not orphan.exists() and not orphan_raw.exists() +@pytest.mark.sqlite_only # 文件锁守卫只在 SQLite(单进程)时有:PostgreSQL 可以多进程,版本存储靠跨进程的锁(app/cluster/mutex) async def test_guard_state_does_not_leak_into_another_data_dir(store, guard_reset, monkeypatch, tmp_path): proc = hold_guard() try: diff --git a/backend/tests/test_task_resume_errors.py b/backend/tests/test_task_resume_errors.py index db9b885e..721a7fd2 100644 --- a/backend/tests/test_task_resume_errors.py +++ b/backend/tests/test_task_resume_errors.py @@ -84,3 +84,32 @@ async def test_any_resume_error_puts_the_task_back(monkeypatch): out = await tasks.respond(task.id, ANN, decision="approve") assert out["resolved"] is True assert (await _wait(run.id, ("succeeded", "failed"))).status == "succeeded" + + +async def test_the_periodic_takeover_leaves_a_task_that_is_being_resolved_alone(monkeypatch): + """复核 R-b2:当班进程隔一会儿的那一次接手(live.trust_mine)撞上一次正在判定、恢复的待办(停在 resolving 的那几秒), + 单进程时也不能把它退回 pending、撤回那一票——只收拾停了两个租约时长以上的。启动时、又没有别的进程的那一次照旧全收拾。""" + from app.cluster.workers import Liveness + + run = await run_manager.start(graph=FLOW, input_payload={"q": "x"}) + await _wait(run.id, ("interrupted",)) + task = await _pending(run.id) + real = run_manager.resume + reaped = asyncio.Event() + + async def slow(*a, **k): + await asyncio.sleep(0.3) + await tasks.startup(live=Liveness().trusting()) # 判定到一半,当班进程来了一次 + reaped.set() + return await real(*a, **k) + + monkeypatch.setattr(run_manager, "resume", slow) + out = await tasks.respond(task.id, ANN, decision="approve") + assert reaped.is_set() and out["resolved"] is True + assert (await _wait(run.id, ("succeeded", "failed"))).status == "succeeded" + async with SessionLocal() as session: + row = await session.get(Task, task.id) + mine = (await session.execute(select(TaskAssignment).where(TaskAssignment.task_id == task.id))).scalars().all() + assert row.status == "resolved", row.status + assert mine and all(a.responded_by == "ann" and a.status not in ("pending", "withdrawn") for a in mine), \ + [(a.status, a.responded_by) for a in mine] diff --git a/deploy/README.md b/deploy/README.md index 4570f34c..a528316f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -10,6 +10,7 @@ | 文件 | 用途 | | --- | --- | | `image.tar` | Docker 镜像(`docker save` 导出),镜像名 `agentlab:__AGENTLAB_VERSION__` | +| `postgres-image.tar` | 内置 PostgreSQL 的镜像(`install --postgres` 时用,见「用 PostgreSQL」)。包因此多约 160 MB,导入后在 docker 里占约 0.7 GB;只用 SQLite 或外接 PostgreSQL 时用不上,打包时用 `--no-postgres` 就没有这个文件 | | `deploy.sh` | 一键部署脚本:安装、启动、停止、升级、卸载 | | `agentlab.env` | 配置:端口、监听地址、数据目录、模型 Key、代理、沙箱 | | `README.md` | 本说明 | @@ -21,7 +22,8 @@ - **架构要一致**:这个包是 `linux/__AGENTLAB_ARCH__`。在目标机上运行 `uname -m`: `aarch64` 对应 arm64(ARM 单板机要装 64 位系统),`x86_64` 对应 amd64。不一致的镜像跑不起来, deploy.sh 会在导入前就报出来; -- 磁盘:镜像导入后约 0.8~1 GB(看 Docker 用的存储方式),另留数据目录的空间; +- 磁盘:镜像导入后约 0.8~1 GB(看 Docker 用的存储方式),另留数据目录的空间;用内置 PostgreSQL 再加约 0.7 GB 的镜像 + 和库本身的空间(在 docker 的数据卷里); - 可选:宿主机支持 KVM(有 `/dev/kvm`)时,代码会在 microVM 里执行,隔离最强,见下面的「代码沙箱」; - 当前用户能用 docker(在 `docker` 组里),或者用 `sudo` 运行 deploy.sh。 @@ -41,6 +43,7 @@ nano agentlab.env deploy.sh 依次:检查 docker 和架构 → 导入镜像(`docker load -i image.tar`,本机已有就跳过)→ 创建数据目录 → 启动容器(`--restart unless-stopped`,开机自动起来)→ 等健康检查通过 → 打印访问地址。 +库缺省是数据目录里的 SQLite;要用 PostgreSQL 的新部署把第 3 步换成 `./deploy.sh install --postgres`(见「用 PostgreSQL」)。 ## 访问 @@ -58,7 +61,26 @@ deploy.sh 依次:检查 docker 和架构 → 导入镜像(`docker load -i im 所有数据都在 `DATA_DIR`(默认是包目录下的 `data/`):数据库 `agentlab.db`、执行断点 `checkpoints.db`、 工件、上传文件、沙箱工作区,以及加密密钥 `.secret_key`。**要备份的就是这个目录**, -丢了 `.secret_key`,存进去的 API Key 就解不开了。 +丢了 `.secret_key`,存进去的 API Key 就解不开了。目录里的 `.agentlab-data` 是服务启动时建的标记,deploy.sh 改属主前按它 +(或 `agentlab.db`、`.secret_key`、`.postgres/state`)认出这是 AgentLab 的数据目录,别删。数据库放在 PostgreSQL 时(下面「用 PostgreSQL」)没有这两个库文件, +另用 `pg_dump` 备份数据库。 + +`./deploy.sh upgrade` 换版本之前会自动备份一次库:SQLite 拷到数据目录的 `backups/sqlite/<时间>/`(SQLite 的备份接口, +一致的一份),内置 PostgreSQL 用 `pg_dump` 存到 `backups/postgres/`,各只留最近 5 份(`agentlab.env` 里 `BACKUP_KEEP=` 改)。 +备份里是整个库(含加密存着的密钥),所以 `backups/` 和下面每一层目录都是 0700、备份文件 0600,属主同数据目录(uid 10001): +宿主机上要看、要拷走都得用 sudo(`sudo ls -l data/backups/sqlite/`、`sudo cp -a data/backups /别处/`)。 +这只是升级的保险,不代替下面的整目录备份:工件、上传文件和 `.secret_key` 不在里面。 + +用 SQLite 的部署要从 `backups/sqlite/` 恢复(以缺省的 `./data` 为例,`<时间>` 换成目录名): + +```bash +./deploy.sh stop +sudo rm -f data/agentlab.db-wal data/agentlab.db-shm data/checkpoints.db-wal data/checkpoints.db-shm +sudo cp -p data/backups/sqlite/<时间>/agentlab.db data/backups/sqlite/<时间>/checkpoints.db data/ # -p:属主仍是 10001 +./deploy.sh start +``` + +`-wal`、`-shm` 要先删:留着的话 SQLite 会把它们当成换回来的库的一部分。 deploy.sh 会把数据目录的属主改成容器里的用户(uid 10001),所以备份、删除要用 sudo。 停下来备份最稳妥(以默认的 `./data` 为例,在包目录里执行): @@ -71,6 +93,162 @@ sudo tar -czf ../agentlab-data-$(date +%Y%m%d).tar.gz data 恢复:停掉容器,把备份解回 `DATA_DIR`,再 `./deploy.sh start`。 +## 用 PostgreSQL + +库有三种放法,选一种: + +| | 怎么用 | 库在哪里 | 数据目录里 | +| --- | --- | --- | --- | +| SQLite(缺省) | `./deploy.sh` | 数据目录的 `agentlab.db`、`checkpoints.db` | 全部数据 | +| 内置 PostgreSQL | 新部署 `./deploy.sh install --postgres`;已经在用 SQLite 的 `./deploy.sh migrate-to-postgres` | docker 数据卷 `-pgdata` | 工件、上传文件、`.secret_key`、库的密码 `.postgres/` 和备份 `backups/` | +| 外接 PostgreSQL | `agentlab.env` 里设 `AGENTLAB_DATABASE_URL` | 你自己的 PostgreSQL | 工件、上传文件、`.secret_key` | + +内置和外接只能选一种:`agentlab.env` 里设了 `AGENTLAB_DATABASE_URL`(或 `AGENTLAB_CHECKPOINT_URL`)又加 `--postgres`, +或者数据目录已经是内置的,deploy.sh 照实报错、不起任何容器。不管哪种,数据目录都照样要挂、要备份。 + +### 内置 PostgreSQL + +`./deploy.sh install --postgres` 在 AgentLab 旁边再起一个 PostgreSQL 容器: + +- 镜像缺省 `postgres:16.15`(小版本固定在 deploy.sh 里,每台机器起的都一样;离线包里带着 `postgres-image.tar`,本机已有同名镜像 + 就不导入)。要换在 `agentlab.env` 里写 `POSTGRES_IMAGE=postgres:16.x`,再 `./deploy.sh install`:服务先停、库的容器按新镜像重建, + 数据卷不动;新镜像起不来(跨大版本时数据格式不通用)就换回原来的镜像、照实报错。 +- 两个容器在同一个 docker 网络 `-net` 里,PostgreSQL 的端口**不对外开**,只有 AgentLab 的容器连得到。 +- 库的数据在 docker 数据卷 `-pgdata` 里(不在数据目录:数据目录会被改属主,PostgreSQL 的文件属主不能动)。 +- 密码第一次部署时自动生成,存在数据目录的 `.postgres/password` 和 `.postgres/pgpass`(都是 0600),不进镜像、不进日志、 + 不进 `docker inspect`,`agentlab.env` 里什么都不用填。AgentLab 的容器按 `PGPASSFILE` 读它。**丢了数据目录就连不上这个库**, + 所以数据目录照样要整份备份。 +- 用过一次之后这个数据目录就一直是内置 PostgreSQL(记在 `.postgres/state`),之后的 `install`、`upgrade`、`start` 不用再加 + `--postgres`。 +- `start` 先起库、等 `pg_isready` 通过再起服务;`stop` 先停服务再停库。`restart` 缺省只重启服务(库停着的话先把库起来), + `restart --postgres` 连库一起重启(先停服务再停库,再先起库后起服务)。开机时两个容器一起被 docker 拉起来,库还没就绪时 + 服务按退避重试(日志里写着「应用库还没就绪……秒后重试」),最多等 90 秒(`AGENTLAB_DB_STARTUP_TIMEOUT`),不会一上来就崩。 +- 表结构和默认数据(内置模板、技能)每次启动自动补齐;几个工作进程(`AGENTLAB_WORKERS`)同时起也只做一次(库里的锁)。 +- `./deploy.sh status` 显示两个容器的状态,`./deploy.sh logs --postgres` 看库的日志。 + +可以在 `agentlab.env` 里改的(都有缺省值,一般不用写): + +| 项 | 缺省 | 说明 | +| --- | --- | --- | +| `POSTGRES_IMAGE` | `postgres:16.15` | 库的镜像 | +| `POSTGRES_CONTAINER` | `-postgres` | 库的容器名 | +| `DOCKER_NETWORK` | `-net` | 两个容器共用的网络 | +| `POSTGRES_VOLUME` | `-pgdata` | 库的数据卷 | +| `BACKUP_KEEP` | `5` | 升级前自动备份留几份(SQLite 也按它) | + +环境变量 `DEPLOY_<名字>` 盖过 `agentlab.env` 里的同名项(这几项加上 `HOST_PORT`、`BIND_ADDR`、`DATA_DIR`、`CONTAINER_NAME`), +一台机器上并排部署两套、或者测试时用。 + +**备份。** `upgrade` 前自动 `pg_dump` 一份到 `backups/postgres/`(0600,见「数据和备份」)。平时要手动备份(服务开着也行, +`pg_dump` 读的是一致的快照;`umask 077` 让导出的文件只有自己读得到): + +```bash +(umask 077; docker exec agentlab-postgres pg_dump -U agentlab -d agentlab -Fc > agentlab-$(date +%Y%m%d).dump) +``` + +再加上数据目录(见「数据和备份」),两样合起来才是完整的一份。 + +**恢复。** 用 `backups/postgres/` 里的备份把库换回去(以缺省的名字、在包目录里为例,`<备份>` 换成文件名,`sudo ls data/backups/postgres/` +看有哪些)。备份文件是 0600、属主 10001,宿主机上的人读不到,所以最后一步在一个以 10001 运行的临时容器里读它,不用 sudo: + +```bash +./deploy.sh stop # 两个都停 +docker start agentlab-postgres # 只起库 +until docker exec agentlab-postgres pg_isready -q -h 127.0.0.1 -U agentlab; do sleep 1; done +docker exec agentlab-postgres psql -U agentlab -d postgres \ + -c 'DROP DATABASE agentlab WITH (FORCE)' -c 'CREATE DATABASE agentlab OWNER agentlab' +docker run --rm --network agentlab-net -v "$(pwd)/data:/data" --user 10001:10001 -e PGPASSFILE=/data/.postgres/pgpass \ + --entrypoint pg_restore postgres:16.15 -h agentlab-postgres -U agentlab -d agentlab --no-owner /data/backups/postgres/<备份> +./deploy.sh start +``` + +自己手动导出、放在别处的备份(上面「备份」那条命令导出的,属主是你自己)直接从宿主机喂进去: +`docker exec -i agentlab-postgres pg_restore -U agentlab -d agentlab --no-owner < agentlab-<日期>.dump`。 + +数据目录要换回同一时间的备份,工件、上传文件和库里的记录才对得上。 + +**从 SQLite 切过来。** 正在用 SQLite 的部署运行 `./deploy.sh migrate-to-postgres`,它依次: + +1. 停服务; +2. 备份 SQLite 库到 `backups/sqlite/<时间>/`; +3. 起内置 PostgreSQL(第一次生成密码); +4. 用镜像里的 `python -m app.db copy-to-postgres` 先核对(`--dry-run`,只数行数); +5. 正式导入,逐表按主键核对、断点按条数核对(停在审批上的、中断了的运行导过去之后照样能接着跑); +6. 在数据目录里记下「已切到内置 PostgreSQL」,按内置库起服务、等健康检查通过。 + +记下之前任何一步出错,都停在原地:服务按 SQLite 原样恢复(原来在跑就起回来),库的容器停掉,屏幕上是那一步的原话。 +处理好之后重新运行就行,重复执行是安全的:已经切过的什么都不做;上次没切成、留在内置库里的半份数据先清空再导, +不会和 SQLite 上后来改过的混在一起(`status` 里会写「上次 migrate-to-postgres 没切成」)。数据卷已经存在、又不是这个 +数据目录上次没切成留下的(比如以前卸载时留下的另一份库),它不会去清空,照实报错让你决定。 + +切完之后 `agentlab.db`、`checkpoints.db` 原样留在数据目录里、不再使用,`backups/sqlite/` 里还有一份。**什么时候能删**: +在 PostgreSQL 上用一段时间、做过一次完整备份(`pg_dump` 加数据目录)之后。删之前它们是回退的路:`./deploy.sh stop`、 +`sudo rm <数据目录>/.postgres/state`、`./deploy.sh install`,就回到切换那一刻的 SQLite(切换之后在 PostgreSQL 上新产生的东西 +不会带回来;库的容器停着不用,`docker rm -f agentlab-postgres` 可删)。 + +**首次在 Linux 上切换前的检查。** 第一次在一台机器上 `install --postgres` 或 `migrate-to-postgres` 之前,花几分钟看一遍 +(以缺省的 `./data` 为例,在包目录里执行): + +- **docker 版本**:`docker version --format '{{.Server.Version}}'`,要 20.10 以上(健康检查、自建网络、`--stop-timeout` 都要用); + 运行 deploy.sh 的用户在 `docker` 组里(`id -nG` 里有 docker),否则整条命令前加 sudo。 +- **属主**:`ls -lnd data` 第三列应是 `10001`(deploy.sh 改的)。数据目录不要放在 NFS、SMB 这类不认 uid 的挂载上:属主改不过去, + 服务写不进去。 +- **权限**:切换(或第一次起库)之后看一眼——`ls -ld data/.postgres` 应是 `drwx--x--x`(0711),`cat data/.postgres/state` + 不用 sudo 就能读到(deploy.sh 在宿主机上直接读它,读不到时会退回去用容器读,只是慢一点);`sudo ls -ln data/.postgres` + 里 `password`、`pgpass` 是 `-rw-------`、属主 10001;`sudo ls -ld data/backups` 是 `drwx------`。 +- **磁盘空间**:切换时数据目录里要放得下两份库(`backups/sqlite/` 里的一份备份,加上导入时的临时快照,用完就删), + 另外 docker 的存储目录要放得下 PostgreSQL 的镜像(约 0.7 GB)和库本身(大致和 SQLite 的库一样大,索引多的表会大一些)。 + 看大小:`sudo du -sh data/agentlab.db data/checkpoints.db`;看剩余:`df -h data` 和 + `df -h "$(docker info --format '{{.DockerRootDir}}')"`。 +- **整目录备份**:切换会先自动备份一份库,但还是建议先按「数据和备份」停服务整份打包一次数据目录。切换期间服务是停着的, + 时长大致是导入的时间(每秒几千到几万行,看机器)。 + +**卸载。** `./deploy.sh uninstall` 连库的容器和网络一起删,数据卷和数据目录都留着;同一个数据目录再 `./deploy.sh install` +就接着用原来的库。确定不要了:`docker volume rm agentlab-pgdata`(不可恢复)。 + +### 外接 PostgreSQL + +库在你自己的 PostgreSQL 上(要先建好),在 `agentlab.env` 里加一行再 `./deploy.sh install`: + +```bash +AGENTLAB_DATABASE_URL=postgresql+asyncpg://agentlab:<密码>@<数据库地址>:5432/agentlab +``` + +容器里的 `127.0.0.1` 是容器自己:数据库在别的机器上就写那台的地址;就在这台机器上,写宿主机在 docker 网桥上的地址 +(一般是 `172.17.0.1`),并让 PostgreSQL 监听这个地址、`pg_hba.conf` 放行容器的网段。执行断点也进同一个库。 +表结构和默认数据同样在每次启动时自动补齐,库暂时连不上时同样按退避重试。`agentlab.env` 末尾「外接 PostgreSQL 与多进程(可选)」 +一段列着这几项(地址、断点库、PgBouncer、工作进程、连接池、启动时等库),都注释着,要用的去掉行首的 `#`;它们和别的 `AGENTLAB_*` +一样经 `--env-file` 原样进容器。 + +已经在用 SQLite 的,先把数据导过去(同一个镜像里带着导入命令;`--dry-run` 只核对、不写): + +```bash +./deploy.sh stop +sudo tar -czf ../agentlab-data-$(date +%Y%m%d).tar.gz data # 先备份 +docker run --rm --user 10001:10001 -v "$(pwd)/data:/data" --env-file agentlab.env --entrypoint python \ + agentlab:<版本> -m app.db copy-to-postgres /data --dry-run +docker run --rm --user 10001:10001 -v "$(pwd)/data:/data" --env-file agentlab.env --entrypoint python \ + agentlab:<版本> -m app.db copy-to-postgres /data +./deploy.sh install # agentlab.env 里已经有 AGENTLAB_DATABASE_URL +``` + +导入结束时逐表打印来源和目标的行数、断点的条数,对不上的列出来、退出码非零;重跑只补没有的行,不重复、不覆盖。 +导完 `data/agentlab.db` 可以留着当回退:把那一行删掉、重新部署就回到 SQLite(导入之后在 PostgreSQL 上新产生的东西不会回来)。 +外接的库 deploy.sh 不备份,自己用 `pg_dump` 备份。 + +### 多进程 + +**多进程。** 换成 PostgreSQL 之后可以在容器里起几个工作进程:`agentlab.env` 里加 `AGENTLAB_WORKERS=4`(缺省 1)再 +`./deploy.sh upgrade`。几个进程共用一个端口,各自在库里登记、续租约;一个进程没了,别的进程在租约到期后接手它留下的运行、 +问数据和后台任务(规矩见仓库 README「多进程」)。SQLite 时设大于 1 会启动失败(照实报错:SQLite 只支持单进程)。 +几台机器、几个容器指着同一个库也行,但数据目录要放在共享存储上、每台挂在同一个路径,限制见仓库 README「多进程」。 + +内置 PostgreSQL 是直连(中间没有 PgBouncer),多进程时什么都不用另设(不用 `AGENTLAB_DATABASE_DIRECT_URL`;设了它、或者 +`AGENTLAB_DB_PGBOUNCER=1`,deploy.sh 照实拦下)。库还是 SQLite 时 `AGENTLAB_WORKERS` 大于 1,deploy.sh 在起容器之前就拦下。新旧版本的进程不能同时 +连着一个库(代码里拦着:新版本发现库里还有活着的旧版本进程就拒绝启动)。`./deploy.sh upgrade` 就是按这个顺序做的:先停掉旧容器 +(里面的全部工作进程一起停)、备份,再起新版本;旧进程要是被强杀、登记还没过期,新版本启动时会等它过期(最多半分钟左右)。 +同一个库还接着别处的 AgentLab 进程(另一台机器、另一个容器)的,升级前先把它们也停掉。 + ## 升级 把新版本的离线包拷到机器上,解到一个新目录,然后: @@ -83,7 +261,9 @@ cp ../agentlab-<旧版本>-linux-__AGENTLAB_ARCH__/agentlab.env . # 沿用原 ``` `upgrade` 导入新镜像、用新镜像替换容器,数据目录不动。deploy.sh 会检查新配置指向的数据目录和 -正在用的是不是同一个,不是就停下来提醒,不会让新版本悄悄从一个空库开始。 +正在用的是不是同一个,不是就停下来提醒,不会让新版本悄悄从一个空库开始。换之前先停旧服务、自动备份一次库 +(见「数据和备份」),备份没做成就把旧服务起回来、不升级。新版本启动时自动补表结构和默认数据,不用手动迁移。 +用内置 PostgreSQL 的,新包里的 `POSTGRES_IMAGE` 换了小版本时库的容器跟着换(数据卷不动)。 新容器没通过健康检查时,原来的容器会保留为 `agentlab-previous`,屏幕上会打印回退命令。 旧镜像也留着,确认新版本没问题后可以 `docker rmi agentlab:<旧版本>` 删掉。 @@ -99,6 +279,7 @@ cp ../agentlab-<旧版本>-linux-__AGENTLAB_ARCH__/agentlab.env . # 沿用原 ./deploy.sh logs # 最后 200 行 ./deploy.sh logs -f # 持续跟随,Ctrl-C 退出 ./deploy.sh logs --tail 1000 +./deploy.sh logs --postgres # 内置 PostgreSQL 的日志 ``` 日志由 docker 自动轮转(单个 10 MB、保留 3 个),不会塞满存储卡。 @@ -108,12 +289,16 @@ cp ../agentlab-<旧版本>-linux-__AGENTLAB_ARCH__/agentlab.env . # 沿用原 ```bash ./deploy.sh stop # 停止(开机也不会自动起来) ./deploy.sh start # 再启动 -./deploy.sh restart # 重启 +./deploy.sh restart # 重启服务(内置 PostgreSQL 不跟着重启) +./deploy.sh restart --postgres # 连内置 PostgreSQL 一起重启 ./deploy.sh uninstall # 删除容器,保留数据和镜像 -./deploy.sh uninstall --purge-image # 连镜像一起删 +./deploy.sh uninstall --purge-image # 连镜像一起删(PostgreSQL 的镜像不删,别的东西可能也在用) ``` -卸载**永远不删数据**。确定不要了再手动删除数据目录(不可恢复):`sudo rm -rf <数据目录>`。 +内置 PostgreSQL 时这几个命令按顺序管两个容器(起的时候先起库,停的时候先停服务;`restart` 不加 `--postgres` 只动服务), +卸载连库的容器和网络一起删。 +卸载**永远不删数据**。确定不要了再手动删除数据目录(不可恢复):`sudo rm -rf <数据目录>`;内置 PostgreSQL 的数据卷另删: +`docker volume rm -pgdata`。 改了 `agentlab.env` 之后运行 `./deploy.sh install` 让新配置生效(`restart` 不会重读配置)。 diff --git a/deploy/agentlab.env b/deploy/agentlab.env index 8431c5e9..97d6335d 100644 --- a/deploy/agentlab.env +++ b/deploy/agentlab.env @@ -111,3 +111,46 @@ ENABLE_MICROVM=auto # sudo sysctl kernel.apparmor_restrict_unprivileged_userns=0 # 这一步会放宽整台机器的限制,自行权衡。 ENABLE_BWRAP=0 + +# ---- 内置 PostgreSQL(可选)---- +# 缺省库是数据目录里的 SQLite。新部署要用内置的 PostgreSQL 运行 ./deploy.sh install --postgres,已经在用 SQLite 的运行 +# ./deploy.sh migrate-to-postgres(见 README.md「用 PostgreSQL」)。密码自动生成在数据目录的 .postgres/ 里,这里不用填。 +# 和外接 PostgreSQL(AGENTLAB_DATABASE_URL)只能选一种。下面几项是给 deploy.sh 用的,都有缺省值,一般不用改;要改去掉 +# 行首的 # 再 ./deploy.sh install。 +# +# 库的镜像(小版本固定;离线包里带着同一个标签的 postgres-image.tar) +# POSTGRES_IMAGE=postgres:16.15 +# 库的容器名,缺省 -postgres +# POSTGRES_CONTAINER=agentlab-postgres +# 两个容器共用的 docker 网络,缺省 -net(库的端口不对外,只在这个网络里可见) +# DOCKER_NETWORK=agentlab-net +# 库的数据放在这个 docker 数据卷里,缺省 -pgdata(卸载不删) +# POSTGRES_VOLUME=agentlab-pgdata +# upgrade 前自动备份库(SQLite 拷到 backups/sqlite/,内置 PostgreSQL 用 pg_dump 存到 backups/postgres/),只留最近几份 +# BACKUP_KEEP=5 + +# ---- 外接 PostgreSQL 与多进程(可选)---- +# 库放在你自己的 PostgreSQL 上(库要先建好):去掉 AGENTLAB_DATABASE_URL 行首的 # 填好地址,再 ./deploy.sh install。 +# 和上面的内置 PostgreSQL 只能选一种:两个都用时 deploy.sh 照实报错、不起容器。容器里的 127.0.0.1 是容器自己:库就在这台机器上时 +# 写宿主机在 docker 网桥上的地址(一般是 172.17.0.1),并让 PostgreSQL 监听这个地址、pg_hba.conf 放行容器的网段(一般是 +# 172.17.0.0/16);库在别的机器上就写那台的地址。下面这些 AGENTLAB_* 都原样传进容器(docker run --env-file 这个文件) +# AGENTLAB_DATABASE_URL=postgresql+asyncpg://agentlab:密码@172.17.0.1:5432/agentlab +# 断点放另一个库时才设(不设就和应用库同一个库) +# AGENTLAB_CHECKPOINT_URL=postgresql://agentlab:密码@172.17.0.1:5432/agentlab_checkpoints +# 前面有 PgBouncer 的事务模式时:AGENTLAB_DB_PGBOUNCER 设成 1,并另给直连 PostgreSQL 的地址(会话模式、直连都不用这两项)。 +# 内置 PostgreSQL 是直连,设了 deploy.sh 会拦下 +# AGENTLAB_DB_PGBOUNCER=0 +# AGENTLAB_DATABASE_DIRECT_URL=postgresql+asyncpg://agentlab:密码@172.17.0.1:5432/agentlab +# 容器里起几个工作进程,缺省 1。只有用 PostgreSQL(内置或外接)时才能大于 1:SQLite 时 deploy.sh 先拦下 +# AGENTLAB_WORKERS=1 +# 工作进程在库里的租约多长、隔多久续一次(秒,缺省 30、5)。租约到期没续上算进程没了,别的进程接手它的活 +# AGENTLAB_WORKER_LEASE_S=30 +# AGENTLAB_WORKER_HEARTBEAT_S=5 +# 连接池(每个工作进程一套):常驻几条、高峰再多开几条、等空闲连接最多几秒、一条连接用多久换新(缺省 10、20、30、1800 秒)。 +# 进程多了连接数跟着乘,PostgreSQL 的 max_connections 要够(见 README「用 PostgreSQL」的连接数的账) +# AGENTLAB_DB_POOL_SIZE=10 +# AGENTLAB_DB_MAX_OVERFLOW=20 +# AGENTLAB_DB_POOL_TIMEOUT=30 +# AGENTLAB_DB_POOL_RECYCLE=1800 +# 启动时库还没就绪最多等几秒(按退避重试,内置、外接都一样),缺省 90 +# AGENTLAB_DB_STARTUP_TIMEOUT=90 diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 144a55a9..bb2cd5f1 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -11,6 +11,8 @@ IMAGE="agentlab:$VERSION" HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ENV_FILE="$HERE/agentlab.env" IMAGE_TAR="$HERE/image.tar" +# 内置 PostgreSQL 的镜像:离线包里带一份(scripts/package-docker.sh 打进去),本机已经有同名镜像就不导入 +POSTGRES_IMAGE_TAR="$HERE/postgres-image.tar" # 给沙箱用 bubblewrap 时容器要加的权限,说明见 agentlab.env 的沙箱一节 BWRAP_OPTS=(--security-opt seccomp=unconfined --security-opt systempaths=unconfined --security-opt apparmor=unconfined) @@ -19,19 +21,35 @@ usage() { cat < 盖过 agentlab.env 里的同名项(测试、一台机器上并排部署两套时用):HOST_PORT、BIND_ADDR、 +DATA_DIR、CONTAINER_NAME,以及内置 PostgreSQL 的 POSTGRES_IMAGE、POSTGRES_CONTAINER、DOCKER_NETWORK、POSTGRES_VOLUME, +备份份数 BACKUP_KEEP。 + 这个包:AgentLab ${VERSION}(linux/${IMAGE_ARCH}) USAGE } @@ -48,6 +66,9 @@ CMD="install" FOLLOW="" TAIL="200" PURGE_IMAGE="" +WITH_POSTGRES="" +LOGS_POSTGRES="" +RESTART_POSTGRES="" if [ $# -gt 0 ]; then case "$1" in -h|--help) usage; exit 0 ;; @@ -56,7 +77,7 @@ if [ $# -gt 0 ]; then esac fi case "$CMD" in - install|upgrade|start|stop|restart|status|logs|uninstall) ;; + install|upgrade|migrate-to-postgres|start|stop|restart|status|logs|uninstall) ;; *) printf '认不出的子命令:%s\n\n' "$CMD" >&2; usage >&2; exit 2 ;; esac while [ $# -gt 0 ]; do @@ -68,6 +89,13 @@ while [ $# -gt 0 ]; do TAIL="$2"; shift ;; --tail=*) [ "$CMD" = logs ] || bad_arg "$1"; TAIL="${1#--tail=}" ;; --purge-image) [ "$CMD" = uninstall ] || bad_arg "$1"; PURGE_IMAGE=1 ;; + --postgres) + case "$CMD" in + install) WITH_POSTGRES=1 ;; + logs) LOGS_POSTGRES=1 ;; + restart) RESTART_POSTGRES=1 ;; + *) bad_arg "$1" ;; + esac ;; *) bad_arg "$1" ;; esac shift @@ -106,6 +134,23 @@ DATA_DIR="./data" CONTAINER_NAME="agentlab" ENABLE_BWRAP="0" ENABLE_MICROVM="auto" +# 内置 PostgreSQL。小版本固定:同一个包到哪台机器都起同一个版本(pg_dump 和库同版本);要换在 agentlab.env 里写 +# POSTGRES_IMAGE。容器、网络、数据卷的名字缺省跟着 CONTAINER_NAME 走 +POSTGRES_IMAGE="postgres:16.15" +POSTGRES_CONTAINER="" +DOCKER_NETWORK="" +POSTGRES_VOLUME="" +BACKUP_KEEP="5" +# agentlab.env 里设了外接的库(AGENTLAB_DATABASE_URL 等)没有:只记设了哪一项,值(带密码)不读进来 +EXTERNAL_DB="" +EXTERNAL_URL_SET="" +DIRECT_URL_SET="" +# 这两项要看值:PgBouncer 事务模式的开关(内置库是直连,开了就拦下)、工作进程数(SQLite 只能是 1)。都原样经 --env-file 进容器 +PGBOUNCER_VALUE="" +WORKERS_VALUE="" +# 内置 PostgreSQL 里的库名、用户名(PostgreSQL 容器第一次初始化时按它建) +PG_DB="agentlab" +PG_USER="agentlab" read_env() { [ -f "$ENV_FILE" ] || die "找不到 ${ENV_FILE},它应该和 deploy.sh 在同一个目录" @@ -120,8 +165,19 @@ read_env() { key="$(trim "${line%%=*}")" val="${line#*=}" case "$key" in - HOST_PORT|BIND_ADDR|DATA_DIR|CONTAINER_NAME|ENABLE_BWRAP|ENABLE_MICROVM) + HOST_PORT|BIND_ADDR|DATA_DIR|CONTAINER_NAME|ENABLE_BWRAP|ENABLE_MICROVM|POSTGRES_IMAGE|POSTGRES_CONTAINER|\ + DOCKER_NETWORK|POSTGRES_VOLUME|BACKUP_KEEP) printf -v "$key" '%s' "$(unquote "$(trim "$val")")" ;; + AGENTLAB_DATABASE_URL|AGENTLAB_CHECKPOINT_URL|AGENTLAB_DATABASE_DIRECT_URL) + if [ -n "$(trim "$val")" ]; then + [ -n "$EXTERNAL_DB" ] || EXTERNAL_DB="$key" + case "$key" in + AGENTLAB_DATABASE_URL) EXTERNAL_URL_SET=1 ;; + AGENTLAB_DATABASE_DIRECT_URL) DIRECT_URL_SET=1 ;; + esac + fi ;; + AGENTLAB_DB_PGBOUNCER) PGBOUNCER_VALUE="$(unquote "$(trim "$val")")" ;; + AGENTLAB_WORKERS) WORKERS_VALUE="$(unquote "$(trim "$val")")" ;; AGENTLAB_DATA_DIR|AGENTLAB_WEB_DIST|AGENTLAB_PORT|AGENTLAB_HOST) die "agentlab.env 第 $n 行:不要设 ${key}。容器里的路径和端口是固定的,改宿主机这边用 DATA_DIR、HOST_PORT、BIND_ADDR" ;; *) @@ -131,13 +187,30 @@ read_env() { esac done < "$ENV_FILE" + # 环境变量 DEPLOY_<名字> 盖过 agentlab.env(测试、并排部署两套时用) + local k v + for k in HOST_PORT BIND_ADDR DATA_DIR CONTAINER_NAME POSTGRES_IMAGE POSTGRES_CONTAINER DOCKER_NETWORK \ + POSTGRES_VOLUME BACKUP_KEEP; do + v="$(printenv "DEPLOY_$k" 2>/dev/null || true)" + [ -z "$v" ] || printf -v "$k" '%s' "$v" + done + [ -n "$POSTGRES_CONTAINER" ] || POSTGRES_CONTAINER="$CONTAINER_NAME-postgres" + [ -n "$DOCKER_NETWORK" ] || DOCKER_NETWORK="$CONTAINER_NAME-net" + [ -n "$POSTGRES_VOLUME" ] || POSTGRES_VOLUME="$CONTAINER_NAME-pgdata" + case "$HOST_PORT" in ''|*[!0-9]*) die "HOST_PORT 必须是数字:$HOST_PORT" ;; esac if [ "$HOST_PORT" -lt 1 ] || [ "$HOST_PORT" -gt 65535 ]; then die "HOST_PORT 超出范围:$HOST_PORT"; fi [ -n "$BIND_ADDR" ] || die "BIND_ADDR 不能为空(所有网卡写 0.0.0.0,只给本机用写 127.0.0.1)" [ -n "$DATA_DIR" ] || die "DATA_DIR 不能为空" - case "$CONTAINER_NAME" in - ''|[!a-zA-Z0-9]*|*[!a-zA-Z0-9_.-]*) die "CONTAINER_NAME 只能用字母、数字和 _ . -,并以字母或数字开头:$CONTAINER_NAME" ;; - esac + for k in CONTAINER_NAME POSTGRES_CONTAINER DOCKER_NETWORK POSTGRES_VOLUME; do + v="${!k}" + case "$v" in + ''|[!a-zA-Z0-9]*|*[!a-zA-Z0-9_.-]*) die "$k 只能用字母、数字和 _ . -,并以字母或数字开头:$v" ;; + esac + done + [ "$POSTGRES_CONTAINER" != "$CONTAINER_NAME" ] || die "POSTGRES_CONTAINER 不能和 CONTAINER_NAME 同名:$CONTAINER_NAME" + case "$BACKUP_KEEP" in ''|*[!0-9]*|0) die "BACKUP_KEEP 要是正整数:$BACKUP_KEEP" ;; esac + case "$POSTGRES_IMAGE" in ''|*[[:space:]]*) die "POSTGRES_IMAGE 写错了:$POSTGRES_IMAGE" ;; esac case "$ENABLE_BWRAP" in 0|1) ;; *) die "ENABLE_BWRAP 只能是 0 或 1:$ENABLE_BWRAP" ;; esac case "$ENABLE_MICROVM" in auto|0|1) ;; *) die "ENABLE_MICROVM 只能是 auto、0 或 1:$ENABLE_MICROVM" ;; esac @@ -192,19 +265,19 @@ image_present() { docker image inspect "$IMAGE" >/dev/null 2>&1; } verify_tar() { # 拷贝中途断了、U 盘出错,docker load 的报错不好懂:先对一下校验和 + local name="${1:-image.tar}" line [ -f "$HERE/SHA256SUMS" ] || return 0 command -v sha256sum >/dev/null 2>&1 || return 0 - local line - line="$(grep ' image.tar$' "$HERE/SHA256SUMS" || true)" + line="$(grep " $name\$" "$HERE/SHA256SUMS" || true)" [ -n "$line" ] || return 0 - say "校验 image.tar" + say "校验 $name" (cd "$HERE" && printf '%s\n' "$line" | sha256sum -c --quiet -) \ - || die "image.tar 校验和对不上,文件不完整或损坏。重新拷一份离线包再试" + || die "$name 校验和对不上,文件不完整或损坏。重新拷一份离线包再试" } load_image() { [ -f "$IMAGE_TAR" ] || die "本地没有镜像 ${IMAGE},包里也找不到 image.tar" - verify_tar + verify_tar image.tar say "导入镜像:docker load -i image.tar(性能较弱的 ARM 单板机上可能要一两分钟)" docker load -i "$IMAGE_TAR" image_present || die "导入后仍找不到 ${IMAGE},image.tar 可能不是这个版本的" @@ -266,8 +339,12 @@ fix_data_owner() { owner="$(ls -nd "$DATA_ABS" | awk '{print $3}')" [ "$owner" = "$uid" ] && return 0 # 改属主前确认这确实是一个新目录或者 AgentLab 的数据目录,免得 DATA_DIR 写错改了别的东西 - if [ -n "$(ls -A "$DATA_ABS" 2>/dev/null)" ] && [ ! -e "$DATA_ABS/agentlab.db" ]; then - die "DATA_DIR 指向的 $DATA_ABS 不是空目录,也不像 AgentLab 的数据目录(里面没有 agentlab.db)。 + # 认这几样里的一样:agentlab.db(库在 SQLite)、.secret_key(自动生成的加密密钥)、.agentlab-data(服务启动时建的标记, + # 库放在 PostgreSQL、密钥又由 AGENTLAB_SECRET_KEY 给时只有它)、.postgres/state(deploy.sh 记的用哪种库:内置库起来了、 + # 服务还没起过时只有它)。只有 uploads/ 这类普通目录名的不认,免得 chown 错了别人的目录 + if [ -n "$(ls -A "$DATA_ABS" 2>/dev/null)" ] && [ ! -e "$DATA_ABS/agentlab.db" ] \ + && [ ! -e "$DATA_ABS/.secret_key" ] && [ ! -e "$DATA_ABS/.agentlab-data" ] && [ ! -e "$DATA_ABS/.postgres/state" ]; then + die "DATA_DIR 指向的 $DATA_ABS 不是空目录,也不像 AgentLab 的数据目录(里面没有 agentlab.db、.secret_key、.agentlab-data)。 为免改错别的文件的属主,请指向一个新目录,或者原来的 AgentLab 数据目录" fi say "把数据目录的属主改成容器里的用户(uid ${uid}),容器里的服务才写得进去" @@ -395,12 +472,13 @@ run_container() { --log-opt max-size=10m --log-opt max-file=3 \ ${extra[@]+"${extra[@]}"} \ ${MICROVM_OPTS[@]+"${MICROVM_OPTS[@]}"} \ + ${DB_OPTS[@]+"${DB_OPTS[@]}"} \ "$IMAGE" >/dev/null } wait_healthy() { local name="$1" state health i - say "等健康检查通过" + say "等 $name 通过健康检查" for i in $(seq 1 90); do state="$(container_state "$name")" health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$name" 2>/dev/null || true)" @@ -487,6 +565,271 @@ deploy_container() { die "容器没通过健康检查。看上面的日志;常见原因见 README 的常见问题" } + +# --------------------------------------------------------------------------- +# 用的哪种库:sqlite(缺省)、builtin(内置 PostgreSQL,数据目录里记着)、external(agentlab.env 里设了 +# AGENTLAB_DATABASE_URL)。内置和外接互斥 +# --------------------------------------------------------------------------- +PG_STATE_REL=".postgres/state" +# 应用容器里连内置库:不带密码,密码在数据目录的 pgpass 里(PGPASSFILE),不进容器的环境变量、不进日志 +PG_PASSFILE="/data/.postgres/pgpass" +MODE="sqlite" +DB_OPTS=() + +# 数据目录里记的状态:sqlite、builtin,或者 migrating(migrate-to-postgres 开始往内置库里导了、还没切成,用的仍是 +# SQLite)。.postgres/ 是 0711、state 是 0644,一般宿主机上直接读得到;读不到(权限)就交给容器里的 app.ops 读 +stored_state() { + local file="$DATA_ABS/$PG_STATE_REL" dir="$DATA_ABS/.postgres" line + if [ -r "$file" ]; then + line="$(grep '^mode=' "$file" 2>/dev/null | head -n 1 || true)" + case "${line#mode=}" in builtin|migrating) echo "${line#mode=}" ;; *) echo sqlite ;; esac + elif [ -e "$file" ] || { [ -e "$dir" ] && [ ! -x "$dir" ]; } || [ ! -x "$DATA_ABS" ]; then + ops state | tail -n 1 + else + echo sqlite + fi +} + +# 实际用的:只有 builtin 算内置,migrating 还是 SQLite +stored_mode() { + case "$(stored_state)" in builtin) echo builtin ;; *) echo sqlite ;; esac +} + +volume_exists() { docker volume inspect "$1" >/dev/null 2>&1; } + +resolve_mode() { + local stored + stored="$(stored_mode)" + if [ -n "$EXTERNAL_DB" ]; then + if [ "$stored" = builtin ] || [ -n "$WITH_POSTGRES" ]; then + die "两种库只能选一种:agentlab.env 里设了 ${EXTERNAL_DB}(外接 PostgreSQL),这个数据目录又是内置 PostgreSQL$( \ + [ -n "$WITH_POSTGRES" ] && [ "$stored" != builtin ] && printf '(--postgres)')。 + - 用内置的:把 agentlab.env 里的 $EXTERNAL_DB 那一行删掉; + - 用外接的:去掉 --postgres;数据目录已经是内置 PostgreSQL 的,先用 pg_dump 把内置库的数据导出来,换一个新的数据目录再部署" + fi + MODE="external" + elif [ "$stored" = builtin ]; then + MODE="builtin" + elif [ -n "$WITH_POSTGRES" ]; then + if [ -e "$DATA_ABS/agentlab.db" ]; then + die "数据目录 $DATA_ABS 里已经有 SQLite 的库(agentlab.db)。install --postgres 只给新部署用;要把现有数据搬进内置 +PostgreSQL,运行 ./deploy.sh migrate-to-postgres" + fi + MODE="builtin" + else + MODE="sqlite" + fi + check_db_settings + DB_OPTS=() + if [ "$MODE" = builtin ]; then + DB_OPTS=(--network "$DOCKER_NETWORK" + -e "AGENTLAB_DATABASE_URL=postgresql+asyncpg://$PG_USER@$POSTGRES_CONTAINER:5432/$PG_DB" + -e "PGPASSFILE=$PG_PASSFILE") + fi +} + +# agentlab.env 里和库有关的几项是否和用的哪种库对得上(resolve_mode 定了 MODE 之后查;migrate-to-postgres 按切过去的内置库查) +pgbouncer_on() { + case "$(printf '%s' "$PGBOUNCER_VALUE" | tr '[:upper:]' '[:lower:]')" in 1|true|yes|on) return 0 ;; *) return 1 ;; esac +} + +check_db_settings() { + local mode="${1:-$MODE}" + if [ -n "$DIRECT_URL_SET" ] && [ -z "$EXTERNAL_URL_SET" ]; then + die "agentlab.env 里设了 AGENTLAB_DATABASE_DIRECT_URL,却没设 AGENTLAB_DATABASE_URL:直连地址是给外接 PostgreSQL 配 PgBouncer 用的, +要和 AGENTLAB_DATABASE_URL 一起设。不用外接的库就把它删掉" + fi + if [ "$mode" = builtin ] && pgbouncer_on; then + die "agentlab.env 里 AGENTLAB_DB_PGBOUNCER=${PGBOUNCER_VALUE},但内置 PostgreSQL 是直连、中间没有 PgBouncer。把这一行删掉或改成 0" + fi + case "$WORKERS_VALUE" in + ''|*[!0-9]*) ;; + *) + if [ "$mode" = sqlite ] && [ "$WORKERS_VALUE" -gt 1 ]; then + die "agentlab.env 里 AGENTLAB_WORKERS=${WORKERS_VALUE},但库是 SQLite:SQLite 只支持单进程,服务会拒绝启动。 + - 改回 1(或删掉这一行); + - 要多进程,先换成 PostgreSQL:./deploy.sh migrate-to-postgres(内置)或设 AGENTLAB_DATABASE_URL(外接)" + fi ;; + esac +} + +mode_label() { + case "$MODE" in + builtin) echo "内置 PostgreSQL(容器 ${POSTGRES_CONTAINER},数据卷 ${POSTGRES_VOLUME})" ;; + external) echo "外接 PostgreSQL(agentlab.env 里的 ${EXTERNAL_DB})" ;; + *) echo "SQLite(数据目录里的 agentlab.db)" ;; + esac +} + +# 在一个用完就删的容器里、以容器里的用户跑 python -m app.ops(数据目录归它,宿主机上的人不一定写得进去)。 +# start / stop 这类用容器正在用的镜像(这个包的镜像可能已经删了) +OPS_IMAGE="" +ops() { + docker run --rm --network none -v "$DATA_ABS:/data" --entrypoint python "${OPS_IMAGE:-$IMAGE}" -m app.ops "$@" +} + +# --------------------------------------------------------------------------- +# 内置 PostgreSQL:镜像、网络、密码、容器、备份 +# --------------------------------------------------------------------------- +pg_image_present() { docker image inspect "$POSTGRES_IMAGE" >/dev/null 2>&1; } + +ensure_pg_image() { + if pg_image_present; then + return 0 + fi + [ -f "$POSTGRES_IMAGE_TAR" ] || die "本地没有 PostgreSQL 的镜像 ${POSTGRES_IMAGE},包里也没有 postgres-image.tar。 + - 能联网:docker pull ${POSTGRES_IMAGE} + - 离线:用 scripts/package-docker.sh 重新打包(缺省会带上它),或者在能联网的机器上 docker save 一份拷过来 docker load" + verify_tar postgres-image.tar + say "导入 PostgreSQL 镜像:docker load -i postgres-image.tar" + docker load -i "$POSTGRES_IMAGE_TAR" + pg_image_present || die "导入后仍找不到 ${POSTGRES_IMAGE}:postgres-image.tar 里的不是这个标签。agentlab.env 里的 POSTGRES_IMAGE 要和包里的一致" +} + +ensure_network() { + docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1 && return 0 + say "创建 docker 网络 ${DOCKER_NETWORK}(PostgreSQL 只在这个网络里可见,端口不对外)" + docker network create "$DOCKER_NETWORK" >/dev/null +} + +pg_secrets() { + # 第一次生成密码(0600),之后沿用;pgpass 按容器名重写。密码只在数据目录里,不打印 + ops postgres-secrets --host "$POSTGRES_CONTAINER" --port 5432 --db "$PG_DB" --user "$PG_USER" >/dev/null \ + || die "没能在数据目录里生成 PostgreSQL 的密码文件($DATA_ABS/.postgres/)" +} + +check_pg_is_ours() { + local role + role="$(docker inspect -f '{{index .Config.Labels "agentlab.role"}}' "$POSTGRES_CONTAINER" 2>/dev/null || true)" + [ "$role" = postgres ] || die "已经有一个叫 $POSTGRES_CONTAINER 的容器,但它不是 deploy.sh 起的内置 PostgreSQL。 +改一下 agentlab.env 里的 POSTGRES_CONTAINER(缺省是 -postgres)" +} + +run_pg_container() { + docker run -d \ + --name "$POSTGRES_CONTAINER" \ + --label agentlab.role=postgres \ + --restart unless-stopped \ + --network "$DOCKER_NETWORK" \ + -e "POSTGRES_USER=$PG_USER" \ + -e "POSTGRES_DB=$PG_DB" \ + -e POSTGRES_PASSWORD_FILE=/run/agentlab/password \ + -v "$POSTGRES_VOLUME:/var/lib/postgresql/data" \ + -v "$DATA_ABS/.postgres/password:/run/agentlab/password:ro" \ + --health-cmd "pg_isready -h 127.0.0.1 -U $PG_USER -d $PG_DB" \ + --health-interval 5s --health-timeout 5s --health-retries 5 --health-start-period 60s \ + --stop-timeout 60 \ + --log-opt max-size=10m --log-opt max-file=3 \ + "$POSTGRES_IMAGE" >/dev/null +} + +pg_logs_to_stderr() { + echo >&2; echo "---- PostgreSQL 最后 40 行日志 ----" >&2 + docker logs --tail 40 "$POSTGRES_CONTAINER" >&2 2>&1 || true + echo "----" >&2 +} + +# install / upgrade 时 POSTGRES_IMAGE 和现有容器的不一样(新包换了小版本,或者 agentlab.env 里改了):服务先停,库的容器 +# 按新镜像重建,数据卷不动。新镜像起不来(多半是跨了大版本,数据格式不认)就换回原来的镜像,什么都不改 +PG_APPLY_IMAGE="" + +replace_pg_image() { + local cur="$1" was_running="" + [ "$(container_state "$CONTAINER_NAME")" = running ] && was_running=1 + say "内置 PostgreSQL 换镜像:$cur → ${POSTGRES_IMAGE}(数据卷 $POSTGRES_VOLUME 不动)" + stop_app + docker stop "$POSTGRES_CONTAINER" >/dev/null 2>&1 || true + docker rm -f "$POSTGRES_CONTAINER" >/dev/null + if run_pg_container && wait_healthy "$POSTGRES_CONTAINER"; then + return 0 + fi + pg_logs_to_stderr + docker rm -f "$POSTGRES_CONTAINER" >/dev/null 2>&1 || true + local wanted="$POSTGRES_IMAGE" + POSTGRES_IMAGE="$cur" + if run_pg_container && wait_healthy "$POSTGRES_CONTAINER"; then + [ -z "$was_running" ] || docker start "$CONTAINER_NAME" >/dev/null 2>&1 || true + die "内置 PostgreSQL 用 $wanted 起不来(见上面的日志;跨大版本时数据格式不通用,要 pg_dump 导出、新版本里导回), +已换回原来的 ${cur},什么都没改。agentlab.env 里的 POSTGRES_IMAGE 改回 $cur 再运行" + fi + die "内置 PostgreSQL 用 $wanted 起不来,换回 $cur 也没起来。看 ./deploy.sh logs --postgres" +} + +# 起内置库(没有就建,停着就起),等 pg_isready 通过 +start_pg() { + ensure_network + if container_exists "$POSTGRES_CONTAINER"; then + check_pg_is_ours + local cur + cur="$(docker inspect -f '{{.Config.Image}}' "$POSTGRES_CONTAINER")" + if [ -n "$PG_APPLY_IMAGE" ] && [ "$cur" != "$POSTGRES_IMAGE" ]; then + replace_pg_image "$cur" + return 0 + fi + if [ "$(container_state "$POSTGRES_CONTAINER")" != running ]; then + say "启动内置 PostgreSQL(${POSTGRES_CONTAINER})" + docker start "$POSTGRES_CONTAINER" >/dev/null || die "内置 PostgreSQL 的容器起不来,见上面 docker 的报错" + fi + else + say "启动内置 PostgreSQL(${POSTGRES_CONTAINER},${POSTGRES_IMAGE},数据卷 ${POSTGRES_VOLUME})" + run_pg_container || die "PostgreSQL 容器启动失败,见上面 docker 的报错" + fi + if ! wait_healthy "$POSTGRES_CONTAINER"; then + pg_logs_to_stderr + die "内置 PostgreSQL 没就绪(pg_isready 不通过)。看上面的日志" + fi +} + +stop_pg() { + container_exists "$POSTGRES_CONTAINER" || return 0 + [ "$(container_state "$POSTGRES_CONTAINER")" = running ] || return 0 + say "停止内置 PostgreSQL(${POSTGRES_CONTAINER},数据保留)" + docker stop "$POSTGRES_CONTAINER" >/dev/null +} + +stop_app() { + container_exists "$CONTAINER_NAME" || return 0 + [ "$(container_state "$CONTAINER_NAME")" = running ] || return 0 + say "停止 ${CONTAINER_NAME}(数据保留)" + docker stop "$CONTAINER_NAME" >/dev/null +} + +image_user() { docker image inspect -f '{{.Config.User}}' "$IMAGE"; } + +# pg_dump 一份到数据目录的 backups/postgres/,只留最近 BACKUP_KEEP 份。用 PostgreSQL 镜像里的 pg_dump(和库同版本), +# 以数据目录的属主写文件。备份里是整个库:目录 0700(app.ops backup-dir)、文件 0600(umask 077 下建,没有别人读得到的那一刻) +backup_pg() { + local stamp file + stamp="$(date -u +%Y%m%dT%H%M%SZ)" + file="backups/postgres/$PG_DB-$stamp.dump" + say "备份内置 PostgreSQL:pg_dump 到 $DATA_ABS/$file" + ops backup-dir postgres >/dev/null || { warn "没能在数据目录里建备份目录 backups/postgres/"; return 1; } + docker run --rm --network "$DOCKER_NETWORK" -v "$DATA_ABS:/data" --user "$(image_user)" -e "PGPASSFILE=$PG_PASSFILE" \ + --entrypoint sh "$POSTGRES_IMAGE" -c 'umask 077 && exec pg_dump "$@"' pg_dump \ + -h "$POSTGRES_CONTAINER" -U "$PG_USER" -d "$PG_DB" -Fc -f "/data/$file" \ + || { warn "pg_dump 失败"; return 1; } + ops prune --dir /data/backups/postgres --keep "$BACKUP_KEEP" >/dev/null || warn "旧备份没删掉,不影响这次升级" +} + +backup_sqlite() { + local out + say "备份 SQLite 库到 $DATA_ABS/backups/sqlite/" + out="$(ops backup-sqlite --keep "$BACKUP_KEEP")" || { warn "SQLite 库备份失败"; return 1; } + # 容器里看到的是 /data/…,换成宿主机上的路径 + case "$out" in + /data/*) say "备份好了:${out/#\/data/$DATA_ABS}" ;; + *) say "$out" ;; + esac +} + +backup_before_upgrade() { + case "$MODE" in + builtin) backup_pg ;; + sqlite) backup_sqlite ;; + external) say "外接 PostgreSQL:升级前请自己备份那个库(pg_dump),这里不替你做" ;; + esac +} + # --------------------------------------------------------------------------- # 子命令 # --------------------------------------------------------------------------- @@ -498,6 +841,26 @@ cmd_install() { prepare_data check_existing fix_data_owner + resolve_mode + say "应用库:$(mode_label)" + PG_APPLY_IMAGE=1 + if [ "$MODE" = builtin ]; then + ensure_pg_image + if [ "$(stored_mode)" != builtin ]; then + # 新部署:数据卷要是已经有了,里面的库是按别的密码初始化的(多半是以前 uninstall 留下的),连不上,也不该拿来用 + volume_exists "$POSTGRES_VOLUME" && die "docker 数据卷 $POSTGRES_VOLUME 已经存在,不是这个数据目录建的(多半是以前卸载时留下的 +内置 PostgreSQL 的数据)。 + - 要接着用那份数据:把 DATA_DIR 指回原来的数据目录再 ./deploy.sh install; + - 确定不要了:docker volume rm ${POSTGRES_VOLUME}(不可恢复),再运行; + - 两份都留着:agentlab.env 里另起一个 POSTGRES_VOLUME 的名字" + pg_secrets + # 先记下再起库:新数据目录里没有 SQLite 的库,没什么可「切」的;起库失败了重来也认得出数据卷是自己的 + ops state --set builtin >/dev/null || die "没能在数据目录里记下「用内置 PostgreSQL」" + else + pg_secrets + fi + start_pg + fi decide_microvm deploy_container } @@ -519,39 +882,190 @@ cmd_upgrade() { prepare_data check_existing fix_data_owner + resolve_mode + say "应用库:$(mode_label)" + PG_APPLY_IMAGE=1 + if [ "$MODE" = builtin ]; then + ensure_pg_image + pg_secrets + start_pg + fi decide_microvm + # 先停旧服务再备份:备份的是升级前最后的样子。备份失败就把旧服务起回来、不升级 + stop_app + if ! backup_before_upgrade; then + docker start "$CONTAINER_NAME" >/dev/null 2>&1 || true + die "升级前的备份没做成,没有升级,原来的服务已经起回来了" + fi + # 新版本启动时自动补表结构(迁移)和默认数据,几个工作进程同时起也只做一次 deploy_container if [ "$old_image" != "$IMAGE" ] && docker image inspect "$old_image" >/dev/null 2>&1; then echo "旧镜像 $old_image 还留着,方便回退。确认新版本没问题后可以删:docker rmi $old_image" fi } -cmd_start() { - check_docker; read_env; require_container - docker start "$CONTAINER_NAME" >/dev/null +# 把正在用 SQLite 的部署切到内置 PostgreSQL。记下「已切换」之前任何一步出错:服务按 SQLite 原样恢复(原来在跑就起回来)、 +# 内置库的容器停掉,数据目录里只多了 backups/ 里的一份备份和 .postgres/ 里的密码、状态(migrating)。重复执行:已经切过的 +# 什么都不做;上次没切成留在内置库里的半份数据先清空再导,不会和 SQLite 里后来改过的混在一起 +MIGRATE_WAS_RUNNING="" +MIGRATE_HAD_APP="" +MIGRATE_DEPLOYING="" + +migrate_rollback() { + local prev="$CONTAINER_NAME-previous" + echo >&2 + warn "$1" + if [ -n "$MIGRATE_DEPLOYING" ]; then + # 按内置库起的新容器不能留着(它带着 --restart,会一直去连内置库):换回原来的,原来就没有就删掉 + if container_exists "$prev"; then + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + docker rename "$prev" "$CONTAINER_NAME" >/dev/null 2>&1 || warn "没能把原来的容器改回 ${CONTAINER_NAME}:docker rename $prev $CONTAINER_NAME" + elif [ -z "$MIGRATE_HAD_APP" ]; then + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + fi + # 已经记成 builtin 的改回 migrating:仍用 SQLite,数据卷还认得出是这个数据目录的 + ops state --set migrating >/dev/null 2>&1 || warn "没能把数据目录的状态改回去:手动把 $DATA_ABS/$PG_STATE_REL 里的 mode=builtin 改成 mode=migrating" + fi + if container_exists "$CONTAINER_NAME"; then + if [ -n "$MIGRATE_WAS_RUNNING" ]; then + docker start "$CONTAINER_NAME" >/dev/null 2>&1 && say "服务已按 SQLite 起回来" || warn "服务没能起回来:./deploy.sh start" + else + docker stop "$CONTAINER_NAME" >/dev/null 2>&1 || true + fi + fi + stop_pg + die "没有切换,仍用 SQLite(数据目录里的 agentlab.db 没动过)。照上面的报错处理好之后重新运行 ./deploy.sh migrate-to-postgres, +重复执行是安全的:内置库里这次导了一半的会先清空再导" +} + +copy_to_pg() { + # --assume-stopped:服务是这里刚停的(强杀留下的 -wal 也不挡);快照放数据目录里(和库同一块盘,空间够)。 + # 内置库是直连(没有 PgBouncer),断点跟着目标地址走,不用 --direct-url。 + # 逐表的结果里来源是空表的几十行不列(没什么可核对的),别的原样打出来;退出码照 copy-to-postgres 的(pipefail) + docker run --rm --network "$DOCKER_NETWORK" -v "$DATA_ABS:/data" -e "PGPASSFILE=$PG_PASSFILE" \ + --entrypoint python "$IMAGE" -m app.db copy-to-postgres /data \ + "postgresql+asyncpg://$PG_USER@$POSTGRES_CONTAINER:5432/$PG_DB" --assume-stopped --work-dir /data "$@" \ + | grep -v ' 空表$' +} + +cmd_migrate_to_postgres() { + check_docker + check_arch + read_env + ensure_image + prepare_data + check_existing + [ -z "$EXTERNAL_DB" ] || die "agentlab.env 里设了 ${EXTERNAL_DB}(外接 PostgreSQL)。内置和外接只能选一种:要切到内置的,先删掉那一行; +要搬进外接的库,用镜像里的 python -m app.db copy-to-postgres(见 README「用 PostgreSQL」)" + # 切过去就是内置库:和它对不上的设置(PgBouncer 开关)先拦下,别停了服务才发现 + check_db_settings builtin + local state + state="$(stored_state)" + if [ "$state" = builtin ]; then + say "这个数据目录已经切到内置 PostgreSQL 了,什么都不用做(./deploy.sh status 看状态)" + return 0 + fi + [ -e "$DATA_ABS/agentlab.db" ] || die "数据目录 $DATA_ABS 里没有 SQLite 的库(agentlab.db),没有可搬的。新部署直接用 ./deploy.sh install --postgres" + # 数据卷已经有了、又不是这个数据目录上次没切成留下的:里面可能是别人的数据,下面要清空它,不能碰 + if [ "$state" != migrating ] && volume_exists "$POSTGRES_VOLUME"; then + die "docker 数据卷 $POSTGRES_VOLUME 已经存在,不是这个数据目录上次没切成留下的(多半是以前卸载时留下的内置 PostgreSQL +的数据)。切换要清空内置库,为免删错数据,没有动它。 + - 确定不要了:docker volume rm ${POSTGRES_VOLUME}(不可恢复),再运行; + - 要留着:agentlab.env 里另起一个 POSTGRES_VOLUME 的名字" + fi + if container_exists "$POSTGRES_CONTAINER"; then check_pg_is_ours; fi + fix_data_owner + ensure_pg_image + PG_APPLY_IMAGE=1 + container_exists "$CONTAINER_NAME" && MIGRATE_HAD_APP=1 + [ "$(container_state "$CONTAINER_NAME")" = running ] && MIGRATE_WAS_RUNNING=1 + + say "1/6 停服务(导入要在服务停着时做)" + stop_app + say "2/6 备份 SQLite 库" + backup_sqlite || migrate_rollback "SQLite 库备份失败" + say "3/6 起内置 PostgreSQL" + ( pg_secrets ) || migrate_rollback "没能生成内置 PostgreSQL 的密码" + ops state --set migrating >/dev/null || migrate_rollback "没能在数据目录里记下「开始切换」" + ( start_pg ) || migrate_rollback "内置 PostgreSQL 没起来" + # 内置库里要是有东西,只能是上次没切成留下的半份(上面查过数据卷),清空重来 + docker exec "$POSTGRES_CONTAINER" psql -q -U "$PG_USER" -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS \"$PG_DB\" WITH (FORCE)" -c "CREATE DATABASE \"$PG_DB\" OWNER \"$PG_USER\"" >/dev/null \ + || migrate_rollback "没能清空内置库里上次留下的数据" + say "4/6 先核对(只数行数,不写)" + copy_to_pg --dry-run || migrate_rollback "核对没过" + say "5/6 导入并逐表核对" + copy_to_pg || migrate_rollback "导入或核对没过" + MIGRATE_DEPLOYING=1 + ops state --set builtin >/dev/null || migrate_rollback "没能在数据目录里记下「已切到内置 PostgreSQL」" + say "6/6 按内置 PostgreSQL 起服务" + resolve_mode + decide_microvm + ( deploy_container ) || migrate_rollback "按内置 PostgreSQL 起的服务没通过健康检查" + echo + say "已切到内置 PostgreSQL。SQLite 的库文件(agentlab.db、checkpoints.db)原样留在 $DATA_ABS 里、不再使用," + echo " 另有一份备份在 backups/sqlite/。用一段时间确认没问题后再删(见 README「用 PostgreSQL」)" +} + +# start / stop / restart / status / logs / uninstall:数据目录看容器实际挂的,没有容器再看配置 +# 这几个只管容器,不改配置:模式照数据目录里记的认,agentlab.env 和它冲突(内置的又设了外接地址)也不在这里拦,留给 install +locate_data() { + DATA_ABS="" + if container_exists "$CONTAINER_NAME"; then + DATA_ABS="$(data_source_of "$CONTAINER_NAME")" + OPS_IMAGE="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER_NAME")" + fi + if [ -z "$DATA_ABS" ] && [ -d "$DATA_PATH" ]; then DATA_ABS="$(cd "$DATA_PATH" && pwd -P)"; fi + MODE="sqlite" + if [ -n "$DATA_ABS" ] && [ "$(stored_mode)" = builtin ]; then + MODE="builtin" + elif [ -n "$EXTERNAL_DB" ]; then + MODE="external" + fi +} + +wait_app() { if wait_healthy "$CONTAINER_NAME"; then - say "已启动"; print_access + say "$1"; print_access else show_failure; die "容器没通过健康检查" fi } +cmd_start() { + check_docker; read_env; require_container; locate_data + [ "$MODE" != builtin ] || start_pg + docker start "$CONTAINER_NAME" >/dev/null + wait_app "已启动" +} + cmd_stop() { - check_docker; read_env; require_container + check_docker; read_env; require_container; locate_data + # 先停服务再停库:服务收尾时还要写库 say "停止 ${CONTAINER_NAME}(数据保留)" docker stop "$CONTAINER_NAME" >/dev/null + [ "$MODE" != builtin ] || stop_pg say "已停止。开机也不会自动起来,要用时运行 ./deploy.sh start" } +# 缺省只重启服务:库不用跟着断一次(库停着的话先把它起来,服务才连得上)。--postgres 连库一起重启 cmd_restart() { - check_docker; read_env; require_container - say "重启 $CONTAINER_NAME" - docker restart "$CONTAINER_NAME" >/dev/null - if wait_healthy "$CONTAINER_NAME"; then - say "已重启"; print_access + check_docker; read_env; require_container; locate_data + if [ -n "$RESTART_POSTGRES" ]; then + [ "$MODE" = builtin ] || die "这个部署没用内置 PostgreSQL,--postgres 没有可重启的库。只重启服务:./deploy.sh restart" + say "重启 $CONTAINER_NAME 和内置 PostgreSQL(${POSTGRES_CONTAINER})" + stop_app + stop_pg + start_pg + docker start "$CONTAINER_NAME" >/dev/null else - show_failure; die "容器没通过健康检查" + say "重启 $CONTAINER_NAME" + if [ "$MODE" = builtin ] && [ "$(container_state "$POSTGRES_CONTAINER")" != running ]; then + start_pg + fi + docker restart "$CONTAINER_NAME" >/dev/null fi + wait_app "已重启" } cmd_status() { @@ -560,6 +1074,7 @@ cmd_status() { echo "没有名为 $CONTAINER_NAME 的容器(还没部署,或已卸载)。部署:./deploy.sh install" exit 1 fi + locate_data local state health image src state="$(container_state "$CONTAINER_NAME")" health="" @@ -572,22 +1087,42 @@ cmd_status() { echo "镜像 $image" echo "端口 $(docker port "$CONTAINER_NAME" 8000/tcp 2>/dev/null | tr '\n' ' ')" echo "数据目录 ${src:-(未挂载)}" + echo "应用库 $(mode_label)" + if [ -n "$DATA_ABS" ] && [ "$MODE" = sqlite ] && [ "$(stored_state)" = migrating ]; then + echo " 上次 migrate-to-postgres 没切成(仍用 SQLite),数据卷 $POSTGRES_VOLUME 里留着导了一半的;重新运行会清空重来" + fi + if [ "$MODE" = builtin ]; then + local pstate phealth pimage + pstate="$(container_state "$POSTGRES_CONTAINER")" + phealth="" + [ "$pstate" != running ] || phealth="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$POSTGRES_CONTAINER")" + pimage="$(docker inspect -f '{{.Config.Image}}' "$POSTGRES_CONTAINER" 2>/dev/null || echo "$POSTGRES_IMAGE")" + echo "内置库 $POSTGRES_CONTAINER ${pstate}${phealth:+(pg_isready:${phealth})},镜像 ${pimage},网络 ${DOCKER_NETWORK}(端口不对外)" + echo "库的备份 ${src:-$DATA_ABS}/backups/postgres/(upgrade 前自动 pg_dump,留最近 $BACKUP_KEEP 份)" + fi [ "$state" != running ] || echo "代码沙箱 $(sandbox_summary)" echo "这个包 $IMAGE" if [ "$state" = running ]; then print_access; fi } cmd_logs() { - check_docker; read_env; require_container + check_docker; read_env + local target="$CONTAINER_NAME" + if [ -n "$LOGS_POSTGRES" ]; then + target="$POSTGRES_CONTAINER" + container_exists "$target" || die "没有名为 $target 的容器:这个部署没用内置 PostgreSQL" + else + require_container + fi local args=(--tail "$TAIL") [ -z "$FOLLOW" ] || args+=(--follow) # exec:Ctrl-C 直接交给 docker logs,不留一个还在跟随的子进程 - exec docker logs "${args[@]}" "$CONTAINER_NAME" + exec docker logs "${args[@]}" "$target" } cmd_uninstall() { check_docker; read_env - local src="" name + local src="" name pg="" if container_exists "$CONTAINER_NAME"; then check_is_agentlab src="$(data_source_of "$CONTAINER_NAME")" @@ -598,23 +1133,40 @@ cmd_uninstall() { docker rm -f "$name" >/dev/null fi done + if container_exists "$POSTGRES_CONTAINER"; then + check_pg_is_ours + say "删除内置 PostgreSQL 的容器 ${POSTGRES_CONTAINER}(数据卷 $POSTGRES_VOLUME 留着)" + docker rm -f "$POSTGRES_CONTAINER" >/dev/null + pg=1 + fi + if docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1; then + docker network rm "$DOCKER_NETWORK" >/dev/null 2>&1 && say "删除 docker 网络 $DOCKER_NETWORK" \ + || warn "docker 网络 $DOCKER_NETWORK 没删掉(可能还有别的容器连着)" + fi if [ -n "$PURGE_IMAGE" ]; then if image_present; then say "删除镜像 $IMAGE" docker rmi "$IMAGE" >/dev/null || warn "镜像 $IMAGE 没删掉(可能还有别的容器在用)" fi + # PostgreSQL 的镜像不删:本机别的东西可能也在用同一个镜像 else echo "镜像 $IMAGE 保留着,重新部署不用再导入。要删:./deploy.sh uninstall --purge-image" fi [ -n "$src" ] || src="$DATA_PATH" echo echo "数据没有删,还在:$src" - echo "确定不要了再手动删除(不可恢复):sudo rm -rf '$src'" + if [ -n "$pg" ] || docker volume inspect "$POSTGRES_VOLUME" >/dev/null 2>&1; then + echo "内置 PostgreSQL 的数据在 docker 数据卷 $POSTGRES_VOLUME 里,也没有删" + echo "确定不要了再手动删除(不可恢复):sudo rm -rf '$src' && docker volume rm $POSTGRES_VOLUME" + else + echo "确定不要了再手动删除(不可恢复):sudo rm -rf '$src'" + fi } case "$CMD" in install) cmd_install ;; upgrade) cmd_upgrade ;; + migrate-to-postgres) cmd_migrate_to_postgres ;; start) cmd_start ;; stop) cmd_stop ;; restart) cmd_restart ;; diff --git a/environment.yml b/environment.yml index 5d995bdb..f173db38 100644 --- a/environment.yml +++ b/environment.yml @@ -20,4 +20,5 @@ dependencies: # 沙箱会退回 Seatbelt —— 代价是内存限额在 macOS 上限不住。 # [docs] 让 PDF / Word 传得进知识库。约十几兆,但少了它上传这类文件 # 只会得到一句"要先装解析库"——而界面上并不会拦着你去传 - - -e ./backend[microvm,dev,db,docs] + # [postgres] 应用库用 PostgreSQL 时要的驱动和断点库(AGENTLAB_DATABASE_URL),不设那一项就用不上 + - -e ./backend[microvm,dev,db,docs,postgres] diff --git a/frontend/src/api/chatTurns.ts b/frontend/src/api/chatTurns.ts index e3990c9b..34bf8c3d 100644 --- a/frontend/src/api/chatTurns.ts +++ b/frontend/src/api/chatTurns.ts @@ -125,7 +125,12 @@ export function streamChatTurn( controller.abort() return } - if (typeof event.seq !== 'number' || event.seq <= lastSeq) continue // 重连时重叠的那段只要一份 + if (typeof event.seq !== 'number') { + // 助手等模型时的心跳不落库、不带 seq(服务端 chat/log.beat):照收、不动 lastSeq,重连时服务端会补最近的一拍 + if (event.type === 'op' && event.data?.op === 'heartbeat' && !closed) onEvent(event as ChatTurnEvent) + continue + } + if (event.seq <= lastSeq) continue // 重连时重叠的那段只要一份 lastSeq = event.seq got = true if (!closed) onEvent(event as ChatTurnEvent) diff --git a/frontend/src/api/schedules.ts b/frontend/src/api/schedules.ts index d92e3015..98faad1f 100644 --- a/frontend/src/api/schedules.ts +++ b/frontend/src/api/schedules.ts @@ -36,8 +36,12 @@ export const schedules = { */ eventVariables: (event: ScheduleEventKind, opts?: RequestOptions) => get(`/schedules/variables${qs({ trigger_kind: 'event', event })}`, opts), - /** 新建、修改事件触发任务时能选的对象(能不能选、为什么由服务端判)、来源、应用。老服务端没有,回 404 */ - triggerOptions: (opts?: RequestOptions) => get('/schedules/trigger-options', opts), + /** + * 新建、修改事件触发任务时能选的对象(能不能选、为什么由服务端判)、来源、应用。老服务端没有,回 404。对象一页有上限 + * (more 为真时没给完):按事件种类、名字里的关键字再查;selected 是当前选中的那一个,总在结果里 + */ + triggerOptions: (find?: { event?: ScheduleEventKind; q?: string; selected?: string }, opts?: RequestOptions) => + get(`/schedules/trigger-options${qs(find)}`, opts), /** 立即运行一次:记一条手动触发,上一次还没结束时照样按重叠策略跳过 */ run: (id: string) => post(`${at(id)}/run`), /** 触发历史,新的在前。before 是上一页给的 next_before */ diff --git a/frontend/src/canvas/RecallPanel.tsx b/frontend/src/canvas/RecallPanel.tsx index a09532fd..3e29fe6d 100644 --- a/frontend/src/canvas/RecallPanel.tsx +++ b/frontend/src/canvas/RecallPanel.tsx @@ -79,6 +79,8 @@ export function RecallPreviewRow({ workflowId, version }: { workflowId: string; const STATE_COLOR: Record = { changed: 'var(--st-waiting)', failed: 'var(--err)', not_recomputable: 'var(--text-faint)', unchanged: 'var(--ok)', + // 核对不了(行序不定):要人去核对,和「数变了」同一种提醒色 + row_order: 'var(--st-waiting)', } /** 列表先列几组,其余收起 */ const SHOWN = 6 diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 4c4d5056..55bbbfe6 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -5075,6 +5075,8 @@ export const DUAL_CHECK_TEXT = { /** 认不出任何指标时留的出口:驳回后节点按人工驳回失败 */ reject: '驳回并终止', emptyReject: '驳回后节点失败,本次运行终止。可以在备注中说明原因', + /** 节点接了驳回出口(中断内容的 reject_route):驳回后转到那一步 */ + emptyRejectRoute: (goes: string) => `驳回后${goes}。可以在备注中说明原因`, rejected: '已驳回,本次运行终止', }, diff --git a/frontend/src/lib/termsApp.ts b/frontend/src/lib/termsApp.ts index 9f7fb8c4..d17b8762 100644 --- a/frontend/src/lib/termsApp.ts +++ b/frontend/src/lib/termsApp.ts @@ -110,7 +110,7 @@ export const TASK_TEXT = { group: (name: string) => `组「${name}」`, }, assignmentStatus: { pending: '待回复', responded: '已回复', withdrawn: '已撤回' }, - decision: { approve: '同意', reject: '驳回', submit: '已提交', confirmed: '已生效', not_applied_resend: '重新发送', + decision: { approve: '同意', reject: '驳回', expired_reject: '超时驳回', submit: '已提交', confirmed: '已生效', not_applied_resend: '重新发送', not_applied_abandon: '放弃' } as Record, verified: '已登录核实', unverified: '自报署名', @@ -125,9 +125,13 @@ export const TASK_TEXT = { editedContent: '修改后的内容', approveHint: '批准:继续运行;驳回:转入「驳回」分支', inputHint: '提交:以你填写的内容继续运行;驳回:终止本次运行', + /** 驳回之后去哪(中断内容的 reject_route,见 REJECT_ROUTE_TEXT):带了的按它说,老待办没带的照上面两句 */ + approveHintRoute: (goes: string) => `批准:继续运行;驳回:${goes}`, + inputHintRoute: (goes: string) => `提交:以你填写的内容继续运行;驳回:${goes}`, doneApprove: '已批准,运行继续', doneReject: '已驳回,转入驳回分支', doneRejectStop: '已驳回,本次运行终止', + doneRejectRoute: (goes: string) => `已驳回,${goes}`, doneSubmit: '已提交,运行继续', doneVote: '已记下你的意见,等待其他人处理', formInvalid: (n: number) => `有 ${n} 项需要修改`, @@ -154,6 +158,34 @@ export const TASK_TEXT = { code: '代码', } +/** + * 驳回之后去哪:待办的中断内容带着 reject_route(服务端 engine/replay.reject_route,总方案 §3 的驳回出口)。 + * - to:接了驳回出口,驳回后转到这几步(名字是这次运行里的说法:子工作流里的写「子工作流 › x」); + * - ends=run:没接驳回出口(审批设了驳回即终止的也是),驳回后运行就此结束、记为被驳回; + * - ends=path:审批没接驳回出口、没设驳回即终止,这一路到审批为止,不再往下走。 + * 老待办没带的为 null,各处照原来的说法 + */ +export type RejectRoute = { to: string[] } | { ends: 'run' | 'path' } + +export function rejectRouteOf(payload: Record | null | undefined): RejectRoute | null { + const raw = payload?.reject_route + if (!raw || typeof raw !== 'object') return null + const r = raw as { to?: unknown; ends?: unknown } + if (Array.isArray(r.to)) { + const to = r.to.map((t: any) => String(t?.label || t?.node_id || '')).filter(Boolean) + return to.length ? { to } : null + } + return r.ends === 'run' || r.ends === 'path' ? { ends: r.ends } : null +} + +export const REJECT_ROUTE_TEXT = { + /** 「转到「通知申请人」」/「本次运行终止」/「这一路到此为止,不再往下走」 */ + goes: (r: RejectRoute): string => ('to' in r ? `转到${r.to.map((t) => `「${t}」`).join('、')}` + : r.ends === 'run' ? '本次运行终止' : '这一路到此为止,不再往下走'), + /** 处理过的待办「结果」那一格:驳回之后去了哪 */ + outcome: (decision: string, r: RejectRoute): string => `${decision},${REJECT_ROUTE_TEXT.goes(r)}`, +} + /** 动作确认的哈希卡(§3.8:确认时看过的就是发出去的) */ export const ACTION_CONFIRM_TEXT = { head: '确认后发送', @@ -171,6 +203,8 @@ export const ACTION_CONFIRM_TEXT = { noHash: '缺少内容哈希,无法确认发送', done: '已确认,按此内容发送', rejected: '已驳回,不发送', + /** 驳回之后去哪(reject_route):「已驳回,不发送,转到「通知申请人」」 */ + rejectedRoute: (goes: string) => `已驳回,不发送,${goes}`, } /** 核实结果未知的动作:三个选项(§2.5 A5 第 4 条) */ diff --git a/frontend/src/lib/termsConcurrency.ts b/frontend/src/lib/termsConcurrency.ts new file mode 100644 index 00000000..0b192c6c --- /dev/null +++ b/frontend/src/lib/termsConcurrency.ts @@ -0,0 +1,33 @@ +/** + * 设置页「运行并发」一节的叫法(服务端 app/cluster/slots.py:交互、应用、定时、系统四类,各有全站上限和优先级,另有全站总上限)。 + * 叫法同服务端的 CLASS_LABEL;取值范围、合不合规只在服务端判(422 的原话落在这一节下面),界面不另判一套。 + */ +export const CONCURRENCY_TEXT = { + title: '运行并发', + hint: '全站同时执行的运行按来源分四类,各有上限和优先级:名额满了时先给优先级高的一类,已经在执行的不打断。多进程部署时几个进程合起来算', + maxTotal: '全站同时运行的上限', + reserve: '给交互留底', + reserveHint: '全站名额里永远留给交互类的个数,应用、定时、系统用不到这一部分:有人问数据、试运行时总有名额。至多全站上限减 1', + borrow: (n: number) => `应用、定时、系统合起来最多借用 ${n} 个名额(全站上限减留底):没人问数据时评测可以同时多开几题`, + limitHint: '各类的上限可以调小;应用、定时、系统再大也借不到留底那一部分', + limit: '上限', + priority: '优先级', + priorityHint: '数字大的先拿到空出来的名额(0 到 100)', + classes: { + interactive: { label: '交互', hint: '问数据、画布试运行、手动发起的正式运行、蓝图试运行、个人应用的刷新' }, + app: { label: '应用', hint: '应用入口、按需视图、外部系统的钩子、接口调用' }, + schedule: { label: '定时', hint: '定时任务和事件触发' }, + system: { label: '系统', hint: '评测、口径召回、对账、补算、按口径正式出具' }, + } as Record, + now: (running: number, waiting: number) => `现在 ${running} 个在执行${waiting ? `、${waiting} 个在等名额` : ''}`, + save: '保存运行并发设置', + saving: '正在保存…', + saved: '已保存运行并发设置', + discard: '放弃修改', + readOnly: '只有管理员能修改运行并发设置', + leaveTitle: (n: number) => `运行并发设置有 ${n} 项还没保存`, + leaveConsequences: ['离开后这些修改会丢失'], +} + +/** 四类的先后(同服务端 CLASSES) */ +export const CONCURRENCY_CLASSES = ['interactive', 'app', 'schedule', 'system'] as const diff --git a/frontend/src/lib/termsLedger.ts b/frontend/src/lib/termsLedger.ts index 5be516de..29165b0b 100644 --- a/frontend/src/lib/termsLedger.ts +++ b/frontend/src/lib/termsLedger.ts @@ -87,6 +87,9 @@ export const LEDGER_UI = { maskedPlaceholder: '已遮罩,请重新填写', pairs: '选择要对照的指标', pairsHint: '认不出这次的哪一列对应口径的哪个指标,请为要对照的列选一个口径卡上的指标', + /** 有预选的:按逐格比对证明了的那几列预选上,人可以改 */ + pairsSuggested: '已按逐格比对的结论预选了能证明对应的列,请核对,可以改选', + pairSuggestedTag: '预选', pairNone: '不对照', pairEmpty: '至少选一组对照', launch: '发起对账', @@ -130,6 +133,8 @@ export const LEDGER_UI = { recall: { /** 口径升版后在同一份数据上重算过、数没变 */ checked: (caliber: string) => `已按口径${caliber} 核对,不影响`, + /** 部分指标行序不定:那几项没能核对,照实列出 */ + checkedRowOrder: (names: string[]) => `其中${names.map((x) => `「${x}」`).join('、')}行序不定,没能核对`, /** 依据有变:目录里影响数字的项被驳回或修改、出现新码值、处理变化时改了目录(目录类),表或列被删、类型变了(表结构) */ basis: (what: 'catalog' | 'structure' | 'other', items: number) => `${what === 'structure' ? '依据的表结构有变' : what === 'catalog' ? '依据的数据目录有变' : '答案的依据有变'}` diff --git a/frontend/src/lib/termsRecall.ts b/frontend/src/lib/termsRecall.ts index 0cc978be..e15ef124 100644 --- a/frontend/src/lib/termsRecall.ts +++ b/frontend/src/lib/termsRecall.ts @@ -58,6 +58,7 @@ export const RECALL_TEXT = { const parts = [ c.changed ? `数变了 ${n(c.changed)} 组` : '', c.unchanged ? `没变 ${n(c.unchanged)} 组` : '', + c.row_order ? `核对不了(行序不定)${n(c.row_order)} 组` : '', c.running ? `正在重算 ${n(c.running)} 组` : '', c.failed ? `重算失败 ${n(c.failed)} 组` : '', c.not_recomputable ? `无法重算 ${n(c.not_recomputable)} 组` : '', @@ -65,7 +66,8 @@ export const RECALL_TEXT = { return parts.join(',') }, state: { - changed: '数变了', unchanged: '没变', failed: '重算失败', running: '正在重算', pending: '未重算', + changed: '数变了', unchanged: '没变', row_order: '核对不了(行序不定)', failed: '重算失败', running: '正在重算', + pending: '未重算', not_recomputable: '无法重算', } as Record, reason: (state: string, reason: string) => (state === 'not_recomputable' ? `无法重算:${reason}` : reason), @@ -99,6 +101,8 @@ export const RECALL_TEXT = { original: '原答案当时的值', changed: '变了', same: '没变', + rowOrder: '行序不定', + rowOrderHint: '取的是多行结果里按行号的一格,查询没有排序,重新执行时这一行可能是另一行,两版按位置比不出变没变', masked: '已遮罩', missing: '—', answers: '用到这份口径的答案', diff --git a/frontend/src/lib/termsSource.ts b/frontend/src/lib/termsSource.ts index cd04b605..c10fc9be 100644 --- a/frontend/src/lib/termsSource.ts +++ b/frontend/src/lib/termsSource.ts @@ -110,6 +110,11 @@ export const SOURCE_CHANGE_TEXT = { impactNeeds: '人工确认过、还说着被删的列', impactRules: '有效记录条件用到了这一列:取值有变时,按它过滤的结果可能漏掉新状态的行', impactTemplates: (n: number) => `用到这张表的已发布模板 ${formatNumber(n)} 个`, + /** 表结构变了之后没能试算的口径:缺参数的写缺哪几个,别的写原因原话 */ + probesTitle: '没有试算的口径', + probeSkipped: (name: string, version: number, missing: string[], reason: string) => + `「${name}」v${version}:${missing.length ? `缺少参数 ${missing.join('、')},没有试算` : `没有试算(${reason})`}`, + probeFrom: { formal: '参数按最近一次正式运行取', answer: '参数按等价的问数回答取', defaults: '这一版还没有正式运行,只能用入口的缺省值' } as Record, templateImpact: { direct: '写死的 SQL 直接引用', possible: '可能涉及' } as Record, templateVersion: (v: number) => `v${v}`, breaks: (cols: string[]) => (cols.length diff --git a/frontend/src/lib/termsTrigger.ts b/frontend/src/lib/termsTrigger.ts index 2a94ad32..caeff647 100644 --- a/frontend/src/lib/termsTrigger.ts +++ b/frontend/src/lib/termsTrigger.ts @@ -43,6 +43,10 @@ export const TRIGGER_TEXT = { /** 不能选的对象:名字后面接服务端给的原因 */ subjectNotOk: (name: string, reason: string) => `${name}(${reason})`, subjectGone: (name: string) => `${name}(已不存在)`, + /** 对象太多、一页没列完:按名字查找 */ + subjectSearch: (label: string) => `按名称查找${label}`, + subjectSearchHint: '只列出了一部分,输入名称中的字查找', + subjectSearchNone: '没有名称中含这些字的', optionsLoading: '正在读取可选的对象…', optionsFailed: '无法读取可选的对象', optionsUnsupported: '服务端暂不支持事件触发,请升级服务端后再使用', diff --git a/frontend/src/pages/ConcurrencySettings.tsx b/frontend/src/pages/ConcurrencySettings.tsx new file mode 100644 index 00000000..7bcc18f7 --- /dev/null +++ b/frontend/src/pages/ConcurrencySettings.tsx @@ -0,0 +1,159 @@ +import { useEffect, useState } from 'react' +import { api } from '../api/client' +import { Field, Notice, SectionBar, Spinner, confirmDialog } from '../components/ui' +import { errorMessage } from '../lib/errors' +import { useLeaveGuard } from '../lib/leave' +import { CONCURRENCY_CLASSES, CONCURRENCY_TEXT } from '../lib/termsConcurrency' +import { atLeast, usePrincipal } from '../store/auth' + +const T = CONCURRENCY_TEXT +type Klass = (typeof CONCURRENCY_CLASSES)[number] +interface Draft { total: string; reserve: string; limits: Record; priorities: Record } +type Now = Partial> + +const draftOf = (group: Record | undefined): Draft => ({ + total: group?.max_total == null ? '' : String(group.max_total), + reserve: group?.interactive_reserve == null ? '' : String(group.interactive_reserve), + limits: Object.fromEntries(CONCURRENCY_CLASSES.map((k) => [k, String(group?.classes?.[k]?.limit ?? '')])) as Record, + priorities: Object.fromEntries(CONCURRENCY_CLASSES.map((k) => [k, String(group?.classes?.[k]?.priority ?? '')])) as Record, +}) + +const changedCount = (a: Draft, b: Draft): number => + (a.total.trim() !== b.total ? 1 : 0) + + (a.reserve.trim() !== b.reserve ? 1 : 0) + + CONCURRENCY_CLASSES.filter((k) => a.limits[k].trim() !== b.limits[k]).length + + CONCURRENCY_CLASSES.filter((k) => a.priorities[k].trim() !== b.priorities[k]).length + +/** + * 设置页「偏好设置」里「运行并发」一节(docs/design/orchestration-round §5 W2-b):全站总上限,交互、应用、定时、系统四类各自的 + * 上限和优先级,旁边写各类现在在跑、在等几个(/system 的 cluster.runs)。 + * + * 自己读、自己存(PUT /settings 只带 concurrency 一组),带自己的离开前确认,同评测那一节的做法。搭建者只读、管理员写 + */ +export function ConcurrencySettingsSection() { + const me = usePrincipal() + const admin = atLeast(me, 'admin') + const [saved, setSaved] = useState(null) + const [draft, setDraft] = useState(null) + const [now, setNow] = useState(null) + const [loadError, setLoadError] = useState(null) + const [saving, setSaving] = useState(false) + const [error, setError] = useState(null) + const [flash, setFlash] = useState(false) + + const loadNow = () => api.system().then((s) => setNow(s?.cluster?.runs?.classes ?? null), () => setNow(null)) + useEffect(() => { + api.settings.get().then((v) => { + setSaved(draftOf(v?.concurrency)) + setLoadError(null) + }, setLoadError) + void loadNow() + }, []) + + const cur = draft ?? saved + const changed = !!(cur && saved) && changedCount(cur, saved) > 0 + useLeaveGuard(changed && !saving, () => confirmDialog({ + title: T.leaveTitle(cur && saved ? changedCount(cur, saved) : 0), + consequences: T.leaveConsequences, confirmLabel: T.discard, danger: true, + })) + + const save = async () => { + if (!cur || !changed) return + setSaving(true) + setError(null) + try { + const value = { + max_total: Number(cur.total.trim()), + interactive_reserve: Number(cur.reserve.trim()), + classes: Object.fromEntries(CONCURRENCY_CLASSES.map((k) => [k, { + limit: Number(cur.limits[k].trim()), priority: Number(cur.priorities[k].trim()), + }])), + } + const out = await api.settings.put({ concurrency: value }) + setSaved(draftOf(out?.concurrency ?? value)) + setDraft(null) + setFlash(true) + setTimeout(() => setFlash(false), 1600) + void loadNow() + } catch (e) { + setError(errorMessage(e)) + } finally { + setSaving(false) + } + } + + if (loadError || !cur) return null + // 借用的上限按存着的算(服务端生效的那份),改到一半的输入不跟着跳 + const borrow = saved && saved.total !== '' && saved.reserve !== '' ? Math.max(0, Number(saved.total) - Number(saved.reserve)) : null + const edit = (patch: Partial) => { + setError(null) + setDraft({ ...cur, ...patch, limits: { ...cur.limits, ...(patch.limits ?? {}) }, + priorities: { ...cur.priorities, ...(patch.priorities ?? {}) } }) + } + return ( +
+ +
+
+ + {(p) => ( + edit({ total: e.target.value })} data-concurrency-setting="max_total" /> + )} + + + {(p) => ( + edit({ reserve: e.target.value })} data-concurrency-setting="interactive_reserve" /> + )} + +
+ {borrow != null &&

{T.borrow(borrow)}

} +
+ {CONCURRENCY_CLASSES.map((k) => { + const info = T.classes[k] + const live = now?.[k] + return ( +
+
+
{info.label}
+
{info.hint}
+ {live &&
{T.now(live.running ?? 0, live.waiting ?? 0)}
} +
+ + {(p) => ( + edit({ limits: { [k]: e.target.value } as Record })} + data-concurrency-setting={`${k}.limit`} /> + )} + + + {(p) => ( + edit({ priorities: { [k]: e.target.value } as Record })} + data-concurrency-setting={`${k}.priority`} /> + )} + +
+ ) + })} +
+
+ {!admin &&

{T.readOnly}

} + {error && {error}} + {admin && (changed || flash) && ( +
+ {flash && !changed && {T.saved}} + {changed && ( + <> + + + + )} +
+ )} +
+ ) +} diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index 3d19eb5c..27e32f07 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -34,6 +34,7 @@ import { } from './VerifySettings' import { DraftModelSection } from './DraftModelSettings' import { EvalSettingsSection } from './EvalSettings' +import { ConcurrencySettingsSection } from './ConcurrencySettings' import { GeneralTab } from './settings/GeneralTab' import { IdentityTab } from './settings/IdentityTab' import { NotifyTab } from './settings/NotifyTab' @@ -1204,6 +1205,8 @@ function PrefsTab() { {values.eval && } + {values.concurrency && } + {values.ledger && } {values.catalog_draft && ( diff --git a/frontend/src/pages/catalog/ChangeDetail.tsx b/frontend/src/pages/catalog/ChangeDetail.tsx index 7657db92..979ac9a2 100644 --- a/frontend/src/pages/catalog/ChangeDetail.tsx +++ b/frontend/src/pages/catalog/ChangeDetail.tsx @@ -294,7 +294,8 @@ function Facts({ change: c }: { change: SourceChange }) { /** 影响面:说到这些列的目录项(测到时和现在的状态)、用到这张表的已发布模板(写死的 SQL 会不会坏)、有效记录条件 */ function Impact({ change: c }: { change: SourceChange }) { const { catalog, templates, rules_touched: rules } = c.affects - const empty = !catalog.length && !templates.length && !rules + const probes = c.affects.probes ?? [] + const empty = !catalog.length && !templates.length && !rules && !probes.length return ( {empty &&

{T.impactNone}

} @@ -343,6 +344,21 @@ function Impact({ change: c }: { change: SourceChange }) { )} + {probes.length > 0 && ( +
+

{T.probesTitle}

+
    + {probes.map((p) => ( +
  • + {T.probeSkipped(p.name, p.version, p.missing ?? [], p.reason)} + {p.inputs_from && T.probeFrom[p.inputs_from] && ( + {T.probeFrom[p.inputs_from]} + )} +
  • + ))} +
+
+ )}
) } diff --git a/frontend/src/pages/runs/RunDetailView.tsx b/frontend/src/pages/runs/RunDetailView.tsx index f6e79d55..7d0d50a4 100644 --- a/frontend/src/pages/runs/RunDetailView.tsx +++ b/frontend/src/pages/runs/RunDetailView.tsx @@ -21,6 +21,7 @@ import { signerNow } from '../../store/auth' import { explainRunError, explainStartError } from '../../lib/explain' import { canLeave, leavePass } from '../../lib/leave' import { SIGNER_TEXT, UNSAVED_HINT, isUnsaved, runName } from '../../lib/terms' +import { rejectRouteOf } from '../../lib/termsApp' import { asView, duplicateNames, graphShape, idTail, isLiveRun, runScope, type DetailView, } from './model' @@ -347,7 +348,11 @@ export function RunDetailView({ runId, onChange, onDeleted }: { setFeedback({ kind: 'decided', approved, node: a.node_label ?? labelOf(a.node_id) ?? a.node_id, by: done?.resolved_by, at: done?.resolved_at ?? new Date().toISOString(), - terminates: approved === false && a.mode !== 'approve', + // 驳回之后去哪按中断内容的 reject_route(接了驳回出口的不终止,没接的工具 / 代码审批会终止);老审批没带的照旧按模式猜 + terminates: approved === false && (() => { + const route = rejectRouteOf(a.payload as Record | undefined) + return route ? 'ends' in route && route.ends === 'run' : a.mode !== 'approve' + })(), }) // 批完要能看着它往下跑:接上实时流,时间线继续长 d.follow(next) diff --git a/frontend/src/pages/schedules/TriggerEditor.tsx b/frontend/src/pages/schedules/TriggerEditor.tsx index d36f131b..b307e704 100644 --- a/frontend/src/pages/schedules/TriggerEditor.tsx +++ b/frontend/src/pages/schedules/TriggerEditor.tsx @@ -73,6 +73,28 @@ function useTriggerOptions(): OptionsState { return state } +/** + * 对象太多、一页没列完(服务端的 more)时按名字查:输入停一下再查,结果只换这一种事件的对象;当前选中的那一个总带上 + * (selected),不被分页挤掉、不显示成已不存在 + */ +function useObjectSearch(kind: ScheduleEventKind, selected: string, enabled: boolean) { + const [q, setQ] = useState('') + const [found, setFound] = useState<{ key: string; objects: ScheduleTriggerOptions['events'][number]['objects'] } | null>(null) + const words = q.trim() + const key = `${kind}\u0000${words}` + useEffect(() => { + if (!enabled || !words) return + const controller = new AbortController() + const timer = setTimeout(() => { + api.schedules.triggerOptions({ event: kind, q: words, selected: selected || undefined }, { signal: controller.signal }) + .then((data) => setFound({ key, objects: data.events.find((e) => e.event === kind)?.objects ?? [] })) + .catch(() => {}) + }, 300) + return () => { clearTimeout(timer); controller.abort() } + }, [enabled, kind, words, selected, key]) + return { q, setQ, objects: words && found?.key === key ? found.objects : null } +} + export function EventSection({ value, onChange, timezone, setTimezone, savedName, error, debounceError }: { value: TriggerForm onChange: (t: TriggerForm) => void @@ -91,7 +113,9 @@ export function EventSection({ value, onChange, timezone, setTimezone, savedName const kind = value.event const label = TT.subject[kind] const entry = options.status === 'ok' ? options.data.events.find((e) => e.event === kind) : undefined - const objects = entry?.objects ?? [] + const paged = !!entry?.more + const search = useObjectSearch(kind, value.subjectId, paged) + const objects = search.objects ?? entry?.objects ?? [] const listed = objects.some((o) => o.id === value.subjectId) let picker @@ -112,7 +136,10 @@ export function EventSection({ value, onChange, timezone, setTimezone, savedName aria-describedby={error ? 'schedule-trigger-subject-error' : undefined} onChange={(e) => set({ subjectId: e.target.value })}> - {value.subjectId && !listed && } + {/* 一页没列完时不在列表里不等于不存在:写保存时记下的名字,不标「已不存在」 */} + {value.subjectId && !listed && ( + + )} {objects.map((o) => (