mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
内置 PostgreSQL 的四条取舍:agentlab.env 模板末尾追加注释掉的「内置 PostgreSQL(可选)」一段(POSTGRES_IMAGE、POSTGRES_CONTAINER、DOCKER_NETWORK、POSTGRES_VOLUME、BACKUP_KEEP,缺省值写在注释里,已有的行一行没动);升级前的备份收紧权限——backups/ 起每一层目录 0700、备份文件 0600,属主同数据目录(SQLite 的在 umask 077 下先以 0600 建空文件再交给备份接口写,pg_dump 在 umask 077 下写、目录由 app.ops backup-dir 建和收紧),恢复步骤里写清哪一步要 sudo、内置库的恢复改成在以 10001 运行的临时容器里读备份(不用 sudo);restart 缺省只重启服务(库停着时先把库起来),restart --postgres 连库一起重启,没用内置库时加 --postgres 照实报错;离线包缺省带 PostgreSQL 镜像不变,deploy/README 写清多出约 160 MB、导入后约 0.7 GB;deploy/README 加「首次在 Linux 上切换前的检查」(docker 版本、属主、权限、磁盘空间、整目录备份)。单测核对权限位、restart 两种用法、模板里的注释和 deploy.sh 的缺省值一致
This commit is contained in:
+47
-5
@@ -10,12 +10,17 @@
|
||||
用的仍是 SQLite,只是让下一次 migrate-to-postgres 认得出数据卷里那半份是自己上次留下的
|
||||
backup-sqlite 拷一份 agentlab.db、checkpoints.db 到 <数据目录>/backups/sqlite/<时间>/(SQLite 的备份接口,一致的一份),
|
||||
只留最近 --keep 份
|
||||
backup-dir 建(或收紧)备份目录:从 backups/ 起每一层 0700(pg_dump 写进去之前用)
|
||||
prune 目录里按名字排序只留最近 --keep 个(pg_dump 的备份用)
|
||||
|
||||
备份里是整个库(含加密存的密钥、业务数据),目录一律 0700、文件 0600,属主是容器里的用户(同数据目录);宿主机上要读用 sudo。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import os
|
||||
import secrets
|
||||
import sqlite3
|
||||
import sys
|
||||
@@ -28,6 +33,10 @@ from app.core import atomic
|
||||
POSTGRES_DIR = ".postgres"
|
||||
STATE_FILE = "state"
|
||||
MODES = ("sqlite", "builtin", "migrating")
|
||||
BACKUPS_DIR = "backups"
|
||||
#: 备份的目录、文件的权限:只有属主(容器里的用户)能读
|
||||
BACKUP_DIR_MODE = 0o700
|
||||
BACKUP_FILE_MODE = 0o600
|
||||
|
||||
|
||||
def _stamp() -> str:
|
||||
@@ -77,16 +86,42 @@ def write_state(data: Path, mode: str) -> None:
|
||||
atomic.write_text(root / STATE_FILE, f"mode={mode}\nsince={datetime.now(timezone.utc).isoformat()}\n", mode=0o644)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _private_umask():
|
||||
"""这一段里新建的文件、目录别人读不到(SQLite 备份时建的 -journal 也一样),出来恢复原来的 umask。"""
|
||||
old = os.umask(0o077)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
os.umask(old)
|
||||
|
||||
|
||||
def backup_dir(data: Path, *parts: str) -> Path:
|
||||
"""<数据目录>/backups/<parts…>:从 backups/ 起每一层建出来(已有的收紧)成 0700,交回最里面那层。"""
|
||||
folder = data / BACKUPS_DIR
|
||||
levels = [folder]
|
||||
for part in parts:
|
||||
folder = folder / part
|
||||
levels.append(folder)
|
||||
for level in levels:
|
||||
level.mkdir(exist_ok=True)
|
||||
level.chmod(BACKUP_DIR_MODE)
|
||||
return folder
|
||||
|
||||
|
||||
def _copy_sqlite(src: Path, dst: Path) -> None:
|
||||
# 读写打开、只许读:WAL 模式的库用 mode=ro 打开,关掉时会留下它自己建的 -wal(同 db/to_postgres.snapshot)
|
||||
conn = sqlite3.connect(src)
|
||||
conn.execute("PRAGMA query_only = 1")
|
||||
try:
|
||||
# 先以 0600 建出空文件再交给 SQLite 写:从有这个文件起就只有属主读得到
|
||||
os.close(os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, BACKUP_FILE_MODE))
|
||||
out = sqlite3.connect(dst)
|
||||
try:
|
||||
conn.backup(out)
|
||||
finally:
|
||||
out.close()
|
||||
dst.chmod(BACKUP_FILE_MODE)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -112,11 +147,13 @@ def backup_sqlite(data: Path, *, keep: int) -> Path | None:
|
||||
sources = [p for p in (data / "agentlab.db", data / "checkpoints.db") if p.is_file()]
|
||||
if not sources:
|
||||
return None
|
||||
root = data / "backups" / "sqlite"
|
||||
target = root / _stamp()
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for src in sources:
|
||||
_copy_sqlite(src, target / src.name)
|
||||
with _private_umask():
|
||||
root = backup_dir(data, "sqlite")
|
||||
target = root / _stamp()
|
||||
target.mkdir(exist_ok=True)
|
||||
target.chmod(BACKUP_DIR_MODE)
|
||||
for src in sources:
|
||||
_copy_sqlite(src, target / src.name)
|
||||
prune(root, keep)
|
||||
return target
|
||||
|
||||
@@ -137,6 +174,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
b = sub.add_parser("backup-sqlite")
|
||||
b.add_argument("--data", default="/data")
|
||||
b.add_argument("--keep", type=int, default=5)
|
||||
d = sub.add_parser("backup-dir")
|
||||
d.add_argument("--data", default="/data")
|
||||
d.add_argument("parts", nargs="*", help="backups/ 下面的几层,比如 postgres")
|
||||
r = sub.add_parser("prune")
|
||||
r.add_argument("--dir", required=True)
|
||||
r.add_argument("--keep", type=int, default=5)
|
||||
@@ -153,6 +193,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
elif args.command == "backup-sqlite":
|
||||
target = backup_sqlite(Path(args.data), keep=args.keep)
|
||||
print(target if target is not None else "没有 SQLite 库,不用备份")
|
||||
elif args.command == "backup-dir":
|
||||
print(backup_dir(Path(args.data), *args.parts))
|
||||
elif args.command == "prune":
|
||||
for gone in prune(Path(args.dir), args.keep):
|
||||
print(f"删掉旧备份 {gone.name}")
|
||||
|
||||
@@ -154,7 +154,7 @@ def run_oneoff(state: dict, opts: dict, image: str, command: list[str]) -> int:
|
||||
from app import ops
|
||||
|
||||
rest = [to_host(a, data) for a in command[2:]]
|
||||
if rest and rest[0] in ("postgres-secrets", "state", "backup-sqlite") and "--data" not in rest:
|
||||
if rest and rest[0] in ("postgres-secrets", "state", "backup-sqlite", "backup-dir") and "--data" not in rest:
|
||||
rest[1:1] = ["--data", data or "/data"]
|
||||
return ops.main(rest)
|
||||
if entry == "python" and command[:3] == ["-m", "app.db", "copy-to-postgres"]:
|
||||
@@ -171,12 +171,17 @@ def run_oneoff(state: dict, opts: dict, image: str, command: list[str]) -> int:
|
||||
if not a.startswith("-"):
|
||||
Path(to_host(a, data)).mkdir(parents=True, exist_ok=True)
|
||||
return 0
|
||||
if entry == "pg_dump":
|
||||
if entry == "sh" and command[:1] == ["-c"] and 'pg_dump "$@"' in command[1]:
|
||||
# sh -c '<脚本>' pg_dump <参数…>:照脚本里的 umask 建文件
|
||||
umask = 0o077 if "umask 077" in command[1] else 0o022
|
||||
if os.environ.get("FAKE_PGDUMP_FAIL"):
|
||||
print("pg_dump: fake 的失败", file=sys.stderr)
|
||||
return 1
|
||||
out = command[command.index("-f") + 1]
|
||||
Path(to_host(out, data)).write_bytes(b"PGDMP fake")
|
||||
args = command[3:]
|
||||
out = Path(to_host(args[args.index("-f") + 1], data))
|
||||
fd = os.open(out, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666 & ~umask)
|
||||
with os.fdopen(fd, "wb") as fh:
|
||||
fh.write(b"PGDMP fake")
|
||||
return 0
|
||||
unknown(["run", "--rm", image, *command])
|
||||
return 99
|
||||
|
||||
@@ -127,7 +127,8 @@ def deploy(tmp_path):
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args", [["bogus"], ["start", "--postgres"], ["upgrade", "--postgres"],
|
||||
@pytest.mark.parametrize("args", [["bogus"], ["start", "--postgres"], ["upgrade", "--postgres"], ["status", "--postgres"],
|
||||
["restart", "--all"],
|
||||
["install", "--purge-image"], ["migrate-to-postgres", "-f"], ["status", "--tail", "3"]])
|
||||
def test_unknown_arguments_are_refused_before_doing_anything(deploy, args):
|
||||
out = deploy(*args, ok=False)
|
||||
@@ -138,7 +139,8 @@ def test_unknown_arguments_are_refused_before_doing_anything(deploy, args):
|
||||
|
||||
def test_help_lists_the_postgres_commands(deploy):
|
||||
out = deploy("--help")
|
||||
for word in ("install [--postgres]", "migrate-to-postgres", "logs [-f] [--tail N] [--postgres]", "DEPLOY_<名字>"):
|
||||
for word in ("install [--postgres]", "migrate-to-postgres", "restart [--postgres]", "logs [-f] [--tail N] [--postgres]",
|
||||
"DEPLOY_<名字>"):
|
||||
assert word in out.stdout
|
||||
|
||||
|
||||
@@ -282,14 +284,39 @@ def test_start_stop_restart_order_the_two_containers(deploy):
|
||||
pg_ready = index_of(calls, lambda a: a[:1] == ["inspect"] and a[-1] == PG and ".State.Health" in a[2])
|
||||
assert pg_start < pg_ready < index_of(calls, is_cmd("start", APP)), "先起库、等就绪,再起服务"
|
||||
|
||||
deploy("restart")
|
||||
deploy("restart", "--postgres")
|
||||
calls = deploy.calls()
|
||||
order = [index_of(calls, is_cmd("stop", APP)), index_of(calls, is_cmd("stop", PG)),
|
||||
index_of(calls, is_cmd("start", PG)), index_of(calls, is_cmd("start", APP))]
|
||||
assert order == sorted(order)
|
||||
assert order == sorted(order), "--postgres:先停服务、再停库,先起库、再起服务"
|
||||
assert {c["state"] for c in deploy.containers().values()} == {"running"}
|
||||
|
||||
|
||||
def test_restart_by_default_leaves_the_database_alone(deploy):
|
||||
deploy("install", "--postgres")
|
||||
out = deploy("restart")
|
||||
calls = deploy.calls()
|
||||
assert ["restart", APP] in calls
|
||||
assert not any(a[:1] in (["stop"], ["start"], ["restart"]) and a[-1] == PG for a in calls), "库不跟着重启"
|
||||
assert "已重启" in out.stdout
|
||||
# 库停着时先把库起来,服务才连得上
|
||||
state = deploy.state()
|
||||
state["containers"][PG]["state"] = "exited"
|
||||
deploy.state_file.write_text(json.dumps(state))
|
||||
deploy("restart")
|
||||
calls = deploy.calls()
|
||||
assert index_of(calls, is_cmd("start", PG)) < index_of(calls, lambda a: a == ["restart", APP])
|
||||
assert not any(is_cmd("stop", PG)(a) for a in calls)
|
||||
assert deploy.containers()[PG]["state"] == "running"
|
||||
|
||||
|
||||
def test_restart_postgres_needs_the_builtin_database(deploy):
|
||||
deploy("install")
|
||||
out = deploy("restart", "--postgres", ok=False)
|
||||
assert "没用内置 PostgreSQL" in out.stderr
|
||||
assert not any(a[:1] in (["stop"], ["restart"]) for a in deploy.calls()), "什么都没动"
|
||||
|
||||
|
||||
def test_status_and_logs_show_the_database(deploy):
|
||||
deploy("install", "--postgres")
|
||||
out = deploy("status").stdout
|
||||
@@ -346,14 +373,19 @@ def test_upgrade_builtin_dumps_after_stopping_the_app_and_keeps_the_last_n(deplo
|
||||
deploy("upgrade", DEPLOY_BACKUP_KEEP="3")
|
||||
calls = deploy.calls()
|
||||
stop = index_of(calls, is_cmd("stop", APP))
|
||||
dump = index_of(calls, oneoff("pg_dump"))
|
||||
dump = index_of(calls, oneoff("sh", "pg_dump"))
|
||||
new_app = index_of(calls, runs_named(APP))
|
||||
assert stop < dump < new_app, "停了旧服务再备份,备份完才起新的"
|
||||
dump_args = calls[dump]
|
||||
assert dump_args[dump_args.index("--network") + 1] == NET and "PGPASSFILE=/data/.postgres/pgpass" in dump_args
|
||||
assert dump_args[dump_args.index("-c") + 1].startswith("umask 077 && "), "备份文件从建出来就是 0600"
|
||||
assert dump_args[dump_args.index("--user") + 1] == "10001:10001", "属主照数据目录:容器里的用户"
|
||||
kept = sorted(p.name for p in folder.iterdir())
|
||||
assert len(kept) == 3 and kept[-1].startswith("agentlab-") and kept[-1] > "agentlab-20260103", kept
|
||||
assert (folder / kept[-1]).read_bytes() == b"PGDMP fake"
|
||||
assert stat.S_IMODE((folder / kept[-1]).stat().st_mode) == 0o600
|
||||
for level in (deploy.data / "backups", folder):
|
||||
assert stat.S_IMODE(level.stat().st_mode) == 0o700, level
|
||||
|
||||
|
||||
def test_upgrade_sqlite_backs_up_the_database_files(deploy):
|
||||
@@ -367,6 +399,9 @@ def test_upgrade_sqlite_backs_up_the_database_files(deploy):
|
||||
copy = sqlite3.connect(stamp / "agentlab.db")
|
||||
assert copy.execute("SELECT v FROM t").fetchall() == [("一",)]
|
||||
copy.close()
|
||||
for level in (deploy.data / "backups", deploy.data / "backups" / "sqlite", stamp):
|
||||
assert stat.S_IMODE(level.stat().st_mode) == 0o700, level
|
||||
assert stat.S_IMODE((stamp / "agentlab.db").stat().st_mode) == 0o600
|
||||
|
||||
|
||||
def test_failed_backup_aborts_the_upgrade_and_restarts_the_old_service(deploy):
|
||||
@@ -515,6 +550,28 @@ def test_package_script_takes_the_postgres_image_from_deploy_sh():
|
||||
assert out.stdout.strip() == PG_IMAGE
|
||||
|
||||
|
||||
def test_env_template_lists_the_postgres_settings_commented_out_with_the_defaults():
|
||||
"""agentlab.env 模板末尾的「内置 PostgreSQL(可选)」:几项都注释着(不改任何人的部署),缺省值和 deploy.sh 里的一致。"""
|
||||
template = (REPO / "deploy" / "agentlab.env").read_text(encoding="utf-8")
|
||||
section = template[template.index("# ---- 内置 PostgreSQL(可选)----"):]
|
||||
assert template.rstrip().endswith(section.rstrip()), "追加在文件末尾"
|
||||
commented = dict(line[2:].split("=", 1) for line in section.splitlines()
|
||||
if line.startswith("# ") and "=" in line and line[2:3].isupper())
|
||||
assert commented == {"POSTGRES_IMAGE": PG_IMAGE, "POSTGRES_CONTAINER": "agentlab-postgres",
|
||||
"DOCKER_NETWORK": "agentlab-net", "POSTGRES_VOLUME": "agentlab-pgdata", "BACKUP_KEEP": "5"}
|
||||
assert not any(line.split("=", 1)[0] in commented for line in template.splitlines() if not line.startswith("#")), \
|
||||
"都是注释"
|
||||
|
||||
|
||||
def test_settings_written_in_agentlab_env_take_effect(tmp_path):
|
||||
deploy = Deploy(tmp_path, env_lines=["POSTGRES_CONTAINER=agl-db", "DOCKER_NETWORK=agl-n", "POSTGRES_VOLUME=agl-v",
|
||||
"BACKUP_KEEP=1"])
|
||||
deploy("install", "--postgres")
|
||||
assert set(deploy.containers()) == {APP, "agl-db"}
|
||||
assert deploy.containers()["agl-db"]["mounts"]["/var/lib/postgresql/data"] == "agl-v"
|
||||
assert deploy.containers()[APP]["network"] == "agl-n"
|
||||
|
||||
|
||||
def test_dev_script_refuses_unknown_arguments_and_documents_postgres():
|
||||
env = {**os.environ, "PATH": "/usr/bin:/bin"}
|
||||
out = subprocess.run(["bash", str(REPO / "scripts" / "dev.sh"), "--bogus"], capture_output=True, text=True, env=env)
|
||||
|
||||
@@ -95,6 +95,11 @@ def test_backup_sqlite_copies_a_consistent_snapshot_including_the_wal(tmp_path,
|
||||
assert copy.execute("SELECT v FROM t").fetchall() == [("还在 WAL 里",)]
|
||||
copy.close()
|
||||
assert (stamp / "checkpoints.db").exists()
|
||||
# 备份里是整个库:目录只有属主进得去,文件只有属主读得到
|
||||
for level in (tmp_path / "backups", target, stamp):
|
||||
assert _mode(level) == 0o700, level
|
||||
assert _mode(stamp / "agentlab.db") == 0o600 and _mode(stamp / "checkpoints.db") == 0o600
|
||||
assert sorted(p.name for p in stamp.iterdir()) == ["agentlab.db", "checkpoints.db"], "没留下 -journal 之类的"
|
||||
# 只留最近 keep 份
|
||||
for name in ("20200101T000000Z", "20200102T000000Z"):
|
||||
(target / name).mkdir()
|
||||
@@ -102,6 +107,15 @@ def test_backup_sqlite_copies_a_consistent_snapshot_including_the_wal(tmp_path,
|
||||
assert len(list(target.iterdir())) == 2 and not (target / "20200101T000000Z").exists()
|
||||
|
||||
|
||||
def test_backup_dir_creates_and_tightens_every_level(tmp_path, capsys):
|
||||
(tmp_path / "backups").mkdir(mode=0o755)
|
||||
(tmp_path / "backups").chmod(0o755)
|
||||
assert ops.main(["backup-dir", "--data", str(tmp_path), "postgres"]) == 0
|
||||
assert capsys.readouterr().out.strip() == str(tmp_path / "backups" / "postgres")
|
||||
assert _mode(tmp_path / "backups") == 0o700, "已有的收紧"
|
||||
assert _mode(tmp_path / "backups" / "postgres") == 0o700
|
||||
|
||||
|
||||
def test_backup_sqlite_without_databases_does_nothing(tmp_path, capsys):
|
||||
assert ops.main(["backup-sqlite", "--data", str(tmp_path)]) == 0
|
||||
assert "没有 SQLite 库" in capsys.readouterr().out
|
||||
|
||||
+47
-8
@@ -10,7 +10,7 @@
|
||||
| 文件 | 用途 |
|
||||
| --- | --- |
|
||||
| `image.tar` | Docker 镜像(`docker save` 导出),镜像名 `agentlab:__AGENTLAB_VERSION__` |
|
||||
| `postgres-image.tar` | 内置 PostgreSQL 的镜像(`install --postgres` 时用,见「用 PostgreSQL」)。打包时用了 `--no-postgres` 的包里没有 |
|
||||
| `postgres-image.tar` | 内置 PostgreSQL 的镜像(`install --postgres` 时用,见「用 PostgreSQL」)。包因此多约 160 MB,导入后在 docker 里占约 0.7 GB;只用 SQLite 或外接 PostgreSQL 时用不上,打包时用 `--no-postgres` 就没有这个文件 |
|
||||
| `deploy.sh` | 一键部署脚本:安装、启动、停止、升级、卸载 |
|
||||
| `agentlab.env` | 配置:端口、监听地址、数据目录、模型 Key、代理、沙箱 |
|
||||
| `README.md` | 本说明 |
|
||||
@@ -67,8 +67,21 @@ deploy.sh 依次:检查 docker 和架构 → 导入镜像(`docker load -i im
|
||||
|
||||
`./deploy.sh upgrade` 换版本之前会自动备份一次库:SQLite 拷到数据目录的 `backups/sqlite/<时间>/`(SQLite 的备份接口,
|
||||
一致的一份),内置 PostgreSQL 用 `pg_dump` 存到 `backups/postgres/`,各只留最近 5 份(`agentlab.env` 里 `BACKUP_KEEP=` 改)。
|
||||
备份里是整个库(含加密存着的密钥),所以 `backups/` 和下面每一层目录都是 0700、备份文件 0600,属主同数据目录(uid 10001):
|
||||
宿主机上要看、要拷走都得用 sudo(`sudo ls -l data/backups/sqlite/`、`sudo cp -a data/backups /别处/`)。
|
||||
这只是升级的保险,不代替下面的整目录备份:工件、上传文件和 `.secret_key` 不在里面。
|
||||
|
||||
用 SQLite 的部署要从 `backups/sqlite/` 恢复(以缺省的 `./data` 为例,`<时间>` 换成目录名):
|
||||
|
||||
```bash
|
||||
./deploy.sh stop
|
||||
sudo rm -f data/agentlab.db-wal data/agentlab.db-shm data/checkpoints.db-wal data/checkpoints.db-shm
|
||||
sudo cp -p data/backups/sqlite/<时间>/agentlab.db data/backups/sqlite/<时间>/checkpoints.db data/ # -p:属主仍是 10001
|
||||
./deploy.sh start
|
||||
```
|
||||
|
||||
`-wal`、`-shm` 要先删:留着的话 SQLite 会把它们当成换回来的库的一部分。
|
||||
|
||||
deploy.sh 会把数据目录的属主改成容器里的用户(uid 10001),所以备份、删除要用 sudo。
|
||||
停下来备份最稳妥(以默认的 `./data` 为例,在包目录里执行):
|
||||
|
||||
@@ -107,7 +120,8 @@ sudo tar -czf ../agentlab-data-$(date +%Y%m%d).tar.gz data
|
||||
所以数据目录照样要整份备份。
|
||||
- 用过一次之后这个数据目录就一直是内置 PostgreSQL(记在 `.postgres/state`),之后的 `install`、`upgrade`、`start` 不用再加
|
||||
`--postgres`。
|
||||
- `start`、`restart` 先起库、等 `pg_isready` 通过再起服务;`stop` 先停服务再停库。开机时两个容器一起被 docker 拉起来,库还没就绪时
|
||||
- `start` 先起库、等 `pg_isready` 通过再起服务;`stop` 先停服务再停库。`restart` 缺省只重启服务(库停着的话先把库起来),
|
||||
`restart --postgres` 连库一起重启(先停服务再停库,再先起库后起服务)。开机时两个容器一起被 docker 拉起来,库还没就绪时
|
||||
服务按退避重试(日志里写着「应用库还没就绪……秒后重试」),最多等 90 秒(`AGENTLAB_DB_STARTUP_TIMEOUT`),不会一上来就崩。
|
||||
- 表结构和默认数据(内置模板、技能)每次启动自动补齐;几个工作进程(`AGENTLAB_WORKERS`)同时起也只做一次(库里的锁)。
|
||||
- `./deploy.sh status` 显示两个容器的状态,`./deploy.sh logs --postgres` 看库的日志。
|
||||
@@ -125,15 +139,17 @@ sudo tar -czf ../agentlab-data-$(date +%Y%m%d).tar.gz data
|
||||
环境变量 `DEPLOY_<名字>` 盖过 `agentlab.env` 里的同名项(这几项加上 `HOST_PORT`、`BIND_ADDR`、`DATA_DIR`、`CONTAINER_NAME`),
|
||||
一台机器上并排部署两套、或者测试时用。
|
||||
|
||||
**备份。** `upgrade` 前自动 `pg_dump` 一份到 `backups/postgres/`。平时要手动备份(服务开着也行,`pg_dump` 读的是一致的快照):
|
||||
**备份。** `upgrade` 前自动 `pg_dump` 一份到 `backups/postgres/`(0600,见「数据和备份」)。平时要手动备份(服务开着也行,
|
||||
`pg_dump` 读的是一致的快照;`umask 077` 让导出的文件只有自己读得到):
|
||||
|
||||
```bash
|
||||
docker exec agentlab-postgres pg_dump -U agentlab -d agentlab -Fc > agentlab-$(date +%Y%m%d).dump
|
||||
(umask 077; docker exec agentlab-postgres pg_dump -U agentlab -d agentlab -Fc > agentlab-$(date +%Y%m%d).dump)
|
||||
```
|
||||
|
||||
再加上数据目录(见「数据和备份」),两样合起来才是完整的一份。
|
||||
|
||||
**恢复。** 用 `pg_dump` 的备份把库换回去(以缺省的容器名为例,`<备份>` 换成文件名):
|
||||
**恢复。** 用 `backups/postgres/` 里的备份把库换回去(以缺省的名字、在包目录里为例,`<备份>` 换成文件名,`sudo ls data/backups/postgres/`
|
||||
看有哪些)。备份文件是 0600、属主 10001,宿主机上的人读不到,所以最后一步在一个以 10001 运行的临时容器里读它,不用 sudo:
|
||||
|
||||
```bash
|
||||
./deploy.sh stop # 两个都停
|
||||
@@ -141,10 +157,14 @@ docker start agentlab-postgres # 只起库
|
||||
until docker exec agentlab-postgres pg_isready -q -h 127.0.0.1 -U agentlab; do sleep 1; done
|
||||
docker exec agentlab-postgres psql -U agentlab -d postgres \
|
||||
-c 'DROP DATABASE agentlab WITH (FORCE)' -c 'CREATE DATABASE agentlab OWNER agentlab'
|
||||
docker exec -i agentlab-postgres pg_restore -U agentlab -d agentlab --no-owner < <备份>
|
||||
docker run --rm --network agentlab-net -v "$(pwd)/data:/data" --user 10001:10001 -e PGPASSFILE=/data/.postgres/pgpass \
|
||||
--entrypoint pg_restore postgres:16.15 -h agentlab-postgres -U agentlab -d agentlab --no-owner /data/backups/postgres/<备份>
|
||||
./deploy.sh start
|
||||
```
|
||||
|
||||
自己手动导出、放在别处的备份(上面「备份」那条命令导出的,属主是你自己)直接从宿主机喂进去:
|
||||
`docker exec -i agentlab-postgres pg_restore -U agentlab -d agentlab --no-owner < agentlab-<日期>.dump`。
|
||||
|
||||
数据目录要换回同一时间的备份,工件、上传文件和库里的记录才对得上。
|
||||
|
||||
**从 SQLite 切过来。** 正在用 SQLite 的部署运行 `./deploy.sh migrate-to-postgres`,它依次:
|
||||
@@ -166,6 +186,23 @@ docker exec -i agentlab-postgres pg_restore -U agentlab -d agentlab --no-owner <
|
||||
`sudo rm <数据目录>/.postgres/state`、`./deploy.sh install`,就回到切换那一刻的 SQLite(切换之后在 PostgreSQL 上新产生的东西
|
||||
不会带回来;库的容器停着不用,`docker rm -f agentlab-postgres` 可删)。
|
||||
|
||||
**首次在 Linux 上切换前的检查。** 第一次在一台机器上 `install --postgres` 或 `migrate-to-postgres` 之前,花几分钟看一遍
|
||||
(以缺省的 `./data` 为例,在包目录里执行):
|
||||
|
||||
- **docker 版本**:`docker version --format '{{.Server.Version}}'`,要 20.10 以上(健康检查、自建网络、`--stop-timeout` 都要用);
|
||||
运行 deploy.sh 的用户在 `docker` 组里(`id -nG` 里有 docker),否则整条命令前加 sudo。
|
||||
- **属主**:`ls -lnd data` 第三列应是 `10001`(deploy.sh 改的)。数据目录不要放在 NFS、SMB 这类不认 uid 的挂载上:属主改不过去,
|
||||
服务写不进去。
|
||||
- **权限**:切换(或第一次起库)之后看一眼——`ls -ld data/.postgres` 应是 `drwx--x--x`(0711),`cat data/.postgres/state`
|
||||
不用 sudo 就能读到(deploy.sh 在宿主机上直接读它,读不到时会退回去用容器读,只是慢一点);`sudo ls -ln data/.postgres`
|
||||
里 `password`、`pgpass` 是 `-rw-------`、属主 10001;`sudo ls -ld data/backups` 是 `drwx------`。
|
||||
- **磁盘空间**:切换时数据目录里要放得下两份库(`backups/sqlite/` 里的一份备份,加上导入时的临时快照,用完就删),
|
||||
另外 docker 的存储目录要放得下 PostgreSQL 的镜像(约 0.7 GB)和库本身(大致和 SQLite 的库一样大,索引多的表会大一些)。
|
||||
看大小:`sudo du -sh data/agentlab.db data/checkpoints.db`;看剩余:`df -h data` 和
|
||||
`df -h "$(docker info --format '{{.DockerRootDir}}')"`。
|
||||
- **整目录备份**:切换会先自动备份一份库,但还是建议先按「数据和备份」停服务整份打包一次数据目录。切换期间服务是停着的,
|
||||
时长大致是导入的时间(每秒几千到几万行,看机器)。
|
||||
|
||||
**卸载。** `./deploy.sh uninstall` 连库的容器和网络一起删,数据卷和数据目录都留着;同一个数据目录再 `./deploy.sh install`
|
||||
就接着用原来的库。确定不要了:`docker volume rm agentlab-pgdata`(不可恢复)。
|
||||
|
||||
@@ -244,12 +281,14 @@ cp ../agentlab-<旧版本>-linux-__AGENTLAB_ARCH__/agentlab.env . # 沿用原
|
||||
```bash
|
||||
./deploy.sh stop # 停止(开机也不会自动起来)
|
||||
./deploy.sh start # 再启动
|
||||
./deploy.sh restart # 重启
|
||||
./deploy.sh restart # 重启服务(内置 PostgreSQL 不跟着重启)
|
||||
./deploy.sh restart --postgres # 连内置 PostgreSQL 一起重启
|
||||
./deploy.sh uninstall # 删除容器,保留数据和镜像
|
||||
./deploy.sh uninstall --purge-image # 连镜像一起删(PostgreSQL 的镜像不删,别的东西可能也在用)
|
||||
```
|
||||
|
||||
内置 PostgreSQL 时这几个命令按顺序管两个容器(起的时候先起库,停的时候先停服务),卸载连库的容器和网络一起删。
|
||||
内置 PostgreSQL 时这几个命令按顺序管两个容器(起的时候先起库,停的时候先停服务;`restart` 不加 `--postgres` 只动服务),
|
||||
卸载连库的容器和网络一起删。
|
||||
卸载**永远不删数据**。确定不要了再手动删除数据目录(不可恢复):`sudo rm -rf <数据目录>`;内置 PostgreSQL 的数据卷另删:
|
||||
`docker volume rm <CONTAINER_NAME>-pgdata`。
|
||||
|
||||
|
||||
@@ -111,3 +111,20 @@ ENABLE_MICROVM=auto
|
||||
# sudo sysctl kernel.apparmor_restrict_unprivileged_userns=0
|
||||
# 这一步会放宽整台机器的限制,自行权衡。
|
||||
ENABLE_BWRAP=0
|
||||
|
||||
# ---- 内置 PostgreSQL(可选)----
|
||||
# 缺省库是数据目录里的 SQLite。新部署要用内置的 PostgreSQL 运行 ./deploy.sh install --postgres,已经在用 SQLite 的运行
|
||||
# ./deploy.sh migrate-to-postgres(见 README.md「用 PostgreSQL」)。密码自动生成在数据目录的 .postgres/ 里,这里不用填。
|
||||
# 和外接 PostgreSQL(AGENTLAB_DATABASE_URL)只能选一种。下面几项是给 deploy.sh 用的,都有缺省值,一般不用改;要改去掉
|
||||
# 行首的 # 再 ./deploy.sh install。
|
||||
#
|
||||
# 库的镜像(小版本固定;离线包里带着同一个标签的 postgres-image.tar)
|
||||
# POSTGRES_IMAGE=postgres:16.15
|
||||
# 库的容器名,缺省 <CONTAINER_NAME>-postgres
|
||||
# POSTGRES_CONTAINER=agentlab-postgres
|
||||
# 两个容器共用的 docker 网络,缺省 <CONTAINER_NAME>-net(库的端口不对外,只在这个网络里可见)
|
||||
# DOCKER_NETWORK=agentlab-net
|
||||
# 库的数据放在这个 docker 数据卷里,缺省 <CONTAINER_NAME>-pgdata(卸载不删)
|
||||
# POSTGRES_VOLUME=agentlab-pgdata
|
||||
# upgrade 前自动备份库(SQLite 拷到 backups/sqlite/,内置 PostgreSQL 用 pg_dump 存到 backups/postgres/),只留最近几份
|
||||
# BACKUP_KEEP=5
|
||||
|
||||
+18
-9
@@ -34,8 +34,10 @@ usage() {
|
||||
migrate-to-postgres
|
||||
把正在用 SQLite 的部署切到内置 PostgreSQL:停服务 → 备份 → 起 PostgreSQL → 先核对再导入 → 记下
|
||||
「已切换」→ 起服务。哪一步出错都停在原地、仍用 SQLite。重复执行是安全的;SQLite 的库文件原样留着。
|
||||
start | stop | restart
|
||||
启动 / 停止 / 重启。内置 PostgreSQL 时先起库、等它就绪再起服务;停的时候先停服务再停库。
|
||||
start | stop 启动 / 停止。内置 PostgreSQL 时先起库、等它就绪再起服务;停的时候先停服务再停库。
|
||||
restart [--postgres]
|
||||
重启服务(库停着的话先把库起来)。--postgres:内置 PostgreSQL 也一起重启(先停服务、再停库,
|
||||
起的时候先起库)。
|
||||
status 查看容器状态、健康检查、数据目录、用的哪种库、代码沙箱和访问地址。
|
||||
logs [-f] [--tail N] [--postgres]
|
||||
查看日志(默认最后 200 行),-f 持续跟随,--postgres 看内置 PostgreSQL 的。
|
||||
@@ -66,6 +68,7 @@ TAIL="200"
|
||||
PURGE_IMAGE=""
|
||||
WITH_POSTGRES=""
|
||||
LOGS_POSTGRES=""
|
||||
RESTART_POSTGRES=""
|
||||
if [ $# -gt 0 ]; then
|
||||
case "$1" in
|
||||
-h|--help) usage; exit 0 ;;
|
||||
@@ -90,6 +93,7 @@ while [ $# -gt 0 ]; do
|
||||
case "$CMD" in
|
||||
install) WITH_POSTGRES=1 ;;
|
||||
logs) LOGS_POSTGRES=1 ;;
|
||||
restart) RESTART_POSTGRES=1 ;;
|
||||
*) bad_arg "$1" ;;
|
||||
esac ;;
|
||||
*) bad_arg "$1" ;;
|
||||
@@ -754,17 +758,16 @@ stop_app() {
|
||||
image_user() { docker image inspect -f '{{.Config.User}}' "$IMAGE"; }
|
||||
|
||||
# pg_dump 一份到数据目录的 backups/postgres/,只留最近 BACKUP_KEEP 份。用 PostgreSQL 镜像里的 pg_dump(和库同版本),
|
||||
# 以数据目录的属主写文件
|
||||
# 以数据目录的属主写文件。备份里是整个库:目录 0700(app.ops backup-dir)、文件 0600(umask 077 下建,没有别人读得到的那一刻)
|
||||
backup_pg() {
|
||||
local stamp file
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
file="backups/postgres/$PG_DB-$stamp.dump"
|
||||
say "备份内置 PostgreSQL:pg_dump 到 $DATA_ABS/$file"
|
||||
docker run --rm --network none -v "$DATA_ABS:/data" --user "$(image_user)" --entrypoint mkdir "$POSTGRES_IMAGE" \
|
||||
-p /data/backups/postgres >/dev/null \
|
||||
|| { warn "没能在数据目录里建备份目录 backups/postgres/"; return 1; }
|
||||
ops backup-dir postgres >/dev/null || { warn "没能在数据目录里建备份目录 backups/postgres/"; return 1; }
|
||||
docker run --rm --network "$DOCKER_NETWORK" -v "$DATA_ABS:/data" --user "$(image_user)" -e "PGPASSFILE=$PG_PASSFILE" \
|
||||
--entrypoint pg_dump "$POSTGRES_IMAGE" -h "$POSTGRES_CONTAINER" -U "$PG_USER" -d "$PG_DB" -Fc -f "/data/$file" \
|
||||
--entrypoint sh "$POSTGRES_IMAGE" -c 'umask 077 && exec pg_dump "$@"' pg_dump \
|
||||
-h "$POSTGRES_CONTAINER" -U "$PG_USER" -d "$PG_DB" -Fc -f "/data/$file" \
|
||||
|| { warn "pg_dump 失败"; return 1; }
|
||||
ops prune --dir /data/backups/postgres --keep "$BACKUP_KEEP" >/dev/null || warn "旧备份没删掉,不影响这次升级"
|
||||
}
|
||||
@@ -1003,15 +1006,21 @@ cmd_stop() {
|
||||
say "已停止。开机也不会自动起来,要用时运行 ./deploy.sh start"
|
||||
}
|
||||
|
||||
# 缺省只重启服务:库不用跟着断一次(库停着的话先把它起来,服务才连得上)。--postgres 连库一起重启
|
||||
cmd_restart() {
|
||||
check_docker; read_env; require_container; locate_data
|
||||
say "重启 $CONTAINER_NAME"
|
||||
if [ "$MODE" = builtin ]; then
|
||||
if [ -n "$RESTART_POSTGRES" ]; then
|
||||
[ "$MODE" = builtin ] || die "这个部署没用内置 PostgreSQL,--postgres 没有可重启的库。只重启服务:./deploy.sh restart"
|
||||
say "重启 $CONTAINER_NAME 和内置 PostgreSQL(${POSTGRES_CONTAINER})"
|
||||
stop_app
|
||||
stop_pg
|
||||
start_pg
|
||||
docker start "$CONTAINER_NAME" >/dev/null
|
||||
else
|
||||
say "重启 $CONTAINER_NAME"
|
||||
if [ "$MODE" = builtin ] && [ "$(container_state "$POSTGRES_CONTAINER")" != running ]; then
|
||||
start_pg
|
||||
fi
|
||||
docker restart "$CONTAINER_NAME" >/dev/null
|
||||
fi
|
||||
wait_app "已重启"
|
||||
|
||||
Reference in New Issue
Block a user