生产部署:prod.sh 直接运行,package-docker.sh 打 Docker 离线包(镜像 + 一键部署脚本 + 配置 + 说明)

- 服务端可以托管前端构建产物(AGENTLAB_WEB_DIST):页面路由回退到 index.html,/api 下不存在的路径照旧是 JSON 404,
  挡住路径穿越;index.html 不缓存,assets/ 带哈希的文件长缓存;没设时行为完全不变
- scripts/prod.sh:不用 Docker 的生产运行。按需构建前端,单进程、不热重载,默认只听本机,--host 开放到局域网并提示没有登录
- Dockerfile(多阶段构建,Linux 镜像):前端在构建机架构上构建,运行阶段用 python:3.12-slim,依赖装进独立的 venv,
  以非 root 用户(uid 10001)运行,数据在 /data,带健康检查;.dockerignore 排除 .env、data/、各种库文件,镜像里实测没有密钥和数据库
- scripts/package-docker.sh:构建指定平台的镜像(默认按本机架构,可选 linux/amd64),打成
  release/agentlab-<版本>-linux-<架构>.tar,里面有 image.tar、deploy.sh、agentlab.env、README.md、SHA256SUMS
- deploy/deploy.sh:目标机上只依赖 Docker,离线可用。install / start / stop / restart / status / logs / upgrade / uninstall;
  检查架构和配置,自动导入镜像,替换容器时保留旧容器以便回退,数据永远保留,日志轮转
- 修正 bubblewrap 的可用性判断:以前只看有没有装 bwrap,容器里和 Ubuntu 24.04 默认配置下装了却建不了命名空间,
  每次执行代码都失败;现在真起一次试,不能用就退回下一个后端,并写明原因
- README 增加「生产部署」一节,部署细节见 deploy/README.md
This commit is contained in:
ThearchyHelios
2026-09-30 09:48:16 +08:00
parent 03c18c4d78
commit 6af4e396b9
16 changed files with 1742 additions and 7 deletions
+20
View File
@@ -44,6 +44,11 @@ class Settings(BaseSettings):
# --- 存储 ---
data_dir: Path = PROJECT_DIR / "data"
# --- 前端构建产物(生产部署用)---
# 设了就由服务端一并托管页面(见 app/web.py),页面和接口同源、只起一个进程。
# 不设(开发模式)什么都不变:页面由 Vite 开发服务器提供。相对路径按仓库根目录算
web_dist: Path | None = None
# --- 密钥加密 ---
# 用于加密落库的 provider api key。留空则自动生成并存到 data_dir/.secret_key。
secret_key: str = ""
@@ -112,6 +117,21 @@ class Settings(BaseSettings):
return [part.strip() for part in text.split(",") if part.strip()]
return value
@field_validator("web_dist", mode="before")
@classmethod
def _blank_is_unset(cls, value: object) -> object:
# AGENTLAB_WEB_DIST= 写成空值等于没设,别变成 Path("") 也就是当前目录
if isinstance(value, str) and not value.strip():
return None
return value
@field_validator("web_dist")
@classmethod
def _relative_to_project(cls, value: Path | None) -> Path | None:
if value is None or value.is_absolute():
return value
return PROJECT_DIR / value
@field_validator("http_tool_fake_ip_ranges")
@classmethod
def _valid_ranges(cls, value: list[str]) -> list[str]:
+8
View File
@@ -120,3 +120,11 @@ async def value_error_handler(_: Request, exc: ValueError) -> JSONResponse:
@app.get("/api/health")
async def health() -> dict[str, str]:
return {"status": "ok", "service": "agentlab"}
# 生产部署:前端构建产物由服务端一并托管(见 app/web.py)。页面回退是通配路由,
# 必须在所有 /api 路由之后注册。没设 AGENTLAB_WEB_DIST 时什么都不挂,开发模式照旧
if settings.web_dist is not None:
from app.web import mount_web
mount_web(app, settings.web_dist)
+52 -3
View File
@@ -5,8 +5,10 @@ import logging
import os
import resource
import shutil
import subprocess
import sys
import time
from functools import lru_cache
from pathlib import Path
from app.core.config import settings, sanitize_session
@@ -30,6 +32,37 @@ _LANG = {
_RO_BINDS = ["/usr", "/lib", "/lib64", "/bin", "/sbin", "/etc/alternatives", "/etc/ssl"]
def _on_linux() -> bool:
return sys.platform.startswith("linux")
@lru_cache(maxsize=4)
def _probe(bwrap: str) -> str:
"""真起一次 bwrap,返回失败原因;空串表示能用。
装了 bwrap 不等于建得了隔离环境:Docker 默认的 seccomp 配置不许非特权进程建
user namespace,Ubuntu 24.04 的 AppArmor 默认也限制它。以前 available() 只看
二进制在不在,于是容器里 auto 选中 bubblewrap,每次执行都报 "No permissions to
create a new namespace"——而本地子进程明明能用。
探测带上执行时同样要用的命名空间、/proc 和 /dev,任何一样建不起来都算不可用。
结果缓存:内核和容器权限在进程生命周期里不会变。
"""
argv = [
bwrap, "--unshare-all", "--die-with-parent", "--new-session",
"--ro-bind", "/", "/", "--proc", "/proc", "--dev", "/dev",
"--", shutil.which("true") or "/bin/true",
]
try:
proc = subprocess.run(argv, capture_output=True, text=True, timeout=10)
except (OSError, subprocess.SubprocessError) as e:
return raw(e)
if proc.returncode == 0:
return ""
lines = [line.strip() for line in (proc.stderr or "").splitlines() if line.strip()]
return lines[-1] if lines else f"退出码 {proc.returncode}"
class BubblewrapSandbox(Sandbox):
"""Linux 上的对等方案:bubblewrap(Flatpak 的沙箱内核)。
@@ -46,7 +79,23 @@ class BubblewrapSandbox(Sandbox):
@staticmethod
def available() -> bool:
return sys.platform.startswith("linux") and shutil.which("bwrap") is not None
if not _on_linux():
return False
bwrap = shutil.which("bwrap")
return bwrap is not None and not _probe(bwrap)
@staticmethod
def probe_failure() -> str:
"""装了 bwrap 却建不起隔离环境时的原因;没装或者能用都返回空串。"""
bwrap = shutil.which("bwrap") if _on_linux() else None
return _probe(bwrap) if bwrap else ""
@classmethod
def unavailable_reason(cls) -> str:
failure = cls.probe_failure()
if failure:
return f"已安装 bwrap,但无法创建隔离环境(容器默认权限或系统策略不允许):{failure}"
return "未找到 bwrap,可执行 apt install bubblewrap 安装"
async def health(self) -> dict[str, object]:
ok = self.available()
@@ -59,7 +108,7 @@ class BubblewrapSandbox(Sandbox):
"enforced": {"timeout": True, "cpu": True, "memory": True, "network": True, "fsize": True},
"root": str(self._root),
"interpreter": interpreter.describe(),
**({} if ok else {"error": "未找到 bwrap,可执行 apt install bubblewrap 安装"}),
**({} if ok else {"error": self.unavailable_reason()}),
}
def _session_dir(self, session_id: str) -> Path:
@@ -119,7 +168,7 @@ class BubblewrapSandbox(Sandbox):
if language not in _LANG:
return ExecResult(ok=False, backend=self.name, error=f"不支持的语言:{language}")
if not self.available():
return ExecResult(ok=False, backend=self.name, error="bwrap 不可用")
return ExecResult(ok=False, backend=self.name, error=self.unavailable_reason())
ext, cmd_prefix = _LANG[language]
# python 的命令前缀运行时才定:要挑一个看不见后端依赖的解释器
+6 -1
View File
@@ -127,7 +127,12 @@ class SandboxManager:
if warm is not None and not warm():
self._resolved_from = f"{name} 可用但镜像未缓存,自动选择时暂不使用"
continue
self._resolved_from = f"自动选择 {name}({_why(name)})"
why = _why(name)
if name == "local" and BubblewrapSandbox.probe_failure():
# 装了 bwrap 却建不起隔离环境(容器默认权限下就是这样)要说清楚,
# 不然看着装了 bwrap 却退到本地子进程,没人知道为什么
why = "已安装 bwrap,但无法创建隔离环境,已降级为本地子进程"
self._resolved_from = f"自动选择 {name}({why})"
return impl()
self._resolved_from = "没有可用的隔离方式,已降级为本地子进程"
+96
View File
@@ -0,0 +1,96 @@
"""生产部署:由服务端直接托管前端构建产物(AGENTLAB_WEB_DIST)。
开发时页面由 Vite 开发服务器提供、/api 由它转发,用不到这里。生产部署只起一个进程,
页面和接口同源,省掉 CORS,也不用另配静态服务器。
规则:
- 构建产物里有这个文件就返回它;
- 其余非 /api 的 GET 回退到 index.html:前端是单页应用,/studio 这类地址由前端路由处理;
- /api 下不存在的路径照旧是 JSON 404。排除 /api 写在路由的路径规则里,不在处理函数里判断——
否则 POST /api/不存在 会因为「路径对上了、方法不对」变成 405;
- 只注册 GET / HEAD 的 HTTP 路由,WebSocket 不经过这里;
- index.html 不缓存,发版后刷新即生效;assets/ 下的文件名带内容哈希,长缓存。
"""
from __future__ import annotations
import mimetypes
from pathlib import Path
from fastapi import FastAPI, HTTPException
from fastapi.responses import FileResponse
from starlette.convertors import Convertor, register_url_convertor
_NO_CACHE = "no-cache"
_IMMUTABLE = "public, max-age=31536000, immutable"
# 精简的 Linux 镜像里没有 /etc/mime.types,mimetypes 的内置表又缺几种(woff2 之类)。
# 构建产物里出现的类型写死在这里,别的再交给 mimetypes 猜
_TYPES = {
".html": "text/html; charset=utf-8",
".js": "text/javascript; charset=utf-8",
".mjs": "text/javascript; charset=utf-8",
".css": "text/css; charset=utf-8",
".json": "application/json",
".map": "application/json",
".svg": "image/svg+xml",
".png": "image/png",
".ico": "image/x-icon",
".woff": "font/woff",
".woff2": "font/woff2",
".wasm": "application/wasm",
".txt": "text/plain; charset=utf-8",
}
class _WebPath(Convertor[str]):
"""任意路径,但不以 api 这一段开头:/api、/api/… 留给接口路由和它们自己的 404。"""
regex = r"(?!api(?:/|$)).*"
def convert(self, value: str) -> str:
return value
def to_string(self, value: str) -> str:
return value
register_url_convertor("agentlab_web", _WebPath())
def _media_type(path: Path) -> str | None:
return _TYPES.get(path.suffix.lower()) or mimetypes.guess_type(path.name)[0]
def mount_web(app: FastAPI, dist: Path) -> None:
"""在 app 上挂页面托管。必须在所有 /api 路由注册之后调用。"""
root = dist.resolve()
index = root / "index.html"
if not index.is_file():
# 配了却没有产物:启动时就报出来,别等打开页面才看到一个 404
raise RuntimeError(
f"AGENTLAB_WEB_DIST 指向的 {root} 下没有 index.html。"
"先构建前端(cd frontend && pnpm build),或者去掉这个设置"
)
def respond(target: Path, *, immutable: bool = False) -> FileResponse:
return FileResponse(
target,
media_type=_media_type(target),
headers={"Cache-Control": _IMMUTABLE if immutable else _NO_CACHE},
)
async def serve(path: str) -> FileResponse:
if path:
target = (root / path).resolve()
# resolve 之后还得在 dist 里面:挡住 %2e%2e 这类穿越和指向外面的软链
if target.is_file() and target.is_relative_to(root):
return respond(target, immutable=path.startswith("assets/"))
if path.startswith("assets/"):
# 发版后旧页面还在请求旧哈希的文件。回退成 index.html 的话浏览器会把它当脚本
# 解析、报一个看不懂的 MIME 错误,所以明确给 404
raise HTTPException(status_code=404)
return respond(index)
app.add_api_route(
"/{path:agentlab_web}", serve, methods=["GET", "HEAD"], include_in_schema=False,
)
+105
View File
@@ -0,0 +1,105 @@
"""装了 bwrap 不等于建得了隔离环境。
Docker 默认的 seccomp 配置不许非特权进程建 user namespace,Ubuntu 24.04 的 AppArmor 默认也限制它。
以前 available() 只看 bwrap 在不在:容器里 auto 选中 bubblewrap,每次执行都报
「No permissions to create a new namespace」,而本地子进程明明能用。现在真起一次 bwrap 试,
起不来就算不可用,auto 往下退。
"""
from __future__ import annotations
import subprocess
from typing import Any
import pytest
from app.sandbox import bubblewrap_sandbox as bw
from app.sandbox import manager as mgr
from app.sandbox.local_sandbox import LocalSandbox
DENIED = (
"bwrap: No permissions to create a new namespace, likely because the kernel does not "
"allow non-privileged user namespaces.\n"
)
@pytest.fixture(autouse=True)
def _fresh_probe(monkeypatch: pytest.MonkeyPatch):
bw._probe.cache_clear()
monkeypatch.setattr(bw, "_on_linux", lambda: True)
yield
bw._probe.cache_clear()
def _fake_run(returncode: int, stderr: str = "", calls: list[Any] | None = None):
def run(argv, **kwargs): # noqa: ANN001, ANN003, ANN202
if calls is not None:
calls.append(argv)
return subprocess.CompletedProcess(argv, returncode, stdout="", stderr=stderr)
return run
def test_binary_present_but_namespaces_denied(monkeypatch: pytest.MonkeyPatch) -> None:
calls: list[Any] = []
monkeypatch.setattr(bw.shutil, "which", lambda name: f"/usr/bin/{name}")
monkeypatch.setattr(bw.subprocess, "run", _fake_run(1, DENIED, calls))
assert bw.BubblewrapSandbox.available() is False
assert "No permissions" in bw.BubblewrapSandbox.probe_failure()
# 探测真的在起 bwrap,并且带上执行时同样要用的命名空间和 /proc
argv = calls[0]
assert argv[0] == "/usr/bin/bwrap"
assert "--unshare-all" in argv and "--proc" in argv
# 结果缓存:内核和容器权限在进程生命周期里不会变
bw.BubblewrapSandbox.available()
assert len(calls) == 1
async def test_health_explains_why(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(bw.shutil, "which", lambda name: f"/usr/bin/{name}")
monkeypatch.setattr(bw.subprocess, "run", _fake_run(1, DENIED))
info = await bw.BubblewrapSandbox().health()
assert info["available"] is False
assert "No permissions" in info["error"]
def test_binary_present_and_working(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(bw.shutil, "which", lambda name: f"/usr/bin/{name}")
monkeypatch.setattr(bw.subprocess, "run", _fake_run(0))
assert bw.BubblewrapSandbox.available() is True
assert bw.BubblewrapSandbox.probe_failure() == ""
def test_probe_that_cannot_start_counts_as_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
def boom(argv, **kwargs): # noqa: ANN001, ANN003, ANN202
raise PermissionError(13, "Permission denied")
monkeypatch.setattr(bw.shutil, "which", lambda name: f"/usr/bin/{name}")
monkeypatch.setattr(bw.subprocess, "run", boom)
assert bw.BubblewrapSandbox.available() is False
def test_no_binary_skips_probe(monkeypatch: pytest.MonkeyPatch) -> None:
calls: list[Any] = []
monkeypatch.setattr(bw.shutil, "which", lambda name: None)
monkeypatch.setattr(bw.subprocess, "run", _fake_run(0, calls=calls))
assert bw.BubblewrapSandbox.available() is False
assert bw.BubblewrapSandbox.probe_failure() == ""
assert calls == []
def test_auto_falls_back_to_local_and_says_why(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(bw.shutil, "which", lambda name: f"/usr/bin/{name}")
monkeypatch.setattr(bw.subprocess, "run", _fake_run(1, DENIED))
monkeypatch.setattr(mgr.MicroVMSandbox, "available", staticmethod(lambda: False))
monkeypatch.setattr(mgr.SeatbeltSandbox, "available", staticmethod(lambda: False))
monkeypatch.setattr(mgr.settings, "sandbox_backend", "auto")
manager = mgr.SandboxManager()
backend = manager._resolve()
assert isinstance(backend, LocalSandbox)
assert "bwrap" in manager._resolved_from
assert "本地子进程" in manager._resolved_from
# 节点上选 fast 档也不能点到一个起不来的 bubblewrap
monkeypatch.setattr(mgr.sys, "platform", "linux")
assert manager._resolve_named("fast") is None
+221
View File
@@ -0,0 +1,221 @@
"""生产模式下服务端托管前端构建产物(AGENTLAB_WEB_DIST)。
用临时目录伪造一个 dist,覆盖这几条:
- 文件存在就返回文件;其余非 /api 的 GET 回退到 index.html(单页应用);
- /api 下不存在的路径照旧是 JSON 404,不能被回退吞掉——POST 也一样,不能变成 405;
- WebSocket 不受影响;
- index.html 不缓存,assets/ 下带哈希的文件长缓存;
- 没设 AGENTLAB_WEB_DIST 时行为完全不变。
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
import httpx
import pytest
from fastapi import APIRouter, FastAPI, WebSocket
from fastapi.testclient import TestClient
from starlette.websockets import WebSocketDisconnect
from app.core.config import BACKEND_DIR, PROJECT_DIR, Settings
INDEX = "<!doctype html><title>fake-agentlab</title><div id=root></div>"
JS = "console.log('fake bundle')"
@pytest.fixture
def dist(tmp_path: Path) -> Path:
root = tmp_path / "dist"
(root / "assets").mkdir(parents=True)
(root / "index.html").write_text(INDEX)
(root / "favicon.svg").write_text("<svg xmlns='http://www.w3.org/2000/svg'/>")
(root / "assets" / "index-AbC123.js").write_text(JS)
(root / "assets" / "index-AbC123.css").write_text("body{}")
# dist 外面的文件:路径穿越不能读到它
(tmp_path / "secret.txt").write_text("TOP-SECRET")
return root
def _app(dist: Path) -> FastAPI:
from app.web import mount_web
app = FastAPI()
api = APIRouter(prefix="/api")
@api.get("/ping")
async def ping() -> dict[str, str]:
return {"pong": "1"}
@api.websocket("/echo")
async def echo(ws: WebSocket) -> None:
await ws.accept()
await ws.send_text(await ws.receive_text())
await ws.close()
app.include_router(api)
mount_web(app, dist)
return app
async def _get(app: FastAPI, path: str, method: str = "GET") -> httpx.Response:
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:
return await client.request(method, path)
async def test_index_and_spa_fallback(dist: Path) -> None:
app = _app(dist)
for path in ("/", "/index.html", "/studio", "/runs/abc/evidence", "/studio/"):
r = await _get(app, path)
assert r.status_code == 200, path
assert r.text == INDEX, path
assert r.headers["content-type"].startswith("text/html"), path
assert "no-cache" in r.headers["cache-control"], path
async def test_existing_files_are_served(dist: Path) -> None:
app = _app(dist)
r = await _get(app, "/assets/index-AbC123.js")
assert r.status_code == 200
assert r.text == JS
assert "javascript" in r.headers["content-type"]
assert "immutable" in r.headers["cache-control"]
assert "max-age=31536000" in r.headers["cache-control"]
r = await _get(app, "/assets/index-AbC123.css")
assert r.status_code == 200 and r.headers["content-type"].startswith("text/css")
r = await _get(app, "/favicon.svg")
assert r.status_code == 200
assert r.headers["content-type"].startswith("image/svg+xml")
assert "immutable" not in r.headers.get("cache-control", "")
async def test_missing_asset_is_404_not_index(dist: Path) -> None:
# 发版后旧页面请求旧哈希的文件:回退成 index.html 会被当脚本解析,还会被长缓存
r = await _get(_app(dist), "/assets/index-old999.js")
assert r.status_code == 404
assert r.text != INDEX
async def test_api_routes_win_and_unknown_api_is_json_404(dist: Path) -> None:
app = _app(dist)
r = await _get(app, "/api/ping")
assert r.status_code == 200 and r.json() == {"pong": "1"}
for path in ("/api/nope", "/api", "/api/", "/api/runs/xyz/nope"):
r = await _get(app, path)
assert r.status_code == 404, path
assert r.headers["content-type"].startswith("application/json"), path
assert r.json() == {"detail": "Not Found"}, path
# 路径规则里就排除了 /api:否则「路径对上、方法不对」会让 POST 变成 405
r = await _get(app, "/api/nope", method="POST")
assert r.status_code == 404
assert r.json() == {"detail": "Not Found"}
# 只保留 api 这一段,别的前缀照常回退
r = await _get(app, "/apiary")
assert r.status_code == 200 and r.text == INDEX
async def test_head_and_non_get(dist: Path) -> None:
app = _app(dist)
r = await _get(app, "/studio", method="HEAD")
assert r.status_code == 200
r = await _get(app, "/studio", method="POST")
assert r.status_code == 405
async def test_path_traversal_stays_inside_dist(dist: Path) -> None:
app = _app(dist)
for path in ("/%2e%2e/secret.txt", "/..%2fsecret.txt", "/assets/%2e%2e/%2e%2e/secret.txt"):
r = await _get(app, path)
assert "TOP-SECRET" not in r.text, path
def test_websocket_unaffected(dist: Path) -> None:
client = TestClient(_app(dist))
with client.websocket_connect("/api/echo") as ws:
ws.send_text("hi")
assert ws.receive_text() == "hi"
# 没有这条 WebSocket 路由的地址不会被页面回退接走
with pytest.raises(WebSocketDisconnect):
with client.websocket_connect("/studio") as ws:
ws.receive_text()
def test_missing_index_fails_fast(tmp_path: Path) -> None:
from app.web import mount_web
with pytest.raises(RuntimeError, match="index.html"):
mount_web(FastAPI(), tmp_path)
def test_setting_parsing(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("AGENTLAB_WEB_DIST", raising=False)
assert Settings().web_dist is None
monkeypatch.setenv("AGENTLAB_WEB_DIST", "")
assert Settings().web_dist is None
# 相对路径按仓库根目录算,和从哪个目录启动无关
monkeypatch.setenv("AGENTLAB_WEB_DIST", "frontend/dist")
assert Settings().web_dist == PROJECT_DIR / "frontend" / "dist"
monkeypatch.setenv("AGENTLAB_WEB_DIST", "/srv/web")
assert Settings().web_dist == Path("/srv/web")
async def test_default_app_unchanged_without_setting() -> None:
from app.core.config import settings
from app.main import app
assert settings.web_dist is None
r = await _get(app, "/studio")
assert r.status_code == 404
assert r.json() == {"detail": "Not Found"}
_PROBE = r"""
import asyncio, json, httpx
from app.main import app
async def main():
t = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=t, base_url="http://t") as c:
out = {}
for m, p in (("GET", "/api/health"), ("GET", "/"), ("GET", "/studio"),
("GET", "/assets/index-AbC123.js"), ("GET", "/api/nope"), ("POST", "/api/nope")):
r = await c.request(m, p)
out[f"{m} {p}"] = [r.status_code, r.text[:200]]
print(json.dumps(out))
asyncio.run(main())
"""
def test_main_app_mounts_when_setting_present(dist: Path, tmp_path: Path) -> None:
"""真实的 app.main 在设了 AGENTLAB_WEB_DIST 时托管页面、接口照旧。
settings 和 app 都是模块级单例,同一进程里改不了,所以起一个子进程导入。
"""
env = {
**os.environ,
"AGENTLAB_DATA_DIR": str(tmp_path / "data"),
"AGENTLAB_WEB_DIST": str(dist),
"AGENTLAB_SECRET_KEY": "test-only-not-a-real-key",
}
proc = subprocess.run(
[sys.executable, "-c", _PROBE], cwd=BACKEND_DIR, env=env,
capture_output=True, text=True, timeout=120,
)
assert proc.returncode == 0, proc.stderr[-2000:]
out = json.loads(proc.stdout.strip().splitlines()[-1])
assert out["GET /api/health"][0] == 200 and '"agentlab"' in out["GET /api/health"][1]
assert out["GET /"] == [200, INDEX]
assert out["GET /studio"] == [200, INDEX]
assert out["GET /assets/index-AbC123.js"] == [200, JS]
assert out["GET /api/nope"][0] == 404 and json.loads(out["GET /api/nope"][1]) == {"detail": "Not Found"}
assert out["POST /api/nope"][0] == 404