AgentLab/backend/tests/test_guardrails.py

"""三道"配了、显示了、却没人执行"的护栏。

- max_run_seconds：配置里有，没有任何代码执行它。一个挂住的模型或工具能把运行
  和它占着的并发名额一直拖着。
- 模型调用超时：ModelSpec.timeout 从没被赋过值，所有调用都走 SDK 默认的 10 分钟
  外加重试。
- manifest 哈希：只写不验，而且是在 run.finished 落库之前算的——清单里恰恰少了
  那条宣布"跑完了、成果是什么"的事件。

显示成生效却没生效，比明说没有更危险：它让人以为有东西在兜底。
"""
from __future__ import annotations

import asyncio
import time

import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import select, update

from app.core.artifact_store import manifest_hash
from app.core.config import settings
from app.core.crypto import encrypt
from app.db.base import SessionLocal
from app.db.models import Provider, Run, RunEvent
from app.engine import compiler
from app.engine.runner import run_manager, verify_manifest
from app.engine.schema import NodeType
from app.main import app
from app.providers.factory import ModelSpec, build_chat_model


@pytest.fixture(autouse=True)
async def engine_up():
    await run_manager.setup()
    yield
    await run_manager.shutdown()


def node(nid, ntype, **config):
    return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0},
            "data": {"label": nid, "config": config}}


def chain(*nodes):
    return {"nodes": list(nodes),
            "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}


SIMPLE = chain(
    node("start", "input", fields=[{"name": "question"}]),
    node("t", "transform", mode="template", template="好", assign_to="x"),
    node("out", "output", fields=[{"name": "结果", "value": "{{ vars.x }}"}]),
)


async def _run(graph, *, timeout=20.0) -> Run:
    run = await run_manager.start(graph=graph, input_payload={"question": "随便"})
    deadline = asyncio.get_event_loop().time() + timeout
    while asyncio.get_event_loop().time() < deadline:
        async with SessionLocal() as session:
            row = await session.get(Run, run.id)
            # 终态先落库、清单随后另一个事务封上（runner._finalize）：只等状态的话，负载一高就会
            # 读到「跑完了但还没封」的那一瞬。中断态不封存
            if row and (row.status == "interrupted"
                        or row.status in ("succeeded", "failed", "cancelled") and row.manifest_seq is not None):
                return row
        await asyncio.sleep(0.05)
    raise AssertionError("运行没有按时结束")


# --------------------------------------------------------------------------
# max_run_seconds
# --------------------------------------------------------------------------


async def test_a_run_past_its_time_limit_is_stopped(monkeypatch):
    async def stuck(state, ctx):
        await asyncio.sleep(30)        # 一个不响应的模型或工具
        return {}

    monkeypatch.setitem(compiler.RUNNERS, NodeType.TRANSFORM, stuck)
    monkeypatch.setattr(settings, "max_run_seconds", 1)

    started = time.perf_counter()
    run = await _run(SIMPLE)
    assert run.status == "failed"
    assert "1 秒的时长上限" in (run.error or "")
    assert time.perf_counter() - started < 10, "上限没起作用，等到了节点自己结束"


# --------------------------------------------------------------------------
# 模型调用超时
# --------------------------------------------------------------------------


def test_models_get_a_timeout_even_when_the_node_sets_none():
    provider = Provider(name="t", kind="anthropic", api_key=encrypt("sk-test"), models=[],
                        default_model="claude-sonnet-5", enabled=True, extra={})
    model = build_chat_model(provider, ModelSpec())
    assert model.default_request_timeout == settings.model_timeout_seconds
    # 节点显式配了的照旧优先
    assert build_chat_model(provider, ModelSpec(timeout=42)).default_request_timeout == 42


# --------------------------------------------------------------------------
# manifest
# --------------------------------------------------------------------------


async def _events(run_id):
    async with SessionLocal() as session:
        return list((await session.execute(
            select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq)
        )).scalars())


async def test_the_seal_covers_run_finished_and_verifies():
    run = await _run(SIMPLE)
    assert run.status == "succeeded", run.error
    events = await _events(run.id)
    finished = next(e for e in events if e.type == "run.finished")
    assert run.manifest_seq == finished.seq, "清单没有封到 run.finished"

    report = await verify_manifest(run.id)
    assert report["sealed"] and report["ok"], report


async def test_tampering_with_the_output_is_caught():
    """改成果——以前 run.finished 不在清单里，这一改清单照样对得上。"""
    run = await _run(SIMPLE)
    async with SessionLocal() as session:
        await session.execute(
            update(RunEvent)
            .where(RunEvent.run_id == run.id, RunEvent.type == "run.finished")
            .values(data={"output": {"结果": "被改过的结论"}})
        )
        await session.commit()
    report = await verify_manifest(run.id)
    assert report["ok"] is False


async def test_events_appended_after_the_seal_are_not_tampering():
    """封存之后追加的事件（复核批注之类）不在范围内，不该让核对失败。"""
    run = await _run(SIMPLE)
    async with SessionLocal() as session:
        row = await session.get(Run, run.id)
        session.add(RunEvent(run_id=run.id, seq=row.manifest_seq + 5, type="log", ts=time.time(),
                             node_id=None, data={"message": "事后追加的批注"}))
        await session.commit()
    assert (await verify_manifest(run.id))["ok"] is True


async def test_runs_sealed_before_manifest_seq_existed_verify_by_the_old_cut():
    """老运行的哈希是在 run.finished 落库之前算的，没有 manifest_seq——按当时的口径核。"""
    run = await _run(SIMPLE)
    rows = [(e.seq, e.type, e.node_id, e.data) for e in await _events(run.id)
            if e.type != "run.finished"]
    async with SessionLocal() as session:
        await session.execute(update(Run).where(Run.id == run.id)
                              .values(manifest_hash=manifest_hash(rows), manifest_seq=None))
        await session.commit()
    report = await verify_manifest(run.id)
    assert report["ok"] is True and report["legacy"] is True


async def test_a_run_waiting_for_a_human_is_not_sealed_yet():
    graph = chain(
        node("start", "input", fields=[{"name": "question"}]),
        node("h", "human", mode="approve", title="批吗"),
        node("out", "output", fields=[]),
    )
    run = await _run(graph)
    assert run.status == "interrupted"
    report = await verify_manifest(run.id)
    assert report["sealed"] is False and report["ok"] is None


async def test_verify_through_the_api():
    run = await _run(SIMPLE)
    transport = ASGITransport(app=app)
    async with AsyncClient(transport=transport, base_url="http://test") as client:
        ok = await client.get(f"/api/runs/{run.id}/verify")
        assert ok.status_code == 200 and ok.json()["ok"] is True
        assert (await client.get("/api/runs/没有这个/verify")).status_code == 404