AgentLab/backend/tests/test_approval_report_check.py

"""人工审批审的是报告时，审批卡附上这份报告的核对概况（human 节点的请求里带 reports）。

真实运行里：报告里有一句结论和证据相矛盾，审批人批准时完全不知道——报告节点只把正文交给了审批卡，几个数字有出处、
结论句判没判过，一个字都没有。用户事后在证据面板点开那句，裁判才判它相矛盾。

现在 human 节点挂起时，上游的报告节点（直接连着的，或者标题、说明里引用了它正文的）各带一份核对概况：
数字核对的统计（report.checked 的 stats）、结论句策略、裁判摘要（几句判了、几句相矛盾），以及报告文档的工件 id，
审批卡据此写「N 个数字都有出处」「结论句已裁判 X 句，证据相矛盾 Y 句」或「结论句尚未核对」，并给出打开证据的入口。

表名、指标、数字都是虚构的。
"""
from __future__ import annotations

import asyncio
import json
import re

import pytest
from langchain_core.messages import AIMessage
from sqlalchemy import delete, select

from app.db.base import SessionLocal
from app.db.models import Approval, Run, RunEvent, Setting
from app.engine import judge as judge_mod
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
from app.engine.runner import run_manager
from app.providers.mock_model import MockChatModel

PRICED = "claude-sonnet-5"
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
TEXT = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。"


@pytest.fixture(autouse=True)
async def engine_up():
    await run_manager.setup()
    async with SessionLocal() as session:
        await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
        await session.commit()
    yield
    await run_manager.shutdown()


@pytest.fixture(autouse=True)
def models(monkeypatch):
    """写作者回 TEXT；裁判把「新客」那句判成证据相矛盾，其余有依据。"""
    def decide(model, messages):
        if JUDGE_ROLE in str(messages[0].content):
            request = str(messages[-1].content)
            items = [{"unit": uid, "verdict": "contradicted" if "新客" in line else "supported",
                      "rationale": "虚构的理由", "used": []}
                     for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)]
            return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
        return AIMessage(content=TEXT)

    monkeypatch.setattr(MockChatModel, "_decide", decide)

    async def judge_model_of(session, spec):
        return MockChatModel(model_name=PRICED), PRICED

    monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)


def node(nid, ntype, label=None, **config):
    return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}}


def graph(*, gate: dict, wire_report_to_gate: bool = True, **report):
    nodes = [
        node("start", "input"),
        node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
        node("caliber", "metrics", caliber="周报口径", caliber_version="v1", metrics=METRICS),
        node("write", "report", "写周报", instructions="写周报", **report),
        node("note", "transform", mode="expression", expression="'ok'", assign_to="note"),
        node("gate", "human", "主管审批", mode="approve", **gate),
        node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
    ]
    chain = ["start", "fetch", "caliber", "write"] + (["gate"] if wire_report_to_gate else ["note", "gate"])
    edges = [{"source": a, "target": b} for a, b in zip(chain, chain[1:])]
    edges.append({"source": "gate", "target": "out", "sourceHandle": "approved"})
    return {"nodes": nodes, "edges": edges}


async def paused(g, *, run_class: str) -> tuple[Run, Approval]:
    started = await run_manager.start(graph=g, input_payload={}, run_class=run_class)
    for _ in range(400):
        await asyncio.sleep(0.05)
        async with SessionLocal() as session:
            row = await session.get(Run, started.id)
            if row.status in ("interrupted", "failed", "succeeded"):
                approval = (await session.execute(select(Approval).where(Approval.run_id == row.id))).scalar_one_or_none()
                if row.status != "interrupted" or approval is not None:
                    break
    assert row.status == "interrupted", row.error
    assert approval is not None
    return row, approval


async def test_formal_run_carries_the_judged_claims_to_the_approval():
    row, approval = await paused(graph(gate={"title": "审核周报", "message": "{{ nodes.write.text }}"},
                                       claims="judge"), run_class="formal")
    [report] = approval.payload["reports"]
    assert report["node_id"] == "write" and report["label"] == "写周报"
    assert isinstance(report["doc_artifact"], str) and report["doc_artifact"]
    assert report["stats"]["numbers"] == report["stats"]["numbers_cited"] == 1
    assert report["claims"] == "judge"
    judge = report["judge"]
    assert judge["mode"] == "inline"
    assert judge["counts"]["contradicted"] == 1 and judge["counts"]["supported"] == 1
    assert judge["counts"]["unjudged"] == 0
    # 和 report.checked 是同一份：审批卡写的和时间线那一行对得上
    async with SessionLocal() as session:
        [checked] = list((await session.execute(select(RunEvent).where(
            RunEvent.run_id == row.id, RunEvent.type == "report.checked"))).scalars())
    assert checked.data["doc_artifact"] == report["doc_artifact"]
    assert {k: checked.data["stats"][k] for k in report["stats"]} == report["stats"]
    assert checked.data["judge"]["counts"] == judge["counts"]
    # human.requested 事件里也带着，回放时界面一样读得到
    async with SessionLocal() as session:
        [requested] = list((await session.execute(select(RunEvent).where(
            RunEvent.run_id == row.id, RunEvent.type == "human.requested"))).scalars())
    assert requested.data["reports"] == approval.payload["reports"]


async def test_exploratory_report_under_approval_is_judged_in_the_node():
    """要交给人审批的报告，探索运行也在节点里判：审批人据此拍板，批准前就该知道哪句结论站不住。
    真实运行里探索运行的报告有一句和证据相矛盾，审批卡只能写「结论句尚未核对」——按需裁判要等封存
    （运行结束）之后才能判，而封存前放开按需裁判会动证据清单的完整性约束，所以改在节点里当场判。
    没有审批的探索运行照旧按需判，不在节点里花这笔钱（test_contract_judge.test_explore_runs_only_annotate）。"""
    _, approval = await paused(graph(gate={"title": "审核周报"}, claims="judge"), run_class="exploratory")
    [report] = approval.payload["reports"]
    assert report["claims"] == "judge" and report["judge"]["mode"] == "inline"
    assert report["judge"]["counts"]["contradicted"] == 1 and report["judge"]["counts"]["unjudged"] == 0


async def test_report_quoted_in_the_approval_message_is_judged_in_the_node_too():
    _, approval = await paused(graph(gate={"title": "审核", "message": "请看：{{ nodes.write.text }}"},
                                     wire_report_to_gate=False, claims="judge"), run_class="exploratory")
    [report] = approval.payload["reports"]
    assert report["judge"]["mode"] == "inline" and report["judge"]["counts"]["contradicted"] == 1


async def test_report_without_claims_policy_still_reports_numbers():
    _, approval = await paused(graph(gate={"title": "审核周报"}), run_class="exploratory")
    [report] = approval.payload["reports"]
    assert report["claims"] == "off" and "judge" not in report
    assert report["stats"]["numbers"] == 1


async def test_a_report_quoted_in_the_message_counts_even_when_not_wired_directly():
    _, approval = await paused(graph(gate={"title": "审核", "message": "请看：{{ nodes.write.text }}"},
                                     wire_report_to_gate=False), run_class="exploratory")
    assert [r["node_id"] for r in approval.payload["reports"]] == ["write"]


async def test_a_report_quoted_through_its_variable_counts_too():
    _, approval = await paused(graph(gate={"title": "审核", "message": "{{ vars.weekly }}"}, wire_report_to_gate=False,
                                     assign_to="weekly"), run_class="exploratory")
    assert [r["node_id"] for r in approval.payload["reports"]] == ["write"]


async def test_approvals_unrelated_to_a_report_carry_nothing():
    _, approval = await paused(graph(gate={"title": "确认", "message": "继续吗"}, wire_report_to_gate=False),
                               run_class="exploratory")
    assert "reports" not in approval.payload