Files
AgentLab/backend/tests/test_judge_rework_grading.py

300 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""结论句裁判改造(JR)的判档:contradicted 沿用原来 unsupported 的处置,insufficient 按缺口处理。
- contradicted(证据相矛盾):按 on_unsupported 降档(degrade)或不予出具(withhold),和原来的 unsupported 一样
- insufficient(证据不足):按缺口处理,最多降档——不会不予出具,也不会判完整出具
- 不许比现在更松:受管正式运行里,原来会因「证据不支持」降档的句子,改判成 contradicted 或 insufficient
都不能完整出具
- 旧文档里的 unsupported 判档照旧当 contradicted 处理
- 统计键(stats、counts、出具横幅的结论句计数)都有 contradicted、insufficient,unsupported 保留
- 报告节点:改写一次把证据相矛盾、证据不足的句子都交回写作者;文档的裁判摘要记下写作模型
所有模型调用都是 mock 剧本,不碰真接口。
"""
from __future__ import annotations
import asyncio
import json
import re
import uuid
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import delete, select
from app.core import artifact_store
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent, Setting, Workflow, WorkflowVersion
from app.engine import judge as judge_mod
from app.engine.evidence import compose_doc, iter_units
from app.engine.issuance import decide_tier
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
from app.engine.nodes.io import _claims
from app.engine.runner import run_manager
from app.main import app
from app.providers.mock_model import MockChatModel
JUDGE_MODEL = "claude-sonnet-5"
WRITER = "mock-fast"
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
CITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]下面看细节。"
SUPPORTED, CAUSAL = "本周销售额 45,678.5元。", "增长主要来自新客首单。"
# --------------------------------------------------------------------------
# 出口判档(io._claims):直接喂文档
# --------------------------------------------------------------------------
def judged_doc(statuses: dict[str, dict], *, on_unsupported="degrade") -> dict:
"""两句结论:SUPPORTED_TEXT、CAUSAL_TEXT;statuses 按句子给判定。counts 照判定数。"""
doc = compose_doc("本周销售额稳定增长。增长主要来自新客首单。", {})
md = doc["markdown"]
counts = {k: 0 for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")}
for _, unit in iter_units(doc):
text = md[unit["span"][0]:unit["span"][1]].strip()
verdict = {"status": "supported", "rationale": "有依据", "judge": JUDGE_MODEL, "post_seal": False, "used": [],
**statuses.get(text, {})}
unit["verdict"] = verdict
counts[verdict["status"]] = counts.get(verdict["status"], 0) + 1
doc["judge"] = {"mode": "inline", "model": JUDGE_MODEL, "on_unsupported": on_unsupported, "counts": counts,
"unjudged": {}, "limits_hit": [], "complete": True, "gaps": []}
return doc
def grade(doc: dict, *, governed=False, contract=None) -> tuple[str, dict]:
out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None, "unsupported": None}
_claims(out, contract or {}, {"claims": "judge", "text": doc["markdown"]}, evidence_on=True, governed=governed,
uncited=[], doc=doc, title="write")
tier = decide_tier(missing_required=[], missing_expected=[], unmatched=[], strict=True, gaps=out["gaps"],
unsupported=out["unsupported"], uncited_claims=out["uncited"],
claims_policy=out["claims_policy"])
return tier, out
TEXT2 = "增长主要来自新客首单。"
def test_all_supported_is_still_a_full_issuance():
tier, out = grade(judged_doc({}))
assert tier == "formal" and out["gaps"] == []
@pytest.mark.parametrize(("on_unsupported", "tier"), [("degrade", "degraded"), ("withhold", "withheld")])
def test_contradicted_follows_on_unsupported(on_unsupported, tier):
doc = judged_doc({TEXT2: {"status": "contradicted", "rationale": "证据里新客占比下降"}},
on_unsupported=on_unsupported)
got, out = grade(doc)
assert got == tier
claims = out["claims"]
[item] = claims["contradicted"]
assert item["text"] == TEXT2 and item["rationale"] == "证据里新客占比下降"
assert claims["counts"]["contradicted"] == 1 and claims["counts"]["insufficient"] == 0
assert out["gaps"] == [], "判档来自判定,不来自缺口"
def test_insufficient_is_a_gap_and_never_withholds():
doc = judged_doc({TEXT2: {"status": "insufficient", "rationale": "摘录里没有新客维度", "missing": "新客维度的查询"}},
on_unsupported="withhold")
tier, out = grade(doc, governed=True, contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}})
assert tier == "degraded", "证据不足最多降档:不能不予出具,也不能完整出具"
[gap] = [g for g in out["gaps"] if "证据不足" in g]
assert gap.startswith("报告「write」") and "1 句" in gap and "新客维度的查询" in gap
[item] = out["claims"]["insufficient"]
assert item["text"] == TEXT2 and item["missing"] == "新客维度的查询"
assert out["claims"]["counts"]["insufficient"] == 1 and out["unsupported"] == []
def test_an_old_unsupported_verdict_is_graded_as_contradicted():
doc = judged_doc({TEXT2: {"status": "unsupported", "rationale": "证据不支持"}}, on_unsupported="withhold")
tier, out = grade(doc)
assert tier == "withheld"
assert out["claims"]["counts"]["unsupported"] == 1
assert [i["text"] for i in out["claims"]["unsupported"]] == [TEXT2]
assert [i["text"] for i in out["unsupported"]] == [TEXT2]
def test_the_issuance_counts_carry_every_key():
_, out = grade(judged_doc({}))
assert set(out["claims"]["counts"]) == set(judge_mod.COUNT_KEYS)
# --------------------------------------------------------------------------
# 端到端:正式运行、受管正式运行
# --------------------------------------------------------------------------
@pytest.fixture
async def engine_up():
await run_manager.setup()
async with SessionLocal() as session:
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
await session.commit()
yield
await run_manager.shutdown()
@pytest.fixture
async def client(engine_up):
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def weekly(*, report=None, contract=None) -> dict:
nodes = [
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
# 这里测的是裁判:写作者那句「增长主要来自新客首单」没有分析结论可挂,分析说法核对关掉,免得多重写一次
node("write", "report", instructions="写周报", **{"claims": "judge", "judge": {"max_cost_usd": 0.05},
"analysis_claims": "off", **(report or {})}),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
contract={"report_from": "write", "metrics_from": ["caliber"], "required": ["gmv"], "strict": True,
**(contract or {})}),
]
ids = [n["id"] for n in nodes]
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
class Models:
"""写作者依次回 writer 里的几段;裁判按句子里的字给判定:verdicts = {句中片段: (判定, missing)}。"""
def __init__(self, monkeypatch, writer=(CITED,), verdicts=None):
self.writes: list[list] = []
self.judges: list[str] = []
verdicts = verdicts or {}
def decide(model, messages):
if JUDGE_ROLE in str(messages[0].content):
request = str(messages[-1].content)
self.judges.append(request)
items = []
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M):
status, missing = next((v for frag, v in verdicts.items() if frag in line), ("supported", ""))
items.append({"unit": uid, "verdict": status, "rationale": f"判的是:{line.strip()[:12]}",
"missing": missing, "used": []})
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
self.writes.append(list(messages))
return AIMessage(content=writer[min(len(self.writes), len(writer)) - 1])
monkeypatch.setattr(MockChatModel, "_decide", decide)
async def judge_model_of(session, spec):
return MockChatModel(model_name=JUDGE_MODEL), JUDGE_MODEL
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
async def wait(run_id: str) -> Run:
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
return row
raise AssertionError(f"run {run_id} 没跑完:{row.status}")
async def run(graph, *, run_class="formal") -> Run:
started = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
row = await wait(started.id)
assert row.status == "succeeded", row.error
return row
async def checked_of(run_id: str) -> tuple[dict, dict]:
async with SessionLocal() as session:
[event] = (await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == "report.checked"))).scalars()
return event.data, artifact_store.load(event.data["doc_artifact"])
async def issuance_event(run_id: str) -> dict:
async with SessionLocal() as session:
[event] = (await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == "issuance"))).scalars()
return event.data
async def test_a_formal_run_withholds_a_contradicted_claim(engine_up, monkeypatch):
Models(monkeypatch, verdicts={"新客": ("contradicted", "")})
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
issuance = row.output["_issuance"]
assert issuance["tier"] == "withheld"
assert [i["text"] for i in issuance["claims"]["contradicted"]] == [CAUSAL]
event = await issuance_event(row.id)
assert event["claims"]["counts"] == {"supported": 1, "partial": 0, "contradicted": 1, "insufficient": 0,
"not_a_claim": 0, "unjudged": 0, "unsupported": 0}
async def test_a_formal_run_only_degrades_an_insufficient_claim(engine_up, monkeypatch):
Models(monkeypatch, verdicts={"新客": ("insufficient", "新客维度的数据")})
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
issuance = row.output["_issuance"]
assert issuance["tier"] == "degraded", "证据不足:最多降档,不因 withhold 不予出具"
assert any("证据不足" in g and "新客维度的数据" in g for g in issuance["gaps"]), issuance["gaps"]
[item] = issuance["claims"]["insufficient"]
assert item["text"] == CAUSAL and item["missing"] == "新客维度的数据"
assert issuance["claims"]["counts"]["insufficient"] == 1
checked, doc = await checked_of(row.id)
assert doc["stats"]["insufficient"] == 1 and doc["stats"]["contradicted"] == 0 and doc["stats"]["unsupported"] == 0
assert checked["stats"]["insufficient"] == 1
assert doc["judge"]["writer_model"] == WRITER and checked["judge"]["writer_model"] == WRITER
[unit] = [u for _, u in iter_units(doc) if u.get("verdict", {}).get("status") == "insufficient"]
assert unit["verdict"]["missing"] == "新客维度的数据"
async def governed(client, graph) -> Run:
async with SessionLocal() as session:
wf = Workflow(name=f"jr-{uuid.uuid4().hex[:6]}", graph=graph, status="governed", published_version=1)
session.add(wf)
await session.flush()
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph))
await session.commit()
wf_id = wf.id
res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"})
assert res.status_code == 201, res.text
row = await wait(res.json()["id"])
assert row.status == "succeeded", row.error
return row
@pytest.mark.parametrize("status", ["contradicted", "insufficient"])
async def test_a_governed_formal_run_is_never_fully_issued_on_a_split_verdict(client, monkeypatch, status):
"""不许比现在更松:原来判「证据不支持」要降档的句子,改判成哪一档都不能完整出具。"""
Models(monkeypatch, verdicts={"新客": (status, "新客维度的数据" if status == "insufficient" else "")})
graph = weekly(report={"numbers": "strict", "on_violation": "fail", "judge": {"max_cost_usd": None}})
row = await governed(client, graph)
assert row.output["_issuance"]["tier"] == "degraded"
async def test_rewrite_once_hands_back_contradicted_and_insufficient_sentences(engine_up, monkeypatch):
draft = ("本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]"
"华东贡献了大部分增长。[[see:m:gmv]]下面看细节。")
rewritten = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客、华东的贡献要等分维度的数据再看。下面看细节。"
models = Models(monkeypatch, writer=(draft, rewritten),
verdicts={"新客首单": ("contradicted", ""), "华东贡献": ("insufficient", "分区域的查询"),
"要等分维度": ("not_a_claim", "")})
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "rewrite_once": True}}))
assert len(models.writes) == 2
request = str(models.writes[1][-1].content)
assert "增长主要来自新客首单。" in request and "华东贡献了大部分增长。" in request
assert "分区域的查询" in request, "证据不足的句子连同缺的是什么一起交回"
assert SUPPORTED not in request
checked, _ = await checked_of(row.id)
assert checked["judge"]["rewrite"]["applied"] is True and len(checked["judge"]["rewrite"]["sentences"]) == 2
async def test_an_explore_report_records_the_writer_model_too(engine_up, monkeypatch):
models = Models(monkeypatch)
row = await run(weekly(), run_class="exploratory")
assert models.judges == []
checked, doc = await checked_of(row.id)
assert doc["judge"]["mode"] == "on_demand" and doc["judge"]["writer_model"] == WRITER
assert set(judge_mod.COUNT_KEYS) <= set(doc["stats"])