Files
AgentLab/backend/tests/test_contract_judge.py
T
ThearchyHelios 9727c21d8d 可点击证据第四期:结论句由模型裁判;只引单元格的问数据图能发成受管
一、结论句裁判

- 报告节点新增 claims: judge:写完、核对完之后,把结论句和它引用的证据摘录交给裁判模型,
  判为支持 / 部分支持 / 不支持 / 不是结论,附一句理由;判定只标注,不改写正文
- 送裁判的句子按固定规则预筛:带引用的、含方向词或因果词的送去判,短句、过渡句不送
- 证据摘录:指标写名称、值和原式;查询给 SQL 前 300 字和被引用的行,被引用的列一定给到;遮罩列一律不给;
  规则里写明「数字已经由系统核对过,不要复算」
- 一份报告合并成一次调用,超出句数就分批;结构化输出失败时退回 JSON 解析,再失败记为未裁判
- 裁判模型的优先级:节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型;思考关闭;
  和写作模型相同时给警告(等于自己审自己)
- rewrite_once(默认关):把不支持的句子和理由交回写作者只改这几句,再裁一次;
  改写稿冒出原稿没有的问题就不采用,保留原稿和原来的判定

二、成本上限

- 每份报告的句数、金额、时长,探索运行每次点击的金额,全局每日金额,每一项都可以自定义,也都可以设成不限;
  默认 40 句、$0.05、30 秒,每次点击 $0.01,每天 $2
- 调用前先估算,超出上限按优先级截断:先判带引用的,再判含方向词、因果词的,其余记为未裁判并写明「已到上限」;
  不报错,也不让报告失败
- 每日花费按本地日期计数,持久化在设置表里;重跑同一次运行不重复计费;价格目录里没有的模型只计次数,并写明估不出金额

三、正式运行与探索运行

- 正式运行在报告节点内裁判,判定写进报告文档、随文档封存;不支持的按 on_unsupported(degrade / withhold)降档或不予出具,
  裁判没跑成记缺口,不能判完整出具
- 探索运行按需裁判:POST /api/runs/{id}/evidence/judge,点开哪句判哪句;判定作为 evidence.judged 事件追加在封存之后,
  标「封存后追加」,封存核对仍然一致;只读封存范围内的工件;同一句不重复判;运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409,写明「正式运行的裁判在节点内完成」

四、治理与自动修复

- G3 放开 judge:受管级别写 judge 必须写预算(judge.max_cost_usd,或显式写 null 表示不限)
- 发布前自动修复:缺预算时给选项(默认金额 / 不限),不替人选;
  从 require_citation 改成 judge 不算降低要求,judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册:受管模板可以写 judge 并带预算;问数据按需裁判,不替人打开结论句策略

五、界面

- 结论句句末放徽标:支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?;不用表示「已核对」的绿色;n / N 也跳到不支持的句子
- 证据面板里「模型的解释」:带「模型判断 · 模型名 · 非确定」,封存后追加的另外标出;
  「请模型判断这句」只在探索运行、还没判过时出现,触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数;设置页新增「证据裁判」一组(裁判模型、各项上限、今日花费),设成不限时写明费用受什么约束;
  报告节点可以配 judge;运行轨迹里裁判相关的事件都有中文标题

六、门禁修正

- 引用模式下,图里没有口径卡(只引查询单元格)的,出具契约不再要求 metrics_from,受管门禁也不再要求 required:
  strict 加 cells 已经保证每个数都点得开、对不上就不予出具;有口径卡的照旧要求。
  以前问数据的图怎么都发不成受管,Copilot 补上契约也会被判「冒出新的问题」作废
2026-09-29 11:10:53 +08:00

324 lines
16 KiB
Python

"""出口契约(引用模式)按结论句裁判的判定判档(四期)。
- 报告节点 claims: judge 在正式运行里当场裁判,判定随文档封存。出口按文档里的判定判档:证据不支持的结论句
按 on_unsupported 降档(degrade,默认)或不予出具(withhold);裁判没跑成、触顶了没判完的,裁判摘要里的缺口
照抄进出具声明——不能判完整出具,也不因此不予出具
- 判定是模型给的,出口复算不了:信按哈希取回的文档
- judge 在挂引用这件事上和 require_citation 一样严:没挂依据的结论句照样计入缺口
- 探索运行按需裁判:出口只标注(候选句都是「未裁判 · 按需」),不因判定改档位
- 契约也能写 claims: judge:只能收紧(on_unsupported 取更严的);报告节点没开 judge 的,照实记缺口
- 升级前的运行照旧:契约写了也记「这一版还不支持」
"""
from __future__ import annotations
import asyncio
import json
import re
import uuid
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import delete, select
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent, Setting, Workflow, WorkflowVersion
from app.engine import judge as judge_mod
from app.engine import runner as runner_mod
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
from app.engine.runner import run_manager
from app.main import app
from app.providers.mock_model import MockChatModel
JUDGE_MODEL = "claude-sonnet-5"
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
#: 两句结论都挂了依据、一句连接性的话
CITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]下面看细节。"
#: 第二句结论没挂依据
UNCITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。下面看细节。"
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
async with SessionLocal() as session:
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
await session.commit()
yield
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def weekly(*, report=None, contract=None) -> dict:
nodes = [
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
node("write", "report", instructions="写周报", **{"claims": "judge", "judge": {"max_cost_usd": 0.05},
**(report or {})}),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
contract={"report_from": "write", "metrics_from": ["caliber"], "required": ["gmv"], "strict": True,
**(contract or {})}),
]
ids = [n["id"] for n in nodes]
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
class Models:
"""写作者回 text;裁判按句子里的字给判定(verdicts:{句中片段: 判定},没对上的是 supported)。"""
def __init__(self, monkeypatch, text=CITED, verdicts=None, *, fail_judge=None):
self.judges: list[str] = []
verdicts = verdicts or {}
def decide(model, messages):
if JUDGE_ROLE in str(messages[0].content):
request = str(messages[-1].content)
self.judges.append(request)
if fail_judge:
raise RuntimeError(fail_judge)
items = [{"unit": uid,
"verdict": next((s for frag, s in verdicts.items() if frag in line), "supported"),
"rationale": "摘录里有对应的指标" if "销售额" in line else "证据里没有新客维度", "used": []}
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)]
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
return AIMessage(content=text)
monkeypatch.setattr(MockChatModel, "_decide", decide)
async def judge_model_of(session, spec):
return MockChatModel(model_name=JUDGE_MODEL), JUDGE_MODEL
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
async def wait(run_id: str) -> Run:
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
return row
raise AssertionError(f"run {run_id} 没跑完:{row.status}")
async def issued(graph, *, run_class="formal") -> dict:
run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
row = await wait(run.id)
assert row.status == "succeeded", row.error
return row.output["_issuance"]
async def governed(client, graph) -> Run:
"""把这张图发布成受管级别,从发布版本发起正式运行。"""
async with SessionLocal() as session:
wf = Workflow(name=f"judge-{uuid.uuid4().hex[:6]}", graph=graph, status="governed", published_version=1)
session.add(wf)
await session.flush()
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph))
await session.commit()
wf_id = wf.id
res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"})
assert res.status_code == 201, res.text
row = await wait(res.json()["id"])
assert row.status == "succeeded", row.error
return row
async def issuance_event(run_id: str) -> dict:
async with SessionLocal() as session:
[event] = (await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == "issuance"))).scalars()
return event.data
# --------------------------------------------------------------------------
# 正式运行:按封存的判定判档
# --------------------------------------------------------------------------
async def test_every_claim_supported_is_a_full_issuance(monkeypatch):
Models(monkeypatch)
issuance = await issued(weekly())
assert issuance["tier"] == "formal" and issuance["gaps"] == [], issuance
claims = issuance["claims"]
assert claims["policy"] == "judge" and claims["mode"] == "inline" and claims["complete"] is True
assert claims["counts"] == {"supported": 2, "partial": 0, "unsupported": 0, "not_a_claim": 0, "unjudged": 0}
assert claims["on_unsupported"] == "degrade" and claims["uncited_claims"] == 0
assert claims["unsupported"] == [] and claims["model"] == JUDGE_MODEL
async def test_an_unsupported_claim_degrades_by_default(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
issuance = await issued(weekly())
assert issuance["tier"] == "degraded", "不支持按缺口降档,不是不予出具(契约是 strict 的)"
assert issuance["gaps"] == [], "判了就不是「没核」:档位来自判定,不来自缺口"
[flagged] = issuance["claims"]["unsupported"]
assert flagged["text"] == "增长主要来自新客首单。" and flagged["rationale"] == "证据里没有新客维度"
assert flagged["unit"] and len(flagged["span"]) == 2
assert issuance["claims"]["counts"]["unsupported"] == 1
async def test_withhold_withholds(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold"
async def test_partial_support_is_listed_but_does_not_cost_a_grade(monkeypatch):
Models(monkeypatch, verdicts={"新客": "partial"})
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
assert issuance["tier"] == "formal"
[item] = issuance["claims"]["partial"]
assert item["text"] == "增长主要来自新客首单。" and issuance["claims"]["counts"]["partial"] == 1
async def test_a_failed_judge_is_a_gap_not_a_withheld_report(monkeypatch):
Models(monkeypatch, fail_judge="网关 502")
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
assert issuance["tier"] == "degraded", "没判成不能盖完整出具,也不能当成「不支持」不予出具"
[gap] = [g for g in issuance["gaps"] if "裁判" in g]
assert gap.startswith("报告「write」") and "没裁判" in gap and "网关 502" in gap
assert issuance["claims"]["complete"] is False and issuance["claims"]["counts"]["unjudged"] == 2
async def test_hitting_a_limit_is_a_gap(monkeypatch):
Models(monkeypatch)
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "max_claims": 1}}))
assert issuance["tier"] == "degraded"
[gap] = [g for g in issuance["gaps"] if "裁判" in g]
assert "已到上限" in gap and "1 句" in gap
assert issuance["claims"]["limits_hit"] == ["max_claims"] and issuance["claims"]["counts"]["unjudged"] == 1
async def test_an_uncited_claim_still_counts_under_judge(monkeypatch):
"""judge 在挂引用这件事上和 require_citation 一样严:从 require_citation 换成 judge 不能让没挂依据的句子过关。"""
Models(monkeypatch, text=UNCITED)
issuance = await issued(weekly())
assert issuance["tier"] == "degraded" and issuance["claims"]["uncited_claims"] == 1
assert issuance["claims"]["uncited"][0]["text"] == "增长主要来自新客首单。"
async def test_the_issuance_event_carries_the_counts(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
run = await run_manager.start(graph=weekly(), input_payload={}, run_class="formal")
row = await wait(run.id)
event = await issuance_event(row.id)
assert event["tier"] == "degraded"
assert event["claims"] == {"policy": "judge", "uncited_claims": 0, "on_unsupported": "degrade",
"mode": "inline", "counts": {"supported": 1, "partial": 0, "unsupported": 1,
"not_a_claim": 0, "unjudged": 0}}
async def test_a_governed_formal_run_grades_by_the_verdicts(client, monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
graph = weekly(report={"numbers": "strict", "on_violation": "fail",
"judge": {"max_cost_usd": None, "on_unsupported": "withhold"}})
row = await governed(client, graph)
issuance = row.output["_issuance"]
assert issuance["tier"] == "withheld" and issuance["claims"]["unsupported"][0]["text"] == "增长主要来自新客首单。"
# --------------------------------------------------------------------------
# 探索运行:只标注
# --------------------------------------------------------------------------
async def test_explore_runs_only_annotate(monkeypatch):
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}),
run_class="exploratory")
assert models.judges == [], "探索运行节点里不花这笔钱"
assert issuance["tier"] == "formal" and issuance["gaps"] == []
claims = issuance["claims"]
assert claims["mode"] == "on_demand" and claims["complete"] is False
assert claims["counts"]["unjudged"] == 2 and claims["unjudged"] == {"on_demand": 2}
assert claims["unsupported"] == []
async def test_explore_runs_still_count_uncited_claims(monkeypatch):
Models(monkeypatch, text=UNCITED)
issuance = await issued(weekly(), run_class="exploratory")
assert issuance["tier"] == "degraded" and issuance["claims"]["uncited_claims"] == 1
# --------------------------------------------------------------------------
# 契约里的 claims: judge
# --------------------------------------------------------------------------
async def test_the_contract_can_tighten_on_unsupported(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
issuance = await issued(weekly(contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}}))
assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold"
assert not any("不支持" in g and "这一版" in g for g in issuance["gaps"])
async def test_the_contract_cannot_loosen_on_unsupported(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}},
contract={"claims": {"policy": "judge", "on_unsupported": "degrade"}}))
assert issuance["tier"] == "withheld"
async def test_a_contract_asking_for_judge_on_a_report_without_it_is_a_gap(monkeypatch):
Models(monkeypatch)
issuance = await issued(weekly(report={"claims": "require_citation"}, contract={"claims": "judge"}))
assert issuance["tier"] == "degraded"
[gap] = [g for g in issuance["gaps"] if "judge" in g]
assert "「write」" in gap and "没有开" in gap and "这一版" not in gap
assert issuance["claims"]["policy"] == "require_citation"
def test_a_run_in_flight_across_the_upgrade_keeps_the_old_gap():
"""升级前写的报告(产出里没有 claims)碰上写了 judge 的契约:照旧记「这一版还不支持」,不判档。"""
from app.engine.nodes.io import _claims
out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None, "unsupported": None}
_claims(out, {"claims": {"policy": "judge", "on_unsupported": "withhold"}}, {"text": "x"}, evidence_on=True,
uncited=[{"unit": "u1", "span": [0, 4], "text": "增长很快"}])
assert out["claims_policy"] is None and out["claims"] is None and out["unsupported"] is None
assert len(out["gaps"]) == 1 and "这一版还不支持" in out["gaps"][0]
async def test_runs_from_before_the_upgrade_ignore_judge(monkeypatch):
async def no_snapshot(session):
return None
monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot)
models = Models(monkeypatch, text=UNCITED, verdicts={"新客": "unsupported"})
issuance = await issued(weekly())
assert models.judges == [] and issuance["tier"] == "formal" and "claims" not in issuance
# --------------------------------------------------------------------------
# 答案复核(review.scan)认得裁判的几条警告
# --------------------------------------------------------------------------
@pytest.mark.parametrize(("code", "shown"), [
("judge_limit", True), ("judge_failed", True), ("report_rewrite", True), ("report_rewrite_rejected", True),
("judge_unpriced", False), ("judge_same_model", False),
])
def test_review_knows_the_judge_warnings(code, shown):
"""没判完、没跑成、改写过的是缺口(要说明);估不出金额、自己审自己只是配置上的提醒,轨迹里有就够了,
不值得为它再叫一次复核模型。"""
from app.engine.review import DEGRADED, scan
signals = scan([{"type": "log", "data": {"level": "warn", "code": code, "message": f"{code} 的原话"}}],
{"answer": "本周销售额 45,678.5元。"})
if shown:
[signal] = signals
assert signal.kind == code and signal.severity == DEGRADED and signal.detail == f"{code} 的原话"
else:
assert signals == []