mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 14:42:00 +00:00
ThearchyHelios
bd8306bf95
- 判定取值改为：支持 / 部分支持 / 证据相矛盾 / 证据不足 / 不是结论。只有证据和句子冲突才判相矛盾； 证据里没有相关信息判证据不足，并写明缺什么（比如 SQL、字段清单） - 出具判档不变松：证据相矛盾按 on_unsupported 降档或不予出具；证据不足记一条缺口，最多降档、不能判完整出具； 改造前的「证据不支持」照旧当相矛盾处理 - 补全证据摘录：表名附字段清单和类型（最多 60 个，快照不全时注明，遮罩列不列）；口径卡指标附上每个输入 取自哪次查询、哪一格，以及那次查询的 SQL；一律只读封存范围内的工件 - 理由上限从 60 字放宽到 120 字，在句子边界截断，不再截在半句中间 - 裁判模型：按需裁判也判断是否和写作模型相同、是否在价格表里，证据面板里醒目提示； 设置页没配裁判模型时写明「未单独配置，将使用助手的模型」 - 新判定带规则版本：改造前的判定在界面上标「按旧规则判定」，可以请模型按当前规则重新判断 - 界面：证据相矛盾用红色；证据不足用中性色和 ○，写明缺什么；讲方法、讲结构的句子在证据面板里直接列出 对应的 SQL 和字段清单；出具横幅、审计表、报告节点印章的计数都分开这两档 - 顺带写稳 check-publish 里一条等按钮状态的断言（并行时偶发）
326 lines
16 KiB
Python
326 lines
16 KiB
Python
"""出口契约（引用模式）按结论句裁判的判定判档（四期）。
|
|
|
|
- 报告节点 claims: judge 在正式运行里当场裁判，判定随文档封存。出口按文档里的判定判档：证据不支持的结论句
|
|
按 on_unsupported 降档（degrade，默认）或不予出具（withhold）；裁判没跑成、触顶了没判完的，裁判摘要里的缺口
|
|
照抄进出具声明——不能判完整出具，也不因此不予出具
|
|
- 判定是模型给的，出口复算不了：信按哈希取回的文档
|
|
- judge 在挂引用这件事上和 require_citation 一样严：没挂依据的结论句照样计入缺口
|
|
- 探索运行按需裁判：出口只标注（候选句都是「未裁判 · 按需」），不因判定改档位
|
|
- 契约也能写 claims: judge：只能收紧（on_unsupported 取更严的）；报告节点没开 judge 的，照实记缺口
|
|
- 升级前的运行照旧：契约写了也只记一条缺口「报告生成于系统升级前，结论句未核对」
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import re
|
|
import uuid
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from langchain_core.messages import AIMessage
|
|
from sqlalchemy import delete, select
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Run, RunEvent, Setting, Workflow, WorkflowVersion
|
|
from app.engine import judge as judge_mod
|
|
from app.engine import runner as runner_mod
|
|
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
|
|
from app.engine.runner import run_manager
|
|
from app.main import app
|
|
from app.providers.mock_model import MockChatModel
|
|
|
|
JUDGE_MODEL = "claude-sonnet-5"
|
|
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
|
#: 两句结论都挂了依据、一句连接性的话
|
|
CITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]下面看细节。"
|
|
#: 第二句结论没挂依据
|
|
UNCITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。下面看细节。"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
async with SessionLocal() as session:
|
|
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
|
|
await session.commit()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
|
|
|
|
|
def weekly(*, report=None, contract=None) -> dict:
|
|
nodes = [
|
|
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
|
|
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
|
|
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
|
|
node("write", "report", instructions="写周报", **{"claims": "judge", "judge": {"max_cost_usd": 0.05},
|
|
**(report or {})}),
|
|
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
|
|
contract={"report_from": "write", "metrics_from": ["caliber"], "required": ["gmv"], "strict": True,
|
|
**(contract or {})}),
|
|
]
|
|
ids = [n["id"] for n in nodes]
|
|
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
|
|
|
|
|
class Models:
|
|
"""写作者回 text；裁判按句子里的字给判定（verdicts：{句中片段: 判定}，没对上的是 supported）。"""
|
|
|
|
def __init__(self, monkeypatch, text=CITED, verdicts=None, *, fail_judge=None):
|
|
self.judges: list[str] = []
|
|
verdicts = verdicts or {}
|
|
|
|
def decide(model, messages):
|
|
if JUDGE_ROLE in str(messages[0].content):
|
|
request = str(messages[-1].content)
|
|
self.judges.append(request)
|
|
if fail_judge:
|
|
raise RuntimeError(fail_judge)
|
|
items = [{"unit": uid,
|
|
"verdict": next((s for frag, s in verdicts.items() if frag in line), "supported"),
|
|
"rationale": "摘录里有对应的指标" if "销售额" in line else "证据里没有新客维度", "used": []}
|
|
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)]
|
|
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
|
|
return AIMessage(content=text)
|
|
|
|
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
|
|
|
async def judge_model_of(session, spec):
|
|
return MockChatModel(model_name=JUDGE_MODEL), JUDGE_MODEL
|
|
|
|
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
|
|
|
|
|
|
async def wait(run_id: str) -> Run:
|
|
for _ in range(400):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run_id)
|
|
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
|
return row
|
|
raise AssertionError(f"run {run_id} 没跑完：{row.status}")
|
|
|
|
|
|
async def issued(graph, *, run_class="formal") -> dict:
|
|
run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
|
|
row = await wait(run.id)
|
|
assert row.status == "succeeded", row.error
|
|
return row.output["_issuance"]
|
|
|
|
|
|
async def governed(client, graph) -> Run:
|
|
"""把这张图发布成受管级别，从发布版本发起正式运行。"""
|
|
async with SessionLocal() as session:
|
|
wf = Workflow(name=f"judge-{uuid.uuid4().hex[:6]}", graph=graph, status="governed", published_version=1)
|
|
session.add(wf)
|
|
await session.flush()
|
|
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph))
|
|
await session.commit()
|
|
wf_id = wf.id
|
|
res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"})
|
|
assert res.status_code == 201, res.text
|
|
row = await wait(res.json()["id"])
|
|
assert row.status == "succeeded", row.error
|
|
return row
|
|
|
|
|
|
async def issuance_event(run_id: str) -> dict:
|
|
async with SessionLocal() as session:
|
|
[event] = (await session.execute(select(RunEvent).where(
|
|
RunEvent.run_id == run_id, RunEvent.type == "issuance"))).scalars()
|
|
return event.data
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 正式运行：按封存的判定判档
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_every_claim_supported_is_a_full_issuance(monkeypatch):
|
|
Models(monkeypatch)
|
|
issuance = await issued(weekly())
|
|
assert issuance["tier"] == "formal" and issuance["gaps"] == [], issuance
|
|
claims = issuance["claims"]
|
|
assert claims["policy"] == "judge" and claims["mode"] == "inline" and claims["complete"] is True
|
|
assert claims["counts"] == {"supported": 2, "partial": 0, "contradicted": 0, "insufficient": 0, "not_a_claim": 0,
|
|
"unjudged": 0, "unsupported": 0}
|
|
assert claims["on_unsupported"] == "degrade" and claims["uncited_claims"] == 0
|
|
assert claims["contradicted"] == [] and claims["insufficient"] == [] and claims["model"] == JUDGE_MODEL
|
|
|
|
|
|
async def test_a_contradicted_claim_degrades_by_default(monkeypatch):
|
|
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
|
issuance = await issued(weekly())
|
|
assert issuance["tier"] == "degraded", "不支持按缺口降档，不是不予出具（契约是 strict 的）"
|
|
assert issuance["gaps"] == [], "判了就不是「没核」：档位来自判定，不来自缺口"
|
|
[flagged] = issuance["claims"]["contradicted"]
|
|
assert flagged["text"] == "增长主要来自新客首单。" and flagged["rationale"] == "证据里没有新客维度"
|
|
assert flagged["unit"] and len(flagged["span"]) == 2
|
|
assert issuance["claims"]["counts"]["contradicted"] == 1
|
|
|
|
|
|
async def test_withhold_withholds(monkeypatch):
|
|
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
|
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
|
|
assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold"
|
|
|
|
|
|
async def test_partial_support_is_listed_but_does_not_cost_a_grade(monkeypatch):
|
|
Models(monkeypatch, verdicts={"新客": "partial"})
|
|
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
|
|
assert issuance["tier"] == "formal"
|
|
[item] = issuance["claims"]["partial"]
|
|
assert item["text"] == "增长主要来自新客首单。" and issuance["claims"]["counts"]["partial"] == 1
|
|
|
|
|
|
async def test_a_failed_judge_is_a_gap_not_a_withheld_report(monkeypatch):
|
|
Models(monkeypatch, fail_judge="网关 502")
|
|
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
|
|
assert issuance["tier"] == "degraded", "没判成不能盖完整出具，也不能当成「不支持」不予出具"
|
|
[gap] = [g for g in issuance["gaps"] if "裁判" in g]
|
|
assert gap.startswith("报告「write」") and "未裁判" in gap and "网关 502" in gap
|
|
assert issuance["claims"]["complete"] is False and issuance["claims"]["counts"]["unjudged"] == 2
|
|
|
|
|
|
async def test_hitting_a_limit_is_a_gap(monkeypatch):
|
|
Models(monkeypatch)
|
|
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "max_claims": 1}}))
|
|
assert issuance["tier"] == "degraded"
|
|
[gap] = [g for g in issuance["gaps"] if "裁判" in g]
|
|
assert "已达上限" in gap and "1 句" in gap
|
|
assert issuance["claims"]["limits_hit"] == ["max_claims"] and issuance["claims"]["counts"]["unjudged"] == 1
|
|
|
|
|
|
async def test_an_uncited_claim_still_counts_under_judge(monkeypatch):
|
|
"""judge 在挂引用这件事上和 require_citation 一样严：从 require_citation 换成 judge 不能让没挂依据的句子过关。"""
|
|
Models(monkeypatch, text=UNCITED)
|
|
issuance = await issued(weekly())
|
|
assert issuance["tier"] == "degraded" and issuance["claims"]["uncited_claims"] == 1
|
|
assert issuance["claims"]["uncited"][0]["text"] == "增长主要来自新客首单。"
|
|
|
|
|
|
async def test_the_issuance_event_carries_the_counts(monkeypatch):
|
|
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
|
run = await run_manager.start(graph=weekly(), input_payload={}, run_class="formal")
|
|
row = await wait(run.id)
|
|
event = await issuance_event(row.id)
|
|
assert event["tier"] == "degraded"
|
|
assert event["claims"] == {"policy": "judge", "uncited_claims": 0, "on_unsupported": "degrade",
|
|
"mode": "inline", "counts": {"supported": 1, "partial": 0, "contradicted": 1,
|
|
"insufficient": 0, "unsupported": 0,
|
|
"not_a_claim": 0, "unjudged": 0}}
|
|
|
|
|
|
async def test_a_governed_formal_run_grades_by_the_verdicts(client, monkeypatch):
|
|
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
|
graph = weekly(report={"numbers": "strict", "on_violation": "fail",
|
|
"judge": {"max_cost_usd": None, "on_unsupported": "withhold"}})
|
|
row = await governed(client, graph)
|
|
issuance = row.output["_issuance"]
|
|
assert issuance["tier"] == "withheld" and issuance["claims"]["contradicted"][0]["text"] == "增长主要来自新客首单。"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 探索运行：只标注
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_explore_runs_only_annotate(monkeypatch):
|
|
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
|
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}),
|
|
run_class="exploratory")
|
|
assert models.judges == [], "探索运行节点里不花这笔钱"
|
|
assert issuance["tier"] == "formal" and issuance["gaps"] == []
|
|
claims = issuance["claims"]
|
|
assert claims["mode"] == "on_demand" and claims["complete"] is False
|
|
assert claims["counts"]["unjudged"] == 2 and claims["unjudged"] == {"on_demand": 2}
|
|
assert claims["contradicted"] == [] and claims["insufficient"] == []
|
|
|
|
|
|
async def test_explore_runs_still_count_uncited_claims(monkeypatch):
|
|
Models(monkeypatch, text=UNCITED)
|
|
issuance = await issued(weekly(), run_class="exploratory")
|
|
assert issuance["tier"] == "degraded" and issuance["claims"]["uncited_claims"] == 1
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 契约里的 claims: judge
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_the_contract_can_tighten_on_unsupported(monkeypatch):
|
|
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
|
issuance = await issued(weekly(contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}}))
|
|
assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold"
|
|
assert not any("不支持" in g and "这一版" in g for g in issuance["gaps"])
|
|
|
|
|
|
async def test_the_contract_cannot_loosen_on_unsupported(monkeypatch):
|
|
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
|
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}},
|
|
contract={"claims": {"policy": "judge", "on_unsupported": "degrade"}}))
|
|
assert issuance["tier"] == "withheld"
|
|
|
|
|
|
async def test_a_contract_asking_for_judge_on_a_report_without_it_is_a_gap(monkeypatch):
|
|
Models(monkeypatch)
|
|
issuance = await issued(weekly(report={"claims": "require_citation"}, contract={"claims": "judge"}))
|
|
assert issuance["tier"] == "degraded"
|
|
[gap] = [g for g in issuance["gaps"] if "模型裁判" in g]
|
|
assert "「write」" in gap and "未开启" in gap and "这一版" not in gap
|
|
assert issuance["claims"]["policy"] == "require_citation"
|
|
|
|
|
|
def test_a_run_in_flight_across_the_upgrade_keeps_the_old_gap():
|
|
"""升级前写的报告（产出里没有 claims）碰上写了 judge 的契约：照旧只记一条缺口，不判档。"""
|
|
from app.engine.nodes.io import _claims
|
|
|
|
out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None, "unsupported": None}
|
|
_claims(out, {"claims": {"policy": "judge", "on_unsupported": "withhold"}}, {"text": "x"}, evidence_on=True,
|
|
uncited=[{"unit": "u1", "span": [0, 4], "text": "增长很快"}])
|
|
assert out["claims_policy"] is None and out["claims"] is None and out["unsupported"] is None
|
|
assert len(out["gaps"]) == 1 and "生成于系统升级前" in out["gaps"][0]
|
|
|
|
|
|
async def test_runs_from_before_the_upgrade_ignore_judge(monkeypatch):
|
|
async def no_snapshot(session):
|
|
return None
|
|
|
|
monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot)
|
|
models = Models(monkeypatch, text=UNCITED, verdicts={"新客": "contradicted"})
|
|
issuance = await issued(weekly())
|
|
assert models.judges == [] and issuance["tier"] == "formal" and "claims" not in issuance
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 答案复核（review.scan）认得裁判的几条警告
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(("code", "shown"), [
|
|
("judge_limit", True), ("judge_failed", True), ("report_rewrite", True), ("report_rewrite_rejected", True),
|
|
("judge_unpriced", False), ("judge_same_model", False),
|
|
])
|
|
def test_review_knows_the_judge_warnings(code, shown):
|
|
"""没判完、没跑成、改写过的是缺口（要说明）；估不出金额、自己审自己只是配置上的提醒，轨迹里有就够了，
|
|
不值得为它再叫一次复核模型。"""
|
|
from app.engine.review import DEGRADED, scan
|
|
|
|
signals = scan([{"type": "log", "data": {"level": "warn", "code": code, "message": f"{code} 的原话"}}],
|
|
{"answer": "本周销售额 45,678.5元。"})
|
|
if shown:
|
|
[signal] = signals
|
|
assert signal.kind == code and signal.severity == DEGRADED and signal.detail == f"{code} 的原话"
|
|
else:
|
|
assert signals == []
|