mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 06:41:58 +00:00
ThearchyHelios
bd8306bf95
- 判定取值改为：支持 / 部分支持 / 证据相矛盾 / 证据不足 / 不是结论。只有证据和句子冲突才判相矛盾； 证据里没有相关信息判证据不足，并写明缺什么（比如 SQL、字段清单） - 出具判档不变松：证据相矛盾按 on_unsupported 降档或不予出具；证据不足记一条缺口，最多降档、不能判完整出具； 改造前的「证据不支持」照旧当相矛盾处理 - 补全证据摘录：表名附字段清单和类型（最多 60 个，快照不全时注明，遮罩列不列）；口径卡指标附上每个输入 取自哪次查询、哪一格，以及那次查询的 SQL；一律只读封存范围内的工件 - 理由上限从 60 字放宽到 120 字，在句子边界截断，不再截在半句中间 - 裁判模型：按需裁判也判断是否和写作模型相同、是否在价格表里，证据面板里醒目提示； 设置页没配裁判模型时写明「未单独配置，将使用助手的模型」 - 新判定带规则版本：改造前的判定在界面上标「按旧规则判定」，可以请模型按当前规则重新判断 - 界面：证据相矛盾用红色；证据不足用中性色和 ○，写明缺什么；讲方法、讲结构的句子在证据面板里直接列出 对应的 SQL 和字段清单；出具横幅、审计表、报告节点印章的计数都分开这两档 - 顺带写稳 check-publish 里一条等按钮状态的断言（并行时偶发）
299 lines
15 KiB
Python
299 lines
15 KiB
Python
"""结论句裁判改造（JR）的判档：contradicted 沿用原来 unsupported 的处置，insufficient 按缺口处理。
|
||
|
||
- contradicted（证据相矛盾）：按 on_unsupported 降档（degrade）或不予出具（withhold），和原来的 unsupported 一样
|
||
- insufficient（证据不足）：按缺口处理，最多降档——不会不予出具，也不会判完整出具
|
||
- 不许比现在更松：受管正式运行里，原来会因「证据不支持」降档的句子，改判成 contradicted 或 insufficient
|
||
都不能完整出具
|
||
- 旧文档里的 unsupported 判档照旧当 contradicted 处理
|
||
- 统计键（stats、counts、出具横幅的结论句计数）都有 contradicted、insufficient，unsupported 保留
|
||
- 报告节点：改写一次把证据相矛盾、证据不足的句子都交回写作者；文档的裁判摘要记下写作模型
|
||
|
||
所有模型调用都是 mock 剧本，不碰真接口。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import re
|
||
import uuid
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessage
|
||
from sqlalchemy import delete, select
|
||
|
||
from app.core import artifact_store
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Run, RunEvent, Setting, Workflow, WorkflowVersion
|
||
from app.engine import judge as judge_mod
|
||
from app.engine.evidence import compose_doc, iter_units
|
||
from app.engine.issuance import decide_tier
|
||
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
|
||
from app.engine.nodes.io import _claims
|
||
from app.engine.runner import run_manager
|
||
from app.main import app
|
||
from app.providers.mock_model import MockChatModel
|
||
|
||
JUDGE_MODEL = "claude-sonnet-5"
|
||
WRITER = "mock-fast"
|
||
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
||
CITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]下面看细节。"
|
||
SUPPORTED, CAUSAL = "本周销售额 45,678.5元。", "增长主要来自新客首单。"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 出口判档（io._claims）：直接喂文档
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def judged_doc(statuses: dict[str, dict], *, on_unsupported="degrade") -> dict:
|
||
"""两句结论：SUPPORTED_TEXT、CAUSAL_TEXT；statuses 按句子给判定。counts 照判定数。"""
|
||
doc = compose_doc("本周销售额稳定增长。增长主要来自新客首单。", {})
|
||
md = doc["markdown"]
|
||
counts = {k: 0 for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")}
|
||
for _, unit in iter_units(doc):
|
||
text = md[unit["span"][0]:unit["span"][1]].strip()
|
||
verdict = {"status": "supported", "rationale": "有依据", "judge": JUDGE_MODEL, "post_seal": False, "used": [],
|
||
**statuses.get(text, {})}
|
||
unit["verdict"] = verdict
|
||
counts[verdict["status"]] = counts.get(verdict["status"], 0) + 1
|
||
doc["judge"] = {"mode": "inline", "model": JUDGE_MODEL, "on_unsupported": on_unsupported, "counts": counts,
|
||
"unjudged": {}, "limits_hit": [], "complete": True, "gaps": []}
|
||
return doc
|
||
|
||
|
||
def grade(doc: dict, *, governed=False, contract=None) -> tuple[str, dict]:
|
||
out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None, "unsupported": None}
|
||
_claims(out, contract or {}, {"claims": "judge", "text": doc["markdown"]}, evidence_on=True, governed=governed,
|
||
uncited=[], doc=doc, title="write")
|
||
tier = decide_tier(missing_required=[], missing_expected=[], unmatched=[], strict=True, gaps=out["gaps"],
|
||
unsupported=out["unsupported"], uncited_claims=out["uncited"],
|
||
claims_policy=out["claims_policy"])
|
||
return tier, out
|
||
|
||
|
||
TEXT2 = "增长主要来自新客首单。"
|
||
|
||
|
||
def test_all_supported_is_still_a_full_issuance():
|
||
tier, out = grade(judged_doc({}))
|
||
assert tier == "formal" and out["gaps"] == []
|
||
|
||
|
||
@pytest.mark.parametrize(("on_unsupported", "tier"), [("degrade", "degraded"), ("withhold", "withheld")])
|
||
def test_contradicted_follows_on_unsupported(on_unsupported, tier):
|
||
doc = judged_doc({TEXT2: {"status": "contradicted", "rationale": "证据里新客占比下降"}},
|
||
on_unsupported=on_unsupported)
|
||
got, out = grade(doc)
|
||
assert got == tier
|
||
claims = out["claims"]
|
||
[item] = claims["contradicted"]
|
||
assert item["text"] == TEXT2 and item["rationale"] == "证据里新客占比下降"
|
||
assert claims["counts"]["contradicted"] == 1 and claims["counts"]["insufficient"] == 0
|
||
assert out["gaps"] == [], "判档来自判定，不来自缺口"
|
||
|
||
|
||
def test_insufficient_is_a_gap_and_never_withholds():
|
||
doc = judged_doc({TEXT2: {"status": "insufficient", "rationale": "摘录里没有新客维度", "missing": "新客维度的查询"}},
|
||
on_unsupported="withhold")
|
||
tier, out = grade(doc, governed=True, contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}})
|
||
assert tier == "degraded", "证据不足最多降档：不能不予出具，也不能完整出具"
|
||
[gap] = [g for g in out["gaps"] if "证据不足" in g]
|
||
assert gap.startswith("报告「write」") and "1 句" in gap and "新客维度的查询" in gap
|
||
[item] = out["claims"]["insufficient"]
|
||
assert item["text"] == TEXT2 and item["missing"] == "新客维度的查询"
|
||
assert out["claims"]["counts"]["insufficient"] == 1 and out["unsupported"] == []
|
||
|
||
|
||
def test_an_old_unsupported_verdict_is_graded_as_contradicted():
|
||
doc = judged_doc({TEXT2: {"status": "unsupported", "rationale": "证据不支持"}}, on_unsupported="withhold")
|
||
tier, out = grade(doc)
|
||
assert tier == "withheld"
|
||
assert out["claims"]["counts"]["unsupported"] == 1
|
||
assert [i["text"] for i in out["claims"]["unsupported"]] == [TEXT2]
|
||
assert [i["text"] for i in out["unsupported"]] == [TEXT2]
|
||
|
||
|
||
def test_the_issuance_counts_carry_every_key():
|
||
_, out = grade(judged_doc({}))
|
||
assert set(out["claims"]["counts"]) == set(judge_mod.COUNT_KEYS)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 端到端：正式运行、受管正式运行
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
|
||
await session.commit()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def client(engine_up):
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
def node(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
def weekly(*, report=None, contract=None) -> dict:
|
||
nodes = [
|
||
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
|
||
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
|
||
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
|
||
node("write", "report", instructions="写周报", **{"claims": "judge", "judge": {"max_cost_usd": 0.05},
|
||
**(report or {})}),
|
||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
|
||
contract={"report_from": "write", "metrics_from": ["caliber"], "required": ["gmv"], "strict": True,
|
||
**(contract or {})}),
|
||
]
|
||
ids = [n["id"] for n in nodes]
|
||
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||
|
||
|
||
class Models:
|
||
"""写作者依次回 writer 里的几段；裁判按句子里的字给判定：verdicts = {句中片段: (判定, missing)}。"""
|
||
|
||
def __init__(self, monkeypatch, writer=(CITED,), verdicts=None):
|
||
self.writes: list[list] = []
|
||
self.judges: list[str] = []
|
||
verdicts = verdicts or {}
|
||
|
||
def decide(model, messages):
|
||
if JUDGE_ROLE in str(messages[0].content):
|
||
request = str(messages[-1].content)
|
||
self.judges.append(request)
|
||
items = []
|
||
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M):
|
||
status, missing = next((v for frag, v in verdicts.items() if frag in line), ("supported", ""))
|
||
items.append({"unit": uid, "verdict": status, "rationale": f"判的是：{line.strip()[:12]}",
|
||
"missing": missing, "used": []})
|
||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
|
||
self.writes.append(list(messages))
|
||
return AIMessage(content=writer[min(len(self.writes), len(writer)) - 1])
|
||
|
||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||
|
||
async def judge_model_of(session, spec):
|
||
return MockChatModel(model_name=JUDGE_MODEL), JUDGE_MODEL
|
||
|
||
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
|
||
|
||
|
||
async def wait(run_id: str) -> Run:
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run_id)
|
||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||
return row
|
||
raise AssertionError(f"run {run_id} 没跑完：{row.status}")
|
||
|
||
|
||
async def run(graph, *, run_class="formal") -> Run:
|
||
started = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
|
||
row = await wait(started.id)
|
||
assert row.status == "succeeded", row.error
|
||
return row
|
||
|
||
|
||
async def checked_of(run_id: str) -> tuple[dict, dict]:
|
||
async with SessionLocal() as session:
|
||
[event] = (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type == "report.checked"))).scalars()
|
||
return event.data, artifact_store.load(event.data["doc_artifact"])
|
||
|
||
|
||
async def issuance_event(run_id: str) -> dict:
|
||
async with SessionLocal() as session:
|
||
[event] = (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type == "issuance"))).scalars()
|
||
return event.data
|
||
|
||
|
||
async def test_a_formal_run_withholds_a_contradicted_claim(engine_up, monkeypatch):
|
||
Models(monkeypatch, verdicts={"新客": ("contradicted", "")})
|
||
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
|
||
issuance = row.output["_issuance"]
|
||
assert issuance["tier"] == "withheld"
|
||
assert [i["text"] for i in issuance["claims"]["contradicted"]] == [CAUSAL]
|
||
event = await issuance_event(row.id)
|
||
assert event["claims"]["counts"] == {"supported": 1, "partial": 0, "contradicted": 1, "insufficient": 0,
|
||
"not_a_claim": 0, "unjudged": 0, "unsupported": 0}
|
||
|
||
|
||
async def test_a_formal_run_only_degrades_an_insufficient_claim(engine_up, monkeypatch):
|
||
Models(monkeypatch, verdicts={"新客": ("insufficient", "新客维度的数据")})
|
||
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
|
||
issuance = row.output["_issuance"]
|
||
assert issuance["tier"] == "degraded", "证据不足：最多降档，不因 withhold 不予出具"
|
||
assert any("证据不足" in g and "新客维度的数据" in g for g in issuance["gaps"]), issuance["gaps"]
|
||
[item] = issuance["claims"]["insufficient"]
|
||
assert item["text"] == CAUSAL and item["missing"] == "新客维度的数据"
|
||
assert issuance["claims"]["counts"]["insufficient"] == 1
|
||
|
||
checked, doc = await checked_of(row.id)
|
||
assert doc["stats"]["insufficient"] == 1 and doc["stats"]["contradicted"] == 0 and doc["stats"]["unsupported"] == 0
|
||
assert checked["stats"]["insufficient"] == 1
|
||
assert doc["judge"]["writer_model"] == WRITER and checked["judge"]["writer_model"] == WRITER
|
||
[unit] = [u for _, u in iter_units(doc) if u.get("verdict", {}).get("status") == "insufficient"]
|
||
assert unit["verdict"]["missing"] == "新客维度的数据"
|
||
|
||
|
||
async def governed(client, graph) -> Run:
|
||
async with SessionLocal() as session:
|
||
wf = Workflow(name=f"jr-{uuid.uuid4().hex[:6]}", graph=graph, status="governed", published_version=1)
|
||
session.add(wf)
|
||
await session.flush()
|
||
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph))
|
||
await session.commit()
|
||
wf_id = wf.id
|
||
res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"})
|
||
assert res.status_code == 201, res.text
|
||
row = await wait(res.json()["id"])
|
||
assert row.status == "succeeded", row.error
|
||
return row
|
||
|
||
|
||
@pytest.mark.parametrize("status", ["contradicted", "insufficient"])
|
||
async def test_a_governed_formal_run_is_never_fully_issued_on_a_split_verdict(client, monkeypatch, status):
|
||
"""不许比现在更松：原来判「证据不支持」要降档的句子，改判成哪一档都不能完整出具。"""
|
||
Models(monkeypatch, verdicts={"新客": (status, "新客维度的数据" if status == "insufficient" else "")})
|
||
graph = weekly(report={"numbers": "strict", "on_violation": "fail", "judge": {"max_cost_usd": None}})
|
||
row = await governed(client, graph)
|
||
assert row.output["_issuance"]["tier"] == "degraded"
|
||
|
||
|
||
async def test_rewrite_once_hands_back_contradicted_and_insufficient_sentences(engine_up, monkeypatch):
|
||
draft = ("本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]"
|
||
"华东贡献了大部分增长。[[see:m:gmv]]下面看细节。")
|
||
rewritten = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客、华东的贡献要等分维度的数据再看。下面看细节。"
|
||
models = Models(monkeypatch, writer=(draft, rewritten),
|
||
verdicts={"新客首单": ("contradicted", ""), "华东贡献": ("insufficient", "分区域的查询"),
|
||
"要等分维度": ("not_a_claim", "")})
|
||
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "rewrite_once": True}}))
|
||
assert len(models.writes) == 2
|
||
request = str(models.writes[1][-1].content)
|
||
assert "增长主要来自新客首单。" in request and "华东贡献了大部分增长。" in request
|
||
assert "分区域的查询" in request, "证据不足的句子连同缺的是什么一起交回"
|
||
assert SUPPORTED not in request
|
||
checked, _ = await checked_of(row.id)
|
||
assert checked["judge"]["rewrite"]["applied"] is True and len(checked["judge"]["rewrite"]["sentences"]) == 2
|
||
|
||
|
||
async def test_an_explore_report_records_the_writer_model_too(engine_up, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
row = await run(weekly(), run_class="exploratory")
|
||
assert models.judges == []
|
||
checked, doc = await checked_of(row.id)
|
||
assert doc["judge"]["mode"] == "on_demand" and doc["judge"]["writer_model"] == WRITER
|
||
assert set(judge_mod.COUNT_KEYS) <= set(doc["stats"])
|