M2 片段接口的按需裁判说法照实写:正式运行的报告没开结论句裁判时,on_demand 不再说「已在报告撰写节点内完成裁判」,改说这份报告没有开启结论句裁判;开了裁判却因报告没通过核对而没判的也照实说;不是结论句的(标题、连接性文字)先答 not_a_claim;按需裁判接口对正式运行回 409 时说同一句话

验收里正式运行(claims: off)的片段接口对结论句、标题都答「已在节点内完成裁判」,节点里其实没判过。现在按文档里有没有当场裁判的摘要(mode inline)和写作时的结论句策略认,片段接口和 POST …/evidence/judge 共用 _formal_message。
This commit is contained in:
ThearchyHelios
2026-10-07 00:03:58 +08:00
parent 58a6759fe2
commit a45ca28c6b
2 changed files with 223 additions and 9 deletions
@@ -0,0 +1,185 @@
"""片段接口的 on_demand(这句能不能请模型判断)在正式运行里照实说。
验收发现:正式运行的片段接口一律答「正式运行的结论句已在报告撰写节点内完成裁判」,可那次报告的结论句策略是不裁判
(report.checked 记 claims: off,文档里没有裁判摘要),节点里根本没判过;标题这种不是结论句的片段也答同一句。现在:
- 不是结论句的先说(not_a_claim):标题、连接性文字不论哪类运行都无需裁判
- 正式运行、节点里确实判过(文档里有当场裁判的摘要):照旧说已在节点内完成裁判
- 正式运行、没开结论句裁判(不裁判、只要求附依据,或者升级前的运行):照实说这份报告没有开启结论句裁判
- 正式运行、开了裁判节点里却没判(报告没通过核对,节点紧接着判失败,不再花钱裁判):照实说没有在节点内完成裁判
- 按需裁判接口(POST …/evidence/judge)对正式运行回 409 时说同一句话
写作、裁判都用 MockChatModel 照剧本,不调真接口。
"""
from __future__ import annotations
import asyncio
import json
import re
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from app.core import artifact_store
from app.db.base import SessionLocal
from app.db.models import Run
from app.engine import judge as judge_mod
from app.engine.evidence import compose_doc, iter_units
from app.engine.judge import JUDGE_ROLE
from app.engine.runner import run_manager
from app.main import app
from app.providers.mock_model import MockChatModel
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
#: 一个标题、一句挂了依据的结论、一句没挂依据的归因
TEXT = "## 本周\n\n本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。"
#: 标题那一句(正文里去掉了「## 」),和没挂依据的归因
HEADING, CAUSAL = "本周", "增长主要来自新客首单。"
JUDGED_WORDS = "已在报告撰写节点内完成裁判"
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def weekly(**report) -> dict:
nodes = [
node("start", "input"),
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
node("write", "report", instructions="写周报", **report),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
]
ids = [n["id"] for n in nodes]
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
def models(monkeypatch) -> list[str]:
"""写作者回 TEXT;裁判(claims: judge 时节点里当场判)一律判有依据。返回交给裁判的请求原文。"""
judged: list[str] = []
def decide(chat, messages):
if JUDGE_ROLE in str(messages[0].content):
request = str(messages[-1].content)
judged.append(request)
items = [{"unit": uid, "verdict": "supported", "rationale": "有依据", "used": []}
for uid, _ in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)]
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
return AIMessage(content=TEXT)
monkeypatch.setattr(MockChatModel, "_decide", decide)
async def judge_model_of(session, spec):
return MockChatModel(model_name=spec.model or "judge-x"), spec.model or "judge-x"
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
return judged
async def finish(graph, *, run_class="formal") -> Run:
run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run.id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
assert row.status == "succeeded", row.error
return row
raise AssertionError(f"run {run.id} 没跑完:{row.status}")
def unit_of(doc: dict, text: str) -> str:
md = doc["markdown"]
return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip() == text)
async def on_demand(client, row: Run, text: str) -> dict:
doc = artifact_store.load(row.output["_evidence"]["doc_artifact"])
uid = unit_of(doc, text)
seg = next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"] if s["kind"] != "structural")
res = await client.get(f"/api/runs/{row.id}/evidence/segments/{seg}?report=write")
assert res.status_code == 200, res.text
return res.json()["unit"]["on_demand"]
# --------------------------------------------------------------------------
# 片段接口
# --------------------------------------------------------------------------
@pytest.mark.parametrize("claims", [None, "off", "require_citation"])
async def test_a_formal_report_without_claim_judging_says_it_was_not_judged(client, monkeypatch, claims):
"""验收原样:正式运行、结论句策略不裁判(没写也一样,或者只要求附依据)。不能说「已在节点内完成裁判」。"""
judged = models(monkeypatch)
row = await finish(weekly(**({"claims": claims} if claims else {})))
got = await on_demand(client, row, CAUSAL)
assert got["available"] is False and got["reason"] == "formal", got
assert JUDGED_WORDS not in got["message"] and "没有开启结论句裁判" in got["message"], got["message"]
assert judged == [], "节点里确实没请裁判"
async def test_a_heading_in_a_formal_run_is_not_a_claim(client, monkeypatch):
"""标题不是结论句:不论哪类运行都答 not_a_claim,不答「已在节点内完成裁判」。"""
models(monkeypatch)
row = await finish(weekly())
got = await on_demand(client, row, HEADING)
assert got["available"] is False and got["reason"] == "not_a_claim" and JUDGED_WORDS not in got["message"], got
async def test_a_formal_report_judged_in_the_node_says_so(client, monkeypatch):
"""开了裁判、节点里当场判过:照旧说已完成裁判;标题照样是 not_a_claim。"""
judged = models(monkeypatch)
row = await finish(weekly(claims="judge"))
assert judged, "正式运行在节点里当场裁判"
got = await on_demand(client, row, CAUSAL)
assert got["available"] is False and got["reason"] == "formal" and JUDGED_WORDS in got["message"], got
assert (await on_demand(client, row, HEADING))["reason"] == "not_a_claim"
async def test_the_judge_endpoint_says_the_same_for_a_formal_report_without_judging(client, monkeypatch):
"""按需裁判接口对正式运行照旧 409,说的话和片段接口一致:没开裁判的照实说没开。"""
models(monkeypatch)
row = await finish(weekly())
doc = artifact_store.load(row.output["_evidence"]["doc_artifact"])
res = await client.post(f"/api/runs/{row.id}/evidence/judge", json={"units": [unit_of(doc, CAUSAL)]})
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal", res.text
detail = res.json()["detail"]
assert JUDGED_WORDS not in detail and "没有开启结论句裁判" in detail, detail
assert detail == (await on_demand(client, row, CAUSAL))["message"]
# --------------------------------------------------------------------------
# 手搭:开了裁判、节点里却没判(报告没通过核对,节点紧接着判失败)
# --------------------------------------------------------------------------
def test_a_formal_report_that_opted_in_but_was_never_judged_says_so():
"""claims: judge 的正式运行,报告没通过核对、节点判失败之前不再花钱裁判:文档里没有裁判摘要。不能说已完成裁判。"""
from app.api.evidence import _on_demand, _Sealed
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r-unjudged")
[unit] = [u for _, u in iter_units(doc)]
sealed = _Sealed(run_id="r-unjudged", graph={}, events=[], run_class="formal", status="failed", ledger_on=True,
seal={"sealed": True, "ok": True, "manifest_seq": 3, "legacy": False})
got = _on_demand(sealed, doc, unit["id"], None, claims="judge")
assert got["available"] is False and got["reason"] == "formal", got
assert JUDGED_WORDS not in got["message"] and "没有在报告撰写节点内完成裁判" in got["message"], got["message"]
# 同一份文档带上当场裁判的摘要:照旧说已完成裁判
judged = {**doc, "judge": {"mode": "inline", "counts": {}}}
assert JUDGED_WORDS in _on_demand(sealed, judged, unit["id"], None, claims="judge")["message"]
# 不传策略(老调用方):看文档里有没有当场裁判的摘要
assert "没有开启结论句裁判" in _on_demand(sealed, doc, unit["id"], None)["message"]