mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 01:01:59 +00:00
M2 片段接口的按需裁判说法照实写：正式运行的报告没开结论句裁判时，on_demand 不再说「已在报告撰写节点内完成裁判」，改说这份报告没有开启结论句裁判；开了裁判却因报告没通过核对而没判的也照实说；不是结论句的（标题、连接性文字）先答 not_a_claim；按需裁判接口对正式运行回 409 时说同一句话
验收里正式运行（claims: off）的片段接口对结论句、标题都答「已在节点内完成裁判」，节点里其实没判过。现在按文档里有没有当场裁判的摘要（mode inline）和写作时的结论句策略认，片段接口和 POST …/evidence/judge 共用 _formal_message。
This commit is contained in:
ThearchyHelios
@@ -108,7 +108,7 @@ from app.engine.evidence import (
|
||||
)
|
||||
from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated
|
||||
from app.engine.merge_query import MERGE_SOURCE, trace_cell
|
||||
from app.engine.schema import GraphSpec, challenge_report, dual_check_card
|
||||
from app.engine.schema import CLAIMS_JUDGE, GraphSpec, challenge_report, dual_check_card
|
||||
from app.engine.toolcalls import QUERY_PREFIX
|
||||
|
||||
router = APIRouter(prefix="/api/runs", tags=["evidence"])
|
||||
@@ -750,7 +750,7 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No
|
||||
**({"verdict": verdict} if verdict is not None else {}),
|
||||
# 期 4：反方审稿对这一句的判断，没审到这一句的没有这个键
|
||||
**({"challenge": challenged} if challenged is not None else {}),
|
||||
"on_demand": _on_demand(sealed, doc, str(unit.get("id")), verdict)},
|
||||
"on_demand": _on_demand(sealed, doc, str(unit.get("id")), verdict, chosen.claims)},
|
||||
"block": {"id": block.get("id"), "type": block.get("type")},
|
||||
"chain": chain,
|
||||
"note": _note(seg, unit),
|
||||
@@ -2281,6 +2281,10 @@ JUDGE_MAX_UNITS = 200
|
||||
_TERMINAL = ("succeeded", "failed", "cancelled")
|
||||
|
||||
FORMAL_JUDGED = "正式运行的结论句已在报告撰写节点内完成裁判，结果随报告一起封存，不能再按需裁判"
|
||||
#: 正式运行、报告没开结论句裁判（结论句策略是不裁判或只要求附依据，升级前的运行也一样）：节点里没判过，照实说
|
||||
FORMAL_NO_JUDGE = "这份报告没有开启结论句裁判，正式运行的结论句不能按需裁判"
|
||||
#: 正式运行、开了裁判节点里却没判：报告没通过核对、节点按设置紧接着判失败时，不再花钱裁判
|
||||
FORMAL_UNJUDGED = "这份报告开启了结论句裁判，但报告未通过核对，结论句没有在报告撰写节点内完成裁判；正式运行的结论句不能按需裁判"
|
||||
UNSEALED_JUDGE = "运行尚未结束并封存（仍在运行、停在人工审批或正在继续运行），请在运行结束后再请模型裁判"
|
||||
LEGACY_JUDGE = "这次运行发起时系统尚未记录证据，不支持按需裁判"
|
||||
BROKEN_JUDGE = "封存核对未通过：封存后有事件被修改、删除或插入，报告和证据已不可信，无法再请模型裁判"
|
||||
@@ -2374,17 +2378,21 @@ def _judge_meta(report: _Report, sealed: _Sealed) -> dict[str, Any] | None:
|
||||
"priced": data.get("priced") if isinstance(data.get("priced"), bool) else None, "writer_model": writer}
|
||||
|
||||
|
||||
def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""这句能不能「请模型判断」：{available, reason, message}。和 POST …/evidence/judge 同一套条件。"""
|
||||
def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str, Any] | None,
|
||||
claims: str | None = None) -> dict[str, Any]:
|
||||
"""这句能不能「请模型判断」：{available, reason, message}。和 POST …/evidence/judge 同一套条件。
|
||||
|
||||
claims 是写这份报告时的结论句策略（_Report.claims）。不是结论句的先说：标题、连接性文字不论哪类运行都无需裁判。
|
||||
以前正式运行一律先答「已在节点内完成裁判」，标题、没开裁判的报告也这么说（_formal_message）。"""
|
||||
def no(reason: str, message: str) -> dict[str, Any]:
|
||||
return {"available": False, "reason": reason, "message": message}
|
||||
|
||||
if sealed.run_class == "formal":
|
||||
return no("formal", FORMAL_JUDGED)
|
||||
if not sealed.ledger_on:
|
||||
return no("legacy", LEGACY_JUDGE)
|
||||
if not candidates(doc, [uid]):
|
||||
return no("not_a_claim", f"这句{NOT_A_CLAIM}")
|
||||
if sealed.run_class == "formal":
|
||||
return no("formal", _formal_message(doc, claims))
|
||||
if not sealed.ledger_on:
|
||||
return no("legacy", LEGACY_JUDGE)
|
||||
stale = _stale(verdict)
|
||||
if isinstance(verdict, dict) and verdict.get("status") in SETTLED and not stale:
|
||||
return no("judged", JUDGED_ALREADY)
|
||||
@@ -2404,6 +2412,27 @@ def _stale(verdict: Any) -> bool:
|
||||
return isinstance(verdict, dict) and bool(verdict.get("post_seal")) and outdated(verdict)
|
||||
|
||||
|
||||
def _formal_message(doc: dict[str, Any] | None, claims: str | None) -> str:
|
||||
"""正式运行为什么不能按需裁判，照实说节点里判没判过：文档里有当场裁判的摘要（mode inline）才是判过了；开了裁判却
|
||||
没有摘要的，是报告没通过核对、节点紧接着判失败，不再花钱裁判；别的是没开结论句裁判（策略为不裁判、只要求附依据，
|
||||
升级前的运行没记策略，也没有裁判摘要）。"""
|
||||
judge = (doc or {}).get("judge")
|
||||
if isinstance(judge, dict) and judge.get("mode") == "inline":
|
||||
return FORMAL_JUDGED
|
||||
return FORMAL_UNJUDGED if claims == CLAIMS_JUDGE else FORMAL_NO_JUDGE
|
||||
|
||||
|
||||
def _formal_refusal(sealed: _Sealed, wanted: str | None) -> str:
|
||||
"""按需裁判接口拒绝正式运行时的那句话，和片段接口说的一致：按请求要的那份报告（没指定取成果里标注的那一份）说。
|
||||
选不出报告、文档读不出来的，按没有裁判摘要说。"""
|
||||
reports = _reports(sealed)
|
||||
try:
|
||||
chosen = _choose(reports, wanted, sealed) if reports else None
|
||||
except CodedHTTPException:
|
||||
chosen = None
|
||||
return _formal_message(chosen.doc if chosen else None, chosen.claims if chosen else None)
|
||||
|
||||
|
||||
def _sealed_loader(sealed: _Sealed, *, chain: bool = False) -> Callable[[str], Any]:
|
||||
"""给裁判取证据的 loader：只认封存范围内的事件交回过的工件（查询、表结构、检索快照和台账里记过的），
|
||||
取回时照旧复验哈希。别的一律当取不到——人为插进工件表的行、封存之后才追加的事件引用的快照，
|
||||
@@ -2515,7 +2544,7 @@ async def judge_units(run_id: str, payload: Any = Body(None), report: str | None
|
||||
async with _JUDGING.hold(run_id):
|
||||
sealed = await _sealed(run_id)
|
||||
if sealed.run_class == "formal":
|
||||
raise CodedHTTPException(409, FORMAL_JUDGED, JUDGE_FORMAL)
|
||||
raise CodedHTTPException(409, _formal_refusal(sealed, wanted), JUDGE_FORMAL)
|
||||
if not sealed.ledger_on:
|
||||
raise CodedHTTPException(409, LEGACY_JUDGE, JUDGE_LEGACY)
|
||||
if not sealed.sealed or sealed.status not in _TERMINAL:
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""片段接口的 on_demand（这句能不能请模型判断）在正式运行里照实说。
|
||||
|
||||
验收发现：正式运行的片段接口一律答「正式运行的结论句已在报告撰写节点内完成裁判」，可那次报告的结论句策略是不裁判
|
||||
（report.checked 记 claims: off，文档里没有裁判摘要），节点里根本没判过；标题这种不是结论句的片段也答同一句。现在：
|
||||
|
||||
- 不是结论句的先说（not_a_claim）：标题、连接性文字不论哪类运行都无需裁判
|
||||
- 正式运行、节点里确实判过（文档里有当场裁判的摘要）：照旧说已在节点内完成裁判
|
||||
- 正式运行、没开结论句裁判（不裁判、只要求附依据，或者升级前的运行）：照实说这份报告没有开启结论句裁判
|
||||
- 正式运行、开了裁判节点里却没判（报告没通过核对，节点紧接着判失败，不再花钱裁判）：照实说没有在节点内完成裁判
|
||||
- 按需裁判接口（POST …/evidence/judge）对正式运行回 409 时说同一句话
|
||||
|
||||
写作、裁判都用 MockChatModel 照剧本，不调真接口。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run
|
||||
from app.engine import judge as judge_mod
|
||||
from app.engine.evidence import compose_doc, iter_units
|
||||
from app.engine.judge import JUDGE_ROLE
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from app.providers.mock_model import MockChatModel
|
||||
|
||||
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
||||
#: 一个标题、一句挂了依据的结论、一句没挂依据的归因
|
||||
TEXT = "## 本周\n\n本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。"
|
||||
#: 标题那一句（正文里去掉了「## 」），和没挂依据的归因
|
||||
HEADING, CAUSAL = "本周", "增长主要来自新客首单。"
|
||||
JUDGED_WORDS = "已在报告撰写节点内完成裁判"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def weekly(**report) -> dict:
|
||||
nodes = [
|
||||
node("start", "input"),
|
||||
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
|
||||
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
|
||||
node("write", "report", instructions="写周报", **report),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||||
]
|
||||
ids = [n["id"] for n in nodes]
|
||||
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||||
|
||||
|
||||
def models(monkeypatch) -> list[str]:
|
||||
"""写作者回 TEXT；裁判（claims: judge 时节点里当场判）一律判有依据。返回交给裁判的请求原文。"""
|
||||
judged: list[str] = []
|
||||
|
||||
def decide(chat, messages):
|
||||
if JUDGE_ROLE in str(messages[0].content):
|
||||
request = str(messages[-1].content)
|
||||
judged.append(request)
|
||||
items = [{"unit": uid, "verdict": "supported", "rationale": "有依据", "used": []}
|
||||
for uid, _ in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)]
|
||||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
|
||||
return AIMessage(content=TEXT)
|
||||
|
||||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||||
|
||||
async def judge_model_of(session, spec):
|
||||
return MockChatModel(model_name=spec.model or "judge-x"), spec.model or "judge-x"
|
||||
|
||||
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
|
||||
return judged
|
||||
|
||||
|
||||
async def finish(graph, *, run_class="formal") -> Run:
|
||||
run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
|
||||
for _ in range(400):
|
||||
await asyncio.sleep(0.05)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run.id)
|
||||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||||
assert row.status == "succeeded", row.error
|
||||
return row
|
||||
raise AssertionError(f"run {run.id} 没跑完：{row.status}")
|
||||
|
||||
|
||||
def unit_of(doc: dict, text: str) -> str:
|
||||
md = doc["markdown"]
|
||||
return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip() == text)
|
||||
|
||||
|
||||
async def on_demand(client, row: Run, text: str) -> dict:
|
||||
doc = artifact_store.load(row.output["_evidence"]["doc_artifact"])
|
||||
uid = unit_of(doc, text)
|
||||
seg = next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"] if s["kind"] != "structural")
|
||||
res = await client.get(f"/api/runs/{row.id}/evidence/segments/{seg}?report=write")
|
||||
assert res.status_code == 200, res.text
|
||||
return res.json()["unit"]["on_demand"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 片段接口
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("claims", [None, "off", "require_citation"])
|
||||
async def test_a_formal_report_without_claim_judging_says_it_was_not_judged(client, monkeypatch, claims):
|
||||
"""验收原样：正式运行、结论句策略不裁判（没写也一样，或者只要求附依据）。不能说「已在节点内完成裁判」。"""
|
||||
judged = models(monkeypatch)
|
||||
row = await finish(weekly(**({"claims": claims} if claims else {})))
|
||||
got = await on_demand(client, row, CAUSAL)
|
||||
assert got["available"] is False and got["reason"] == "formal", got
|
||||
assert JUDGED_WORDS not in got["message"] and "没有开启结论句裁判" in got["message"], got["message"]
|
||||
assert judged == [], "节点里确实没请裁判"
|
||||
|
||||
|
||||
async def test_a_heading_in_a_formal_run_is_not_a_claim(client, monkeypatch):
|
||||
"""标题不是结论句：不论哪类运行都答 not_a_claim，不答「已在节点内完成裁判」。"""
|
||||
models(monkeypatch)
|
||||
row = await finish(weekly())
|
||||
got = await on_demand(client, row, HEADING)
|
||||
assert got["available"] is False and got["reason"] == "not_a_claim" and JUDGED_WORDS not in got["message"], got
|
||||
|
||||
|
||||
async def test_a_formal_report_judged_in_the_node_says_so(client, monkeypatch):
|
||||
"""开了裁判、节点里当场判过：照旧说已完成裁判；标题照样是 not_a_claim。"""
|
||||
judged = models(monkeypatch)
|
||||
row = await finish(weekly(claims="judge"))
|
||||
assert judged, "正式运行在节点里当场裁判"
|
||||
got = await on_demand(client, row, CAUSAL)
|
||||
assert got["available"] is False and got["reason"] == "formal" and JUDGED_WORDS in got["message"], got
|
||||
assert (await on_demand(client, row, HEADING))["reason"] == "not_a_claim"
|
||||
|
||||
|
||||
async def test_the_judge_endpoint_says_the_same_for_a_formal_report_without_judging(client, monkeypatch):
|
||||
"""按需裁判接口对正式运行照旧 409，说的话和片段接口一致：没开裁判的照实说没开。"""
|
||||
models(monkeypatch)
|
||||
row = await finish(weekly())
|
||||
doc = artifact_store.load(row.output["_evidence"]["doc_artifact"])
|
||||
res = await client.post(f"/api/runs/{row.id}/evidence/judge", json={"units": [unit_of(doc, CAUSAL)]})
|
||||
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal", res.text
|
||||
detail = res.json()["detail"]
|
||||
assert JUDGED_WORDS not in detail and "没有开启结论句裁判" in detail, detail
|
||||
assert detail == (await on_demand(client, row, CAUSAL))["message"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 手搭：开了裁判、节点里却没判（报告没通过核对，节点紧接着判失败）
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_formal_report_that_opted_in_but_was_never_judged_says_so():
|
||||
"""claims: judge 的正式运行，报告没通过核对、节点判失败之前不再花钱裁判：文档里没有裁判摘要。不能说已完成裁判。"""
|
||||
from app.api.evidence import _on_demand, _Sealed
|
||||
|
||||
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r-unjudged")
|
||||
[unit] = [u for _, u in iter_units(doc)]
|
||||
sealed = _Sealed(run_id="r-unjudged", graph={}, events=[], run_class="formal", status="failed", ledger_on=True,
|
||||
seal={"sealed": True, "ok": True, "manifest_seq": 3, "legacy": False})
|
||||
got = _on_demand(sealed, doc, unit["id"], None, claims="judge")
|
||||
assert got["available"] is False and got["reason"] == "formal", got
|
||||
assert JUDGED_WORDS not in got["message"] and "没有在报告撰写节点内完成裁判" in got["message"], got["message"]
|
||||
# 同一份文档带上当场裁判的摘要：照旧说已完成裁判
|
||||
judged = {**doc, "judge": {"mode": "inline", "counts": {}}}
|
||||
assert JUDGED_WORDS in _on_demand(sealed, judged, unit["id"], None, claims="judge")["message"]
|
||||
# 不传策略（老调用方）：看文档里有没有当场裁判的摘要
|
||||
assert "没有开启结论句裁判" in _on_demand(sealed, doc, unit["id"], None)["message"]
|
||||
Reference in New Issue
Block a user