From a45ca28c6bbdeb752fb60e3ed5ef3103fde6de19 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 7 Oct 2026 00:03:58 +0800 Subject: [PATCH] =?UTF-8?q?M2=20=E7=89=87=E6=AE=B5=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E7=9A=84=E6=8C=89=E9=9C=80=E8=A3=81=E5=88=A4=E8=AF=B4=E6=B3=95?= =?UTF-8?q?=E7=85=A7=E5=AE=9E=E5=86=99=EF=BC=9A=E6=AD=A3=E5=BC=8F=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=9A=84=E6=8A=A5=E5=91=8A=E6=B2=A1=E5=BC=80=E7=BB=93?= =?UTF-8?q?=E8=AE=BA=E5=8F=A5=E8=A3=81=E5=88=A4=E6=97=B6=EF=BC=8Con=5Fdema?= =?UTF-8?q?nd=20=E4=B8=8D=E5=86=8D=E8=AF=B4=E3=80=8C=E5=B7=B2=E5=9C=A8?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E6=92=B0=E5=86=99=E8=8A=82=E7=82=B9=E5=86=85?= =?UTF-8?q?=E5=AE=8C=E6=88=90=E8=A3=81=E5=88=A4=E3=80=8D=EF=BC=8C=E6=94=B9?= =?UTF-8?q?=E8=AF=B4=E8=BF=99=E4=BB=BD=E6=8A=A5=E5=91=8A=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E5=BC=80=E5=90=AF=E7=BB=93=E8=AE=BA=E5=8F=A5=E8=A3=81=E5=88=A4?= =?UTF-8?q?=EF=BC=9B=E5=BC=80=E4=BA=86=E8=A3=81=E5=88=A4=E5=8D=B4=E5=9B=A0?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E6=B2=A1=E9=80=9A=E8=BF=87=E6=A0=B8=E5=AF=B9?= =?UTF-8?q?=E8=80=8C=E6=B2=A1=E5=88=A4=E7=9A=84=E4=B9=9F=E7=85=A7=E5=AE=9E?= =?UTF-8?q?=E8=AF=B4=EF=BC=9B=E4=B8=8D=E6=98=AF=E7=BB=93=E8=AE=BA=E5=8F=A5?= =?UTF-8?q?=E7=9A=84=EF=BC=88=E6=A0=87=E9=A2=98=E3=80=81=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E6=80=A7=E6=96=87=E5=AD=97=EF=BC=89=E5=85=88=E7=AD=94=20not=5F?= =?UTF-8?q?a=5Fclaim=EF=BC=9B=E6=8C=89=E9=9C=80=E8=A3=81=E5=88=A4=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E5=AF=B9=E6=AD=A3=E5=BC=8F=E8=BF=90=E8=A1=8C=E5=9B=9E?= =?UTF-8?q?=20409=20=E6=97=B6=E8=AF=B4=E5=90=8C=E4=B8=80=E5=8F=A5=E8=AF=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验收里正式运行（claims: off）的片段接口对结论句、标题都答「已在节点内完成裁判」，节点里其实没判过。现在按文档里有没有当场裁判的摘要（mode inline）和写作时的结论句策略认，片段接口和 POST …/evidence/judge 共用 _formal_message。 --- backend/app/api/evidence.py | 47 ++++- .../tests/test_evidence_on_demand_formal.py | 185 ++++++++++++++++++ 2 files changed, 223 insertions(+), 9 deletions(-) create mode 100644 backend/tests/test_evidence_on_demand_formal.py diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index a67d4e20..ead0c107 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -108,7 +108,7 @@ from app.engine.evidence import ( ) from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated from app.engine.merge_query import MERGE_SOURCE, trace_cell -from app.engine.schema import GraphSpec, challenge_report, dual_check_card +from app.engine.schema import CLAIMS_JUDGE, GraphSpec, challenge_report, dual_check_card from app.engine.toolcalls import QUERY_PREFIX router = APIRouter(prefix="/api/runs", tags=["evidence"]) @@ -750,7 +750,7 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No **({"verdict": verdict} if verdict is not None else {}), # 期 4：反方审稿对这一句的判断，没审到这一句的没有这个键 **({"challenge": challenged} if challenged is not None else {}), - "on_demand": _on_demand(sealed, doc, str(unit.get("id")), verdict)}, + "on_demand": _on_demand(sealed, doc, str(unit.get("id")), verdict, chosen.claims)}, "block": {"id": block.get("id"), "type": block.get("type")}, "chain": chain, "note": _note(seg, unit), @@ -2281,6 +2281,10 @@ JUDGE_MAX_UNITS = 200 _TERMINAL = ("succeeded", "failed", "cancelled") FORMAL_JUDGED = "正式运行的结论句已在报告撰写节点内完成裁判，结果随报告一起封存，不能再按需裁判" +#: 正式运行、报告没开结论句裁判（结论句策略是不裁判或只要求附依据，升级前的运行也一样）：节点里没判过，照实说 +FORMAL_NO_JUDGE = "这份报告没有开启结论句裁判，正式运行的结论句不能按需裁判" +#: 正式运行、开了裁判节点里却没判：报告没通过核对、节点按设置紧接着判失败时，不再花钱裁判 +FORMAL_UNJUDGED = "这份报告开启了结论句裁判，但报告未通过核对，结论句没有在报告撰写节点内完成裁判；正式运行的结论句不能按需裁判" UNSEALED_JUDGE = "运行尚未结束并封存（仍在运行、停在人工审批或正在继续运行），请在运行结束后再请模型裁判" LEGACY_JUDGE = "这次运行发起时系统尚未记录证据，不支持按需裁判" BROKEN_JUDGE = "封存核对未通过：封存后有事件被修改、删除或插入，报告和证据已不可信，无法再请模型裁判" @@ -2374,17 +2378,21 @@ def _judge_meta(report: _Report, sealed: _Sealed) -> dict[str, Any] | None: "priced": data.get("priced") if isinstance(data.get("priced"), bool) else None, "writer_model": writer} -def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str, Any] | None) -> dict[str, Any]: - """这句能不能「请模型判断」：{available, reason, message}。和 POST …/evidence/judge 同一套条件。""" +def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str, Any] | None, + claims: str | None = None) -> dict[str, Any]: + """这句能不能「请模型判断」：{available, reason, message}。和 POST …/evidence/judge 同一套条件。 + + claims 是写这份报告时的结论句策略（_Report.claims）。不是结论句的先说：标题、连接性文字不论哪类运行都无需裁判。 + 以前正式运行一律先答「已在节点内完成裁判」，标题、没开裁判的报告也这么说（_formal_message）。""" def no(reason: str, message: str) -> dict[str, Any]: return {"available": False, "reason": reason, "message": message} - if sealed.run_class == "formal": - return no("formal", FORMAL_JUDGED) - if not sealed.ledger_on: - return no("legacy", LEGACY_JUDGE) if not candidates(doc, [uid]): return no("not_a_claim", f"这句{NOT_A_CLAIM}") + if sealed.run_class == "formal": + return no("formal", _formal_message(doc, claims)) + if not sealed.ledger_on: + return no("legacy", LEGACY_JUDGE) stale = _stale(verdict) if isinstance(verdict, dict) and verdict.get("status") in SETTLED and not stale: return no("judged", JUDGED_ALREADY) @@ -2404,6 +2412,27 @@ def _stale(verdict: Any) -> bool: return isinstance(verdict, dict) and bool(verdict.get("post_seal")) and outdated(verdict) +def _formal_message(doc: dict[str, Any] | None, claims: str | None) -> str: + """正式运行为什么不能按需裁判，照实说节点里判没判过：文档里有当场裁判的摘要（mode inline）才是判过了；开了裁判却 + 没有摘要的，是报告没通过核对、节点紧接着判失败，不再花钱裁判；别的是没开结论句裁判（策略为不裁判、只要求附依据， + 升级前的运行没记策略，也没有裁判摘要）。""" + judge = (doc or {}).get("judge") + if isinstance(judge, dict) and judge.get("mode") == "inline": + return FORMAL_JUDGED + return FORMAL_UNJUDGED if claims == CLAIMS_JUDGE else FORMAL_NO_JUDGE + + +def _formal_refusal(sealed: _Sealed, wanted: str | None) -> str: + """按需裁判接口拒绝正式运行时的那句话，和片段接口说的一致：按请求要的那份报告（没指定取成果里标注的那一份）说。 + 选不出报告、文档读不出来的，按没有裁判摘要说。""" + reports = _reports(sealed) + try: + chosen = _choose(reports, wanted, sealed) if reports else None + except CodedHTTPException: + chosen = None + return _formal_message(chosen.doc if chosen else None, chosen.claims if chosen else None) + + def _sealed_loader(sealed: _Sealed, *, chain: bool = False) -> Callable[[str], Any]: """给裁判取证据的 loader：只认封存范围内的事件交回过的工件（查询、表结构、检索快照和台账里记过的）， 取回时照旧复验哈希。别的一律当取不到——人为插进工件表的行、封存之后才追加的事件引用的快照， @@ -2515,7 +2544,7 @@ async def judge_units(run_id: str, payload: Any = Body(None), report: str | None async with _JUDGING.hold(run_id): sealed = await _sealed(run_id) if sealed.run_class == "formal": - raise CodedHTTPException(409, FORMAL_JUDGED, JUDGE_FORMAL) + raise CodedHTTPException(409, _formal_refusal(sealed, wanted), JUDGE_FORMAL) if not sealed.ledger_on: raise CodedHTTPException(409, LEGACY_JUDGE, JUDGE_LEGACY) if not sealed.sealed or sealed.status not in _TERMINAL: diff --git a/backend/tests/test_evidence_on_demand_formal.py b/backend/tests/test_evidence_on_demand_formal.py new file mode 100644 index 00000000..4fcef955 --- /dev/null +++ b/backend/tests/test_evidence_on_demand_formal.py @@ -0,0 +1,185 @@ +"""片段接口的 on_demand（这句能不能请模型判断）在正式运行里照实说。 + +验收发现：正式运行的片段接口一律答「正式运行的结论句已在报告撰写节点内完成裁判」，可那次报告的结论句策略是不裁判 +（report.checked 记 claims: off，文档里没有裁判摘要），节点里根本没判过；标题这种不是结论句的片段也答同一句。现在： + +- 不是结论句的先说（not_a_claim）：标题、连接性文字不论哪类运行都无需裁判 +- 正式运行、节点里确实判过（文档里有当场裁判的摘要）：照旧说已在节点内完成裁判 +- 正式运行、没开结论句裁判（不裁判、只要求附依据，或者升级前的运行）：照实说这份报告没有开启结论句裁判 +- 正式运行、开了裁判节点里却没判（报告没通过核对，节点紧接着判失败，不再花钱裁判）：照实说没有在节点内完成裁判 +- 按需裁判接口（POST …/evidence/judge）对正式运行回 409 时说同一句话 + +写作、裁判都用 MockChatModel 照剧本，不调真接口。 +""" +from __future__ import annotations + +import asyncio +import json +import re + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage + +from app.core import artifact_store +from app.db.base import SessionLocal +from app.db.models import Run +from app.engine import judge as judge_mod +from app.engine.evidence import compose_doc, iter_units +from app.engine.judge import JUDGE_ROLE +from app.engine.runner import run_manager +from app.main import app +from app.providers.mock_model import MockChatModel + +METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}] +#: 一个标题、一句挂了依据的结论、一句没挂依据的归因 +TEXT = "## 本周\n\n本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。" +#: 标题那一句（正文里去掉了「## 」），和没挂依据的归因 +HEADING, CAUSAL = "本周", "增长主要来自新客首单。" +JUDGED_WORDS = "已在报告撰写节点内完成裁判" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def weekly(**report) -> dict: + nodes = [ + node("start", "input"), + node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"), + node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS), + node("write", "report", instructions="写周报", **report), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + ] + ids = [n["id"] for n in nodes] + return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]} + + +def models(monkeypatch) -> list[str]: + """写作者回 TEXT；裁判（claims: judge 时节点里当场判）一律判有依据。返回交给裁判的请求原文。""" + judged: list[str] = [] + + def decide(chat, messages): + if JUDGE_ROLE in str(messages[0].content): + request = str(messages[-1].content) + judged.append(request) + items = [{"unit": uid, "verdict": "supported", "rationale": "有依据", "used": []} + for uid, _ in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)] + return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False)) + return AIMessage(content=TEXT) + + monkeypatch.setattr(MockChatModel, "_decide", decide) + + async def judge_model_of(session, spec): + return MockChatModel(model_name=spec.model or "judge-x"), spec.model or "judge-x" + + monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of) + return judged + + +async def finish(graph, *, run_class="formal") -> Run: + run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + assert row.status == "succeeded", row.error + return row + raise AssertionError(f"run {run.id} 没跑完：{row.status}") + + +def unit_of(doc: dict, text: str) -> str: + md = doc["markdown"] + return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip() == text) + + +async def on_demand(client, row: Run, text: str) -> dict: + doc = artifact_store.load(row.output["_evidence"]["doc_artifact"]) + uid = unit_of(doc, text) + seg = next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"] if s["kind"] != "structural") + res = await client.get(f"/api/runs/{row.id}/evidence/segments/{seg}?report=write") + assert res.status_code == 200, res.text + return res.json()["unit"]["on_demand"] + + +# -------------------------------------------------------------------------- +# 片段接口 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("claims", [None, "off", "require_citation"]) +async def test_a_formal_report_without_claim_judging_says_it_was_not_judged(client, monkeypatch, claims): + """验收原样：正式运行、结论句策略不裁判（没写也一样，或者只要求附依据）。不能说「已在节点内完成裁判」。""" + judged = models(monkeypatch) + row = await finish(weekly(**({"claims": claims} if claims else {}))) + got = await on_demand(client, row, CAUSAL) + assert got["available"] is False and got["reason"] == "formal", got + assert JUDGED_WORDS not in got["message"] and "没有开启结论句裁判" in got["message"], got["message"] + assert judged == [], "节点里确实没请裁判" + + +async def test_a_heading_in_a_formal_run_is_not_a_claim(client, monkeypatch): + """标题不是结论句：不论哪类运行都答 not_a_claim，不答「已在节点内完成裁判」。""" + models(monkeypatch) + row = await finish(weekly()) + got = await on_demand(client, row, HEADING) + assert got["available"] is False and got["reason"] == "not_a_claim" and JUDGED_WORDS not in got["message"], got + + +async def test_a_formal_report_judged_in_the_node_says_so(client, monkeypatch): + """开了裁判、节点里当场判过：照旧说已完成裁判；标题照样是 not_a_claim。""" + judged = models(monkeypatch) + row = await finish(weekly(claims="judge")) + assert judged, "正式运行在节点里当场裁判" + got = await on_demand(client, row, CAUSAL) + assert got["available"] is False and got["reason"] == "formal" and JUDGED_WORDS in got["message"], got + assert (await on_demand(client, row, HEADING))["reason"] == "not_a_claim" + + +async def test_the_judge_endpoint_says_the_same_for_a_formal_report_without_judging(client, monkeypatch): + """按需裁判接口对正式运行照旧 409，说的话和片段接口一致：没开裁判的照实说没开。""" + models(monkeypatch) + row = await finish(weekly()) + doc = artifact_store.load(row.output["_evidence"]["doc_artifact"]) + res = await client.post(f"/api/runs/{row.id}/evidence/judge", json={"units": [unit_of(doc, CAUSAL)]}) + assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal", res.text + detail = res.json()["detail"] + assert JUDGED_WORDS not in detail and "没有开启结论句裁判" in detail, detail + assert detail == (await on_demand(client, row, CAUSAL))["message"] + + +# -------------------------------------------------------------------------- +# 手搭：开了裁判、节点里却没判（报告没通过核对，节点紧接着判失败） +# -------------------------------------------------------------------------- + + +def test_a_formal_report_that_opted_in_but_was_never_judged_says_so(): + """claims: judge 的正式运行，报告没通过核对、节点判失败之前不再花钱裁判：文档里没有裁判摘要。不能说已完成裁判。""" + from app.api.evidence import _on_demand, _Sealed + + doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r-unjudged") + [unit] = [u for _, u in iter_units(doc)] + sealed = _Sealed(run_id="r-unjudged", graph={}, events=[], run_class="formal", status="failed", ledger_on=True, + seal={"sealed": True, "ok": True, "manifest_seq": 3, "legacy": False}) + got = _on_demand(sealed, doc, unit["id"], None, claims="judge") + assert got["available"] is False and got["reason"] == "formal", got + assert JUDGED_WORDS not in got["message"] and "没有在报告撰写节点内完成裁判" in got["message"], got["message"] + # 同一份文档带上当场裁判的摘要：照旧说已完成裁判 + judged = {**doc, "judge": {"mode": "inline", "counts": {}}} + assert JUDGED_WORDS in _on_demand(sealed, judged, unit["id"], None, claims="judge")["message"] + # 不传策略（老调用方）：看文档里有没有当场裁判的摘要 + assert "没有开启结论句裁判" in _on_demand(sealed, doc, unit["id"], None)["message"]