"""报告节点的结论句策略 claims:off(默认,和以前一样)/ require_citation(没挂依据的结论句计入缺口)。 这里测确定性的一半(模型裁判 judge 见 test_report_judge.py): - require_citation 时写作提示明确要求每句结论挂 [[see:…]];没挂的照常产出、计数, 出口按出具档位降档(io.py),报告节点自己不为它失败、不为它重写 - report.checked 事件和节点产出都带上 claims,出口据此判档 - 写了不认识的值节点直接报错,不能悄悄当成 off - 升级前发起的运行一个字都不变 """ from __future__ import annotations import asyncio import pytest from langchain_core.messages import AIMessage from sqlalchemy import select from app.core import artifact_store from app.db.base import SessionLocal from app.db.models import Run, RunEvent from app.engine import runner as runner_mod from app.engine.evidence import CLAIMS_RULE, build_catalog, compose_doc, uncited_claims from app.engine.runner import run_manager from app.engine.schema import CLAIMS_VALUES, claims_problem @pytest.fixture(autouse=True) async def engine_up(): await run_manager.setup() yield await run_manager.shutdown() def node(nid, ntype, **config): return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}] #: 第一句挂了依据,第二句是没挂依据的结论(有方向词),第三句是连接性的话 TEXT = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客。下面看细节。" def weekly(**report): nodes = [ node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]), node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"), node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS), node("write", "report", instructions="写周报", **report), node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), ] ids = [n["id"] for n in nodes] return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]} def script(monkeypatch, reply=TEXT) -> list: from app.providers import mock_model seen: list = [] def _decide(self, messages): seen.append(list(messages)) return AIMessage(content=reply) monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide) return seen def lenient(monkeypatch): """装作画布校验没拦住(校验以后放宽、或者绕过校验发起的运行):节点执行时自己也要守住这一关, 报的是同一句话。""" original = runner_mod.validate_graph def permissive(spec): report = original(spec) report.issues = [i for i in report.issues if i.field != "claims"] report.ok = not any(i.level == "error" for i in report.issues) return report monkeypatch.setattr(runner_mod, "validate_graph", permissive) async def finish(graph) -> Run: run = await run_manager.start(graph=graph, input_payload={}) for _ in range(400): await asyncio.sleep(0.05) async with SessionLocal() as session: row = await session.get(Run, run.id) if row.status in ("succeeded", "failed") and row.manifest_seq is not None: return row raise AssertionError(f"run {run.id} 没跑完:{row.status}") async def events(run_id: str, etype: str) -> list[RunEvent]: async with SessionLocal() as session: q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype).order_by(RunEvent.seq) return list((await session.execute(q)).scalars()) async def checked_and_output(run_id: str) -> tuple[dict, dict]: [checked] = await events(run_id, "report.checked") finished = [e for e in await events(run_id, "node.finished") if e.node_id == "write"][0] return checked.data, artifact_store.load(finished.data["artifact"]) def prompt_of(seen: list) -> str: return "\n".join(str(m.content) for m in seen[0]) async def test_default_is_off_and_changes_nothing(monkeypatch): seen = script(monkeypatch) row = await finish(weekly()) assert row.status == "succeeded", row.error checked, output = await checked_and_output(row.id) assert checked["claims"] == "off" and output["claims"] == "off" assert CLAIMS_RULE not in prompt_of(seen) assert checked["stats"]["uncited_claims"] == 1 async def test_require_citation_asks_for_support_but_does_not_block(monkeypatch): seen = script(monkeypatch) row = await finish(weekly(claims="require_citation", on_violation="fail")) assert row.status == "succeeded", row.error assert len(seen) == 1, "没挂依据的结论句不触发重写" assert CLAIMS_RULE in prompt_of(seen) checked, output = await checked_and_output(row.id) assert checked["claims"] == "require_citation" and output["claims"] == "require_citation" assert checked["ok"] is True and checked["violations"] == [] assert checked["stats"]["uncited_claims"] == output["stats"]["uncited_claims"] == 1 doc = artifact_store.load(checked["doc_artifact"]) [flagged] = uncited_claims(doc) assert flagged["text"] == "增长主要来自新客。" and doc["markdown"][slice(*flagged["span"])] == flagged["text"] async def test_a_bad_value_is_refused_before_any_model_call(monkeypatch): seen = script(monkeypatch) with pytest.raises(ValueError) as info: # 画布校验先拦 await run_manager.start(graph=weekly(claims="sometimes"), input_payload={}) assert claims_problem("sometimes") in str(info.value) lenient(monkeypatch) row = await finish(weekly(claims="sometimes")) assert row.status == "failed" and claims_problem("sometimes") in (row.error or ""), row.error assert seen == [], "不支持的策略不该先花一次模型调用" @pytest.mark.parametrize("value", ["sometimes", 1, ["require_citation"]]) async def test_other_values_are_refused(monkeypatch, value): script(monkeypatch) lenient(monkeypatch) row = await finish(weekly(claims=value)) assert row.status == "failed", row.error assert claims_problem(value) in row.error and "「计入缺口」" in row.error async def test_runs_from_before_the_upgrade_are_untouched(monkeypatch): async def no_snapshot(session): return None monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot) seen = script(monkeypatch) row = await finish(weekly()) assert row.status == "succeeded", row.error checked, output = await checked_and_output(row.id) assert "claims" not in checked and "claims" not in output assert CLAIMS_RULE not in prompt_of(seen) # -------------------------------------------------------------------------- # 出口和界面共用的纯函数 # -------------------------------------------------------------------------- def test_the_supported_policies_match_the_validator(): from app.engine.nodes.report import CLAIMS, report_claims from app.engine.schema import GraphSpec assert CLAIMS == CLAIMS_VALUES == ("off", "require_citation", "judge") spec = GraphSpec.model_validate(weekly(claims="require_citation")) assert report_claims(spec, spec.node_map()["write"]) == "require_citation" spec = GraphSpec.model_validate(weekly()) assert report_claims(spec, spec.node_map()["write"]) == "off" spec.defaults["claims"] = "require_citation" # 图级 defaults 同样生效,和执行时一个规则 assert report_claims(spec, spec.node_map()["write"]) == "require_citation" def test_uncited_claims_lists_claim_units_without_support(): catalog = build_catalog(nodes={"caliber": {"kind": "metric_set", "caliber": "c", "caliber_version": "v1", "artifact": "f" * 64, "metrics": [{"id": "gmv", "name": "销售额", "value": 1, "unit": ""}]}}) doc = compose_doc("销售额 [[m:gmv]]。环比下降。[[see:m:gmv]]主要来自东区。连接的话。", catalog) assert [u["text"] for u in uncited_claims(doc)] == ["主要来自东区。"] assert all(set(u) == {"unit", "span", "text"} for u in uncited_claims(doc))