Files
AgentLab/backend/tests/test_evidence_on_demand_formal.py
T
ThearchyHelios ce4be9cc8f M2 复核第 2 轮:报告自查按值认出正文里写着的遮罩值,只数个数的列不再判违规;遮没遮核对不了的输入写「无法核对」;整句面板挂的依据按证据图的标签写
- 自查按值认(engine/evidence.py 的 masked_values,report.py 的 _MaskedCells):写作目录里确定取自遮罩列的值出现在正文里——照抄成文字(姓名这类文字列数字核对管不到)、写成数字(换了写法的长号码组装时被拆成几个数字片段,按整句认)、经指标带出来([[m:phone_raw]] 的值就是号码)——记一条 masked_cell(by_value),不带值,交回写作者时附上值换成「已遮罩」的那一句。只收确定取自遮罩列的列、不收分档标签;运行输入里写着的值是发起运行的人自己填的,不算。只读数据源现在设了遮罩的快照,认引用时读过的不再读,没设遮罩的运行一份都不读
- 时间线:日志 report_masked_cell 多带 written(其中按值认出的几处),标题分「引用了…格子」「写进了…值」「引用、写进了…」;缺口、日志按两种分别说;违规清单、CSV 的叫法改成「写进了遮罩列的值」
- 只数个数的列(COUNT(*) FILTER (WHERE phone IS NULL)、SUM(phone IS NULL)、AVG(phone IS NOT NULL)、SUM(amount) FILTER (WHERE phone IS NOT NULL)):认遮罩列的那一套把 FILTER 条件、汇总里的真假条件当成值的来历,数据质量报告引用「手机号缺失的人数」在正式运行里判失败。自查认之前先改写(_counted:FILTER 同 WHERE 去掉,汇总里的真假条件改成 CASE WHEN … THEN 1 ELSE 0 END),不判违规;证据面板、推演照可验证报告的认法多遮(只影响展示),根上的改法在 masking 里
- 证据面板:遮没遮核对不了的输入(快照、取值链上的产出读不出来),值和代入式里写「无法核对」,不写「已遮罩」,数据源可能根本没设遮罩
- 整句面板「挂的依据」不再直接读文档目录里的「名称 = 渲染值」:按证据图的标签写(服务端已把用到遮罩列的指标值换成「已遮罩」),证据图没回来时指标只写名字
- on_demand:当场裁判过、文档里却压根没有这句的判定时,不说「原因见判定」
- 测试:合并查询自查的用例补上拼接、upper、WITH;推演对整行、下标不是字面量的一格说「用到了遮罩列」(顺着取值链认出来的,不是按值猜的);计数指标 cnt 的原值照给;SQL 认不准、只数个数的反例
2026-10-12 10:48:19 +08:00

223 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""片段接口的 on_demand(这句能不能请模型判断)在正式运行里照实说。
验收发现:正式运行的片段接口一律答「正式运行的结论句已在报告撰写节点内完成裁判」,可那次报告的结论句策略是不裁判
(report.checked 记 claims: off,文档里没有裁判摘要),节点里根本没判过;标题这种不是结论句的片段也答同一句。现在:
- 不是结论句的先说(not_a_claim):标题、连接性文字不论哪类运行都无需裁判
- 正式运行、节点里确实判过(文档里有当场裁判的摘要):照旧说已在节点内完成裁判
- 正式运行、没开结论句裁判(不裁判、只要求附依据,或者升级前的运行):照实说这份报告没有开启结论句裁判
- 正式运行、开了裁判节点里却没判(报告没通过核对,节点紧接着判失败,不再花钱裁判):照实说没有在节点内完成裁判
- 按需裁判接口(POST …/evidence/judge)对正式运行回 409 时说同一句话
写作、裁判都用 MockChatModel 照剧本,不调真接口。
"""
from __future__ import annotations
import asyncio
import json
import re
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from app.core import artifact_store
from app.db.base import SessionLocal
from app.db.models import Run
from app.engine import judge as judge_mod
from app.engine.evidence import compose_doc, iter_units
from app.engine.judge import JUDGE_ROLE
from app.engine.runner import run_manager
from app.main import app
from app.providers.mock_model import MockChatModel
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
#: 一个标题、一句挂了依据的结论、一句没挂依据的归因
TEXT = "## 本周\n\n本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。"
#: 标题那一句(正文里去掉了「## 」),和没挂依据的归因
HEADING, CAUSAL = "本周", "增长主要来自新客首单。"
JUDGED_WORDS = "已在报告撰写节点内完成裁判"
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def weekly(**report) -> dict:
nodes = [
node("start", "input"),
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
node("write", "report", instructions="写周报", **report),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
]
ids = [n["id"] for n in nodes]
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
def models(monkeypatch) -> list[str]:
"""写作者回 TEXT;裁判(claims: judge 时节点里当场判)一律判有依据。返回交给裁判的请求原文。"""
judged: list[str] = []
def decide(chat, messages):
if JUDGE_ROLE in str(messages[0].content):
request = str(messages[-1].content)
judged.append(request)
items = [{"unit": uid, "verdict": "supported", "rationale": "有依据", "used": []}
for uid, _ in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)]
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
return AIMessage(content=TEXT)
monkeypatch.setattr(MockChatModel, "_decide", decide)
async def judge_model_of(session, spec):
return MockChatModel(model_name=spec.model or "judge-x"), spec.model or "judge-x"
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
return judged
async def finish(graph, *, run_class="formal") -> Run:
run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run.id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
assert row.status == "succeeded", row.error
return row
raise AssertionError(f"run {run.id} 没跑完:{row.status}")
def unit_of(doc: dict, text: str) -> str:
md = doc["markdown"]
return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip() == text)
async def on_demand(client, row: Run, text: str) -> dict:
doc = artifact_store.load(row.output["_evidence"]["doc_artifact"])
uid = unit_of(doc, text)
seg = next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"] if s["kind"] != "structural")
res = await client.get(f"/api/runs/{row.id}/evidence/segments/{seg}?report=write")
assert res.status_code == 200, res.text
return res.json()["unit"]["on_demand"]
# --------------------------------------------------------------------------
# 片段接口
# --------------------------------------------------------------------------
@pytest.mark.parametrize("claims", [None, "off", "require_citation"])
async def test_a_formal_report_without_claim_judging_says_it_was_not_judged(client, monkeypatch, claims):
"""验收原样:正式运行、结论句策略不裁判(没写也一样,或者只要求附依据)。不能说「已在节点内完成裁判」。"""
judged = models(monkeypatch)
row = await finish(weekly(**({"claims": claims} if claims else {})))
got = await on_demand(client, row, CAUSAL)
assert got["available"] is False and got["reason"] == "formal", got
assert JUDGED_WORDS not in got["message"] and "没有开启结论句裁判" in got["message"], got["message"]
assert judged == [], "节点里确实没请裁判"
async def test_a_heading_in_a_formal_run_is_not_a_claim(client, monkeypatch):
"""标题不是结论句:不论哪类运行都答 not_a_claim,不答「已在节点内完成裁判」。"""
models(monkeypatch)
row = await finish(weekly())
got = await on_demand(client, row, HEADING)
assert got["available"] is False and got["reason"] == "not_a_claim" and JUDGED_WORDS not in got["message"], got
async def test_a_formal_report_judged_in_the_node_says_so(client, monkeypatch):
"""开了裁判、节点里当场判过:照旧说已完成裁判;标题照样是 not_a_claim。"""
judged = models(monkeypatch)
row = await finish(weekly(claims="judge"))
assert judged, "正式运行在节点里当场裁判"
got = await on_demand(client, row, CAUSAL)
assert got["available"] is False and got["reason"] == "formal" and JUDGED_WORDS in got["message"], got
assert (await on_demand(client, row, HEADING))["reason"] == "not_a_claim"
async def test_the_judge_endpoint_says_the_same_for_a_formal_report_without_judging(client, monkeypatch):
"""按需裁判接口对正式运行照旧 409,说的话和片段接口一致:没开裁判的照实说没开。"""
models(monkeypatch)
row = await finish(weekly())
doc = artifact_store.load(row.output["_evidence"]["doc_artifact"])
res = await client.post(f"/api/runs/{row.id}/evidence/judge", json={"units": [unit_of(doc, CAUSAL)]})
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal", res.text
detail = res.json()["detail"]
assert JUDGED_WORDS not in detail and "没有开启结论句裁判" in detail, detail
assert detail == (await on_demand(client, row, CAUSAL))["message"]
# 点名的不是报告撰写节点:认不出是哪份报告,不替它说判没判过,只说正式运行不能按需裁判
res = await client.post(f"/api/runs/{row.id}/evidence/judge", json={"units": ["u1"], "report": "caliber"})
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal", res.text
assert res.json()["detail"] == "正式运行的结论句只在报告撰写节点内裁判,不能按需裁判"
# --------------------------------------------------------------------------
# 手搭:开了裁判、节点里却没判(报告没通过核对,节点紧接着判失败)
# --------------------------------------------------------------------------
def test_a_formal_report_that_opted_in_but_was_never_judged_says_so():
"""claims: judge 的正式运行,报告没通过核对、节点判失败之前不再花钱裁判:文档里没有裁判摘要。不能说已完成裁判。"""
from app.api.evidence import _on_demand, _Sealed
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r-unjudged")
[unit] = [u for _, u in iter_units(doc)]
sealed = _Sealed(run_id="r-unjudged", graph={}, events=[], run_class="formal", status="failed", ledger_on=True,
seal={"sealed": True, "ok": True, "manifest_seq": 3, "legacy": False})
got = _on_demand(sealed, doc, unit["id"], None, claims="judge")
assert got["available"] is False and got["reason"] == "formal", got
assert JUDGED_WORDS not in got["message"] and "没有在报告撰写节点内完成裁判" in got["message"], got["message"]
# 同一份文档带上当场裁判的摘要、这句判过了:照旧说已完成裁判
judged = {**doc, "judge": {"mode": "inline", "counts": {}}}
assert JUDGED_WORDS in _on_demand(sealed, judged, unit["id"], {"status": "supported"}, claims="judge")["message"]
# 不传策略(老调用方):看文档里有没有当场裁判的摘要
assert "没有开启结论句裁判" in _on_demand(sealed, doc, unit["id"], None)["message"]
@pytest.mark.parametrize("verdict", [
# 触顶:这句没判成,判定里记着原因
{"status": "unjudged", "reason": "max_claims", "rationale": "已达上限(每份报告最多裁判 1 句),这句未裁判"},
# 裁判出错、格式不对、被截断都一样是未裁判
{"status": "unjudged", "reason": "error", "rationale": "裁判失败:连接超时"},
# 文档里压根没有这句的判定
None,
])
def test_an_inline_judged_report_that_did_not_judge_this_sentence_says_so(verdict):
"""正式运行在节点里当场裁判过(文档里有裁判摘要),可这一句没判成:触顶、出错,或者没有判定。不能说已完成裁判,
照实说这句没判成、原因在判定里;判过的句子照旧说已完成裁判。按需裁判接口一次点了几句、有没判成的,也照实说。"""
from app.api.evidence import FORMAL_INLINE_MISSED_SOME, _formal_message, _on_demand, _Sealed
doc = compose_doc("增长主要来自新客首单。复购率也在上升。", {}, node_id="write", run_id="r-capped")
doc["judge"] = {"mode": "inline", "complete": False, "counts": {"unjudged": 1}, "limits_hit": ["max_claims"]}
first, second = [u["id"] for _, u in iter_units(doc)]
sealed = _Sealed(run_id="r-capped", graph={}, events=[], run_class="formal", status="succeeded", ledger_on=True,
seal={"sealed": True, "ok": True, "manifest_seq": 3, "legacy": False})
got = _on_demand(sealed, doc, first, verdict, claims="judge")
assert got["available"] is False and got["reason"] == "formal", got
assert JUDGED_WORDS not in got["message"] and "不能按需裁判" in got["message"], got["message"]
if verdict is None:
# 没有判定可看:不说「原因见判定」
assert "这句没有判定" in got["message"] and "原因见判定" not in got["message"], got["message"]
else:
assert "这句没有判成" in got["message"] and "原因见判定" in got["message"], got["message"]
# 判过的那一句照旧说已完成裁判
assert JUDGED_WORDS in _on_demand(sealed, doc, second, {"status": "supported"}, claims="judge")["message"]
# 一次问几句(按需裁判接口):有一句没判成就照实说
assert _formal_message(doc, "judge", [verdict, {"status": "supported"}]) == FORMAL_INLINE_MISSED_SOME
assert _formal_message(doc, "judge", [{"status": "supported"}, {"status": "partial"}]).startswith("正式运行的结论句已在")