Files
AgentLab/backend/tests/test_verify_rationale_rules.py
T
ThearchyHelios 3d85a43301 M1 复核修正(第 2 轮):去掉依据之后按真正挨上的两个字判粘连——两个只装着依据的括号连着写、数字包着粗体时,两边的数不再并成一个;依据标记的「]」不当数字;一片里只补一个空格
渲染(evidence._tidy):
- 上一轮判粘连只看原文里紧挨着括号的两个字(往后只越过 [[see:]])。两个只装着依据的括号连着写(「[[v:Q1.r0.销售额]]
  ([[see:Q1]])([[see:m:gmv]])2 月」)、数字包着粗体(「**[[v:…]]**([[see:Q1]])2 月」,组装文档时星号又被去掉)
  时,去掉之后真正挨上的是两边的数,读者看到「4,919.52 月」「212212 笔」,compose_doc、verify_doc 都不报。现在先把依据
  标记和只装着依据的括号(套着的、几个连着写的都算)认成渲染出来什么都没有的一段(_voids),连着写的几个并成一组,外面的
  停顿标点、空白照原规矩收拾;所有切口去掉之后,再往两边越过去掉的字、依据、粗体斜体的星号和下划线,找真正挨上的两个看得
  见的字,会读成一串的补回一个空格(_glue)
- 依据标记的「]」越过去、不当数字:「华南居首。[[see:Q1]]([[see:Q2]])2 月」不再多出空格,「[[see:Q1]]([[see:Q1]])
  2 月」不再以空格开头
- 同一片里最多补一个空格:「212,[[see:Q1]],[[see:m:gmv]],5」→「212, 5」(上一轮是两个空格);挨上的是写作者自己留着的
  空白的,不补
- 半角标点后面补空格并进同一条规则:只在下文是数字、字母或渲染出字的标记时补(「a,[[see:Q1]],b」→「a, b」),下文是中文
  的照写作者原样
- 流式渲染把末尾的星号、下划线也攒到后面的字来了再放,和整篇渲染判得一样
- 粗体把括号包在里面时(「**[[v:…]]([[see:Q1]])**2 月」)空格落在收尾的星号前面:前端行内 Markdown 照样认这对星号,
  组装出的文档去掉了包着标记的星号,读者看到的一样;挪到星号外面得先知道这对星号从哪开始,流式渲染的一块里不一定有。
  测试里用严格的 xfail 写明这条已知限制

测试:两个括号连着写(全角半角混用、套着的)、粗体在括号前、后、里面、下划线和单个星号、依据的「]」、一片只补一个的用例;
结构化组合(数、依据、只装依据的括号、空白、星号、标点)逐份查写作者隔开了的两边没有挨上、没多出连着的两个空格,逐字和
每 4 个字分块渲染都等于整篇渲染;审批质询、核查 agent、反方审稿、双盲复算各补一个端到端用例。11 处改动逐一去掉实现,
相关测试都变红。另在测试之外抽查:结构化组合 15.9 万份和基线 58a6759 逐份比数字串,只剩复核说可以不管的「),5」这类半角
标点紧贴数字的写法;随机 8000 份不含依据的文档,内容哈希、渲染结果和基线逐份一致;随机 1.6 万份含依据的文档流式和整篇
一致。
2026-10-12 10:35:13 +08:00

229 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""核查 agent 交理由的写法和收尾超时记的上限(期 5 真实模型验收 M1)。
- 交结果的提示(VERIFY_FINAL_PROMPT,双盲复算、反方审稿、审批质询共用)和写作者一致:依据写在句末,不放进括号或句子
中间。以前写的是「提到某次查询时写 [[see:Q1]]」,模型就写成「结果一致([[see:Q3]],原报告…)」,反方审稿的系统提示
自己也把它写在括号里(「([[v:Qn.rX.列名]] 或 [[see:Qn]])」)
- 写进了括号的依据,理由文档里照样渲染干净(evidence._tidy),这句仍挂着那条依据
- 模型自己停了工具调用、收尾那次调用超过了时长上限:limit 照实记 timeout_s。以前是 None——质询的报错写着「回答超过
时长上限(180 秒)」,limit 却是空的,界面上缺了「已达时长上限」。不走结构化输出的模型(普通调用收尾)也一样
- 交回的理由是空的、只有标点,或者只是一个占位词(字面的 placeholder、N/A):当作没给理由,不把占位词当理由展示。
按渲染出来的正文判:只写了依据的([[see:Q1]]、「([[see:Q1]])」)显示出来是空的,占位词后面挂着依据的显示出来只剩
占位词,都算没给。理由就是结果本身时(审批质询,结构里只有理由一项),「无」「none」是回答,不算占位
- 依据写在「依据」「见」「取自」这类引出语后面(渲染时去不掉,引出语悬空):记一条日志,验收时查得到
模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,数字都是虚构的。
"""
from __future__ import annotations
import asyncio
import logging
import re
import time
import pytest
from app.engine import verify_agent
from app.engine.evidence import iter_units
from app.engine.inquiry import INQUIRY_SCHEMA, INQUIRY_SYSTEM, INQUIRY_TASK_PROMPT
from app.engine.nodes.challenge import CHALLENGE_SYSTEM, CHALLENGE_TASK_PROMPT
from app.engine.nodes.dual_check import BLIND_SYSTEM, BLIND_TASK_PROMPT
from app.engine.verify_agent import VERIFY_FINAL_PROMPT, VERIFY_SYSTEM
from app.providers.mock_model import MockChatModel
from tests.test_verify_agent import VALID_SUM, Script, clean_spend, q, run, shop # noqa: F401 - 夹具按名字引入
#: 括号里写着依据标记:「([[v:Qn.rX.列名]] 或 [[see:Qn]])」「([[see:Q1]])」这种示范
_BRACKETED_SEE = re.compile(r"[((][^()()]*\[\[see:[^\]]*\]\][^()()]*[))]")
FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致"}
def slow_final(monkeypatch, seconds: float = 5.0) -> None:
"""交结果那次调用(消息里带着 VERIFY_FINAL_PROMPT:结构化输出、退回普通调用都是)迟迟不返回,查库的几轮照常。"""
original = MockChatModel._agenerate
async def held(chat, messages, stop=None, run_manager=None, **kwargs):
if any(VERIFY_FINAL_PROMPT in str(m.content) for m in messages):
await asyncio.sleep(seconds)
return await original(chat, messages, stop, run_manager, **kwargs)
monkeypatch.setattr(MockChatModel, "_agenerate", held)
# --------------------------------------------------------------------------
# 提示词
# --------------------------------------------------------------------------
def test_the_final_prompt_puts_see_at_the_end_of_the_sentence():
assert "提到某次查询时写 [[see:Q1]]" not in VERIFY_FINAL_PROMPT
assert "句末" in VERIFY_FINAL_PROMPT and "括号" in VERIFY_FINAL_PROMPT
assert re.search(r"。\[\[see:Q\d\]\]", VERIFY_FINAL_PROMPT), "示范的写法:依据挂在句号后面"
assert "渲染后不显示" in VERIFY_FINAL_PROMPT
@pytest.mark.parametrize("prompt", [VERIFY_SYSTEM, VERIFY_FINAL_PROMPT, CHALLENGE_SYSTEM, CHALLENGE_TASK_PROMPT,
INQUIRY_SYSTEM, INQUIRY_TASK_PROMPT, BLIND_SYSTEM, BLIND_TASK_PROMPT])
def test_no_verify_prompt_shows_see_inside_brackets(prompt):
assert not _BRACKETED_SEE.search(prompt), _BRACKETED_SEE.search(prompt).group(0)
async def test_the_final_call_is_told_the_rule(monkeypatch, shop):
name, _ = shop
script = Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": "合计 [[v:Q1.r0.total]]。[[see:Q1]]"})
await run(name)
kind, messages = script.calls[-1]
assert kind == "final" and VERIFY_FINAL_PROMPT in str(messages[-1].content)
# --------------------------------------------------------------------------
# 写进括号的依据
# --------------------------------------------------------------------------
async def test_see_written_in_brackets_renders_clean_in_the_rationale(monkeypatch, shop):
name, _ = shop
said = "未作废订单的金额合计是 [[v:Q1.r0.total]]([[see:Q1]]),口径一致([[see:Q1]],已排除作废单)。"
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
out = await run(name)
shown = "未作废订单的金额合计是 230,口径一致(已排除作废单)。"
assert out.result["rationale"] == shown and out.rationale["markdown"] == shown
assert out.rationale["source"] == said, "原文照存,依据标记还在"
[unit] = [unit for _, unit in iter_units(out.rationale)]
assert [s["alias"] for s in unit["see"]] == ["Q1", "Q1"] and unit["cites"] == ["Q1"]
assert out.rationale["violations"] == []
async def test_see_brackets_written_together_do_not_glue_two_numbers(monkeypatch, shop):
"""两个只装着依据的括号连着写、数字包着粗体(复核第 2 轮):去掉括号之后两个数不能挨成一个(「2303 笔」),句末前
悬着的逗号也去掉。"""
name, _ = shop
said = ("未作废订单合计 **[[v:Q1.r0.total]]**([[see:Q1]])([[see:Q2]])[[v:Q2.r0.n]] 笔都已核对,口径一致,"
"([[see:Q1]])([[see:Q2]])。")
count = "SELECT COUNT(*) AS n FROM orders WHERE is_deleted = 0"
Script(monkeypatch, [q(name, VALID_SUM, "c1"), q(name, count, "c2")], {**FOUND, "rationale": said})
out = await run(name)
shown = "未作废订单合计 230 3 笔都已核对,口径一致。"
assert out.result["rationale"] == shown and out.rationale["markdown"] == shown
[unit] = [unit for _, unit in iter_units(out.rationale)]
assert [s["alias"] for s in unit["see"]] == ["Q1", "Q2", "Q1", "Q2"]
numbers = [seg for seg in unit["segments"] if seg.get("ref")]
assert [(seg["text"], bool(seg.get("strong"))) for seg in numbers] == [("230", True), ("3", False)]
assert out.rationale["violations"] == []
# --------------------------------------------------------------------------
# 收尾超时
# --------------------------------------------------------------------------
@pytest.mark.parametrize("structured", [True, False], ids=["structured", "plain"])
async def test_a_final_call_that_runs_out_of_time_records_the_time_limit(monkeypatch, shop, structured):
"""模型查了一次就自己停下(没撞上任何上限),交结果那次调用却一直不返回:到点放弃,limit 是 timeout_s,和原因一致。"""
name, _ = shop
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
script = Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": "合计 [[v:Q1.r0.total]]。"})
slow_final(monkeypatch)
if not structured:
def refuse(self, schema, **kwargs):
raise NotImplementedError("不支持结构化输出")
monkeypatch.setattr(MockChatModel, "with_structured_output", refuse)
started = time.monotonic()
out = await run(name, timeout_s=0.5)
assert time.monotonic() - started < 3.0, "收尾那次只等到宽限结束"
assert script.kinds == ["step", "step"], "查了一次、自己停下,交结果那次没等到回复"
assert out.answered is True and [qd["alias"] for qd in out.queries] == ["Q1"]
assert out.result is None and out.error == "收尾时超过时长上限,没能交结果"
assert out.limit == "timeout_s"
async def test_a_final_call_that_runs_out_of_time_after_the_query_limit_records_the_time_limit(monkeypatch, shop):
"""循环先撞上查询次数上限、收尾又超时:没交成结果是因为超时,limit 记 timeout_s,和原因一致——记成查询次数上限的话,
质询会显示「未能回答(已达查询次数上限):回答超过时长上限…」,双盲复算会说盲算查满了次数(查满了照样能交结果)。
金额被拒的收尾照旧报循环里先撞上的那个(见 test_verify_truncation)。"""
name, _ = shop
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
Script(monkeypatch, [q(name, VALID_SUM, "c1"), q(name, "SELECT COUNT(*) AS n FROM orders", "c2")],
{**FOUND, "rationale": "x"})
slow_final(monkeypatch)
out = await run(name, timeout_s=0.5, max_queries=1)
assert [qd["alias"] for qd in out.queries] == ["Q1"], "查满了一次"
assert out.result is None and out.error == "收尾时超过时长上限,没能交结果"
assert out.limit == "timeout_s"
# --------------------------------------------------------------------------
# 占位词当理由
# --------------------------------------------------------------------------
@pytest.mark.parametrize("said", ["placeholder", "PLACEHOLDER", " ", "N/A", "(略)", "…", "<rationale>", "TODO", "无",
"暂无。", "None"])
async def test_a_placeholder_rationale_counts_as_none(monkeypatch, shop, said):
name, _ = shop
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
out = await run(name)
assert out.result["value"] == 230, "数照样取"
assert out.result["rationale"] == "" and out.rationale is None
@pytest.mark.parametrize("said", ["[[see:Q1]]", "([[see:Q1]])", " [[see:Q1]]。", "placeholder。[[see:Q1]]",
"N/A([[see:Q1]])", "TODO [[see:Q1]]"])
async def test_a_rationale_that_shows_nothing_counts_as_none(monkeypatch, shop, said):
"""只写了依据、或者占位词后面挂着依据:原文里看着有字,显示出来没有理由。以前按原文判,标记里的字(see、Q1)也算,
理由文档照样交回——审批卡按文档画出一片空白,判「被推翻」的还因为引用到了 Q1 被当成有出处。"""
name, _ = shop
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
out = await run(name)
assert out.result["value"] == 230, "数照样取"
assert out.result["rationale"] == "" and out.rationale is None
@pytest.mark.parametrize("said", ["无异常。", "见 [[v:Q1.r0.total]]。", "略高于上周。", "[[v:Q1.r0.total]]",
"一致。[[see:Q1]]"])
async def test_a_short_real_rationale_is_kept(monkeypatch, shop, said):
name, _ = shop
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
out = await run(name)
assert out.result["rationale"] and out.rationale is not None
@pytest.mark.parametrize(("said", "shown"), [("无。", "无。"), ("None.", "None."), ("暂无。[[see:Q1]]", "暂无。")])
async def test_nothing_is_an_answer_when_the_rationale_is_the_answer(monkeypatch, shop, said, shown):
"""结构里只有理由一项(审批质询):理由就是回答。问「有没有测试账号」,答「无。」是一个回答,不能当成没给、让提问人
重新提问再花一次钱。"""
name, _ = shop
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {"rationale": said})
out = await run(name, output_schema=INQUIRY_SCHEMA)
assert out.result["rationale"] == shown and out.rationale["markdown"] == shown
@pytest.mark.parametrize("said", ["placeholder", "null", "[[see:Q1]]", "N/A。[[see:Q1]]"])
async def test_a_placeholder_answer_still_counts_as_none(monkeypatch, shop, said):
name, _ = shop
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {"rationale": said})
out = await run(name, output_schema=INQUIRY_SCHEMA)
assert out.result["rationale"] == "" and out.rationale is None
# --------------------------------------------------------------------------
# 依据写在引出语后面
# --------------------------------------------------------------------------
async def test_see_after_a_lead_in_word_is_logged(monkeypatch, shop, caplog):
"""「依据 [[see:Q1]]。」「(来源:[[see:Q1]])」渲染后引出语悬空,渲染时去不掉(见 test_evidence_see_brackets 的已知
限制):记一条日志,写明是哪几处,验收时查得到。只记标记和引出语,不记数。"""
name, _ = shop
said = "合计 [[v:Q1.r0.total]],依据 [[see:Q1]]。口径一致(来源:[[see:Q1]])。"
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
with caplog.at_level(logging.WARNING, logger="app.engine.verify_agent"):
out = await run(name, node_id="chal", run_id="run-1")
assert out.rationale is not None, "照常交回"
[record] = [r for r in caplog.records if "引出语" in r.getMessage()]
message = record.getMessage()
assert "依据 [[see:Q1]]" in message and "来源:[[see:Q1]]" in message and "run-1" in message
assert "230" not in message
async def test_see_at_the_end_of_the_sentence_is_not_logged(monkeypatch, shop, caplog):
name, _ = shop
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": "合计 [[v:Q1.r0.total]],结果一致。[[see:Q1]]"})
with caplog.at_level(logging.WARNING, logger="app.engine.verify_agent"):
await run(name)
assert not [r for r in caplog.records if "引出语" in r.getMessage()]