Files
AgentLab/backend/tests/test_challenge_rationale_rules.py
T
ThearchyHelios 3d85a43301 M1 复核修正(第 2 轮):去掉依据之后按真正挨上的两个字判粘连——两个只装着依据的括号连着写、数字包着粗体时,两边的数不再并成一个;依据标记的「]」不当数字;一片里只补一个空格
渲染(evidence._tidy):
- 上一轮判粘连只看原文里紧挨着括号的两个字(往后只越过 [[see:]])。两个只装着依据的括号连着写(「[[v:Q1.r0.销售额]]
  ([[see:Q1]])([[see:m:gmv]])2 月」)、数字包着粗体(「**[[v:…]]**([[see:Q1]])2 月」,组装文档时星号又被去掉)
  时,去掉之后真正挨上的是两边的数,读者看到「4,919.52 月」「212212 笔」,compose_doc、verify_doc 都不报。现在先把依据
  标记和只装着依据的括号(套着的、几个连着写的都算)认成渲染出来什么都没有的一段(_voids),连着写的几个并成一组,外面的
  停顿标点、空白照原规矩收拾;所有切口去掉之后,再往两边越过去掉的字、依据、粗体斜体的星号和下划线,找真正挨上的两个看得
  见的字,会读成一串的补回一个空格(_glue)
- 依据标记的「]」越过去、不当数字:「华南居首。[[see:Q1]]([[see:Q2]])2 月」不再多出空格,「[[see:Q1]]([[see:Q1]])
  2 月」不再以空格开头
- 同一片里最多补一个空格:「212,[[see:Q1]],[[see:m:gmv]],5」→「212, 5」(上一轮是两个空格);挨上的是写作者自己留着的
  空白的,不补
- 半角标点后面补空格并进同一条规则:只在下文是数字、字母或渲染出字的标记时补(「a,[[see:Q1]],b」→「a, b」),下文是中文
  的照写作者原样
- 流式渲染把末尾的星号、下划线也攒到后面的字来了再放,和整篇渲染判得一样
- 粗体把括号包在里面时(「**[[v:…]]([[see:Q1]])**2 月」)空格落在收尾的星号前面:前端行内 Markdown 照样认这对星号,
  组装出的文档去掉了包着标记的星号,读者看到的一样;挪到星号外面得先知道这对星号从哪开始,流式渲染的一块里不一定有。
  测试里用严格的 xfail 写明这条已知限制

测试:两个括号连着写(全角半角混用、套着的)、粗体在括号前、后、里面、下划线和单个星号、依据的「]」、一片只补一个的用例;
结构化组合(数、依据、只装依据的括号、空白、星号、标点)逐份查写作者隔开了的两边没有挨上、没多出连着的两个空格,逐字和
每 4 个字分块渲染都等于整篇渲染;审批质询、核查 agent、反方审稿、双盲复算各补一个端到端用例。11 处改动逐一去掉实现,
相关测试都变红。另在测试之外抽查:结构化组合 15.9 万份和基线 58a6759 逐份比数字串,只剩复核说可以不管的「),5」这类半角
标点紧贴数字的写法;随机 8000 份不含依据的文档,内容哈希、渲染结果和基线逐份一致;随机 1.6 万份含依据的文档流式和整篇
一致。
2026-10-12 10:35:13 +08:00

101 lines
6.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""反方审稿的理由:依据写进括号不再断句,占位词不当理由(期 5 真实模型验收 M1)。
真实模型验收里,反方的理由「先按订单状态做全量概览核对口径([[see:Q1]]):」「复算写作者的原查询(同 SQL 同过滤条件,
[[see:Q2]])」渲染成「():」「(同 SQL 同过滤条件,)」:审稿记录(rationale 和 rationale_doc)随正式运行封存,审批卡上
照片段原文显示。
模型一律是 MockChatModel 剧本,区域、金额都是虚构的。
"""
from __future__ import annotations
from app.core import artifact_store
from tests.test_challenge_node import ( # noqa: F401 - 夹具按名字引入
NET,
REFUNDS,
Models,
Play,
engine_up,
finish,
graph,
output_of,
shop,
)
BROKEN = ("()", "(,", ",)", "()", "(,", ",)")
async def _record(monkeypatch, shop, plays) -> tuple[dict, dict]:
Models(monkeypatch, shop, plays=plays)
row = await finish(graph(shop))
assert row.status == "succeeded", row.error
out = await output_of(row.id, "chal")
return artifact_store.load(out["challenge_artifact"]), row.output["_issuance"]
async def test_see_in_brackets_does_not_break_the_rationale(monkeypatch, shop):
plays = {
"华南销售额超过华东": Play([NET], "refuted", "扣除退款后华南为 [[v:Q1.r1.净销售额]]([[see:Q1]]),"
"仍低于华东的 [[v:Q1.r0.净销售额]](同口径,[[see:Q1]])。"),
"居各区域之首": Play([REFUNDS], "weakened", "先核对退款口径([[see:Q1]]):华东含 [[v:Q1.r0.退款笔数]] 笔退款"
"([[see:Q1]],扣除后仍居首)。"),
}
record, issuance = await _record(monkeypatch, shop, plays)
first, second = record["claims"]
assert first["rationale"] == "先核对退款口径:华东含 1 笔退款(扣除后仍居首)。"
assert second["rationale"] == "扣除退款后华南为 900,仍低于华东的 1,200(同口径)。"
for claim in (first, second):
doc = claim["rationale_doc"]
assert doc["markdown"] == claim["rationale"] and doc["violations"] == []
assert not any(mark in claim["rationale"] for mark in BROKEN)
# 括号里的依据仍挂在句子上:判「被推翻」的引用到了自己的查询,照常采信、出具降档
assert second["verdict"] == "refuted" and issuance["tier"] == "degraded"
async def test_see_brackets_written_together_do_not_glue_two_numbers(monkeypatch, shop):
"""两个只装着依据的括号连着写、数字包着粗体(复核第 2 轮):去掉之后两个数不能挨成一个(「9001,200」),句末前悬着的
逗号也去掉。"""
plays = {
"华南销售额超过华东": Play([NET], "refuted", "扣除退款后华南、华东分别为 **[[v:Q1.r1.净销售额]]**([[see:Q1]])"
"([[see:Q1]])[[v:Q1.r0.净销售额]],华南仍低于华东,([[see:Q1]])([[see:Q1]])。"),
"居各区域之首": Play([REFUNDS], "stands", "华东退款 [[v:Q1.r0.退款笔数]] 笔,扣除后仍居首。[[see:Q1]]"),
}
record, _ = await _record(monkeypatch, shop, plays)
refuted = next(claim for claim in record["claims"] if claim["verdict"] == "refuted")
assert refuted["rationale"] == "扣除退款后华南、华东分别为 900 1,200,华南仍低于华东。"
assert refuted["rationale_doc"]["markdown"] == refuted["rationale"]
assert refuted["rationale_doc"]["violations"] == []
async def test_a_placeholder_rationale_is_recorded_as_no_rationale(monkeypatch, shop):
"""反方交回的理由只是占位词:记录里理由为空(审批卡据此写「反方没有给出理由」),判「被推翻」的照常不予采信。"""
plays = {"华南销售额超过华东": Play([NET], "refuted", "placeholder"),
"居各区域之首": Play([REFUNDS], "stands", "placeholder")}
record, issuance = await _record(monkeypatch, shop, plays)
first, second = record["claims"]
assert (first["verdict"], first["rationale"], first["rationale_doc"]) == ("stands", "", None)
assert second["verdict"] == "inconclusive"
assert second["rationale"] == "理由没有引用到任何查询结果,反方给出的「被推翻」不予采信,记为无法判断。反方没有给出理由"
assert "placeholder" not in str(record)
assert issuance["tier"] == "formal"
async def test_a_rationale_that_shows_nothing_is_recorded_as_no_rationale(monkeypatch, shop):
"""反方的理由只写了依据:显示出来是空的。以前按原文判,理由文档照样记下——审批卡按文档画出一片空白,不说「反方没有
给出理由」;判「被推翻」的因为引用到了 Q1 被当成有出处,照常采信、出具降档。"""
plays = {"华南销售额超过华东": Play([NET], "refuted", "[[see:Q1]]"),
"居各区域之首": Play([REFUNDS], "stands", "([[see:Q1]])")}
record, issuance = await _record(monkeypatch, shop, plays)
first, second = record["claims"]
assert (first["verdict"], first["rationale"], first["rationale_doc"]) == ("stands", "", None)
assert second["verdict"] == "inconclusive" and second["rationale_doc"] is None
assert second["rationale"].endswith("反方给出的「被推翻」不予采信,记为无法判断。反方没有给出理由")
assert issuance["tier"] == "formal", "没有理由的「被推翻」不能让出具降档"
async def test_a_placeholder_with_a_see_is_still_a_placeholder(monkeypatch, shop):
plays = {"华南销售额超过华东": Play([NET], "stands", "placeholder。[[see:Q1]]"),
"居各区域之首": Play([REFUNDS], "stands", "无。")}
record, _ = await _record(monkeypatch, shop, plays)
assert [(c["rationale"], c["rationale_doc"]) for c in record["claims"]] == [("", None), ("", None)]
assert "placeholder" not in str(record)