mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 13:32:00 +00:00
3d85a43301
渲染(evidence._tidy):
- 上一轮判粘连只看原文里紧挨着括号的两个字(往后只越过 [[see:]])。两个只装着依据的括号连着写(「[[v:Q1.r0.销售额]]
([[see:Q1]])([[see:m:gmv]])2 月」)、数字包着粗体(「**[[v:…]]**([[see:Q1]])2 月」,组装文档时星号又被去掉)
时,去掉之后真正挨上的是两边的数,读者看到「4,919.52 月」「212212 笔」,compose_doc、verify_doc 都不报。现在先把依据
标记和只装着依据的括号(套着的、几个连着写的都算)认成渲染出来什么都没有的一段(_voids),连着写的几个并成一组,外面的
停顿标点、空白照原规矩收拾;所有切口去掉之后,再往两边越过去掉的字、依据、粗体斜体的星号和下划线,找真正挨上的两个看得
见的字,会读成一串的补回一个空格(_glue)
- 依据标记的「]」越过去、不当数字:「华南居首。[[see:Q1]]([[see:Q2]])2 月」不再多出空格,「[[see:Q1]]([[see:Q1]])
2 月」不再以空格开头
- 同一片里最多补一个空格:「212,[[see:Q1]],[[see:m:gmv]],5」→「212, 5」(上一轮是两个空格);挨上的是写作者自己留着的
空白的,不补
- 半角标点后面补空格并进同一条规则:只在下文是数字、字母或渲染出字的标记时补(「a,[[see:Q1]],b」→「a, b」),下文是中文
的照写作者原样
- 流式渲染把末尾的星号、下划线也攒到后面的字来了再放,和整篇渲染判得一样
- 粗体把括号包在里面时(「**[[v:…]]([[see:Q1]])**2 月」)空格落在收尾的星号前面:前端行内 Markdown 照样认这对星号,
组装出的文档去掉了包着标记的星号,读者看到的一样;挪到星号外面得先知道这对星号从哪开始,流式渲染的一块里不一定有。
测试里用严格的 xfail 写明这条已知限制
测试:两个括号连着写(全角半角混用、套着的)、粗体在括号前、后、里面、下划线和单个星号、依据的「]」、一片只补一个的用例;
结构化组合(数、依据、只装依据的括号、空白、星号、标点)逐份查写作者隔开了的两边没有挨上、没多出连着的两个空格,逐字和
每 4 个字分块渲染都等于整篇渲染;审批质询、核查 agent、反方审稿、双盲复算各补一个端到端用例。11 处改动逐一去掉实现,
相关测试都变红。另在测试之外抽查:结构化组合 15.9 万份和基线 58a6759 逐份比数字串,只剩复核说可以不管的「),5」这类半角
标点紧贴数字的写法;随机 8000 份不含依据的文档,内容哈希、渲染结果和基线逐份一致;随机 1.6 万份含依据的文档流式和整篇
一致。
115 lines
6.0 KiB
Python
115 lines
6.0 KiB
Python
"""审批质询的回答:依据写进括号不再断句,收尾超时记时长上限,占位词不当回答(期 5 真实模型验收 M1)。
|
||
|
||
真实模型验收里,质询 agent 写的「结果一致([[see:Q3]],原报告 SQL 未按 status 过滤)」「合计 [[v:Q1.r1.金额合计]]
|
||
([[see:Q1]])」渲染成「(,原报告…)」「()」,随正式运行封存;另一次回答查完了,收尾那次调用超过时长上限,报错写着
|
||
「回答超过时长上限(180 秒)」,answer.limit 却是 null,界面上缺了「已达时长上限」。
|
||
|
||
回答只写了依据(「([[see:Q1]])」)时显示出来是空的:以前照样交回,封存之后的追问末尾补上「注:以上回答依据的是数据源
|
||
当前的数据…」,回答正文只剩这句系统说明、却记成回答成功。现在按没给回答记。回答就是一个「无。」的,照常是回答。
|
||
|
||
模型一律是 MockChatModel 剧本,区域、金额都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from app.core import artifact_store
|
||
from app.engine import verify_agent
|
||
from app.engine.inquiry import NO_ANSWER
|
||
from tests.test_inquiry_api import ( # noqa: F401 - 夹具按名字引入
|
||
EAST,
|
||
NO_TEST,
|
||
QUESTION,
|
||
Models,
|
||
Play,
|
||
ask,
|
||
client,
|
||
engine_up,
|
||
events,
|
||
finish,
|
||
pause,
|
||
pending,
|
||
report_doc,
|
||
set_verify,
|
||
shop,
|
||
unit_of,
|
||
)
|
||
from tests.test_verify_rationale_rules import slow_final
|
||
|
||
#: 依据写进了括号:开头跟着逗号的、只有依据的
|
||
BRACKETED = ("仍居各区域之首([[see:Q1]],已排除测试账号):华东为 [[v:Q1.r0.销售额]]([[see:Q1]]),"
|
||
"华南为 [[v:Q1.r1.销售额]](同口径,[[see:Q1]])。")
|
||
BRACKETED_TEXT = "仍居各区域之首(已排除测试账号):华东为 1,200,华南为 900(同口径)。"
|
||
#: 两个只装着依据的括号连着写、数字包着粗体(复核第 2 轮):去掉括号之后两个数不能挨成一个(「1,200900」),句末前悬着的
|
||
#: 逗号也去掉
|
||
TOGETHER = ("华东、华南分别为 **[[v:Q1.r0.销售额]]**([[see:Q1]])([[see:Q1]])[[v:Q1.r1.销售额]],"
|
||
"华东仍居各区域之首,([[see:Q1]])([[see:Q1]])。")
|
||
TOGETHER_TEXT = "华东、华南分别为 1,200 900,华东仍居各区域之首。"
|
||
|
||
|
||
async def _asked(client, shop) -> tuple[dict, list]:
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
return r.json()["inquiry"], await events(paused.id, "approval.inquiry")
|
||
|
||
|
||
@pytest.mark.parametrize(("said", "shown"), [(BRACKETED, BRACKETED_TEXT), (TOGETHER, TOGETHER_TEXT)],
|
||
ids=["bracketed", "together"])
|
||
async def test_see_in_brackets_does_not_break_the_answer(monkeypatch, shop, client, said, shown):
|
||
Models(monkeypatch, shop, plays={"排除测试账号": Play([NO_TEST], said)})
|
||
item, [event] = await _asked(client, shop)
|
||
answer = item["answer"]
|
||
assert answer["ok"] is True and answer["text"] == shown
|
||
doc = artifact_store.load(answer["doc_artifact"])
|
||
assert doc["markdown"] == shown and doc["source"] == said
|
||
assert doc["violations"] == []
|
||
assert event.data["answer_text"] == shown
|
||
|
||
|
||
async def test_a_final_answer_that_runs_out_of_time_reports_the_time_limit(monkeypatch, shop, client):
|
||
"""质询 agent 查完了、自己停下,交回答那次调用超过时长上限:报错说时长上限,limit 也是 timeout_s(事件里同样)。"""
|
||
await set_verify({"inquiry": {"timeout_s": 0.5}})
|
||
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
|
||
models = Models(monkeypatch, shop)
|
||
slow_final(monkeypatch)
|
||
item, [event] = await _asked(client, shop)
|
||
answer = item["answer"]
|
||
assert answer["ok"] is False and answer["text"] == "" and answer["doc_artifact"] is None
|
||
assert answer["error"] == "回答超过时长上限(0.5 秒),未能给出回答,可缩小问题范围后重新提问"
|
||
assert answer["limit"] == "timeout_s"
|
||
assert [q["sql"] for q in item["queries"]] == [NO_TEST], "查到的照记"
|
||
assert (event.data["limit"], event.data["error"]) == ("timeout_s", answer["error"])
|
||
assert models.finals == [], "交回答那次没等到回复"
|
||
|
||
|
||
async def test_a_placeholder_answer_is_no_answer(monkeypatch, shop, client):
|
||
Models(monkeypatch, shop, plays={"排除测试账号": Play([NO_TEST], "placeholder")})
|
||
item, _ = await _asked(client, shop)
|
||
answer = item["answer"]
|
||
assert (answer["ok"], answer["text"], answer["limit"]) == (False, "", None)
|
||
assert answer["error"] == NO_ANSWER
|
||
|
||
|
||
async def test_an_answer_that_shows_nothing_is_no_answer_even_after_the_seal(monkeypatch, shop, client):
|
||
"""封存之后的追问:回答只写了依据,显示出来是空的。以前回答文档照样组装,末尾补上「查的是现在的数据」那句说明,
|
||
回答正文只剩系统说明、却记成回答成功。"""
|
||
Models(monkeypatch, shop, plays={"排除测试账号": Play([NO_TEST], "([[see:Q1]])")})
|
||
done = await finish(shop)
|
||
assert done.status == "succeeded", done.error
|
||
r = await ask(client, done.id, report="write", unit=unit_of(await report_doc(done.id), EAST), question=QUESTION)
|
||
assert r.status_code == 200, r.text
|
||
answer = r.json()["inquiry"]["answer"]
|
||
assert (answer["ok"], answer["text"], answer["doc_artifact"]) == (False, "", None)
|
||
assert answer["error"] == NO_ANSWER
|
||
|
||
|
||
async def test_a_one_word_answer_is_an_answer(monkeypatch, shop, client):
|
||
"""问的是「有没有」,答「无。」:这是回答,不是占位词,不能报「模型没有给出回答内容」让提问人重问、再花一次钱。"""
|
||
Models(monkeypatch, shop, plays={"排除测试账号": Play([NO_TEST], "无。[[see:Q1]]")})
|
||
item, [event] = await _asked(client, shop)
|
||
answer = item["answer"]
|
||
assert (answer["ok"], answer["text"], answer["error"]) == (True, "无。", None)
|
||
assert event.data["answer_text"] == "无。"
|