mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-20 19:21:58 +00:00
c97390511e
渲染(evidence._see_cuts): - 括号是两边唯一的分隔、去掉之后数字字母会粘成一串的(「[[v:Q1.r0.销售额]]([[see:Q1]])2 月」渲染成「4,919.52 月」,片段 本身没错、月份不算裸数字,哪项核对都查不出来),括号换成一个空格;半角逗号夹着依据、去掉后一个之后同理(「212,5」→ 「212, 5」)。切口可以带替换串,连成一片的按并集换 - 只装着依据的括号去掉之后,整对括号当一个依据看外面:括号前悬着的停顿标点(「华南居首,([[see:Q1]])。」)、到句末只隔着 的空白、夹在两个分隔标点中间的后一个,和不带括号的写法收拾得一样 - 依据后面到句末标点只隔着空白的,空白一起去掉(「华南居首 [[see:Q1]] 。」不再剩「华南居首 。」) - 换行不再算句末:列表项末尾的「;」、软换行处连着两个分句的「,」是写作者自己的,依据挂在它后面正合规矩,照旧留着(上一轮 把它们删了) - 没有依据的文档渲染、内容哈希不变(随机两万份和基线逐份比过);流式分块输出仍等于整篇渲染 - 依据写在「依据」「见」「取自」「来源」这类引出语后面时引出语悬空,渲染时去不掉(引出语在依据前面,流式渲染得攒住每个可能 是引出语的字):测试里改成严格的 xfail 写明已知限制,不再把验收里的缺陷钉成预期;核查 agent 遇到这种写法记一条日志 核查 agent(verify_agent): - 有没有给理由按渲染出来的正文判:只写了依据([[see:Q1]]、「([[see:Q1]])」)显示出来是空的,占位词后面挂着依据的显示出来 只剩占位词,都算没给。以前按原文判,标记里的字也算,理由文档照样交回:审批卡画出一片空白,判「被推翻」的因为引用到了 Q1 被当成有出处、出具降档;封存之后的质询回答只剩系统说明却记成回答成功 - 「无」「暂无」「none」只在理由是另一个结果的说明时(双盲复算、反方审稿)算没给;结构里只有理由一项(审批质询)时理由就是 回答,「无。」是一个回答,不再报「模型没有给出回答内容」让提问人重问、再花一次钱 - 收尾超时的原话收成常量 SETTLE_TIMEOUT,调用方据此换成自己的说法 反方审稿:交结论那次超过时长上限的句子,理由写「反查未完成:已达上限(每句时长上限 N 秒)」(收紧过的说运行时长上限将至), 和时间线上的 challenge_limit 一致,不再把核查 agent 的内部原话摆上审批卡。 测试:补节点级用例钉住收尾超时之后两个节点对外的说法(challenge_limit 的 limit、claims_skipped、cause,双盲复算的无法复算 原因和 dual_check_limit,收紧过的两种);理由只有依据、占位词挂依据、「无。」作为回答的各层用例。每处改动都做过去掉实现、 测试变红的验证。
211 lines
12 KiB
Python
211 lines
12 KiB
Python
"""核查 agent 交理由的写法和收尾超时记的上限(期 5 真实模型验收 M1)。
|
||
|
||
- 交结果的提示(VERIFY_FINAL_PROMPT,双盲复算、反方审稿、审批质询共用)和写作者一致:依据写在句末,不放进括号或句子
|
||
中间。以前写的是「提到某次查询时写 [[see:Q1]]」,模型就写成「结果一致([[see:Q3]],原报告…)」,反方审稿的系统提示
|
||
自己也把它写在括号里(「([[v:Qn.rX.列名]] 或 [[see:Qn]])」)
|
||
- 写进了括号的依据,理由文档里照样渲染干净(evidence._tidy),这句仍挂着那条依据
|
||
- 模型自己停了工具调用、收尾那次调用超过了时长上限:limit 照实记 timeout_s。以前是 None——质询的报错写着「回答超过
|
||
时长上限(180 秒)」,limit 却是空的,界面上缺了「已达时长上限」。不走结构化输出的模型(普通调用收尾)也一样
|
||
- 交回的理由是空的、只有标点,或者只是一个占位词(字面的 placeholder、N/A):当作没给理由,不把占位词当理由展示。
|
||
按渲染出来的正文判:只写了依据的([[see:Q1]]、「([[see:Q1]])」)显示出来是空的,占位词后面挂着依据的显示出来只剩
|
||
占位词,都算没给。理由就是结果本身时(审批质询,结构里只有理由一项),「无」「none」是回答,不算占位
|
||
- 依据写在「依据」「见」「取自」这类引出语后面(渲染时去不掉,引出语悬空):记一条日志,验收时查得到
|
||
|
||
模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,数字都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
import re
|
||
import time
|
||
|
||
import pytest
|
||
|
||
from app.engine import verify_agent
|
||
from app.engine.evidence import iter_units
|
||
from app.engine.inquiry import INQUIRY_SCHEMA, INQUIRY_SYSTEM, INQUIRY_TASK_PROMPT
|
||
from app.engine.nodes.challenge import CHALLENGE_SYSTEM, CHALLENGE_TASK_PROMPT
|
||
from app.engine.nodes.dual_check import BLIND_SYSTEM, BLIND_TASK_PROMPT
|
||
from app.engine.verify_agent import VERIFY_FINAL_PROMPT, VERIFY_SYSTEM
|
||
from app.providers.mock_model import MockChatModel
|
||
from tests.test_verify_agent import VALID_SUM, Script, clean_spend, q, run, shop # noqa: F401 - 夹具按名字引入
|
||
|
||
#: 括号里写着依据标记:「([[v:Qn.rX.列名]] 或 [[see:Qn]])」「([[see:Q1]])」这种示范
|
||
_BRACKETED_SEE = re.compile(r"[((][^()()]*\[\[see:[^\]]*\]\][^()()]*[))]")
|
||
FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致"}
|
||
|
||
|
||
def slow_final(monkeypatch, seconds: float = 5.0) -> None:
|
||
"""交结果那次调用(消息里带着 VERIFY_FINAL_PROMPT:结构化输出、退回普通调用都是)迟迟不返回,查库的几轮照常。"""
|
||
original = MockChatModel._agenerate
|
||
|
||
async def held(chat, messages, stop=None, run_manager=None, **kwargs):
|
||
if any(VERIFY_FINAL_PROMPT in str(m.content) for m in messages):
|
||
await asyncio.sleep(seconds)
|
||
return await original(chat, messages, stop, run_manager, **kwargs)
|
||
|
||
monkeypatch.setattr(MockChatModel, "_agenerate", held)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 提示词
|
||
# --------------------------------------------------------------------------
|
||
|
||
def test_the_final_prompt_puts_see_at_the_end_of_the_sentence():
|
||
assert "提到某次查询时写 [[see:Q1]]" not in VERIFY_FINAL_PROMPT
|
||
assert "句末" in VERIFY_FINAL_PROMPT and "括号" in VERIFY_FINAL_PROMPT
|
||
assert re.search(r"。\[\[see:Q\d\]\]", VERIFY_FINAL_PROMPT), "示范的写法:依据挂在句号后面"
|
||
assert "渲染后不显示" in VERIFY_FINAL_PROMPT
|
||
|
||
|
||
@pytest.mark.parametrize("prompt", [VERIFY_SYSTEM, VERIFY_FINAL_PROMPT, CHALLENGE_SYSTEM, CHALLENGE_TASK_PROMPT,
|
||
INQUIRY_SYSTEM, INQUIRY_TASK_PROMPT, BLIND_SYSTEM, BLIND_TASK_PROMPT])
|
||
def test_no_verify_prompt_shows_see_inside_brackets(prompt):
|
||
assert not _BRACKETED_SEE.search(prompt), _BRACKETED_SEE.search(prompt).group(0)
|
||
|
||
|
||
async def test_the_final_call_is_told_the_rule(monkeypatch, shop):
|
||
name, _ = shop
|
||
script = Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": "合计 [[v:Q1.r0.total]]。[[see:Q1]]"})
|
||
await run(name)
|
||
kind, messages = script.calls[-1]
|
||
assert kind == "final" and VERIFY_FINAL_PROMPT in str(messages[-1].content)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 写进括号的依据
|
||
# --------------------------------------------------------------------------
|
||
|
||
async def test_see_written_in_brackets_renders_clean_in_the_rationale(monkeypatch, shop):
|
||
name, _ = shop
|
||
said = "未作废订单的金额合计是 [[v:Q1.r0.total]]([[see:Q1]]),口径一致([[see:Q1]],已排除作废单)。"
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
|
||
out = await run(name)
|
||
shown = "未作废订单的金额合计是 230,口径一致(已排除作废单)。"
|
||
assert out.result["rationale"] == shown and out.rationale["markdown"] == shown
|
||
assert out.rationale["source"] == said, "原文照存,依据标记还在"
|
||
[unit] = [unit for _, unit in iter_units(out.rationale)]
|
||
assert [s["alias"] for s in unit["see"]] == ["Q1", "Q1"] and unit["cites"] == ["Q1"]
|
||
assert out.rationale["violations"] == []
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 收尾超时
|
||
# --------------------------------------------------------------------------
|
||
|
||
@pytest.mark.parametrize("structured", [True, False], ids=["structured", "plain"])
|
||
async def test_a_final_call_that_runs_out_of_time_records_the_time_limit(monkeypatch, shop, structured):
|
||
"""模型查了一次就自己停下(没撞上任何上限),交结果那次调用却一直不返回:到点放弃,limit 是 timeout_s,和原因一致。"""
|
||
name, _ = shop
|
||
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
|
||
script = Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": "合计 [[v:Q1.r0.total]]。"})
|
||
slow_final(monkeypatch)
|
||
if not structured:
|
||
def refuse(self, schema, **kwargs):
|
||
raise NotImplementedError("不支持结构化输出")
|
||
|
||
monkeypatch.setattr(MockChatModel, "with_structured_output", refuse)
|
||
started = time.monotonic()
|
||
out = await run(name, timeout_s=0.5)
|
||
assert time.monotonic() - started < 3.0, "收尾那次只等到宽限结束"
|
||
assert script.kinds == ["step", "step"], "查了一次、自己停下,交结果那次没等到回复"
|
||
assert out.answered is True and [qd["alias"] for qd in out.queries] == ["Q1"]
|
||
assert out.result is None and out.error == "收尾时超过时长上限,没能交结果"
|
||
assert out.limit == "timeout_s"
|
||
|
||
|
||
async def test_a_final_call_that_runs_out_of_time_after_the_query_limit_records_the_time_limit(monkeypatch, shop):
|
||
"""循环先撞上查询次数上限、收尾又超时:没交成结果是因为超时,limit 记 timeout_s,和原因一致——记成查询次数上限的话,
|
||
质询会显示「未能回答(已达查询次数上限):回答超过时长上限…」,双盲复算会说盲算查满了次数(查满了照样能交结果)。
|
||
金额被拒的收尾照旧报循环里先撞上的那个(见 test_verify_truncation)。"""
|
||
name, _ = shop
|
||
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1"), q(name, "SELECT COUNT(*) AS n FROM orders", "c2")],
|
||
{**FOUND, "rationale": "x"})
|
||
slow_final(monkeypatch)
|
||
out = await run(name, timeout_s=0.5, max_queries=1)
|
||
assert [qd["alias"] for qd in out.queries] == ["Q1"], "查满了一次"
|
||
assert out.result is None and out.error == "收尾时超过时长上限,没能交结果"
|
||
assert out.limit == "timeout_s"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 占位词当理由
|
||
# --------------------------------------------------------------------------
|
||
|
||
@pytest.mark.parametrize("said", ["placeholder", "PLACEHOLDER", " ", "N/A", "(略)", "…", "<rationale>", "TODO", "无",
|
||
"暂无。", "None"])
|
||
async def test_a_placeholder_rationale_counts_as_none(monkeypatch, shop, said):
|
||
name, _ = shop
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
|
||
out = await run(name)
|
||
assert out.result["value"] == 230, "数照样取"
|
||
assert out.result["rationale"] == "" and out.rationale is None
|
||
|
||
|
||
@pytest.mark.parametrize("said", ["[[see:Q1]]", "([[see:Q1]])", " [[see:Q1]]。", "placeholder。[[see:Q1]]",
|
||
"N/A([[see:Q1]])", "TODO [[see:Q1]]"])
|
||
async def test_a_rationale_that_shows_nothing_counts_as_none(monkeypatch, shop, said):
|
||
"""只写了依据、或者占位词后面挂着依据:原文里看着有字,显示出来没有理由。以前按原文判,标记里的字(see、Q1)也算,
|
||
理由文档照样交回——审批卡按文档画出一片空白,判「被推翻」的还因为引用到了 Q1 被当成有出处。"""
|
||
name, _ = shop
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
|
||
out = await run(name)
|
||
assert out.result["value"] == 230, "数照样取"
|
||
assert out.result["rationale"] == "" and out.rationale is None
|
||
|
||
|
||
@pytest.mark.parametrize("said", ["无异常。", "见 [[v:Q1.r0.total]]。", "略高于上周。", "[[v:Q1.r0.total]]",
|
||
"一致。[[see:Q1]]"])
|
||
async def test_a_short_real_rationale_is_kept(monkeypatch, shop, said):
|
||
name, _ = shop
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
|
||
out = await run(name)
|
||
assert out.result["rationale"] and out.rationale is not None
|
||
|
||
|
||
@pytest.mark.parametrize(("said", "shown"), [("无。", "无。"), ("None.", "None."), ("暂无。[[see:Q1]]", "暂无。")])
|
||
async def test_nothing_is_an_answer_when_the_rationale_is_the_answer(monkeypatch, shop, said, shown):
|
||
"""结构里只有理由一项(审批质询):理由就是回答。问「有没有测试账号」,答「无。」是一个回答,不能当成没给、让提问人
|
||
重新提问再花一次钱。"""
|
||
name, _ = shop
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {"rationale": said})
|
||
out = await run(name, output_schema=INQUIRY_SCHEMA)
|
||
assert out.result["rationale"] == shown and out.rationale["markdown"] == shown
|
||
|
||
|
||
@pytest.mark.parametrize("said", ["placeholder", "null", "[[see:Q1]]", "N/A。[[see:Q1]]"])
|
||
async def test_a_placeholder_answer_still_counts_as_none(monkeypatch, shop, said):
|
||
name, _ = shop
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {"rationale": said})
|
||
out = await run(name, output_schema=INQUIRY_SCHEMA)
|
||
assert out.result["rationale"] == "" and out.rationale is None
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 依据写在引出语后面
|
||
# --------------------------------------------------------------------------
|
||
|
||
async def test_see_after_a_lead_in_word_is_logged(monkeypatch, shop, caplog):
|
||
"""「依据 [[see:Q1]]。」「(来源:[[see:Q1]])」渲染后引出语悬空,渲染时去不掉(见 test_evidence_see_brackets 的已知
|
||
限制):记一条日志,写明是哪几处,验收时查得到。只记标记和引出语,不记数。"""
|
||
name, _ = shop
|
||
said = "合计 [[v:Q1.r0.total]],依据 [[see:Q1]]。口径一致(来源:[[see:Q1]])。"
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
|
||
with caplog.at_level(logging.WARNING, logger="app.engine.verify_agent"):
|
||
out = await run(name, node_id="chal", run_id="run-1")
|
||
assert out.rationale is not None, "照常交回"
|
||
[record] = [r for r in caplog.records if "引出语" in r.getMessage()]
|
||
message = record.getMessage()
|
||
assert "依据 [[see:Q1]]" in message and "来源:[[see:Q1]]" in message and "run-1" in message
|
||
assert "230" not in message
|
||
|
||
|
||
async def test_see_at_the_end_of_the_sentence_is_not_logged(monkeypatch, shop, caplog):
|
||
name, _ = shop
|
||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": "合计 [[v:Q1.r0.total]],结果一致。[[see:Q1]]"})
|
||
with caplog.at_level(logging.WARNING, logger="app.engine.verify_agent"):
|
||
await run(name)
|
||
assert not [r for r in caplog.records if "引出语" in r.getMessage()]
|