mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 01:01:59 +00:00
核查 agent 的依据照写作者的规矩挂在句末;收尾超时照实记时长上限;占位词不当理由
- 交结果的提示(VERIFY_FINAL_PROMPT,双盲复算、反方审稿、审批质询共用)以前写「提到某次查询时写 [[see:Q1]]」,模型就 把它写进括号、句中;反方审稿的系统提示自己也把它写在括号里(「([[v:Qn.rX.列名]] 或 [[see:Qn]])」)。改成和写作者 一致:依据挂在句末(「……结果一致。[[see:Q3]]」),说明它渲染后不显示,不要放进括号、句中或「依据」「见」「取自」 后面。已经写进括号的由上一个提交的渲染收拾干净 - 模型自己停了查询、收尾那次调用超过时长上限时,limit 以前是 None:质询的报错写着「回答超过时长上限(180 秒)」, answer.limit 却是 null,界面少了「已达时长上限」;双盲复算、反方审稿把它当成别的原因,审批卡上照出内部原话。 现在记 timeout_s;循环里先撞上的上限照旧优先(同金额被拒的收尾),结构化输出、退回普通调用两条路都一样 - 交回的理由是空的、只有标点,或者只是一个占位词(验收里盲算交了字面的「placeholder」,审批卡照着显示):当没给 理由,不组装理由文档。反方审稿的记录里理由为空,审批卡照旧写「反方没有给出理由」;质询照「模型没有给出回答内容」 报错;双盲复算的对照工件里说明为空 测试:交给模型的提示(含三处调用方的系统提示、任务)里不再示范括号里的依据;写进括号的依据在理由文档里渲染干净、 仍挂在句子上;审批质询的回答、反方审稿的理由、双盲复算的说明端到端都没有空括号;收尾超时 limit 为 timeout_s(核查 agent 和质询接口、事件里都是);占位词的理由三处都不当理由。
This commit is contained in:
@@ -117,7 +117,7 @@ CHALLENGE_SYSTEM = (
|
||||
"原因说过了头;refuted 查到的数据和这句结论相矛盾;inconclusive 查不到判断所需的数据\n"
|
||||
"- 没有找到反例不等于结论被推翻:只有数据和结论冲突时才判 refuted\n"
|
||||
"- 结论要有出处:一条查询都没执行成功时,判什么都不会被采信;判 refuted 时,理由里至少要引用一处你自己查询的"
|
||||
"结果([[v:Qn.rX.列名]] 或 [[see:Qn]]),否则同样不会被采信"
|
||||
"结果:写 [[v:Qn.rX.列名]] 引用一格,或者在那句的句末写 [[see:Qn]],否则同样不会被采信"
|
||||
)
|
||||
CHALLENGE_TASK_PROMPT = (
|
||||
"要审的结论(报告「{report}」中的一句{section}):\n「{claim}」\n\n"
|
||||
|
||||
@@ -124,7 +124,10 @@ VERIFY_FINAL_PROMPT = (
|
||||
"row 是行号(从 0 数),column 是列名。系统按引用从查询结果里取值,你自己写的数字不会被采用;查不到就写 null,"
|
||||
"不要估算\n"
|
||||
"- rationale 写理由,简短说清依据。理由里出现的每个数字都写成 [[v:Q1.r0.列名]] 这样的引用,系统会换成查询结果"
|
||||
"里的值;不要直接写数字,提到某次查询时写 [[see:Q1]]\n"
|
||||
"里的值;不要直接写数字\n"
|
||||
"- 要指明一句话依据的是哪次查询,在这句的句末写 [[see:Q1]],比如「……结果一致。[[see:Q3]]」。[[see:…]] 渲染后"
|
||||
"不显示,只能挂在句末:不要放进括号,也不要写进句子中间、写在「依据」「见」「取自」这类字后面,否则会留下「()」"
|
||||
"「(,」「依据。」这样的断句\n"
|
||||
"- 其余字段按各自的说明填写"
|
||||
)
|
||||
_CELL_REF: dict[str, Any] = {
|
||||
@@ -662,6 +665,20 @@ def _parsed(value: Any) -> dict[str, Any] | None:
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
#: 只是占位词的理由(只留字母、数字和汉字,不分大小写):模型照着结构填了个字面的 placeholder、N/A,等于没给理由
|
||||
_PLACEHOLDERS = frozenset({"placeholder", "todo", "tbd", "na", "none", "null", "nil", "rationale", "reason",
|
||||
"无", "暂无", "略", "理由"})
|
||||
|
||||
|
||||
def _no_rationale(value: Any) -> bool:
|
||||
"""交回的理由等于没给:不是文字、空的、只有标点,或者只是一个占位词。当没给理由处理(理由为空、没有理由文档),
|
||||
审批卡照「没有给出理由」说,不把占位词当理由展示。"""
|
||||
if not isinstance(value, str):
|
||||
return True
|
||||
word = "".join(ch for ch in value if ch.isalnum()).casefold()
|
||||
return not word or word in _PLACEHOLDERS
|
||||
|
||||
|
||||
@dataclass
|
||||
class _State:
|
||||
"""一次核查过程中攒下的东西:出意外时也照这些交回,用了多少、查了什么不丢。"""
|
||||
@@ -1007,10 +1024,13 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
if parsed is None:
|
||||
if settle_cut:
|
||||
return await report_cut("verify_result", settler.max_tokens)
|
||||
why = "收尾时超过时长上限,没能交结果" if isinstance(failure, asyncio.TimeoutError) else \
|
||||
timed_out = isinstance(failure, asyncio.TimeoutError)
|
||||
why = "收尾时超过时长上限,没能交结果" if timed_out else \
|
||||
f"核查模型没有按结构交结果({explain_model_error(failure, model_id)})" if failure is not None else \
|
||||
"核查模型没有按结构交结果"
|
||||
return outcome(limit=stopped, answered=answered, error=why)
|
||||
# 收尾超时也是触了时长上限:模型自己停了查询(stopped 为 None)时照实记 timeout_s,和原因对得上(质询的报错说
|
||||
# 时长上限,界面按 limit 标「已达时长上限」)。循环里先撞上的上限照旧优先,同上面金额被拒的收尾
|
||||
return outcome(limit=stopped or ("timeout_s" if timed_out else None), answered=answered, error=why)
|
||||
declared = list((resolved.get("properties") or {}) if _is_object(resolved) else {})
|
||||
if declared and not any(k in parsed for k in declared):
|
||||
# 一个要求的字段都没交:不是「查不到」,是没按结构交。不这么分,字段名对不上的回复会被当成什么都没查到
|
||||
@@ -1023,7 +1043,7 @@ async def _verify(state: _State, *, spec: ModelSpec, system: str, task: str, sou
|
||||
result = cells.walk(resolved, parsed) if _is_object(resolved) else {}
|
||||
rationale = parsed.get(RATIONALE)
|
||||
doc = None
|
||||
if isinstance(rationale, str) and rationale.strip():
|
||||
if not _no_rationale(rationale):
|
||||
doc = compose_doc(rationale, build_catalog(nodes={}, ledger=ledger), node_id=owner, run_id=run_id or None)
|
||||
result[RATIONALE] = doc["markdown"] if doc else ""
|
||||
return outcome(result=result, cells=cells.cells, rationale=doc, limit=stopped, answered=answered)
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""反方审稿的理由:依据写进括号不再断句,占位词不当理由(期 5 真实模型验收 M1)。
|
||||
|
||||
真实模型验收里,反方的理由「先按订单状态做全量概览核对口径([[see:Q1]]):」「复算写作者的原查询(同 SQL 同过滤条件,
|
||||
[[see:Q2]])」渲染成「():」「(同 SQL 同过滤条件,)」:审稿记录(rationale 和 rationale_doc)随正式运行封存,审批卡上
|
||||
照片段原文显示。
|
||||
|
||||
模型一律是 MockChatModel 剧本,区域、金额都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core import artifact_store
|
||||
from tests.test_challenge_node import ( # noqa: F401 - 夹具按名字引入
|
||||
NET,
|
||||
REFUNDS,
|
||||
Models,
|
||||
Play,
|
||||
engine_up,
|
||||
finish,
|
||||
graph,
|
||||
output_of,
|
||||
shop,
|
||||
)
|
||||
|
||||
BROKEN = ("()", "(,", ",)", "()", "(,", ",)")
|
||||
|
||||
|
||||
async def _record(monkeypatch, shop, plays) -> tuple[dict, dict]:
|
||||
Models(monkeypatch, shop, plays=plays)
|
||||
row = await finish(graph(shop))
|
||||
assert row.status == "succeeded", row.error
|
||||
out = await output_of(row.id, "chal")
|
||||
return artifact_store.load(out["challenge_artifact"]), row.output["_issuance"]
|
||||
|
||||
|
||||
async def test_see_in_brackets_does_not_break_the_rationale(monkeypatch, shop):
|
||||
plays = {
|
||||
"华南销售额超过华东": Play([NET], "refuted", "扣除退款后华南为 [[v:Q1.r1.净销售额]]([[see:Q1]]),"
|
||||
"仍低于华东的 [[v:Q1.r0.净销售额]](同口径,[[see:Q1]])。"),
|
||||
"居各区域之首": Play([REFUNDS], "weakened", "先核对退款口径([[see:Q1]]):华东含 [[v:Q1.r0.退款笔数]] 笔退款"
|
||||
"([[see:Q1]],扣除后仍居首)。"),
|
||||
}
|
||||
record, issuance = await _record(monkeypatch, shop, plays)
|
||||
first, second = record["claims"]
|
||||
assert first["rationale"] == "先核对退款口径:华东含 1 笔退款(扣除后仍居首)。"
|
||||
assert second["rationale"] == "扣除退款后华南为 900,仍低于华东的 1,200(同口径)。"
|
||||
for claim in (first, second):
|
||||
doc = claim["rationale_doc"]
|
||||
assert doc["markdown"] == claim["rationale"] and doc["violations"] == []
|
||||
assert not any(mark in claim["rationale"] for mark in BROKEN)
|
||||
# 括号里的依据仍挂在句子上:判「被推翻」的引用到了自己的查询,照常采信、出具降档
|
||||
assert second["verdict"] == "refuted" and issuance["tier"] == "degraded"
|
||||
|
||||
|
||||
async def test_a_placeholder_rationale_is_recorded_as_no_rationale(monkeypatch, shop):
|
||||
"""反方交回的理由只是占位词:记录里理由为空(审批卡据此写「反方没有给出理由」),判「被推翻」的照常不予采信。"""
|
||||
plays = {"华南销售额超过华东": Play([NET], "refuted", "placeholder"),
|
||||
"居各区域之首": Play([REFUNDS], "stands", "placeholder")}
|
||||
record, issuance = await _record(monkeypatch, shop, plays)
|
||||
first, second = record["claims"]
|
||||
assert (first["verdict"], first["rationale"], first["rationale_doc"]) == ("stands", "", None)
|
||||
assert second["verdict"] == "inconclusive"
|
||||
assert second["rationale"] == "理由没有引用到任何查询结果,反方给出的「被推翻」不予采信,记为无法判断。反方没有给出理由"
|
||||
assert "placeholder" not in str(record)
|
||||
assert issuance["tier"] == "formal"
|
||||
@@ -0,0 +1,42 @@
|
||||
"""双盲复算的说明:依据写进括号不再断句,占位词不当说明(期 5 真实模型验收 M1)。
|
||||
|
||||
盲算 agent 和反方审稿、审批质询共用核查 agent 交结果的提示,同样会把 [[see:Q1]] 写进括号;真实模型验收里还见过盲算交回
|
||||
字面的「placeholder」当说明,审批卡上照着显示。
|
||||
|
||||
模型一律是 MockChatModel 剧本,订单库是虚构的(tests/fixtures/dual_check_shop.py)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core import artifact_store
|
||||
from tests.fixtures.dual_check_shop import BY_ORDER_DATE, RIGHT, Blind
|
||||
from tests.test_dual_check_node import ( # noqa: F401 - 夹具按名字引入
|
||||
clean_settings,
|
||||
engine_up,
|
||||
graph,
|
||||
pending,
|
||||
shop,
|
||||
start,
|
||||
)
|
||||
|
||||
|
||||
async def _blind_input(monkeypatch, shop, rationale: str) -> dict:
|
||||
name, _ = shop
|
||||
Blind(monkeypatch, name, {"销售额": (RIGHT, "gmv")}, rationale=rationale)
|
||||
run = await start(graph(name, BY_ORDER_DATE))
|
||||
assert run.status == "interrupted", run.error
|
||||
card = artifact_store.load((await pending(run.id)).payload["card_artifact"])
|
||||
[mine] = card["metrics"][0]["blind"]["inputs"]
|
||||
return mine
|
||||
|
||||
|
||||
async def test_see_in_brackets_does_not_break_the_blind_rationale(monkeypatch, shop):
|
||||
mine = await _blind_input(monkeypatch, shop, "按口径说明统计([[see:Q1]],已排除退款),结果是 {ref}([[see:Q1]])。")
|
||||
assert mine["rationale"] == "按口径说明统计(已排除退款),结果是 330。"
|
||||
doc = mine["rationale_doc"]
|
||||
assert doc["markdown"] == mine["rationale"] and doc["violations"] == []
|
||||
|
||||
|
||||
async def test_a_placeholder_rationale_is_not_shown_as_the_explanation(monkeypatch, shop):
|
||||
mine = await _blind_input(monkeypatch, shop, "placeholder")
|
||||
assert mine["value"] == 330.0, "数照样采用"
|
||||
assert (mine["rationale"], mine["rationale_doc"]) == ("", None)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""审批质询的回答:依据写进括号不再断句,收尾超时记时长上限,占位词不当回答(期 5 真实模型验收 M1)。
|
||||
|
||||
真实模型验收里,质询 agent 写的「结果一致([[see:Q3]],原报告 SQL 未按 status 过滤)」「合计 [[v:Q1.r1.金额合计]]
|
||||
([[see:Q1]])」渲染成「(,原报告…)」「()」,随正式运行封存;另一次回答查完了,收尾那次调用超过时长上限,报错写着
|
||||
「回答超过时长上限(180 秒)」,answer.limit 却是 null,界面上缺了「已达时长上限」。
|
||||
|
||||
模型一律是 MockChatModel 剧本,区域、金额都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.engine import verify_agent
|
||||
from app.engine.inquiry import NO_ANSWER
|
||||
from tests.test_inquiry_api import ( # noqa: F401 - 夹具按名字引入
|
||||
NO_TEST,
|
||||
QUESTION,
|
||||
Models,
|
||||
Play,
|
||||
ask,
|
||||
client,
|
||||
engine_up,
|
||||
events,
|
||||
pause,
|
||||
pending,
|
||||
set_verify,
|
||||
shop,
|
||||
)
|
||||
from tests.test_verify_rationale_rules import slow_final
|
||||
|
||||
#: 依据写进了括号:开头跟着逗号的、只有依据的
|
||||
BRACKETED = ("仍居各区域之首([[see:Q1]],已排除测试账号):华东为 [[v:Q1.r0.销售额]]([[see:Q1]]),"
|
||||
"华南为 [[v:Q1.r1.销售额]](同口径,[[see:Q1]])。")
|
||||
BRACKETED_TEXT = "仍居各区域之首(已排除测试账号):华东为 1,200,华南为 900(同口径)。"
|
||||
|
||||
|
||||
async def _asked(client, shop) -> tuple[dict, list]:
|
||||
paused = await pause(shop)
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()["inquiry"], await events(paused.id, "approval.inquiry")
|
||||
|
||||
|
||||
async def test_see_in_brackets_does_not_break_the_answer(monkeypatch, shop, client):
|
||||
Models(monkeypatch, shop, plays={"排除测试账号": Play([NO_TEST], BRACKETED)})
|
||||
item, [event] = await _asked(client, shop)
|
||||
answer = item["answer"]
|
||||
assert answer["ok"] is True and answer["text"] == BRACKETED_TEXT
|
||||
doc = artifact_store.load(answer["doc_artifact"])
|
||||
assert doc["markdown"] == BRACKETED_TEXT and doc["source"] == BRACKETED
|
||||
assert doc["violations"] == []
|
||||
assert event.data["answer_text"] == BRACKETED_TEXT
|
||||
|
||||
|
||||
async def test_a_final_answer_that_runs_out_of_time_reports_the_time_limit(monkeypatch, shop, client):
|
||||
"""质询 agent 查完了、自己停下,交回答那次调用超过时长上限:报错说时长上限,limit 也是 timeout_s(事件里同样)。"""
|
||||
await set_verify({"inquiry": {"timeout_s": 0.5}})
|
||||
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
|
||||
models = Models(monkeypatch, shop)
|
||||
slow_final(monkeypatch)
|
||||
item, [event] = await _asked(client, shop)
|
||||
answer = item["answer"]
|
||||
assert answer["ok"] is False and answer["text"] == "" and answer["doc_artifact"] is None
|
||||
assert answer["error"] == "回答超过时长上限(0.5 秒),未能给出回答,可缩小问题范围后重新提问"
|
||||
assert answer["limit"] == "timeout_s"
|
||||
assert [q["sql"] for q in item["queries"]] == [NO_TEST], "查到的照记"
|
||||
assert (event.data["limit"], event.data["error"]) == ("timeout_s", answer["error"])
|
||||
assert models.finals == [], "交回答那次没等到回复"
|
||||
|
||||
|
||||
async def test_a_placeholder_answer_is_no_answer(monkeypatch, shop, client):
|
||||
Models(monkeypatch, shop, plays={"排除测试账号": Play([NO_TEST], "placeholder")})
|
||||
item, _ = await _asked(client, shop)
|
||||
answer = item["answer"]
|
||||
assert (answer["ok"], answer["text"], answer["limit"]) == (False, "", None)
|
||||
assert answer["error"] == NO_ANSWER
|
||||
@@ -0,0 +1,144 @@
|
||||
"""核查 agent 交理由的写法和收尾超时记的上限(期 5 真实模型验收 M1)。
|
||||
|
||||
- 交结果的提示(VERIFY_FINAL_PROMPT,双盲复算、反方审稿、审批质询共用)和写作者一致:依据写在句末,不放进括号或句子
|
||||
中间。以前写的是「提到某次查询时写 [[see:Q1]]」,模型就写成「结果一致([[see:Q3]],原报告…)」,反方审稿的系统提示
|
||||
自己也把它写在括号里(「([[v:Qn.rX.列名]] 或 [[see:Qn]])」)
|
||||
- 写进了括号的依据,理由文档里照样渲染干净(evidence._tidy),这句仍挂着那条依据
|
||||
- 模型自己停了工具调用、收尾那次调用超过了时长上限:limit 照实记 timeout_s。以前是 None——质询的报错写着「回答超过
|
||||
时长上限(180 秒)」,limit 却是空的,界面上缺了「已达时长上限」。不走结构化输出的模型(普通调用收尾)也一样
|
||||
- 交回的理由是空的、只有标点,或者只是一个占位词(字面的 placeholder、N/A):当作没给理由,不把占位词当理由展示
|
||||
|
||||
模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,数字都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from app.engine import verify_agent
|
||||
from app.engine.evidence import iter_units
|
||||
from app.engine.inquiry import INQUIRY_SYSTEM, INQUIRY_TASK_PROMPT
|
||||
from app.engine.nodes.challenge import CHALLENGE_SYSTEM, CHALLENGE_TASK_PROMPT
|
||||
from app.engine.nodes.dual_check import BLIND_SYSTEM, BLIND_TASK_PROMPT
|
||||
from app.engine.verify_agent import VERIFY_FINAL_PROMPT, VERIFY_SYSTEM
|
||||
from app.providers.mock_model import MockChatModel
|
||||
from tests.test_verify_agent import VALID_SUM, Script, clean_spend, q, run, shop # noqa: F401 - 夹具按名字引入
|
||||
|
||||
#: 括号里写着依据标记:「([[v:Qn.rX.列名]] 或 [[see:Qn]])」「([[see:Q1]])」这种示范
|
||||
_BRACKETED_SEE = re.compile(r"[((][^()()]*\[\[see:[^\]]*\]\][^()()]*[))]")
|
||||
FOUND = {"value": {"alias": "Q1", "row": 0, "column": "total"}, "verdict": "一致"}
|
||||
|
||||
|
||||
def slow_final(monkeypatch, seconds: float = 5.0) -> None:
|
||||
"""交结果那次调用(消息里带着 VERIFY_FINAL_PROMPT:结构化输出、退回普通调用都是)迟迟不返回,查库的几轮照常。"""
|
||||
original = MockChatModel._agenerate
|
||||
|
||||
async def held(chat, messages, stop=None, run_manager=None, **kwargs):
|
||||
if any(VERIFY_FINAL_PROMPT in str(m.content) for m in messages):
|
||||
await asyncio.sleep(seconds)
|
||||
return await original(chat, messages, stop, run_manager, **kwargs)
|
||||
|
||||
monkeypatch.setattr(MockChatModel, "_agenerate", held)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 提示词
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def test_the_final_prompt_puts_see_at_the_end_of_the_sentence():
|
||||
assert "提到某次查询时写 [[see:Q1]]" not in VERIFY_FINAL_PROMPT
|
||||
assert "句末" in VERIFY_FINAL_PROMPT and "括号" in VERIFY_FINAL_PROMPT
|
||||
assert re.search(r"。\[\[see:Q\d\]\]", VERIFY_FINAL_PROMPT), "示范的写法:依据挂在句号后面"
|
||||
assert "渲染后不显示" in VERIFY_FINAL_PROMPT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt", [VERIFY_SYSTEM, VERIFY_FINAL_PROMPT, CHALLENGE_SYSTEM, CHALLENGE_TASK_PROMPT,
|
||||
INQUIRY_SYSTEM, INQUIRY_TASK_PROMPT, BLIND_SYSTEM, BLIND_TASK_PROMPT])
|
||||
def test_no_verify_prompt_shows_see_inside_brackets(prompt):
|
||||
assert not _BRACKETED_SEE.search(prompt), _BRACKETED_SEE.search(prompt).group(0)
|
||||
|
||||
|
||||
async def test_the_final_call_is_told_the_rule(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
script = Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": "合计 [[v:Q1.r0.total]]。[[see:Q1]]"})
|
||||
await run(name)
|
||||
kind, messages = script.calls[-1]
|
||||
assert kind == "final" and VERIFY_FINAL_PROMPT in str(messages[-1].content)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 写进括号的依据
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
async def test_see_written_in_brackets_renders_clean_in_the_rationale(monkeypatch, shop):
|
||||
name, _ = shop
|
||||
said = "未作废订单的金额合计是 [[v:Q1.r0.total]]([[see:Q1]]),口径一致([[see:Q1]],已排除作废单)。"
|
||||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
|
||||
out = await run(name)
|
||||
shown = "未作废订单的金额合计是 230,口径一致(已排除作废单)。"
|
||||
assert out.result["rationale"] == shown and out.rationale["markdown"] == shown
|
||||
assert out.rationale["source"] == said, "原文照存,依据标记还在"
|
||||
[unit] = [unit for _, unit in iter_units(out.rationale)]
|
||||
assert [s["alias"] for s in unit["see"]] == ["Q1", "Q1"] and unit["cites"] == ["Q1"]
|
||||
assert out.rationale["violations"] == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 收尾超时
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("structured", [True, False], ids=["structured", "plain"])
|
||||
async def test_a_final_call_that_runs_out_of_time_records_the_time_limit(monkeypatch, shop, structured):
|
||||
"""模型查了一次就自己停下(没撞上任何上限),交结果那次调用却一直不返回:到点放弃,limit 是 timeout_s,和原因一致。"""
|
||||
name, _ = shop
|
||||
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
|
||||
script = Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": "合计 [[v:Q1.r0.total]]。"})
|
||||
slow_final(monkeypatch)
|
||||
if not structured:
|
||||
def refuse(self, schema, **kwargs):
|
||||
raise NotImplementedError("不支持结构化输出")
|
||||
|
||||
monkeypatch.setattr(MockChatModel, "with_structured_output", refuse)
|
||||
started = time.monotonic()
|
||||
out = await run(name, timeout_s=0.5)
|
||||
assert time.monotonic() - started < 3.0, "收尾那次只等到宽限结束"
|
||||
assert script.kinds == ["step", "step"], "查了一次、自己停下,交结果那次没等到回复"
|
||||
assert out.answered is True and [qd["alias"] for qd in out.queries] == ["Q1"]
|
||||
assert out.result is None and out.error == "收尾时超过时长上限,没能交结果"
|
||||
assert out.limit == "timeout_s"
|
||||
|
||||
|
||||
async def test_a_limit_hit_in_the_loop_is_still_the_one_reported(monkeypatch, shop):
|
||||
"""循环先撞上查询次数上限、收尾又超时:触顶原因照旧报先撞上的那个(同金额被拒的收尾,见 test_verify_truncation)。"""
|
||||
name, _ = shop
|
||||
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
|
||||
Script(monkeypatch, [q(name, VALID_SUM, "c1"), q(name, "SELECT COUNT(*) AS n FROM orders", "c2")],
|
||||
{**FOUND, "rationale": "x"})
|
||||
slow_final(monkeypatch)
|
||||
out = await run(name, timeout_s=0.5, max_queries=1)
|
||||
assert out.result is None and out.error == "收尾时超过时长上限,没能交结果"
|
||||
assert out.limit == "max_queries"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 占位词当理由
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("said", ["placeholder", "PLACEHOLDER", " ", "N/A", "(略)", "…", "<rationale>", "TODO"])
|
||||
async def test_a_placeholder_rationale_counts_as_none(monkeypatch, shop, said):
|
||||
name, _ = shop
|
||||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
|
||||
out = await run(name)
|
||||
assert out.result["value"] == 230, "数照样取"
|
||||
assert out.result["rationale"] == "" and out.rationale is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("said", ["无异常。", "见 [[v:Q1.r0.total]]。", "略高于上周。"])
|
||||
async def test_a_short_real_rationale_is_kept(monkeypatch, shop, said):
|
||||
name, _ = shop
|
||||
Script(monkeypatch, [q(name, VALID_SUM, "c1")], {**FOUND, "rationale": said})
|
||||
out = await run(name)
|
||||
assert out.result["rationale"] and out.rationale is not None
|
||||
Reference in New Issue
Block a user