mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 07:21:58 +00:00
735dc32052
- 交结果的提示(VERIFY_FINAL_PROMPT,双盲复算、反方审稿、审批质询共用)以前写「提到某次查询时写 [[see:Q1]]」,模型就 把它写进括号、句中;反方审稿的系统提示自己也把它写在括号里(「([[v:Qn.rX.列名]] 或 [[see:Qn]])」)。改成和写作者 一致:依据挂在句末(「……结果一致。[[see:Q3]]」),说明它渲染后不显示,不要放进括号、句中或「依据」「见」「取自」 后面。已经写进括号的由上一个提交的渲染收拾干净 - 模型自己停了查询、收尾那次调用超过时长上限时,limit 以前是 None:质询的报错写着「回答超过时长上限(180 秒)」, answer.limit 却是 null,界面少了「已达时长上限」;双盲复算、反方审稿把它当成别的原因,审批卡上照出内部原话。 现在记 timeout_s;循环里先撞上的上限照旧优先(同金额被拒的收尾),结构化输出、退回普通调用两条路都一样 - 交回的理由是空的、只有标点,或者只是一个占位词(验收里盲算交了字面的「placeholder」,审批卡照着显示):当没给 理由,不组装理由文档。反方审稿的记录里理由为空,审批卡照旧写「反方没有给出理由」;质询照「模型没有给出回答内容」 报错;双盲复算的对照工件里说明为空 测试:交给模型的提示(含三处调用方的系统提示、任务)里不再示范括号里的依据;写进括号的依据在理由文档里渲染干净、 仍挂在句子上;审批质询的回答、反方审稿的理由、双盲复算的说明端到端都没有空括号;收尾超时 limit 为 timeout_s(核查 agent 和质询接口、事件里都是);占位词的理由三处都不当理由。
77 lines
3.7 KiB
Python
77 lines
3.7 KiB
Python
"""审批质询的回答:依据写进括号不再断句,收尾超时记时长上限,占位词不当回答(期 5 真实模型验收 M1)。
|
||
|
||
真实模型验收里,质询 agent 写的「结果一致([[see:Q3]],原报告 SQL 未按 status 过滤)」「合计 [[v:Q1.r1.金额合计]]
|
||
([[see:Q1]])」渲染成「(,原报告…)」「()」,随正式运行封存;另一次回答查完了,收尾那次调用超过时长上限,报错写着
|
||
「回答超过时长上限(180 秒)」,answer.limit 却是 null,界面上缺了「已达时长上限」。
|
||
|
||
模型一律是 MockChatModel 剧本,区域、金额都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from app.core import artifact_store
|
||
from app.engine import verify_agent
|
||
from app.engine.inquiry import NO_ANSWER
|
||
from tests.test_inquiry_api import ( # noqa: F401 - 夹具按名字引入
|
||
NO_TEST,
|
||
QUESTION,
|
||
Models,
|
||
Play,
|
||
ask,
|
||
client,
|
||
engine_up,
|
||
events,
|
||
pause,
|
||
pending,
|
||
set_verify,
|
||
shop,
|
||
)
|
||
from tests.test_verify_rationale_rules import slow_final
|
||
|
||
#: 依据写进了括号:开头跟着逗号的、只有依据的
|
||
BRACKETED = ("仍居各区域之首([[see:Q1]],已排除测试账号):华东为 [[v:Q1.r0.销售额]]([[see:Q1]]),"
|
||
"华南为 [[v:Q1.r1.销售额]](同口径,[[see:Q1]])。")
|
||
BRACKETED_TEXT = "仍居各区域之首(已排除测试账号):华东为 1,200,华南为 900(同口径)。"
|
||
|
||
|
||
async def _asked(client, shop) -> tuple[dict, list]:
|
||
paused = await pause(shop)
|
||
[approval] = await pending(paused.id)
|
||
r = await ask(client, paused.id, report="write", unit=None, question=QUESTION, approval_id=approval.id)
|
||
assert r.status_code == 200, r.text
|
||
return r.json()["inquiry"], await events(paused.id, "approval.inquiry")
|
||
|
||
|
||
async def test_see_in_brackets_does_not_break_the_answer(monkeypatch, shop, client):
|
||
Models(monkeypatch, shop, plays={"排除测试账号": Play([NO_TEST], BRACKETED)})
|
||
item, [event] = await _asked(client, shop)
|
||
answer = item["answer"]
|
||
assert answer["ok"] is True and answer["text"] == BRACKETED_TEXT
|
||
doc = artifact_store.load(answer["doc_artifact"])
|
||
assert doc["markdown"] == BRACKETED_TEXT and doc["source"] == BRACKETED
|
||
assert doc["violations"] == []
|
||
assert event.data["answer_text"] == BRACKETED_TEXT
|
||
|
||
|
||
async def test_a_final_answer_that_runs_out_of_time_reports_the_time_limit(monkeypatch, shop, client):
|
||
"""质询 agent 查完了、自己停下,交回答那次调用超过时长上限:报错说时长上限,limit 也是 timeout_s(事件里同样)。"""
|
||
await set_verify({"inquiry": {"timeout_s": 0.5}})
|
||
monkeypatch.setattr(verify_agent, "SETTLE_GRACE_S", 0.3)
|
||
models = Models(monkeypatch, shop)
|
||
slow_final(monkeypatch)
|
||
item, [event] = await _asked(client, shop)
|
||
answer = item["answer"]
|
||
assert answer["ok"] is False and answer["text"] == "" and answer["doc_artifact"] is None
|
||
assert answer["error"] == "回答超过时长上限(0.5 秒),未能给出回答,可缩小问题范围后重新提问"
|
||
assert answer["limit"] == "timeout_s"
|
||
assert [q["sql"] for q in item["queries"]] == [NO_TEST], "查到的照记"
|
||
assert (event.data["limit"], event.data["error"]) == ("timeout_s", answer["error"])
|
||
assert models.finals == [], "交回答那次没等到回复"
|
||
|
||
|
||
async def test_a_placeholder_answer_is_no_answer(monkeypatch, shop, client):
|
||
Models(monkeypatch, shop, plays={"排除测试账号": Play([NO_TEST], "placeholder")})
|
||
item, _ = await _asked(client, shop)
|
||
answer = item["answer"]
|
||
assert (answer["ok"], answer["text"], answer["limit"]) == (False, "", None)
|
||
assert answer["error"] == NO_ANSWER
|