mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:31:58 +00:00
验收第二轮的两处说法:审批质询整理回答那一步超时时,不再建议「缩小问题范围」(问题再小也要整理),改说调大时长上限或换用更快的核查模型,查询中途触顶的那句另补上可调大上限;反方判「被推翻」却没给理由时一句话说清「但没有给出理由」,不再先说「理由没有引用到任何查询结果」再接「反方没有给出理由」,固定的那句补上句号
This commit is contained in:
@@ -310,7 +310,10 @@ NO_ANSWER = "模型没有给出回答内容,请重新提问"
|
||||
LEAKED = "回答中含有设了遮罩的值,无法遮住,已不予展示,请换个问法重新提问"
|
||||
ERR_DAILY = "今日核查花费已达上限({cap}),未能给出回答,可在设置的「核查」一节调高上限"
|
||||
ERR_COST = "这次回答的花费已达上限,未能给出回答,可缩小问题范围后重新提问"
|
||||
ERR_TIMEOUT = "回答超过时长上限{seconds},未能给出回答,可缩小问题范围后重新提问"
|
||||
ERR_TIMEOUT = "回答超过时长上限{seconds},未能给出回答,可缩小问题范围,或在设置的「核查」一节调大审批质询的时长上限后重新提问"
|
||||
#: 查完、整理回答那一步(收尾调用)超时:问题再小也一样要整理,缩小范围帮不上,说调大时长上限或换更快的核查模型
|
||||
ERR_SETTLE_TIMEOUT = ("回答超过时长上限{seconds}:整理回答这一步超时,未能给出回答,可在设置的「核查」一节调大审批质询的"
|
||||
"时长上限,或换用更快的核查模型后重新提问")
|
||||
ERR_SHAPE = "模型未能按要求给出回答,请重新提问;如反复出现,请在设置的「核查」一节更换核查模型"
|
||||
ERR_SILENT = "模型没有给出回复,未能给出回答,请重新提问"
|
||||
# 回答末尾的系统说明(with_note:另起一段,不挂出处、不查裸数字)
|
||||
@@ -1896,7 +1899,7 @@ def _error_words(outcome: VerifyOutcome, *, timeout_s: Any, daily_max_usd: Any)
|
||||
return _timeout_words(timeout_s)
|
||||
return ERR_COST
|
||||
if error.startswith("收尾时超过时长上限"):
|
||||
return _timeout_words(timeout_s)
|
||||
return _timeout_words(timeout_s, ERR_SETTLE_TIMEOUT)
|
||||
if error.startswith("核查模型没有按结构交结果"):
|
||||
return ERR_SHAPE
|
||||
if error == "核查模型没有给出回复":
|
||||
@@ -1906,9 +1909,9 @@ def _error_words(outcome: VerifyOutcome, *, timeout_s: Any, daily_max_usd: Any)
|
||||
return error
|
||||
|
||||
|
||||
def _timeout_words(timeout_s: Any) -> str:
|
||||
def _timeout_words(timeout_s: Any, words: str = ERR_TIMEOUT) -> str:
|
||||
seconds = timeout_s if isinstance(timeout_s, (int, float)) and not isinstance(timeout_s, bool) else None
|
||||
return ERR_TIMEOUT.format(seconds=f"({seconds:g} 秒)" if seconds is not None else "")
|
||||
return words.format(seconds=f"({seconds:g} 秒)" if seconds is not None else "")
|
||||
|
||||
|
||||
def _given_numbers(ask: Ask) -> list[str]:
|
||||
|
||||
@@ -382,7 +382,10 @@ def _claim_of(cand: judging.Candidate, outcome: VerifyOutcome) -> tuple[dict[str
|
||||
why = "反查没有执行成功任何查询"
|
||||
return _unfounded(cand, outcome, verdict, why), why
|
||||
if verdict == "refuted" and not _cites(outcome.rationale):
|
||||
return _unfounded(cand, outcome, verdict, "理由没有引用到任何查询结果"), None
|
||||
# 理由是空的(只写了占位词的,核查 agent 已经按没给理由交回,见 verify_agent._no_rationale):照实说没给理由,
|
||||
# 不说「理由没有引用到…」再接「反方没有给出理由」
|
||||
why = "理由没有引用到任何查询结果" if rationale.strip() else None
|
||||
return _unfounded(cand, outcome, verdict, why), None
|
||||
return _claim(cand, verdict, rationale, doc=outcome.rationale, queries=outcome.queries, usage=outcome.usage), None
|
||||
|
||||
|
||||
@@ -486,12 +489,17 @@ def _cites(doc: dict[str, Any] | None) -> bool:
|
||||
return isinstance(doc, dict) and any(unit.get("cites") for _, unit in iter_units(doc))
|
||||
|
||||
|
||||
def _unfounded(cand: judging.Candidate, outcome: VerifyOutcome, verdict: str, why: str) -> dict[str, Any]:
|
||||
"""没有出处的结论:记为无法判断,系统写明原因,反方原本的结论和理由照录在后面(照录的写法见 _said)。
|
||||
def _unfounded(cand: judging.Candidate, outcome: VerifyOutcome, verdict: str, why: str | None) -> dict[str, Any]:
|
||||
"""没有出处的结论:记为无法判断,系统写明原因,反方原本的结论和理由照录在后面(照录的写法见 _said)。why 为 None:
|
||||
反方判了结论却根本没给理由,一句话说清(不再先说「理由没有引用到…」、后面又说「没有给出理由」)。
|
||||
rationale_doc 为 None:这段话是系统说的,界面照 rationale 显示,不拿反方那份文档冒充理由。查过的照记。"""
|
||||
taken = f"反方给出的「{_VERDICT_LABEL[verdict]}」不予采信,记为无法判断" if verdict != "inconclusive" else "记为无法判断"
|
||||
label = _VERDICT_LABEL[verdict]
|
||||
if why is None:
|
||||
text = f"反方判了「{label}」,但没有给出理由,不予采信,记为无法判断。"
|
||||
return _claim(cand, "inconclusive", text, queries=outcome.queries, usage=outcome.usage)
|
||||
taken = f"反方给出的「{label}」不予采信,记为无法判断" if verdict != "inconclusive" else "记为无法判断"
|
||||
rationale = _said(outcome.rationale, str((outcome.result or {}).get(RATIONALE) or "")).strip()
|
||||
said = f"反方的理由:{rationale}" if rationale else "反方没有给出理由"
|
||||
said = f"反方的理由:{rationale}" if rationale else "反方没有给出理由。"
|
||||
return _claim(cand, "inconclusive", f"{why},{taken}。{said}", queries=outcome.queries, usage=outcome.usage)
|
||||
|
||||
|
||||
|
||||
@@ -89,7 +89,7 @@ async def test_a_placeholder_rationale_is_recorded_as_no_rationale(monkeypatch,
|
||||
first, second = record["claims"]
|
||||
assert (first["verdict"], first["rationale"], first["rationale_doc"]) == ("stands", "", None)
|
||||
assert second["verdict"] == "inconclusive"
|
||||
assert second["rationale"] == "理由没有引用到任何查询结果,反方给出的「被推翻」不予采信,记为无法判断。反方没有给出理由"
|
||||
assert second["rationale"] == "反方判了「被推翻」,但没有给出理由,不予采信,记为无法判断。"
|
||||
assert "placeholder" not in str(record)
|
||||
assert issuance["tier"] == "formal"
|
||||
|
||||
@@ -103,7 +103,7 @@ async def test_a_rationale_that_shows_nothing_is_recorded_as_no_rationale(monkey
|
||||
first, second = record["claims"]
|
||||
assert (first["verdict"], first["rationale"], first["rationale_doc"]) == ("stands", "", None)
|
||||
assert second["verdict"] == "inconclusive" and second["rationale_doc"] is None
|
||||
assert second["rationale"].endswith("反方给出的「被推翻」不予采信,记为无法判断。反方没有给出理由")
|
||||
assert second["rationale"] == "反方判了「被推翻」,但没有给出理由,不予采信,记为无法判断。"
|
||||
assert issuance["tier"] == "formal", "没有理由的「被推翻」不能让出具降档"
|
||||
|
||||
|
||||
|
||||
@@ -2174,7 +2174,8 @@ def test_internal_wording_from_the_agent_is_not_shown():
|
||||
return inquiry_mod._error_words(outcome, timeout_s=120, daily_max_usd=2.0)
|
||||
|
||||
assert words("收尾时超过时长上限,没能交结果", limit="timeout_s") == \
|
||||
"回答超过时长上限(120 秒),未能给出回答,可缩小问题范围后重新提问"
|
||||
("回答超过时长上限(120 秒):整理回答这一步超时,未能给出回答,可在设置的「核查」一节调大审批质询的时长上限,"
|
||||
"或换用更快的核查模型后重新提问")
|
||||
assert words("已达全局每日核查金额上限 $2.00,没能交结果", limit="daily_max_usd") == \
|
||||
"今日核查花费已达上限(核查每日金额上限 $2.00),未能给出回答,可在设置的「核查」一节调高上限"
|
||||
assert words("核查模型没有按结构交结果(交回的内容里没有要求的字段:rationale)") == \
|
||||
|
||||
@@ -80,7 +80,8 @@ async def test_a_final_answer_that_runs_out_of_time_reports_the_time_limit(monke
|
||||
item, [event] = await _asked(client, shop)
|
||||
answer = item["answer"]
|
||||
assert answer["ok"] is False and answer["text"] == "" and answer["doc_artifact"] is None
|
||||
assert answer["error"] == "回答超过时长上限(0.5 秒),未能给出回答,可缩小问题范围后重新提问"
|
||||
assert answer["error"] == ("回答超过时长上限(0.5 秒):整理回答这一步超时,未能给出回答,可在设置的「核查」一节调大"
|
||||
"审批质询的时长上限,或换用更快的核查模型后重新提问")
|
||||
assert answer["limit"] == "timeout_s"
|
||||
assert [q["sql"] for q in item["queries"]] == [NO_TEST], "查到的照记"
|
||||
assert (event.data["limit"], event.data["error"]) == ("timeout_s", answer["error"])
|
||||
|
||||
Reference in New Issue
Block a user