mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 06:01:57 +00:00
复核认分析能力一轮的三种日志:report_analysis_claims(判断词没挂对分析结论、原因没依据,重写后仍有)、analysis_refused(一份分析算不了)、metric_comparability_skipped(口径卡指标的两期没做成可比性检查)登记进 WARN_CODES,按自己的 kind 记、都是缺口(DEGRADED),照 00-总方案 §3.7 只写说明、不自动重来:不进 RETRYABLE / REBUILD,一次运行的信号全是这几种(NOTE_ONLY)时复核不叫模型,说明照信号原话写——以前没登记一律按笼统的 warn 叫一次复核模型。「改过一次」的 report_repair 按最后一次核对定档时,只剩只重写一次、留在报告里标注的违规(check 下的分析说法,和同一类的缺失断言 absence_claim)也算改好了:它们各有自己的信号,以前 report.checked 的 ok 为 False 就判没改好、多报一条;复核被别的信号叫起来时照实说「重写后只剩已标出的 1 处分析说法」,一处都没改掉的不说改好了。strict 下的分析说法是出具缺口、带满 20 条的可能还有别的,照旧判没改好。测试:三种信号的 kind 与级别、只有这几种信号时不叫模型(复核接口也是)、和别的信号一起时照常叫、check 与缺失断言的剩余算改好、strict / 还有别的违规 / 带满 20 条照旧
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
@@ -141,6 +142,13 @@ WARN_CODES: dict[str, tuple[str, str]] = {
|
||||
# 缺口降档),是缺口不是断裂;但换一张有这段数据的表再建一次大概率就好,所以能给出重来的指示(REBUILD)
|
||||
"query_no_data": ("query_no_data", DEGRADED),
|
||||
"metric_no_data": ("metric_no_data", DEGRADED),
|
||||
# 分析能力一轮(00-总方案 §3.7):只写说明、不自动重来(NOTE_ONLY)。答案照常交付,缺口照实说一句就够了:
|
||||
# 报告里判断词没挂对分析结论、原因没有依据,重写后仍有(check 标在报告里,strict 另记出具缺口)
|
||||
"report_analysis_claims": ("report_analysis_claims", DEGRADED),
|
||||
# 一份分析算不了(历史不够、本期不完整、截断……):算不了本身是一条结论,写作者照实写原因
|
||||
"analysis_refused": ("analysis_refused", DEGRADED),
|
||||
# 口径卡上用到两次查询的指标没做成可比性检查(探查失败、算不了):指标照常算,两期能不能直接比要人自己核对
|
||||
"metric_comparability_skipped": ("metric_comparability_skipped", DEGRADED),
|
||||
}
|
||||
|
||||
# 这些 kind 重跑一次大概率能好:原因说得清,且不是「问题本身没问明白」。
|
||||
@@ -150,6 +158,10 @@ RETRYABLE = {"step_limit", "tool_error", "empty_completion", "node_failed"}
|
||||
#: 答案照常交付,复核照样可以给出「按复核重来」的指示(should_retry、chat/turn.retry_after_review 都认)
|
||||
REBUILD = {"query_no_data", "metric_no_data"}
|
||||
|
||||
#: 只写说明、不自动重来的信号(00-总方案 §3.7):都是缺口(DEGRADED)、不进 RETRYABLE / REBUILD。一次运行的信号全是
|
||||
#: 这几种时复核不叫模型,说明就照信号原话写(review):答案照常、报告里已经标出,叫一次模型只是把同一句话换个说法
|
||||
NOTE_ONLY = frozenset({"report_analysis_claims", "analysis_refused", "metric_comparability_skipped"})
|
||||
|
||||
#: 串行工具模式下塞给未执行调用的占位说明。它漏到用户面前当过答案,
|
||||
#: 所以复核要认得出来。从 nodes.llm 导入会绕成循环,这里按内容识别。
|
||||
_PLACEHOLDER_MARKS = ("本轮只执行了第一个工具",)
|
||||
@@ -297,6 +309,8 @@ def _step_name(node_id: str, titles: Mapping[str, str]) -> str:
|
||||
|
||||
#: 报告节点「改过一次」的信号:按同一个节点最后一次 report.checked 的结果定档(最终通过的不是缺口)
|
||||
_SETTLED_BY_REPORT = ("report_repair", "report_rewrite")
|
||||
#: report.checked 最多带几条违规(报告节点截到前 20 条):带满了的,后面可能还有,不当作只剩标注
|
||||
_CHECKED_CAP = 20
|
||||
#: 证据不支持的几种判定(engine/judge 的 CONTRADICTED 加证据不足):最终还剩这些,改写就没有改好
|
||||
_UNSUPPORTED = ("contradicted", "unsupported", "insufficient")
|
||||
|
||||
@@ -311,20 +325,38 @@ def _final_checks(rows: list[Any]) -> dict[str, dict[str, Any]]:
|
||||
return out
|
||||
|
||||
|
||||
def _annotations(final: Mapping[str, Any]) -> list[Mapping[str, Any]] | None:
|
||||
"""最后一次核对里剩下的违规都是只重写一次、留在报告里标注的那几类时,返回它们(没剩违规是空列表);还有别的违规返回
|
||||
None。标注的是:缺失断言(absence_claim,另发 report_absence_claim)、check 下的分析说法(engine/analysis_claims,另发
|
||||
report_analysis_claims)——各有自己的信号说清楚,「改过一次」是不是改好了不看它们。strict 下的分析说法是出具缺口,
|
||||
不算标注。"""
|
||||
from app.engine.analysis_claims import CODES as ANALYSIS_CODES
|
||||
|
||||
left = [v for v in final.get("violations") or [] if isinstance(v, Mapping)]
|
||||
marked = {"absence_claim", *(ANALYSIS_CODES if final.get("analysis_claims") == "check" else ())}
|
||||
if final.get("ok") is True and not left:
|
||||
return []
|
||||
if not left or len(left) >= _CHECKED_CAP or any(v.get("code") not in marked for v in left):
|
||||
return None
|
||||
return left
|
||||
|
||||
|
||||
def _settled(code: str, final: Mapping[str, Any] | None) -> bool:
|
||||
"""「改过一次」之后最终是不是好了:report_repair 看最后一次核对有没有违规(ok、violations 为空);report_rewrite
|
||||
看最后的裁判摘要里还有没有证据不支持的句子。找不到最终结果(老运行、节点失败没写出 report.checked)的当没好。"""
|
||||
"""「改过一次」之后最终是不是好了:report_repair 看最后一次核对有没有违规(ok、violations 为空),只剩标注
|
||||
(_annotations)的也算好了;report_rewrite 看最后的裁判摘要里还有没有证据不支持的句子。找不到最终结果(老运行、
|
||||
节点失败没写出 report.checked)的当没好。"""
|
||||
if not final:
|
||||
return False
|
||||
if code == "report_repair":
|
||||
return final.get("ok") is True and not final.get("violations")
|
||||
return _annotations(final) is not None
|
||||
counts = ((final.get("judge") or {}).get("counts") or {}) if isinstance(final.get("judge"), dict) else {}
|
||||
return bool(counts) and not any(int(counts.get(k) or 0) for k in _UNSUPPORTED)
|
||||
|
||||
|
||||
def notes(events: Iterable[Any]) -> list[str]:
|
||||
"""不是异常、但复核模型被别的信号叫起来时该知道的事(review_prompt 的「已经处理好的」):报告首稿没过核对、重写后
|
||||
已全部通过;裁判认为证据不支持的句子改写后都有了依据。只说处理好了的,没好的照旧是信号(scan)。"""
|
||||
已全部通过(只剩标注的照实说剩下几处,那几处各有自己的信号);裁判认为证据不支持的句子改写后都有了依据。只说处理好
|
||||
了的,没好的照旧是信号(scan)。"""
|
||||
rows = list(events)
|
||||
finals = _final_checks(rows)
|
||||
titles: dict[str, str] = {}
|
||||
@@ -349,7 +381,14 @@ def notes(events: Iterable[Any]) -> list[str]:
|
||||
if code == "report_repair":
|
||||
head = f"首稿有 {count} 处没过核对" if isinstance(count, int) and count > 0 else "首稿没过核对"
|
||||
again = f"重写 {times[(node_id, code)]} 次后" if times[(node_id, code)] > 1 else "重写后"
|
||||
out.append(f"{name}{head},{again}已全部通过")
|
||||
left = _annotations(finals.get(node_id) or {}) or []
|
||||
if not left:
|
||||
out.append(f"{name}{head},{again}已全部通过")
|
||||
elif isinstance(count, int) and count > len(left):
|
||||
said = {"absence_claim": "缺失断言"}
|
||||
kinds = Counter(said.get(str(v.get("code")), "分析说法") for v in left)
|
||||
rest = "、".join(f"{n} 处{kind}" for kind, n in kinds.items())
|
||||
out.append(f"{name}{head},{again}只剩已标出的 {rest}")
|
||||
else:
|
||||
head = f"有 {count} 句结论裁判认为证据不支持" if isinstance(count, int) and count > 0 \
|
||||
else "有几句结论裁判认为证据不支持"
|
||||
@@ -486,6 +525,9 @@ async def review(
|
||||
fallback_note = ";".join(s.detail for s in signals if s.severity == BROKEN) or ";".join(
|
||||
s.detail for s in signals
|
||||
)
|
||||
if signals and all(s.kind in NOTE_ONLY for s in signals):
|
||||
# 只写说明、不自动重来(NOTE_ONLY):不叫模型,照信号原话说明
|
||||
return ReviewResult(verdict="annotated", note=fallback_note, retry=False, signals=signals, severity=severity)
|
||||
|
||||
messages = [("system", REVIEW_SYSTEM),
|
||||
("human", review_prompt(question, answer, signals, locked=locked, settled=settled))]
|
||||
|
||||
@@ -476,3 +476,97 @@ def test_a_rewrite_counts_by_the_final_judgement():
|
||||
judge={"counts": {"supported": 2, "contradicted": 1, "insufficient": 0}})]
|
||||
assert rv.scan(good, OK_OUTPUT) == [] and rv.notes(good) == ["报告「写周报」有 1 句结论裁判认为证据不支持,改写后都有了依据"]
|
||||
assert [s.kind for s in rv.scan(bad, OK_OUTPUT)] == ["report_rewrite"] and rv.notes(bad) == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 分析能力一轮的信号:只写说明、不自动重来(00-总方案 §3.7)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: check 下判断词重写一次后仍没挂对分析结论:报告照常产出,标在报告里
|
||||
CLAIMS_LEFT = "报告有 1 处分析说法未通过核对(1 处判断没挂分析结论:「主要来自」),已要求模型重写 1 次仍未改掉:…这几处已标注"
|
||||
REFUSED = "「周报分析」的波动基线没有做成:历史只有 5 期,至少要 8 期"
|
||||
SKIPPED = "指标「销售额环比」用到的两次查询没做成可比性检查:探查超时。指标照常计算,两期能不能直接比请自行核对"
|
||||
|
||||
|
||||
def test_analysis_signals_are_registered_as_gaps_that_only_need_a_note():
|
||||
signals = rv.scan([warn(CLAIMS_LEFT, "report_analysis_claims"), warn(REFUSED, "analysis_refused"),
|
||||
warn(SKIPPED, "metric_comparability_skipped")], OK_OUTPUT)
|
||||
assert [(s.kind, s.severity) for s in signals] == [
|
||||
("report_analysis_claims", rv.DEGRADED), ("analysis_refused", rv.DEGRADED),
|
||||
("metric_comparability_skipped", rv.DEGRADED)]
|
||||
assert {s.kind for s in signals} == rv.NOTE_ONLY
|
||||
assert not rv.should_retry(signals) and not rv.NOTE_ONLY & (rv.RETRYABLE | rv.REBUILD)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_run_with_only_note_only_signals_is_annotated_without_calling_the_model():
|
||||
model = FakeModel(AssertionError("不该叫模型"))
|
||||
signals = rv.scan([warn(CLAIMS_LEFT, "report_analysis_claims"), warn(REFUSED, "analysis_refused")], OK_OUTPUT)
|
||||
result = await rv.review(model, question="q", answer="答案", signals=signals, locked=True)
|
||||
assert model.calls == 0
|
||||
assert result.verdict == "annotated" and result.retry is False and result.severity == rv.DEGRADED
|
||||
assert result.note == f"{CLAIMS_LEFT};{REFUSED}" and result.answer is None
|
||||
# 和别的信号一起出现时照常叫模型
|
||||
mixed = [*signals, rv.Signal("tool_error", "工具 x 调用失败", rv.BROKEN)]
|
||||
model = FakeModel({"note": "取数失败了一次。", "answer": "", "retry": True})
|
||||
assert (await rv.review(model, question="q", answer="答案", signals=mixed)).note == "取数失败了一次。"
|
||||
assert model.calls == 1
|
||||
|
||||
|
||||
def _left_after_rewrite(mode: str, violations: list[dict], count: int = 2) -> list[dict]:
|
||||
return [
|
||||
ev(EventType.NODE_STARTED, node_id="write", label="写周报"),
|
||||
ev(EventType.LOG, node_id="write", level="warn", code="report_repair", count=count, attempt=1,
|
||||
message=f"报告有 {count} 处未通过核对(「12」、「主要来自」),已要求模型重写(第 1 次)"),
|
||||
ev(EventType.REPORT_CHECKED, node_id="write", ok=not violations, violations=violations, repairs=1,
|
||||
analysis_claims=mode),
|
||||
]
|
||||
|
||||
|
||||
CLAIM = {"code": "analysis_claim_uncited", "text": "主要来自", "message": "「主要来自」是分析判断"}
|
||||
|
||||
|
||||
def test_check_mode_leftovers_do_not_make_the_rewrite_count_as_failed():
|
||||
"""check 下重写一次后仍有的说法违规是标注(同缺失断言,各有自己的信号):「改过一次」算改好了,不再另报 report_repair;
|
||||
复核被叫起来时照实说剩下的是哪几处。"""
|
||||
rows = [*_left_after_rewrite("check", [CLAIM]), warn(CLAIMS_LEFT, "report_analysis_claims")]
|
||||
assert [s.kind for s in rv.scan(rows, OK_OUTPUT)] == ["report_analysis_claims"]
|
||||
assert rv.notes(rows) == ["报告「写周报」首稿有 2 处没过核对,重写后只剩已标出的 1 处分析说法"]
|
||||
# 一处都没改掉:不说「改好了」,剩下的由它自己的信号说
|
||||
assert rv.notes(_left_after_rewrite("check", [CLAIM], count=1)) == []
|
||||
|
||||
|
||||
def test_absence_claim_leftovers_are_annotations_too():
|
||||
absent = {"code": "absence_claim", "text": "没有查到", "message": "缺失断言"}
|
||||
rows = _left_after_rewrite("check", [absent, CLAIM], count=3)
|
||||
assert rv.scan(rows, OK_OUTPUT) == []
|
||||
assert rv.notes(rows) == ["报告「写周报」首稿有 3 处没过核对,重写后只剩已标出的 1 处缺失断言、1 处分析说法"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode, violations", [
|
||||
("strict", [CLAIM]), # strict 下是出具缺口,不是标注
|
||||
("check", [CLAIM, {"code": "uncited_number", "text": "12"}]), # 还有别的违规
|
||||
("off", [CLAIM]),
|
||||
("check", [CLAIM] * 20), # 带满了 20 条:后面可能还有
|
||||
])
|
||||
def test_other_leftovers_still_count_as_a_failed_rewrite(mode, violations):
|
||||
rows = _left_after_rewrite(mode, violations)
|
||||
assert [s.kind for s in rv.scan(rows, OK_OUTPUT)] == ["report_repair"] and rv.notes(rows) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_endpoint_does_not_call_the_model_for_a_check_mode_leftover(client, monkeypatch):
|
||||
model = FakeModel(AssertionError("不该叫模型"))
|
||||
|
||||
async def fake_model(*a, **k):
|
||||
return model, "fake"
|
||||
|
||||
monkeypatch.setattr("app.api.copilot.get_chat_model", fake_model)
|
||||
output = {"周报": "周报:销售额 45,678.5元,下降主要来自华东。",
|
||||
"_evidence": {"report_node": "write", "doc_artifact": "a" * 64, "fields": ["周报"]}}
|
||||
run_id = await _make_run(client, output, [*_left_after_rewrite("check", [CLAIM]),
|
||||
warn(CLAIMS_LEFT, "report_analysis_claims")])
|
||||
body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "q"})).json()
|
||||
assert model.calls == 0
|
||||
assert body["verdict"] == "annotated" and body["note"] == CLAIMS_LEFT and body["retry_instruction"] is None
|
||||
assert [s["kind"] for s in body["signals"]] == ["report_analysis_claims"]
|
||||
|
||||
Reference in New Issue
Block a user