diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index f2a52b73..a0387a69 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from collections import Counter from collections.abc import Iterable, Mapping from dataclasses import dataclass from typing import Any @@ -141,6 +142,13 @@ WARN_CODES: dict[str, tuple[str, str]] = { # 缺口降档),是缺口不是断裂;但换一张有这段数据的表再建一次大概率就好,所以能给出重来的指示(REBUILD) "query_no_data": ("query_no_data", DEGRADED), "metric_no_data": ("metric_no_data", DEGRADED), + # 分析能力一轮(00-总方案 §3.7):只写说明、不自动重来(NOTE_ONLY)。答案照常交付,缺口照实说一句就够了: + # 报告里判断词没挂对分析结论、原因没有依据,重写后仍有(check 标在报告里,strict 另记出具缺口) + "report_analysis_claims": ("report_analysis_claims", DEGRADED), + # 一份分析算不了(历史不够、本期不完整、截断……):算不了本身是一条结论,写作者照实写原因 + "analysis_refused": ("analysis_refused", DEGRADED), + # 口径卡上用到两次查询的指标没做成可比性检查(探查失败、算不了):指标照常算,两期能不能直接比要人自己核对 + "metric_comparability_skipped": ("metric_comparability_skipped", DEGRADED), } # 这些 kind 重跑一次大概率能好:原因说得清,且不是「问题本身没问明白」。 @@ -150,6 +158,10 @@ RETRYABLE = {"step_limit", "tool_error", "empty_completion", "node_failed"} #: 答案照常交付,复核照样可以给出「按复核重来」的指示(should_retry、chat/turn.retry_after_review 都认) REBUILD = {"query_no_data", "metric_no_data"} +#: 只写说明、不自动重来的信号(00-总方案 §3.7):都是缺口(DEGRADED)、不进 RETRYABLE / REBUILD。一次运行的信号全是 +#: 这几种时复核不叫模型,说明就照信号原话写(review):答案照常、报告里已经标出,叫一次模型只是把同一句话换个说法 +NOTE_ONLY = frozenset({"report_analysis_claims", "analysis_refused", "metric_comparability_skipped"}) + #: 串行工具模式下塞给未执行调用的占位说明。它漏到用户面前当过答案, #: 所以复核要认得出来。从 nodes.llm 导入会绕成循环,这里按内容识别。 _PLACEHOLDER_MARKS = ("本轮只执行了第一个工具",) @@ -297,6 +309,8 @@ def _step_name(node_id: str, titles: Mapping[str, str]) -> str: #: 报告节点「改过一次」的信号:按同一个节点最后一次 report.checked 的结果定档(最终通过的不是缺口) _SETTLED_BY_REPORT = ("report_repair", "report_rewrite") +#: report.checked 最多带几条违规(报告节点截到前 20 条):带满了的,后面可能还有,不当作只剩标注 +_CHECKED_CAP = 20 #: 证据不支持的几种判定(engine/judge 的 CONTRADICTED 加证据不足):最终还剩这些,改写就没有改好 _UNSUPPORTED = ("contradicted", "unsupported", "insufficient") @@ -311,20 +325,38 @@ def _final_checks(rows: list[Any]) -> dict[str, dict[str, Any]]: return out +def _annotations(final: Mapping[str, Any]) -> list[Mapping[str, Any]] | None: + """最后一次核对里剩下的违规都是只重写一次、留在报告里标注的那几类时,返回它们(没剩违规是空列表);还有别的违规返回 + None。标注的是:缺失断言(absence_claim,另发 report_absence_claim)、check 下的分析说法(engine/analysis_claims,另发 + report_analysis_claims)——各有自己的信号说清楚,「改过一次」是不是改好了不看它们。strict 下的分析说法是出具缺口, + 不算标注。""" + from app.engine.analysis_claims import CODES as ANALYSIS_CODES + + left = [v for v in final.get("violations") or [] if isinstance(v, Mapping)] + marked = {"absence_claim", *(ANALYSIS_CODES if final.get("analysis_claims") == "check" else ())} + if final.get("ok") is True and not left: + return [] + if not left or len(left) >= _CHECKED_CAP or any(v.get("code") not in marked for v in left): + return None + return left + + def _settled(code: str, final: Mapping[str, Any] | None) -> bool: - """「改过一次」之后最终是不是好了:report_repair 看最后一次核对有没有违规(ok、violations 为空);report_rewrite - 看最后的裁判摘要里还有没有证据不支持的句子。找不到最终结果(老运行、节点失败没写出 report.checked)的当没好。""" + """「改过一次」之后最终是不是好了:report_repair 看最后一次核对有没有违规(ok、violations 为空),只剩标注 + (_annotations)的也算好了;report_rewrite 看最后的裁判摘要里还有没有证据不支持的句子。找不到最终结果(老运行、 + 节点失败没写出 report.checked)的当没好。""" if not final: return False if code == "report_repair": - return final.get("ok") is True and not final.get("violations") + return _annotations(final) is not None counts = ((final.get("judge") or {}).get("counts") or {}) if isinstance(final.get("judge"), dict) else {} return bool(counts) and not any(int(counts.get(k) or 0) for k in _UNSUPPORTED) def notes(events: Iterable[Any]) -> list[str]: """不是异常、但复核模型被别的信号叫起来时该知道的事(review_prompt 的「已经处理好的」):报告首稿没过核对、重写后 - 已全部通过;裁判认为证据不支持的句子改写后都有了依据。只说处理好了的,没好的照旧是信号(scan)。""" + 已全部通过(只剩标注的照实说剩下几处,那几处各有自己的信号);裁判认为证据不支持的句子改写后都有了依据。只说处理好 + 了的,没好的照旧是信号(scan)。""" rows = list(events) finals = _final_checks(rows) titles: dict[str, str] = {} @@ -349,7 +381,14 @@ def notes(events: Iterable[Any]) -> list[str]: if code == "report_repair": head = f"首稿有 {count} 处没过核对" if isinstance(count, int) and count > 0 else "首稿没过核对" again = f"重写 {times[(node_id, code)]} 次后" if times[(node_id, code)] > 1 else "重写后" - out.append(f"{name}{head},{again}已全部通过") + left = _annotations(finals.get(node_id) or {}) or [] + if not left: + out.append(f"{name}{head},{again}已全部通过") + elif isinstance(count, int) and count > len(left): + said = {"absence_claim": "缺失断言"} + kinds = Counter(said.get(str(v.get("code")), "分析说法") for v in left) + rest = "、".join(f"{n} 处{kind}" for kind, n in kinds.items()) + out.append(f"{name}{head},{again}只剩已标出的 {rest}") else: head = f"有 {count} 句结论裁判认为证据不支持" if isinstance(count, int) and count > 0 \ else "有几句结论裁判认为证据不支持" @@ -486,6 +525,9 @@ async def review( fallback_note = ";".join(s.detail for s in signals if s.severity == BROKEN) or ";".join( s.detail for s in signals ) + if signals and all(s.kind in NOTE_ONLY for s in signals): + # 只写说明、不自动重来(NOTE_ONLY):不叫模型,照信号原话说明 + return ReviewResult(verdict="annotated", note=fallback_note, retry=False, signals=signals, severity=severity) messages = [("system", REVIEW_SYSTEM), ("human", review_prompt(question, answer, signals, locked=locked, settled=settled))] diff --git a/backend/tests/test_review.py b/backend/tests/test_review.py index d5dbbaf8..fc1c8e20 100644 --- a/backend/tests/test_review.py +++ b/backend/tests/test_review.py @@ -476,3 +476,97 @@ def test_a_rewrite_counts_by_the_final_judgement(): judge={"counts": {"supported": 2, "contradicted": 1, "insufficient": 0}})] assert rv.scan(good, OK_OUTPUT) == [] and rv.notes(good) == ["报告「写周报」有 1 句结论裁判认为证据不支持,改写后都有了依据"] assert [s.kind for s in rv.scan(bad, OK_OUTPUT)] == ["report_rewrite"] and rv.notes(bad) == [] + + +# -------------------------------------------------------------------------- +# 分析能力一轮的信号:只写说明、不自动重来(00-总方案 §3.7) +# -------------------------------------------------------------------------- + +#: check 下判断词重写一次后仍没挂对分析结论:报告照常产出,标在报告里 +CLAIMS_LEFT = "报告有 1 处分析说法未通过核对(1 处判断没挂分析结论:「主要来自」),已要求模型重写 1 次仍未改掉:…这几处已标注" +REFUSED = "「周报分析」的波动基线没有做成:历史只有 5 期,至少要 8 期" +SKIPPED = "指标「销售额环比」用到的两次查询没做成可比性检查:探查超时。指标照常计算,两期能不能直接比请自行核对" + + +def test_analysis_signals_are_registered_as_gaps_that_only_need_a_note(): + signals = rv.scan([warn(CLAIMS_LEFT, "report_analysis_claims"), warn(REFUSED, "analysis_refused"), + warn(SKIPPED, "metric_comparability_skipped")], OK_OUTPUT) + assert [(s.kind, s.severity) for s in signals] == [ + ("report_analysis_claims", rv.DEGRADED), ("analysis_refused", rv.DEGRADED), + ("metric_comparability_skipped", rv.DEGRADED)] + assert {s.kind for s in signals} == rv.NOTE_ONLY + assert not rv.should_retry(signals) and not rv.NOTE_ONLY & (rv.RETRYABLE | rv.REBUILD) + + +@pytest.mark.asyncio +async def test_a_run_with_only_note_only_signals_is_annotated_without_calling_the_model(): + model = FakeModel(AssertionError("不该叫模型")) + signals = rv.scan([warn(CLAIMS_LEFT, "report_analysis_claims"), warn(REFUSED, "analysis_refused")], OK_OUTPUT) + result = await rv.review(model, question="q", answer="答案", signals=signals, locked=True) + assert model.calls == 0 + assert result.verdict == "annotated" and result.retry is False and result.severity == rv.DEGRADED + assert result.note == f"{CLAIMS_LEFT};{REFUSED}" and result.answer is None + # 和别的信号一起出现时照常叫模型 + mixed = [*signals, rv.Signal("tool_error", "工具 x 调用失败", rv.BROKEN)] + model = FakeModel({"note": "取数失败了一次。", "answer": "", "retry": True}) + assert (await rv.review(model, question="q", answer="答案", signals=mixed)).note == "取数失败了一次。" + assert model.calls == 1 + + +def _left_after_rewrite(mode: str, violations: list[dict], count: int = 2) -> list[dict]: + return [ + ev(EventType.NODE_STARTED, node_id="write", label="写周报"), + ev(EventType.LOG, node_id="write", level="warn", code="report_repair", count=count, attempt=1, + message=f"报告有 {count} 处未通过核对(「12」、「主要来自」),已要求模型重写(第 1 次)"), + ev(EventType.REPORT_CHECKED, node_id="write", ok=not violations, violations=violations, repairs=1, + analysis_claims=mode), + ] + + +CLAIM = {"code": "analysis_claim_uncited", "text": "主要来自", "message": "「主要来自」是分析判断"} + + +def test_check_mode_leftovers_do_not_make_the_rewrite_count_as_failed(): + """check 下重写一次后仍有的说法违规是标注(同缺失断言,各有自己的信号):「改过一次」算改好了,不再另报 report_repair; + 复核被叫起来时照实说剩下的是哪几处。""" + rows = [*_left_after_rewrite("check", [CLAIM]), warn(CLAIMS_LEFT, "report_analysis_claims")] + assert [s.kind for s in rv.scan(rows, OK_OUTPUT)] == ["report_analysis_claims"] + assert rv.notes(rows) == ["报告「写周报」首稿有 2 处没过核对,重写后只剩已标出的 1 处分析说法"] + # 一处都没改掉:不说「改好了」,剩下的由它自己的信号说 + assert rv.notes(_left_after_rewrite("check", [CLAIM], count=1)) == [] + + +def test_absence_claim_leftovers_are_annotations_too(): + absent = {"code": "absence_claim", "text": "没有查到", "message": "缺失断言"} + rows = _left_after_rewrite("check", [absent, CLAIM], count=3) + assert rv.scan(rows, OK_OUTPUT) == [] + assert rv.notes(rows) == ["报告「写周报」首稿有 3 处没过核对,重写后只剩已标出的 1 处缺失断言、1 处分析说法"] + + +@pytest.mark.parametrize("mode, violations", [ + ("strict", [CLAIM]), # strict 下是出具缺口,不是标注 + ("check", [CLAIM, {"code": "uncited_number", "text": "12"}]), # 还有别的违规 + ("off", [CLAIM]), + ("check", [CLAIM] * 20), # 带满了 20 条:后面可能还有 +]) +def test_other_leftovers_still_count_as_a_failed_rewrite(mode, violations): + rows = _left_after_rewrite(mode, violations) + assert [s.kind for s in rv.scan(rows, OK_OUTPUT)] == ["report_repair"] and rv.notes(rows) == [] + + +@pytest.mark.asyncio +async def test_review_endpoint_does_not_call_the_model_for_a_check_mode_leftover(client, monkeypatch): + model = FakeModel(AssertionError("不该叫模型")) + + async def fake_model(*a, **k): + return model, "fake" + + monkeypatch.setattr("app.api.copilot.get_chat_model", fake_model) + output = {"周报": "周报:销售额 45,678.5元,下降主要来自华东。", + "_evidence": {"report_node": "write", "doc_artifact": "a" * 64, "fields": ["周报"]}} + run_id = await _make_run(client, output, [*_left_after_rewrite("check", [CLAIM]), + warn(CLAIMS_LEFT, "report_analysis_claims")]) + body = (await client.post("/api/copilot/review", json={"run_id": run_id, "question": "q"})).json() + assert model.calls == 0 + assert body["verdict"] == "annotated" and body["note"] == CLAIMS_LEFT and body["retry_instruction"] is None + assert [s["kind"] for s in body["signals"]] == ["report_analysis_claims"]