From 1e87504e5cfb41addb0ca7dedd982cbd2cd38b4b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 19 Oct 2026 16:43:54 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=8D=E6=A0=B8=E6=8A=8A=E3=80=8C=E9=A6=96?= =?UTF-8?q?=E7=A8=BF=E6=B2=A1=E8=BF=87=E3=80=81=E9=87=8D=E5=86=99=E5=90=8E?= =?UTF-8?q?=E5=B7=B2=E9=80=9A=E8=BF=87=E3=80=8D=E5=BD=93=E7=BC=BA=E5=8F=A3?= =?UTF-8?q?=EF=BC=9Areport=5Frepair=E3=80=81report=5Frewrite=20=E8=BF=99?= =?UTF-8?q?=E7=B1=BB=E3=80=8C=E6=94=B9=E8=BF=87=E4=B8=80=E6=AC=A1=E3=80=8D?= =?UTF-8?q?=E7=9A=84=E4=BF=A1=E5=8F=B7=E6=8C=89=E5=90=8C=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E8=8A=82=E7=82=B9=E6=9C=80=E5=90=8E=E4=B8=80?= =?UTF-8?q?=E6=AC=A1=20report.checked=20=E5=AE=9A=E6=A1=A3=E2=80=94?= =?UTF-8?q?=E2=80=94=E6=9C=80=E7=BB=88=E9=80=9A=E8=BF=87=E7=9A=84=EF=BC=88?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E8=BF=9D=E8=A7=84=EF=BC=9B=E8=A3=81=E5=88=A4?= =?UTF-8?q?=E6=91=98=E8=A6=81=E9=87=8C=E4=B8=8D=E5=86=8D=E6=9C=89=E8=AF=81?= =?UTF-8?q?=E6=8D=AE=E4=B8=8D=E6=94=AF=E6=8C=81=E7=9A=84=E5=8F=A5=E5=AD=90?= =?UTF-8?q?=EF=BC=89=E4=B8=8D=E8=BF=9B=E5=A4=8D=E6=A0=B8=E7=9A=84=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E6=B8=85=E5=8D=95=EF=BC=8C=E5=A4=8D=E6=A0=B8=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E8=A2=AB=E5=88=AB=E7=9A=84=E4=BF=A1=E5=8F=B7=E5=8F=AB?= =?UTF-8?q?=E8=B5=B7=E6=9D=A5=E6=97=B6=EF=BC=8C=E8=AF=B4=E6=98=8E=E9=87=8C?= =?UTF-8?q?=E5=8F=A6=E8=B5=B7=E4=B8=80=E8=8A=82=E3=80=8C=E5=B7=B2=E7=BB=8F?= =?UTF-8?q?=E5=A4=84=E7=90=86=E5=A5=BD=E7=9A=84=E3=80=8D=E5=86=99=E3=80=8C?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E3=80=8Cx=E3=80=8D=E9=A6=96=E7=A8=BF?= =?UTF-8?q?=E6=9C=89=20N=20=E5=A4=84=E6=B2=A1=E8=BF=87=E6=A0=B8=E5=AF=B9?= =?UTF-8?q?=EF=BC=8C=E9=87=8D=E5=86=99=E5=90=8E=E5=B7=B2=E5=85=A8=E9=83=A8?= =?UTF-8?q?=E9=80=9A=E8=BF=87=E3=80=8D=EF=BC=8C=E4=B8=8D=E5=86=8D=E5=86=99?= =?UTF-8?q?=E6=88=90=E3=80=8C=E6=A0=B8=E5=AF=B9=E4=BB=8D=E6=9C=AA=E5=85=A8?= =?UTF-8?q?=E9=83=A8=E9=80=9A=E8=BF=87=E3=80=81=E5=8F=AF=E8=83=BD=E4=B8=8D?= =?UTF-8?q?=E5=87=86=E3=80=8D=EF=BC=9B=E6=9C=80=E7=BB=88=E6=B2=A1=E8=BF=87?= =?UTF-8?q?=E7=9A=84=E7=85=A7=E6=97=A7=E8=AE=B0=E7=BC=BA=E5=8F=A3=E3=80=82?= =?UTF-8?q?report=5Frepair=E3=80=81report=5Frewrite=20=E7=9A=84=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E5=B8=A6=E4=B8=8A=E5=A4=84=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/copilot.py | 2 + backend/app/engine/nodes/report.py | 4 +- backend/app/engine/review.py | 80 ++++++++++++++++++++++++++++-- backend/tests/test_review.py | 48 ++++++++++++++++++ 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index 540c082a..a5d0cb0d 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -2804,5 +2804,7 @@ async def review_of_run(session: AsyncSession, run: Any, *, question: str, provi chat_model, question=question, answer=answer, signals=signals, # 报告撰写节点产出的答案逐段对应着证据文档：只能加说明，不能改写 locked=bool(output and output.get("_evidence")), + # 已经处理好的（首稿没过、重写后已全部通过……）：不是缺口，照实告诉复核模型 + settled=rv.notes(rows), ) return done(result) diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index ca4fff39..efffa7d7 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -791,7 +791,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: while (blocking := _blocking(doc["violations"], numbers)) and repairs < max_repairs \ and not (repairs and not _fatal(blocking)): repairs += 1 - ctx.emit(EventType.LOG, level="warn", code="report_repair", + ctx.emit(EventType.LOG, level="warn", code="report_repair", count=len(blocking), attempt=repairs, message=f"报告有 {len(blocking)} 处未通过核对（{_named(blocking)}），" f"已要求模型重写（第 {repairs} 次）") messages = [*messages, response, HumanMessage(content=_repair_request(blocking, failed_rule))] @@ -979,7 +979,7 @@ async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str if judge_cfg["rewrite_once"] and bad: texts = judging.unit_texts(doc) sentences = [(texts.get(uid, ""), _rewrite_reason(outcome["verdicts"][uid])) for uid in bad] - ctx.emit(EventType.LOG, level="warn", code="report_rewrite", + ctx.emit(EventType.LOG, level="warn", code="report_rewrite", count=len(bad), message=f"裁判认为 {len(bad)} 句结论证据不支持（{_named([{'text': t} for t, _ in sentences])}），" "已退回写作模型仅修改这几句") again, raw = await draft([*messages, HumanMessage(content=_rewrite_request(sentences))]) diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index 26b10939..308d06db 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -206,6 +206,8 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list elif etype == EventType.TOOL_END and tool and tool in failed_at: recovered.add(tool) + #: 每个报告节点最后一次 report.checked：「改过一次」的信号按最终结果定档（_settled） + finals = _final_checks(rows) #: 报过的节点失败 (节点 id, 原话)：子工作流节点跟着子图里那一步失败时认得出是同一件事 failures: list[tuple[str, str]] = [] for e in rows: @@ -257,6 +259,10 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list message = str(data.get("message") or "").strip() code = str(data.get("code") or "") kind, severity = WARN_CODES.get(code, ("warn", DEGRADED)) + if code in _SETTLED_BY_REPORT and _settled(code, finals.get(str(node_id))): + # 首稿没过、改过一次之后最终通过了（同一个报告节点最后一次 report.checked）：不是缺口，不进异常清单。 + # 复核模型被别的信号叫起来时，在 notes() 里照实说「重写后已全部通过」 + continue reason = data.get("reason") if message: add(kind, message[:300], severity, reason if isinstance(reason, str) and reason else None) @@ -282,6 +288,68 @@ def _step_name(node_id: str, titles: Mapping[str, str]) -> str: return " › ".join(titles.get("/".join(parts[:i + 1])) or parts[i] for i in range(len(parts))) +#: 报告节点「改过一次」的信号：按同一个节点最后一次 report.checked 的结果定档（最终通过的不是缺口） +_SETTLED_BY_REPORT = ("report_repair", "report_rewrite") +#: 证据不支持的几种判定（engine/judge 的 CONTRADICTED 加证据不足）：最终还剩这些，改写就没有改好 +_UNSUPPORTED = ("contradicted", "unsupported", "insufficient") + + +def _final_checks(rows: list[Any]) -> dict[str, dict[str, Any]]: + """每个报告节点最后一次 report.checked 的内容（节点 id → data）。""" + out: dict[str, dict[str, Any]] = {} + for e in rows: + if str(_event_field(e, "type", "")) == EventType.REPORT_CHECKED: + data = _event_field(e, "data") or {} + out[str(_event_field(e, "node_id") or "")] = data if isinstance(data, dict) else {} + return out + + +def _settled(code: str, final: Mapping[str, Any] | None) -> bool: + """「改过一次」之后最终是不是好了：report_repair 看最后一次核对有没有违规（ok、violations 为空）；report_rewrite + 看最后的裁判摘要里还有没有证据不支持的句子。找不到最终结果（老运行、节点失败没写出 report.checked）的当没好。""" + if not final: + return False + if code == "report_repair": + return final.get("ok") is True and not final.get("violations") + counts = ((final.get("judge") or {}).get("counts") or {}) if isinstance(final.get("judge"), dict) else {} + return bool(counts) and not any(int(counts.get(k) or 0) for k in _UNSUPPORTED) + + +def notes(events: Iterable[Any]) -> list[str]: + """不是异常、但复核模型被别的信号叫起来时该知道的事（review_prompt 的「已经处理好的」）：报告首稿没过核对、重写后 + 已全部通过；裁判认为证据不支持的句子改写后都有了依据。只说处理好了的，没好的照旧是信号（scan）。""" + rows = list(events) + finals = _final_checks(rows) + titles: dict[str, str] = {} + first: dict[tuple[str, str], dict[str, Any]] = {} + times: dict[tuple[str, str], int] = {} + for e in rows: + etype = str(_event_field(e, "type", "")) + data = _event_field(e, "data") or {} + node_id = str(_event_field(e, "node_id") or "") + if etype == EventType.NODE_STARTED and data.get("label"): + titles.setdefault(node_id, str(data["label"])) + if etype == EventType.LOG and data.get("code") in _SETTLED_BY_REPORT: + key = (node_id, str(data["code"])) + first.setdefault(key, data) + times[key] = times.get(key, 0) + 1 + out: list[str] = [] + for (node_id, code), data in first.items(): + if not _settled(code, finals.get(node_id)): + continue + name = f"报告「{titles.get(node_id) or node_id}」" + count = data.get("count") + if code == "report_repair": + head = f"首稿有 {count} 处没过核对" if isinstance(count, int) and count > 0 else "首稿没过核对" + again = f"重写 {times[(node_id, code)]} 次后" if times[(node_id, code)] > 1 else "重写后" + out.append(f"{name}{head}，{again}已全部通过") + else: + head = f"有 {count} 句结论裁判认为证据不支持" if isinstance(count, int) and count > 0 \ + else "有几句结论裁判认为证据不支持" + out.append(f"{name}{head}，改写后都有了依据") + return out + + def worst(signals: Iterable[Signal]) -> str: """一组信号里最重的那档。没有信号时返回空串。""" ranked = sorted(signals, key=lambda s: _SEVERITY_RANK.get(s.severity, 1), reverse=True) @@ -347,19 +415,23 @@ LOCKED_NOTE = ( ) -def review_prompt(question: str, answer: str, signals: list[Signal], *, locked: bool = False) -> str: +def review_prompt(question: str, answer: str, signals: list[Signal], *, locked: bool = False, + settled: Iterable[str] = ()) -> str: """拼给复核模型的请求。 只给信号清单，不给原始事件流：几百条事件塞进 prompt 既贵又没用，而信号 - 已经是规则层提炼过的结论。 + 已经是规则层提炼过的结论。settled 是已经处理好的（notes()：首稿没过、重写后已全部通过……）： + 照实告诉复核模型，免得它把「改过一次」写成「仍未全部通过、可能不准」。 """ lines = "\n".join( f"- [{'严重' if s.severity == BROKEN else '降级'}] {s.detail}" for s in signals ) + done = "\n".join(f"- {n}" for n in settled) return ( f"用户的问题：\n{question or '（未记录）'}\n\n" f"系统给出的原答案：\n{answer or '（空）'}\n\n" f"检测到的异常：\n{lines}" + + (f"\n\n已经处理好的（不是异常，不要写成缺口或「可能不准」）：\n{done}" if done else "") + (LOCKED_NOTE if locked else "") ) @@ -392,6 +464,7 @@ class ReviewResult: async def review( model: Any, *, question: str, answer: str, signals: list[Signal], locked: bool = False, + settled: Iterable[str] = (), ) -> ReviewResult: """叫一次模型复核。调用方负责确保 signals 非空——空信号根本不该走到这里。 @@ -407,7 +480,8 @@ async def review( s.detail for s in signals ) - messages = [("system", REVIEW_SYSTEM), ("human", review_prompt(question, answer, signals, locked=locked))] + messages = [("system", REVIEW_SYSTEM), + ("human", review_prompt(question, answer, signals, locked=locked, settled=settled))] try: raw = await model.with_structured_output(REVIEW_SCHEMA).ainvoke(messages) if not isinstance(raw, dict): diff --git a/backend/tests/test_review.py b/backend/tests/test_review.py index 90cd3e56..d5dbbaf8 100644 --- a/backend/tests/test_review.py +++ b/backend/tests/test_review.py @@ -428,3 +428,51 @@ def test_agent_field_warnings_are_degraded_signals(): ], OK_OUTPUT) assert [(s.kind, s.severity) for s in signals] == [ ("agent_field_mismatch", rv.DEGRADED), ("agent_field_unverified", rv.DEGRADED)] + + +# -------------------------------------------------------------------------- +# 报告「改过一次」的信号按最终结果定档 +# -------------------------------------------------------------------------- + + +def _repaired(final_ok: bool) -> list[dict]: + violations = [] if final_ok else [{"code": "bare_number", "message": "裸数字 12"}] + return [ + ev(EventType.NODE_STARTED, node_id="write", label="写周报"), + ev(EventType.LOG, node_id="write", level="warn", code="report_repair", count=2, attempt=1, + message="报告有 2 处未通过核对（裸数字 12、裸数字 30），已要求模型重写（第 1 次）"), + ev(EventType.REPORT_CHECKED, node_id="write", ok=final_ok, violations=violations, repairs=1), + ] + + +def test_a_repair_that_finally_passed_is_not_a_gap(): + """首稿没过核对、重写一次之后通过（最后一次 report.checked ok、没有违规）：不进异常清单，复核不为它叫模型； + 复核被别的信号叫起来时，说明里写「首稿有 N 处没过核对，重写后已全部通过」。""" + rows = _repaired(True) + assert rv.scan(rows, OK_OUTPUT) == [] + [note] = rv.notes(rows) + assert note == "报告「写周报」首稿有 2 处没过核对，重写后已全部通过" + prompt = rv.review_prompt("上周产量", "答案", [rv.Signal("tool_error", "工具 x 调用失败", rv.BROKEN)], + settled=rv.notes(rows)) + assert "重写后已全部通过" in prompt and "不要写成缺口" in prompt + assert prompt.index("检测到的异常") < prompt.index("已经处理好的") + assert "首稿有 2 处" not in prompt.split("已经处理好的")[0], "处理好了的不在异常清单里" + + +def test_a_repair_that_still_failed_stays_a_gap(): + rows = _repaired(False) + [signal] = rv.scan(rows, OK_OUTPUT) + assert signal.kind == "report_repair" and signal.severity == rv.DEGRADED + assert rv.notes(rows) == [] + + +def test_a_rewrite_counts_by_the_final_judgement(): + base = [ev(EventType.NODE_STARTED, node_id="write", label="写周报"), + ev(EventType.LOG, node_id="write", level="warn", code="report_rewrite", count=1, + message="裁判认为 1 句结论证据不支持（……），已退回写作模型仅修改这几句")] + good = [*base, ev(EventType.REPORT_CHECKED, node_id="write", ok=True, violations=[], + judge={"counts": {"supported": 3, "contradicted": 0, "insufficient": 0}})] + bad = [*base, ev(EventType.REPORT_CHECKED, node_id="write", ok=True, violations=[], + judge={"counts": {"supported": 2, "contradicted": 1, "insufficient": 0}})] + assert rv.scan(good, OK_OUTPUT) == [] and rv.notes(good) == ["报告「写周报」有 1 句结论裁判认为证据不支持，改写后都有了依据"] + assert [s.kind for s in rv.scan(bad, OK_OUTPUT)] == ["report_rewrite"] and rv.notes(bad) == []