复核把「首稿没过、重写后已通过」当缺口:report_repair、report_rewrite 这类「改过一次」的信号按同一个报告节点最后一次 report.checked 定档——最终通过的(没有违规;裁判摘要里不再有证据不支持的句子)不进复核的异常清单,复核模型被别的信号叫起来时,说明里另起一节「已经处理好的」写「报告「x」首稿有 N 处没过核对,重写后已全部通过」,不再写成「核对仍未全部通过、可能不准」;最终没过的照旧记缺口。report_repair、report_rewrite 的日志带上处数

This commit is contained in:
ThearchyHelios
2026-10-19 16:43:54 +08:00
parent 43909ea3f0
commit 1e87504e5c
4 changed files with 129 additions and 5 deletions
+2
View File
@@ -2804,5 +2804,7 @@ async def review_of_run(session: AsyncSession, run: Any, *, question: str, provi
chat_model, question=question, answer=answer, signals=signals,
# 报告撰写节点产出的答案逐段对应着证据文档:只能加说明,不能改写
locked=bool(output and output.get("_evidence")),
# 已经处理好的(首稿没过、重写后已全部通过……):不是缺口,照实告诉复核模型
settled=rv.notes(rows),
)
return done(result)
+2 -2
View File
@@ -791,7 +791,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
while (blocking := _blocking(doc["violations"], numbers)) and repairs < max_repairs \
and not (repairs and not _fatal(blocking)):
repairs += 1
ctx.emit(EventType.LOG, level="warn", code="report_repair",
ctx.emit(EventType.LOG, level="warn", code="report_repair", count=len(blocking), attempt=repairs,
message=f"报告有 {len(blocking)} 处未通过核对({_named(blocking)}),"
f"已要求模型重写(第 {repairs} 次)")
messages = [*messages, response, HumanMessage(content=_repair_request(blocking, failed_rule))]
@@ -979,7 +979,7 @@ async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str
if judge_cfg["rewrite_once"] and bad:
texts = judging.unit_texts(doc)
sentences = [(texts.get(uid, ""), _rewrite_reason(outcome["verdicts"][uid])) for uid in bad]
ctx.emit(EventType.LOG, level="warn", code="report_rewrite",
ctx.emit(EventType.LOG, level="warn", code="report_rewrite", count=len(bad),
message=f"裁判认为 {len(bad)} 句结论证据不支持({_named([{'text': t} for t, _ in sentences])}),"
"已退回写作模型仅修改这几句")
again, raw = await draft([*messages, HumanMessage(content=_rewrite_request(sentences))])
+77 -3
View File
@@ -206,6 +206,8 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list
elif etype == EventType.TOOL_END and tool and tool in failed_at:
recovered.add(tool)
#: 每个报告节点最后一次 report.checked:「改过一次」的信号按最终结果定档(_settled)
finals = _final_checks(rows)
#: 报过的节点失败 (节点 id, 原话):子工作流节点跟着子图里那一步失败时认得出是同一件事
failures: list[tuple[str, str]] = []
for e in rows:
@@ -257,6 +259,10 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list
message = str(data.get("message") or "").strip()
code = str(data.get("code") or "")
kind, severity = WARN_CODES.get(code, ("warn", DEGRADED))
if code in _SETTLED_BY_REPORT and _settled(code, finals.get(str(node_id))):
# 首稿没过、改过一次之后最终通过了(同一个报告节点最后一次 report.checked):不是缺口,不进异常清单。
# 复核模型被别的信号叫起来时,在 notes() 里照实说「重写后已全部通过」
continue
reason = data.get("reason")
if message:
add(kind, message[:300], severity, reason if isinstance(reason, str) and reason else None)
@@ -282,6 +288,68 @@ def _step_name(node_id: str, titles: Mapping[str, str]) -> str:
return " › ".join(titles.get("/".join(parts[:i + 1])) or parts[i] for i in range(len(parts)))
#: 报告节点「改过一次」的信号:按同一个节点最后一次 report.checked 的结果定档(最终通过的不是缺口)
_SETTLED_BY_REPORT = ("report_repair", "report_rewrite")
#: 证据不支持的几种判定(engine/judge 的 CONTRADICTED 加证据不足):最终还剩这些,改写就没有改好
_UNSUPPORTED = ("contradicted", "unsupported", "insufficient")
def _final_checks(rows: list[Any]) -> dict[str, dict[str, Any]]:
"""每个报告节点最后一次 report.checked 的内容(节点 id → data)。"""
out: dict[str, dict[str, Any]] = {}
for e in rows:
if str(_event_field(e, "type", "")) == EventType.REPORT_CHECKED:
data = _event_field(e, "data") or {}
out[str(_event_field(e, "node_id") or "")] = data if isinstance(data, dict) else {}
return out
def _settled(code: str, final: Mapping[str, Any] | None) -> bool:
"""「改过一次」之后最终是不是好了:report_repair 看最后一次核对有没有违规(ok、violations 为空);report_rewrite
看最后的裁判摘要里还有没有证据不支持的句子。找不到最终结果(老运行、节点失败没写出 report.checked)的当没好。"""
if not final:
return False
if code == "report_repair":
return final.get("ok") is True and not final.get("violations")
counts = ((final.get("judge") or {}).get("counts") or {}) if isinstance(final.get("judge"), dict) else {}
return bool(counts) and not any(int(counts.get(k) or 0) for k in _UNSUPPORTED)
def notes(events: Iterable[Any]) -> list[str]:
"""不是异常、但复核模型被别的信号叫起来时该知道的事(review_prompt 的「已经处理好的」):报告首稿没过核对、重写后
已全部通过;裁判认为证据不支持的句子改写后都有了依据。只说处理好了的,没好的照旧是信号(scan)。"""
rows = list(events)
finals = _final_checks(rows)
titles: dict[str, str] = {}
first: dict[tuple[str, str], dict[str, Any]] = {}
times: dict[tuple[str, str], int] = {}
for e in rows:
etype = str(_event_field(e, "type", ""))
data = _event_field(e, "data") or {}
node_id = str(_event_field(e, "node_id") or "")
if etype == EventType.NODE_STARTED and data.get("label"):
titles.setdefault(node_id, str(data["label"]))
if etype == EventType.LOG and data.get("code") in _SETTLED_BY_REPORT:
key = (node_id, str(data["code"]))
first.setdefault(key, data)
times[key] = times.get(key, 0) + 1
out: list[str] = []
for (node_id, code), data in first.items():
if not _settled(code, finals.get(node_id)):
continue
name = f"报告「{titles.get(node_id) or node_id}」"
count = data.get("count")
if code == "report_repair":
head = f"首稿有 {count} 处没过核对" if isinstance(count, int) and count > 0 else "首稿没过核对"
again = f"重写 {times[(node_id, code)]} 次后" if times[(node_id, code)] > 1 else "重写后"
out.append(f"{name}{head},{again}已全部通过")
else:
head = f"有 {count} 句结论裁判认为证据不支持" if isinstance(count, int) and count > 0 \
else "有几句结论裁判认为证据不支持"
out.append(f"{name}{head},改写后都有了依据")
return out
def worst(signals: Iterable[Signal]) -> str:
"""一组信号里最重的那档。没有信号时返回空串。"""
ranked = sorted(signals, key=lambda s: _SEVERITY_RANK.get(s.severity, 1), reverse=True)
@@ -347,19 +415,23 @@ LOCKED_NOTE = (
)
def review_prompt(question: str, answer: str, signals: list[Signal], *, locked: bool = False) -> str:
def review_prompt(question: str, answer: str, signals: list[Signal], *, locked: bool = False,
settled: Iterable[str] = ()) -> str:
"""拼给复核模型的请求。
只给信号清单,不给原始事件流:几百条事件塞进 prompt 既贵又没用,而信号
已经是规则层提炼过的结论。
已经是规则层提炼过的结论。settled 是已经处理好的(notes():首稿没过、重写后已全部通过……):
照实告诉复核模型,免得它把「改过一次」写成「仍未全部通过、可能不准」。
"""
lines = "\n".join(
f"- [{'严重' if s.severity == BROKEN else '降级'}] {s.detail}" for s in signals
)
done = "\n".join(f"- {n}" for n in settled)
return (
f"用户的问题:\n{question or '(未记录)'}\n\n"
f"系统给出的原答案:\n{answer or '(空)'}\n\n"
f"检测到的异常:\n{lines}"
+ (f"\n\n已经处理好的(不是异常,不要写成缺口或「可能不准」):\n{done}" if done else "")
+ (LOCKED_NOTE if locked else "")
)
@@ -392,6 +464,7 @@ class ReviewResult:
async def review(
model: Any, *, question: str, answer: str, signals: list[Signal], locked: bool = False,
settled: Iterable[str] = (),
) -> ReviewResult:
"""叫一次模型复核。调用方负责确保 signals 非空——空信号根本不该走到这里。
@@ -407,7 +480,8 @@ async def review(
s.detail for s in signals
)
messages = [("system", REVIEW_SYSTEM), ("human", review_prompt(question, answer, signals, locked=locked))]
messages = [("system", REVIEW_SYSTEM),
("human", review_prompt(question, answer, signals, locked=locked, settled=settled))]
try:
raw = await model.with_structured_output(REVIEW_SCHEMA).ainvoke(messages)
if not isinstance(raw, dict):
+48
View File
@@ -428,3 +428,51 @@ def test_agent_field_warnings_are_degraded_signals():
], OK_OUTPUT)
assert [(s.kind, s.severity) for s in signals] == [
("agent_field_mismatch", rv.DEGRADED), ("agent_field_unverified", rv.DEGRADED)]
# --------------------------------------------------------------------------
# 报告「改过一次」的信号按最终结果定档
# --------------------------------------------------------------------------
def _repaired(final_ok: bool) -> list[dict]:
violations = [] if final_ok else [{"code": "bare_number", "message": "裸数字 12"}]
return [
ev(EventType.NODE_STARTED, node_id="write", label="写周报"),
ev(EventType.LOG, node_id="write", level="warn", code="report_repair", count=2, attempt=1,
message="报告有 2 处未通过核对(裸数字 12、裸数字 30),已要求模型重写(第 1 次)"),
ev(EventType.REPORT_CHECKED, node_id="write", ok=final_ok, violations=violations, repairs=1),
]
def test_a_repair_that_finally_passed_is_not_a_gap():
"""首稿没过核对、重写一次之后通过(最后一次 report.checked ok、没有违规):不进异常清单,复核不为它叫模型;
复核被别的信号叫起来时,说明里写「首稿有 N 处没过核对,重写后已全部通过」。"""
rows = _repaired(True)
assert rv.scan(rows, OK_OUTPUT) == []
[note] = rv.notes(rows)
assert note == "报告「写周报」首稿有 2 处没过核对,重写后已全部通过"
prompt = rv.review_prompt("上周产量", "答案", [rv.Signal("tool_error", "工具 x 调用失败", rv.BROKEN)],
settled=rv.notes(rows))
assert "重写后已全部通过" in prompt and "不要写成缺口" in prompt
assert prompt.index("检测到的异常") < prompt.index("已经处理好的")
assert "首稿有 2 处" not in prompt.split("已经处理好的")[0], "处理好了的不在异常清单里"
def test_a_repair_that_still_failed_stays_a_gap():
rows = _repaired(False)
[signal] = rv.scan(rows, OK_OUTPUT)
assert signal.kind == "report_repair" and signal.severity == rv.DEGRADED
assert rv.notes(rows) == []
def test_a_rewrite_counts_by_the_final_judgement():
base = [ev(EventType.NODE_STARTED, node_id="write", label="写周报"),
ev(EventType.LOG, node_id="write", level="warn", code="report_rewrite", count=1,
message="裁判认为 1 句结论证据不支持(……),已退回写作模型仅修改这几句")]
good = [*base, ev(EventType.REPORT_CHECKED, node_id="write", ok=True, violations=[],
judge={"counts": {"supported": 3, "contradicted": 0, "insufficient": 0}})]
bad = [*base, ev(EventType.REPORT_CHECKED, node_id="write", ok=True, violations=[],
judge={"counts": {"supported": 2, "contradicted": 1, "insufficient": 0}})]
assert rv.scan(good, OK_OUTPUT) == [] and rv.notes(good) == ["报告「写周报」有 1 句结论裁判认为证据不支持,改写后都有了依据"]
assert [s.kind for s in rv.scan(bad, OK_OUTPUT)] == ["report_rewrite"] and rv.notes(bad) == []