mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
复核把「首稿没过、重写后已通过」当缺口:report_repair、report_rewrite 这类「改过一次」的信号按同一个报告节点最后一次 report.checked 定档——最终通过的(没有违规;裁判摘要里不再有证据不支持的句子)不进复核的异常清单,复核模型被别的信号叫起来时,说明里另起一节「已经处理好的」写「报告「x」首稿有 N 处没过核对,重写后已全部通过」,不再写成「核对仍未全部通过、可能不准」;最终没过的照旧记缺口。report_repair、report_rewrite 的日志带上处数
This commit is contained in:
@@ -2804,5 +2804,7 @@ async def review_of_run(session: AsyncSession, run: Any, *, question: str, provi
|
||||
chat_model, question=question, answer=answer, signals=signals,
|
||||
# 报告撰写节点产出的答案逐段对应着证据文档:只能加说明,不能改写
|
||||
locked=bool(output and output.get("_evidence")),
|
||||
# 已经处理好的(首稿没过、重写后已全部通过……):不是缺口,照实告诉复核模型
|
||||
settled=rv.notes(rows),
|
||||
)
|
||||
return done(result)
|
||||
|
||||
@@ -791,7 +791,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
while (blocking := _blocking(doc["violations"], numbers)) and repairs < max_repairs \
|
||||
and not (repairs and not _fatal(blocking)):
|
||||
repairs += 1
|
||||
ctx.emit(EventType.LOG, level="warn", code="report_repair",
|
||||
ctx.emit(EventType.LOG, level="warn", code="report_repair", count=len(blocking), attempt=repairs,
|
||||
message=f"报告有 {len(blocking)} 处未通过核对({_named(blocking)}),"
|
||||
f"已要求模型重写(第 {repairs} 次)")
|
||||
messages = [*messages, response, HumanMessage(content=_repair_request(blocking, failed_rule))]
|
||||
@@ -979,7 +979,7 @@ async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str
|
||||
if judge_cfg["rewrite_once"] and bad:
|
||||
texts = judging.unit_texts(doc)
|
||||
sentences = [(texts.get(uid, ""), _rewrite_reason(outcome["verdicts"][uid])) for uid in bad]
|
||||
ctx.emit(EventType.LOG, level="warn", code="report_rewrite",
|
||||
ctx.emit(EventType.LOG, level="warn", code="report_rewrite", count=len(bad),
|
||||
message=f"裁判认为 {len(bad)} 句结论证据不支持({_named([{'text': t} for t, _ in sentences])}),"
|
||||
"已退回写作模型仅修改这几句")
|
||||
again, raw = await draft([*messages, HumanMessage(content=_rewrite_request(sentences))])
|
||||
|
||||
@@ -206,6 +206,8 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list
|
||||
elif etype == EventType.TOOL_END and tool and tool in failed_at:
|
||||
recovered.add(tool)
|
||||
|
||||
#: 每个报告节点最后一次 report.checked:「改过一次」的信号按最终结果定档(_settled)
|
||||
finals = _final_checks(rows)
|
||||
#: 报过的节点失败 (节点 id, 原话):子工作流节点跟着子图里那一步失败时认得出是同一件事
|
||||
failures: list[tuple[str, str]] = []
|
||||
for e in rows:
|
||||
@@ -257,6 +259,10 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list
|
||||
message = str(data.get("message") or "").strip()
|
||||
code = str(data.get("code") or "")
|
||||
kind, severity = WARN_CODES.get(code, ("warn", DEGRADED))
|
||||
if code in _SETTLED_BY_REPORT and _settled(code, finals.get(str(node_id))):
|
||||
# 首稿没过、改过一次之后最终通过了(同一个报告节点最后一次 report.checked):不是缺口,不进异常清单。
|
||||
# 复核模型被别的信号叫起来时,在 notes() 里照实说「重写后已全部通过」
|
||||
continue
|
||||
reason = data.get("reason")
|
||||
if message:
|
||||
add(kind, message[:300], severity, reason if isinstance(reason, str) and reason else None)
|
||||
@@ -282,6 +288,68 @@ def _step_name(node_id: str, titles: Mapping[str, str]) -> str:
|
||||
return " › ".join(titles.get("/".join(parts[:i + 1])) or parts[i] for i in range(len(parts)))
|
||||
|
||||
|
||||
#: 报告节点「改过一次」的信号:按同一个节点最后一次 report.checked 的结果定档(最终通过的不是缺口)
|
||||
_SETTLED_BY_REPORT = ("report_repair", "report_rewrite")
|
||||
#: 证据不支持的几种判定(engine/judge 的 CONTRADICTED 加证据不足):最终还剩这些,改写就没有改好
|
||||
_UNSUPPORTED = ("contradicted", "unsupported", "insufficient")
|
||||
|
||||
|
||||
def _final_checks(rows: list[Any]) -> dict[str, dict[str, Any]]:
|
||||
"""每个报告节点最后一次 report.checked 的内容(节点 id → data)。"""
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for e in rows:
|
||||
if str(_event_field(e, "type", "")) == EventType.REPORT_CHECKED:
|
||||
data = _event_field(e, "data") or {}
|
||||
out[str(_event_field(e, "node_id") or "")] = data if isinstance(data, dict) else {}
|
||||
return out
|
||||
|
||||
|
||||
def _settled(code: str, final: Mapping[str, Any] | None) -> bool:
|
||||
"""「改过一次」之后最终是不是好了:report_repair 看最后一次核对有没有违规(ok、violations 为空);report_rewrite
|
||||
看最后的裁判摘要里还有没有证据不支持的句子。找不到最终结果(老运行、节点失败没写出 report.checked)的当没好。"""
|
||||
if not final:
|
||||
return False
|
||||
if code == "report_repair":
|
||||
return final.get("ok") is True and not final.get("violations")
|
||||
counts = ((final.get("judge") or {}).get("counts") or {}) if isinstance(final.get("judge"), dict) else {}
|
||||
return bool(counts) and not any(int(counts.get(k) or 0) for k in _UNSUPPORTED)
|
||||
|
||||
|
||||
def notes(events: Iterable[Any]) -> list[str]:
|
||||
"""不是异常、但复核模型被别的信号叫起来时该知道的事(review_prompt 的「已经处理好的」):报告首稿没过核对、重写后
|
||||
已全部通过;裁判认为证据不支持的句子改写后都有了依据。只说处理好了的,没好的照旧是信号(scan)。"""
|
||||
rows = list(events)
|
||||
finals = _final_checks(rows)
|
||||
titles: dict[str, str] = {}
|
||||
first: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
times: dict[tuple[str, str], int] = {}
|
||||
for e in rows:
|
||||
etype = str(_event_field(e, "type", ""))
|
||||
data = _event_field(e, "data") or {}
|
||||
node_id = str(_event_field(e, "node_id") or "")
|
||||
if etype == EventType.NODE_STARTED and data.get("label"):
|
||||
titles.setdefault(node_id, str(data["label"]))
|
||||
if etype == EventType.LOG and data.get("code") in _SETTLED_BY_REPORT:
|
||||
key = (node_id, str(data["code"]))
|
||||
first.setdefault(key, data)
|
||||
times[key] = times.get(key, 0) + 1
|
||||
out: list[str] = []
|
||||
for (node_id, code), data in first.items():
|
||||
if not _settled(code, finals.get(node_id)):
|
||||
continue
|
||||
name = f"报告「{titles.get(node_id) or node_id}」"
|
||||
count = data.get("count")
|
||||
if code == "report_repair":
|
||||
head = f"首稿有 {count} 处没过核对" if isinstance(count, int) and count > 0 else "首稿没过核对"
|
||||
again = f"重写 {times[(node_id, code)]} 次后" if times[(node_id, code)] > 1 else "重写后"
|
||||
out.append(f"{name}{head},{again}已全部通过")
|
||||
else:
|
||||
head = f"有 {count} 句结论裁判认为证据不支持" if isinstance(count, int) and count > 0 \
|
||||
else "有几句结论裁判认为证据不支持"
|
||||
out.append(f"{name}{head},改写后都有了依据")
|
||||
return out
|
||||
|
||||
|
||||
def worst(signals: Iterable[Signal]) -> str:
|
||||
"""一组信号里最重的那档。没有信号时返回空串。"""
|
||||
ranked = sorted(signals, key=lambda s: _SEVERITY_RANK.get(s.severity, 1), reverse=True)
|
||||
@@ -347,19 +415,23 @@ LOCKED_NOTE = (
|
||||
)
|
||||
|
||||
|
||||
def review_prompt(question: str, answer: str, signals: list[Signal], *, locked: bool = False) -> str:
|
||||
def review_prompt(question: str, answer: str, signals: list[Signal], *, locked: bool = False,
|
||||
settled: Iterable[str] = ()) -> str:
|
||||
"""拼给复核模型的请求。
|
||||
|
||||
只给信号清单,不给原始事件流:几百条事件塞进 prompt 既贵又没用,而信号
|
||||
已经是规则层提炼过的结论。
|
||||
已经是规则层提炼过的结论。settled 是已经处理好的(notes():首稿没过、重写后已全部通过……):
|
||||
照实告诉复核模型,免得它把「改过一次」写成「仍未全部通过、可能不准」。
|
||||
"""
|
||||
lines = "\n".join(
|
||||
f"- [{'严重' if s.severity == BROKEN else '降级'}] {s.detail}" for s in signals
|
||||
)
|
||||
done = "\n".join(f"- {n}" for n in settled)
|
||||
return (
|
||||
f"用户的问题:\n{question or '(未记录)'}\n\n"
|
||||
f"系统给出的原答案:\n{answer or '(空)'}\n\n"
|
||||
f"检测到的异常:\n{lines}"
|
||||
+ (f"\n\n已经处理好的(不是异常,不要写成缺口或「可能不准」):\n{done}" if done else "")
|
||||
+ (LOCKED_NOTE if locked else "")
|
||||
)
|
||||
|
||||
@@ -392,6 +464,7 @@ class ReviewResult:
|
||||
|
||||
async def review(
|
||||
model: Any, *, question: str, answer: str, signals: list[Signal], locked: bool = False,
|
||||
settled: Iterable[str] = (),
|
||||
) -> ReviewResult:
|
||||
"""叫一次模型复核。调用方负责确保 signals 非空——空信号根本不该走到这里。
|
||||
|
||||
@@ -407,7 +480,8 @@ async def review(
|
||||
s.detail for s in signals
|
||||
)
|
||||
|
||||
messages = [("system", REVIEW_SYSTEM), ("human", review_prompt(question, answer, signals, locked=locked))]
|
||||
messages = [("system", REVIEW_SYSTEM),
|
||||
("human", review_prompt(question, answer, signals, locked=locked, settled=settled))]
|
||||
try:
|
||||
raw = await model.with_structured_output(REVIEW_SCHEMA).ainvoke(messages)
|
||||
if not isinstance(raw, dict):
|
||||
|
||||
@@ -428,3 +428,51 @@ def test_agent_field_warnings_are_degraded_signals():
|
||||
], OK_OUTPUT)
|
||||
assert [(s.kind, s.severity) for s in signals] == [
|
||||
("agent_field_mismatch", rv.DEGRADED), ("agent_field_unverified", rv.DEGRADED)]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 报告「改过一次」的信号按最终结果定档
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _repaired(final_ok: bool) -> list[dict]:
|
||||
violations = [] if final_ok else [{"code": "bare_number", "message": "裸数字 12"}]
|
||||
return [
|
||||
ev(EventType.NODE_STARTED, node_id="write", label="写周报"),
|
||||
ev(EventType.LOG, node_id="write", level="warn", code="report_repair", count=2, attempt=1,
|
||||
message="报告有 2 处未通过核对(裸数字 12、裸数字 30),已要求模型重写(第 1 次)"),
|
||||
ev(EventType.REPORT_CHECKED, node_id="write", ok=final_ok, violations=violations, repairs=1),
|
||||
]
|
||||
|
||||
|
||||
def test_a_repair_that_finally_passed_is_not_a_gap():
|
||||
"""首稿没过核对、重写一次之后通过(最后一次 report.checked ok、没有违规):不进异常清单,复核不为它叫模型;
|
||||
复核被别的信号叫起来时,说明里写「首稿有 N 处没过核对,重写后已全部通过」。"""
|
||||
rows = _repaired(True)
|
||||
assert rv.scan(rows, OK_OUTPUT) == []
|
||||
[note] = rv.notes(rows)
|
||||
assert note == "报告「写周报」首稿有 2 处没过核对,重写后已全部通过"
|
||||
prompt = rv.review_prompt("上周产量", "答案", [rv.Signal("tool_error", "工具 x 调用失败", rv.BROKEN)],
|
||||
settled=rv.notes(rows))
|
||||
assert "重写后已全部通过" in prompt and "不要写成缺口" in prompt
|
||||
assert prompt.index("检测到的异常") < prompt.index("已经处理好的")
|
||||
assert "首稿有 2 处" not in prompt.split("已经处理好的")[0], "处理好了的不在异常清单里"
|
||||
|
||||
|
||||
def test_a_repair_that_still_failed_stays_a_gap():
|
||||
rows = _repaired(False)
|
||||
[signal] = rv.scan(rows, OK_OUTPUT)
|
||||
assert signal.kind == "report_repair" and signal.severity == rv.DEGRADED
|
||||
assert rv.notes(rows) == []
|
||||
|
||||
|
||||
def test_a_rewrite_counts_by_the_final_judgement():
|
||||
base = [ev(EventType.NODE_STARTED, node_id="write", label="写周报"),
|
||||
ev(EventType.LOG, node_id="write", level="warn", code="report_rewrite", count=1,
|
||||
message="裁判认为 1 句结论证据不支持(……),已退回写作模型仅修改这几句")]
|
||||
good = [*base, ev(EventType.REPORT_CHECKED, node_id="write", ok=True, violations=[],
|
||||
judge={"counts": {"supported": 3, "contradicted": 0, "insufficient": 0}})]
|
||||
bad = [*base, ev(EventType.REPORT_CHECKED, node_id="write", ok=True, violations=[],
|
||||
judge={"counts": {"supported": 2, "contradicted": 1, "insufficient": 0}})]
|
||||
assert rv.scan(good, OK_OUTPUT) == [] and rv.notes(good) == ["报告「写周报」有 1 句结论裁判认为证据不支持,改写后都有了依据"]
|
||||
assert [s.kind for s in rv.scan(bad, OK_OUTPUT)] == ["report_rewrite"] and rv.notes(bad) == []
|
||||
|
||||
Reference in New Issue
Block a user