yilunjiang/AgentLab
1
0
Fork 0
mirror of https://github.com/ThearchyHelios/AgentLab.git synced 2026-11-15 00:31:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

反方审稿说清「根本没反查」和「到底降不降档」：产出和审批卡的概况加 unreviewed（模型不可用、触顶、数据源不在了等没能反查的句数，触顶但按查到的收了尾的不算）和 degrades（按降档处置、又接在带出具契约的成果节点上游时才为真，侧枝上、成果节点没带契约时为假）；升级前的产出不带这两项，审批卡照旧规则说

Browse Source
This commit is contained in:
ThearchyHelios
2026-10-05 18:48:35 +08:00
parent 7083d5c457
commit 375f7165e5
2 changed files with 79 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/engine/nodes/challenge.py
+32 -10
View File
@@ -22,8 +22,10 @@
没审全（这两个代码还不在期 3 约定的日志代码表里，界面认不得时照原话显示）
5. 处置：被推翻的按 on_refuted 处置。degrade（默认）在产出里写 issuance_gaps，成果节点的出具契约照单收进缺口、
出具降档（nodes/io.py 的 _upstream_issuance_gaps，所以反方审稿要接在成果节点上游，画布校验会提醒）；fail 判节点
失败；flag 只记录。产出和审批卡里的 on_refuted 是实际生效的那一种（只有 degrade / flag，见 _finish）。审批卡
（human.py 的 _reviewed_reports）附上审稿概况
失败；flag 只记录。产出和审批卡里的 on_refuted 是实际生效的那一种（只有 degrade / flag，见 _finish）；degrades
说被推翻的到底会不会让出具降档（按降档处置、又接在带出具契约的成果节点上游），unreviewed 是没能反查的句数（核查
模型不可用、触顶、数据源不在了……这几句记为无法判断，但根本没查成）。审批卡（human.py 的 _reviewed_reports）附上
审稿概况，据这两项说「出具会降档」「N 句没能反查」
上限：句数、每句查询次数、每句时长、全局每日金额（核查和裁判各记各的账，engine/spend.py）。都按发起时快照的核查
设置（run.options["verify"]），节点上写了的盖过它：审批停下期间改了设置，恢复、接着跑仍按发起时的那份。没有快照
@@ -56,7 +58,7 @@ from app.engine.judge import _usd
from app.engine.merge_query import MERGE_SOURCE
from app.engine.replay import _in_graph
from app.engine.labels import CHALLENGE_LABEL, CHALLENGE_ON_REFUTED_LABEL
from app.engine.schema import GraphSpec, NodeType, challenge_problems, challenge_report
from app.engine.schema import GraphSpec, NodeType, _ancestors, challenge_problems, challenge_report
from app.engine.state import GraphState
from app.engine.verify_agent import RATIONALE, VerifyOutcome, run_verify_agent
from app.tools.datasource import run_versions
@@ -622,6 +624,9 @@ class _Review:
unreviewed.append(why)
counts = {v: sum(1 for c in claims if c["verdict"] == v) for v in VERDICTS}
# 没能反查的句数：没派出去、派出去却没查成的（unreviewed 里各有原因），加上触顶时没能给出结论的。都记为无法判断，
# 可「无法判断 2 句」既可能是查了、数据不够，也可能是根本没查：审批卡要分开说
missed = len(unreviewed) + sum(skipped for _, skipped in hit.values())
artifact = await self._store({"report": self.report_id, "doc_artifact": self.doc_artifact, "claims": claims,
"model": model})
for limit, (touched, skipped) in hit.items():
@@ -651,7 +656,7 @@ class _Review:
ctx.emit(EventType.LOG, level="warn", code="challenge_refuted",
items=[{"unit": c["unit"], "text": c["text"]} for c in refuted[:ITEMS_MAX]],
message=_refuted_words(refuted))
return {"artifact": artifact, "counts": counts, "usage": usage,
return {"artifact": artifact, "counts": counts, "usage": usage, "unreviewed": missed,
"claims": [{k: c[k] for k in ("unit", "text", "verdict")} for c in claims]}
async def _store(self, record: dict[str, Any]) -> str | None:
@@ -680,7 +685,11 @@ def _finish(ctx: NodeContext, state: GraphState, outcome: dict[str, Any], *, on_
f"「{CHALLENGE_LABEL['on_refuted']}」改为「{CHALLENGE_ON_REFUTED_LABEL['flag']}」")
on_refuted = "degrade" if refuted else "flag"
result: dict[str, Any] = {"challenge_artifact": outcome.get("artifact"), "counts": outcome.get("counts") or {},
"items": pick_items(claims), "on_refuted": on_refuted}
"items": pick_items(claims), "on_refuted": on_refuted,
"degrades": on_refuted == "degrade" and feeds_issuance(ctx.run.spec, ctx.node.id)}
# checkpoint 里升级前审的结果没有这一项：不知道就不写，审批卡照旧不说
if isinstance(outcome.get("unreviewed"), int):
result["unreviewed"] = outcome["unreviewed"]
if refuted and on_refuted == "degrade":
result["issuance_gaps"] = [_refuted_gap(refuted)]
updates: dict[str, Any] = {"nodes": {ctx.node.id: result}}
@@ -700,10 +709,19 @@ def _finish(ctx: NodeContext, state: GraphState, outcome: dict[str, Any], *, on_
# --------------------------------------------------------------------------
def feeds_issuance(spec: GraphSpec, node_id: str) -> bool:
"""这个节点在不在某个带出具契约的成果节点上游。成果节点的出具契约只收上游报上来的缺口（nodes/io.py 的
_upstream_issuance_gaps）：接在侧枝上、或者在成果之后，被推翻的结论记了缺口也进不了出具。认法同画布校验
（schema._check_challenge 的「不在成果节点上游」警告）。"""
return any(node_id in _ancestors(spec, n.id) for n in spec.nodes
if n.type == NodeType.OUTPUT and n.config.get("contract"))
def challenge_summary(spec: GraphSpec, outputs: dict[str, Any], report_id: str) -> dict[str, Any] | None:
"""审批卡上这份报告的反方审稿概况 {counts, items, artifact, on_refuted}：审它的、已经审完的反方审稿节点给的。
没有（没加反方审稿、还没审到、直通了）返回 None，审批卡不加这个键。几个反方审稿节点审同一份报告时取画布上
靠后的那个。只看图和状态：重放时算出来的一样。"""
"""审批卡上这份报告的反方审稿概况 {counts, items, artifact, on_refuted, unreviewed, degrades}：审它的、已经审完的
反方审稿节点给的。没有（没加反方审稿、还没审到、直通了）返回 None，审批卡不加这个键。几个反方审稿节点审同一份报告
时取画布上靠后的那个。只看图和状态：重放时算出来的一样。unreviewed、degrades 是期 4 加的：升级前的产出没有，
概况也不带（审批卡按旧规则说）。"""
found: dict[str, Any] | None = None
for node in spec.nodes:
result = outputs.get(node.id)
@@ -713,8 +731,12 @@ def challenge_summary(spec: GraphSpec, outputs: dict[str, Any], report_id: str)
continue
found = {"counts": dict(result["counts"]), "items": list(result.get("items") or [])[:ITEMS_MAX],
"artifact": result.get("challenge_artifact"), "on_refuted": result.get("on_refuted") or "degrade"}
if isinstance(result.get("unreviewed"), int) and not isinstance(result["unreviewed"], bool):
found["unreviewed"] = result["unreviewed"]
if isinstance(result.get("degrades"), bool):
found["degrades"] = result["degrades"]
return found
__all__ = ["CHALLENGE_SYSTEM", "ITEMS_MAX", "VERDICTS", "challenge_summary", "claim_sources", "mark_original",
"pick_claims", "pick_items", "run_challenge", "with_note"]
__all__ = ["CHALLENGE_SYSTEM", "ITEMS_MAX", "VERDICTS", "challenge_summary", "claim_sources", "feeds_issuance",
"mark_original", "pick_claims", "pick_items", "run_challenge", "with_note"]
backend/tests/test_challenge_node.py
+47 -2
View File
@@ -276,6 +276,8 @@ async def test_a_refuted_conclusion_degrades_the_issuance(monkeypatch, shop):
{"unit": south, "text": SOUTH, "verdict": "refuted"}]
assert out["on_refuted"] == "degrade"
assert out["issuance_gaps"] == [f"推翻了 1 句结论：「{SOUTH}」"]
# 两句都真查过了；接在带出具契约的成果节点上游、按降档处置：被推翻的真的会让出具降档
assert out["unreviewed"] == 0 and out["degrades"] is True
record = artifact_store.load(out["challenge_artifact"])
assert set(record) == {"report", "doc_artifact", "claims", "model"}
@@ -337,6 +339,7 @@ async def test_flag_only_records_and_fail_stops_the_run(monkeypatch, shop):
assert row.output["_issuance"]["tier"] == "formal" and row.output["_issuance"]["gaps"] == []
out = await output_of(row.id, "chal")
assert out["on_refuted"] == "flag" and "issuance_gaps" not in out and out["counts"]["refuted"] == 1
assert out["degrades"] is False, "只标注：被推翻的不会让出具降档"
assert len(await logs(row.id, "challenge_refuted")) == 1, "flag 也照实记一条"
Models(monkeypatch, shop)
@@ -408,6 +411,7 @@ async def test_a_verdict_without_any_query_is_not_taken(monkeypatch, shop):
assert east["verdict"] == "inconclusive" and "「经受住」" in east["rationale"]
[warn] = await logs(row.id, "challenge_unreviewed")
assert warn["claims_skipped"] == 2 and "没有执行成功任何查询" in warn["reason"]
assert out["unreviewed"] == 2, "一条查询都没执行成功：两句都是没能反查，不是查过了判断不了"
async def test_a_refutation_that_cites_none_of_its_queries_is_not_taken(monkeypatch, shop):
@@ -425,6 +429,7 @@ async def test_a_refutation_that_cites_none_of_its_queries_is_not_taken(monkeypa
assert "没有引用" in south["rationale"] and "扣除退款后华南仍低于华东。" in south["rationale"]
assert east["verdict"] == "stands", "「经受住」不要求引用：它不改变出具"
assert await logs(row.id, "challenge_unreviewed") == [], "查过了，不算没能反查"
assert out["unreviewed"] == 0
@pytest.mark.parametrize("queries, said, quoted", [
@@ -470,6 +475,7 @@ async def test_claims_whose_sources_are_gone_are_not_sent_out(monkeypatch, shop)
assert all(f"「{shop}」" in c["rationale"] and "不存在或已停用" in c["rationale"] for c in claims)
[warn] = await logs(row.id, "challenge_unreviewed")
assert warn["claims_skipped"] == 2 and f"「{shop}」" in warn["reason"]
assert out["unreviewed"] == 2
async def test_a_claim_with_part_of_its_sources_gone_says_so(monkeypatch, tmp_path, shop):
@@ -665,6 +671,8 @@ async def test_the_time_limit_applies_to_each_claim_and_still_settles(monkeypatc
assert south["verdict"] == "inconclusive" and 1 <= len(south["queries"]) < 20
limits = await logs(row.id, "challenge_limit")
assert [(lm["limit"], lm["claims_skipped"]) for lm in limits] == [("timeout_s", 0)]
# 触顶但按查到的结果收了尾的那句，判断是反方自己给的，不算没能反查；另一句一条查询都没执行，算
assert (await output_of(row.id, "chal"))["unreviewed"] == 1
async def test_the_daily_cap_stops_the_remaining_claims(monkeypatch, shop):
@@ -684,6 +692,7 @@ async def test_the_daily_cap_stops_the_remaining_claims(monkeypatch, shop):
"第一句派出去就撞上上限、一次模型都没调用，和后面不再派出的句子是同一回事，同一种说法"
[limit] = await logs(row.id, "challenge_limit")
assert limit["limit"] == "daily_max_usd" and limit["claims_skipped"] == 2
assert out["unreviewed"] == 2, "撞上每日金额上限没反查的，也是没能反查"
assert len(await logs(row.id, "verify_limit")) == 1, "第一句撞上每日上限之后，后面的句子不再派 agent 去试"
assert row.output["_issuance"]["tier"] == "formal"
@@ -725,13 +734,45 @@ async def test_the_approval_card_carries_the_challenge_summary(monkeypatch, shop
out = await output_of(row.id, "chal")
assert report["node_id"] == "write"
assert report["challenge"] == {"counts": out["counts"], "items": out["items"],
"artifact": out["challenge_artifact"], "on_refuted": "degrade"}
"artifact": out["challenge_artifact"], "on_refuted": "degrade",
"unreviewed": 0, "degrades": False}
assert report["challenge"]["counts"]["refuted"] == 1
# human.requested 事件里也带着：回放时界面一样读得到
[requested] = await events(row.id, "human.requested", "gate")
assert requested.data["reports"] == approval.payload["reports"]
async def test_the_approval_card_says_whether_refutations_degrade(monkeypatch, shop):
"""审批卡上「出具会降档」只在真的会降档时写：degrades 按「按降档处置、又接在带出具契约的成果节点上游」算。
审批之后的成果节点带了出具契约：会降档；没带（gated 的缺省）：不会，见上一个用例"""
Models(monkeypatch, shop)
spec = gated(shop)
next(n for n in spec["nodes"] if n["id"] == "out")["data"]["config"]["contract"] = {"report_from": "write"}
_, approval = await paused(spec)
challenge = approval.payload["reports"][0]["challenge"]
assert challenge["degrades"] is True and challenge["on_refuted"] == "degrade"
async def test_a_challenge_off_the_issuance_path_does_not_degrade(monkeypatch, shop):
"""反方审稿接在侧枝上（成果节点的出具契约收不到它的缺口）：按降档处置也降不了档，产出照实写 degrades 为假"""
Models(monkeypatch, shop)
spec = chain(
node("start", "input"),
node("fetch", "tool", tool=f"db_query__{shop}", args={"sql": QUERY}),
node("write", "report", "写周报", instructions="写周报"),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
contract={"report_from": "write"}),
)
spec["nodes"].append(node("chal", "challenge", "审稿", report="write"))
spec["edges"].append({"source": "write", "target": "chal"})
row = await finish(spec)
assert row.status == "succeeded", row.error
out = await output_of(row.id, "chal")
assert out["on_refuted"] == "degrade" and out["counts"]["refuted"] == 1 and out["issuance_gaps"]
assert out["degrades"] is False
assert row.output["_issuance"]["gaps"] == [], "缺口确实没进出具：和 degrades 说的一致"
async def test_the_approval_card_shows_what_flag_means(monkeypatch, shop):
Models(monkeypatch, shop)
_, approval = await paused(gated(shop, on_refuted="flag"))
@@ -782,8 +823,11 @@ def test_the_card_only_carries_the_review_of_that_report():
items = [{"unit": "u0", "text": "周报甲的一句结论。", "verdict": "stands"}]
outputs = {"chal": {"challenge_artifact": "a1", "counts": counts, "items": items, "on_refuted": "degrade"}}
assert challenge_mod.challenge_summary(spec, outputs, "a") == {
"counts": counts, "items": items, "artifact": "a1", "on_refuted": "degrade"}
"counts": counts, "items": items, "artifact": "a1", "on_refuted": "degrade"}, "老产出没有这两个键：概况也不带"
assert challenge_mod.challenge_summary(spec, outputs, "b") is None
outputs["chal"].update(unreviewed=1, degrades=True)
assert challenge_mod.challenge_summary(spec, outputs, "a") == {
"counts": counts, "items": items, "artifact": "a1", "on_refuted": "degrade", "unreviewed": 1, "degrades": True}
# 直接连着的报告，和直接连着的反方审稿审的那份，都算这个审批节点审的
assert reviewed_report_ids(spec, spec.node_map()["gate"]) == ["a", "b"]
@@ -970,6 +1014,7 @@ async def test_an_unavailable_model_leaves_the_claims_inconclusive(monkeypatch,
# 时间线上看得出根本没审成：不是只有一条「审了 2 句」
[warn] = await logs(row.id, "challenge_unreviewed")
assert warn["level"] == "warn" and warn["claims_skipped"] == 2 and "核查模型不可用" in warn["reason"]
assert out["unreviewed"] == 2
[done] = await logs(row.id, "challenge_done")
assert "2 句未能反查" in done["message"]