From 375f7165e5321ecaeb032df7d327e1fd16fe3ef3 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 5 Oct 2026 18:48:35 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=8D=E6=96=B9=E5=AE=A1=E7=A8=BF=E8=AF=B4?= =?UTF-8?q?=E6=B8=85=E3=80=8C=E6=A0=B9=E6=9C=AC=E6=B2=A1=E5=8F=8D=E6=9F=A5?= =?UTF-8?q?=E3=80=8D=E5=92=8C=E3=80=8C=E5=88=B0=E5=BA=95=E9=99=8D=E4=B8=8D?= =?UTF-8?q?=E9=99=8D=E6=A1=A3=E3=80=8D=EF=BC=9A=E4=BA=A7=E5=87=BA=E5=92=8C?= =?UTF-8?q?=E5=AE=A1=E6=89=B9=E5=8D=A1=E7=9A=84=E6=A6=82=E5=86=B5=E5=8A=A0?= =?UTF-8?q?=20unreviewed=EF=BC=88=E6=A8=A1=E5=9E=8B=E4=B8=8D=E5=8F=AF?= =?UTF-8?q?=E7=94=A8=E3=80=81=E8=A7=A6=E9=A1=B6=E3=80=81=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E4=B8=8D=E5=9C=A8=E4=BA=86=E7=AD=89=E6=B2=A1=E8=83=BD?= =?UTF-8?q?=E5=8F=8D=E6=9F=A5=E7=9A=84=E5=8F=A5=E6=95=B0=EF=BC=8C=E8=A7=A6?= =?UTF-8?q?=E9=A1=B6=E4=BD=86=E6=8C=89=E6=9F=A5=E5=88=B0=E7=9A=84=E6=94=B6?= =?UTF-8?q?=E4=BA=86=E5=B0=BE=E7=9A=84=E4=B8=8D=E7=AE=97=EF=BC=89=E5=92=8C?= =?UTF-8?q?=20degrades=EF=BC=88=E6=8C=89=E9=99=8D=E6=A1=A3=E5=A4=84?= =?UTF-8?q?=E7=BD=AE=E3=80=81=E5=8F=88=E6=8E=A5=E5=9C=A8=E5=B8=A6=E5=87=BA?= =?UTF-8?q?=E5=85=B7=E5=A5=91=E7=BA=A6=E7=9A=84=E6=88=90=E6=9E=9C=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E4=B8=8A=E6=B8=B8=E6=97=B6=E6=89=8D=E4=B8=BA=E7=9C=9F?= =?UTF-8?q?=EF=BC=8C=E4=BE=A7=E6=9E=9D=E4=B8=8A=E3=80=81=E6=88=90=E6=9E=9C?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E6=B2=A1=E5=B8=A6=E5=A5=91=E7=BA=A6=E6=97=B6?= =?UTF-8?q?=E4=B8=BA=E5=81=87=EF=BC=89=EF=BC=9B=E5=8D=87=E7=BA=A7=E5=89=8D?= =?UTF-8?q?=E7=9A=84=E4=BA=A7=E5=87=BA=E4=B8=8D=E5=B8=A6=E8=BF=99=E4=B8=A4?= =?UTF-8?q?=E9=A1=B9=EF=BC=8C=E5=AE=A1=E6=89=B9=E5=8D=A1=E7=85=A7=E6=97=A7?= =?UTF-8?q?=E8=A7=84=E5=88=99=E8=AF=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/nodes/challenge.py | 42 +++++++++++++++++------ backend/tests/test_challenge_node.py | 49 +++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/backend/app/engine/nodes/challenge.py b/backend/app/engine/nodes/challenge.py index 6a02cef1..40286d99 100644 --- a/backend/app/engine/nodes/challenge.py +++ b/backend/app/engine/nodes/challenge.py @@ -22,8 +22,10 @@ 没审全（这两个代码还不在期 3 约定的日志代码表里，界面认不得时照原话显示） 5. 处置：被推翻的按 on_refuted 处置。degrade（默认）在产出里写 issuance_gaps，成果节点的出具契约照单收进缺口、 出具降档（nodes/io.py 的 _upstream_issuance_gaps，所以反方审稿要接在成果节点上游，画布校验会提醒）；fail 判节点 - 失败；flag 只记录。产出和审批卡里的 on_refuted 是实际生效的那一种（只有 degrade / flag，见 _finish）。审批卡 - （human.py 的 _reviewed_reports）附上审稿概况 + 失败；flag 只记录。产出和审批卡里的 on_refuted 是实际生效的那一种（只有 degrade / flag，见 _finish）；degrades + 说被推翻的到底会不会让出具降档（按降档处置、又接在带出具契约的成果节点上游），unreviewed 是没能反查的句数（核查 + 模型不可用、触顶、数据源不在了……这几句记为无法判断，但根本没查成）。审批卡（human.py 的 _reviewed_reports）附上 + 审稿概况，据这两项说「出具会降档」「N 句没能反查」 上限：句数、每句查询次数、每句时长、全局每日金额（核查和裁判各记各的账，engine/spend.py）。都按发起时快照的核查 设置（run.options["verify"]），节点上写了的盖过它：审批停下期间改了设置，恢复、接着跑仍按发起时的那份。没有快照 @@ -56,7 +58,7 @@ from app.engine.judge import _usd from app.engine.merge_query import MERGE_SOURCE from app.engine.replay import _in_graph from app.engine.labels import CHALLENGE_LABEL, CHALLENGE_ON_REFUTED_LABEL -from app.engine.schema import GraphSpec, NodeType, challenge_problems, challenge_report +from app.engine.schema import GraphSpec, NodeType, _ancestors, challenge_problems, challenge_report from app.engine.state import GraphState from app.engine.verify_agent import RATIONALE, VerifyOutcome, run_verify_agent from app.tools.datasource import run_versions @@ -622,6 +624,9 @@ class _Review: unreviewed.append(why) counts = {v: sum(1 for c in claims if c["verdict"] == v) for v in VERDICTS} + # 没能反查的句数：没派出去、派出去却没查成的（unreviewed 里各有原因），加上触顶时没能给出结论的。都记为无法判断， + # 可「无法判断 2 句」既可能是查了、数据不够，也可能是根本没查：审批卡要分开说 + missed = len(unreviewed) + sum(skipped for _, skipped in hit.values()) artifact = await self._store({"report": self.report_id, "doc_artifact": self.doc_artifact, "claims": claims, "model": model}) for limit, (touched, skipped) in hit.items(): @@ -651,7 +656,7 @@ class _Review: ctx.emit(EventType.LOG, level="warn", code="challenge_refuted", items=[{"unit": c["unit"], "text": c["text"]} for c in refuted[:ITEMS_MAX]], message=_refuted_words(refuted)) - return {"artifact": artifact, "counts": counts, "usage": usage, + return {"artifact": artifact, "counts": counts, "usage": usage, "unreviewed": missed, "claims": [{k: c[k] for k in ("unit", "text", "verdict")} for c in claims]} async def _store(self, record: dict[str, Any]) -> str | None: @@ -680,7 +685,11 @@ def _finish(ctx: NodeContext, state: GraphState, outcome: dict[str, Any], *, on_ f"「{CHALLENGE_LABEL['on_refuted']}」改为「{CHALLENGE_ON_REFUTED_LABEL['flag']}」") on_refuted = "degrade" if refuted else "flag" result: dict[str, Any] = {"challenge_artifact": outcome.get("artifact"), "counts": outcome.get("counts") or {}, - "items": pick_items(claims), "on_refuted": on_refuted} + "items": pick_items(claims), "on_refuted": on_refuted, + "degrades": on_refuted == "degrade" and feeds_issuance(ctx.run.spec, ctx.node.id)} + # checkpoint 里升级前审的结果没有这一项：不知道就不写，审批卡照旧不说 + if isinstance(outcome.get("unreviewed"), int): + result["unreviewed"] = outcome["unreviewed"] if refuted and on_refuted == "degrade": result["issuance_gaps"] = [_refuted_gap(refuted)] updates: dict[str, Any] = {"nodes": {ctx.node.id: result}} @@ -700,10 +709,19 @@ def _finish(ctx: NodeContext, state: GraphState, outcome: dict[str, Any], *, on_ # -------------------------------------------------------------------------- +def feeds_issuance(spec: GraphSpec, node_id: str) -> bool: + """这个节点在不在某个带出具契约的成果节点上游。成果节点的出具契约只收上游报上来的缺口（nodes/io.py 的 + _upstream_issuance_gaps）：接在侧枝上、或者在成果之后，被推翻的结论记了缺口也进不了出具。认法同画布校验 + （schema._check_challenge 的「不在成果节点上游」警告）。""" + return any(node_id in _ancestors(spec, n.id) for n in spec.nodes + if n.type == NodeType.OUTPUT and n.config.get("contract")) + + def challenge_summary(spec: GraphSpec, outputs: dict[str, Any], report_id: str) -> dict[str, Any] | None: - """审批卡上这份报告的反方审稿概况 {counts, items, artifact, on_refuted}：审它的、已经审完的反方审稿节点给的。 - 没有（没加反方审稿、还没审到、直通了）返回 None，审批卡不加这个键。几个反方审稿节点审同一份报告时取画布上 - 靠后的那个。只看图和状态：重放时算出来的一样。""" + """审批卡上这份报告的反方审稿概况 {counts, items, artifact, on_refuted, unreviewed, degrades}：审它的、已经审完的 + 反方审稿节点给的。没有（没加反方审稿、还没审到、直通了）返回 None，审批卡不加这个键。几个反方审稿节点审同一份报告 + 时取画布上靠后的那个。只看图和状态：重放时算出来的一样。unreviewed、degrades 是期 4 加的：升级前的产出没有， + 概况也不带（审批卡按旧规则说）。""" found: dict[str, Any] | None = None for node in spec.nodes: result = outputs.get(node.id) @@ -713,8 +731,12 @@ def challenge_summary(spec: GraphSpec, outputs: dict[str, Any], report_id: str) continue found = {"counts": dict(result["counts"]), "items": list(result.get("items") or [])[:ITEMS_MAX], "artifact": result.get("challenge_artifact"), "on_refuted": result.get("on_refuted") or "degrade"} + if isinstance(result.get("unreviewed"), int) and not isinstance(result["unreviewed"], bool): + found["unreviewed"] = result["unreviewed"] + if isinstance(result.get("degrades"), bool): + found["degrades"] = result["degrades"] return found -__all__ = ["CHALLENGE_SYSTEM", "ITEMS_MAX", "VERDICTS", "challenge_summary", "claim_sources", "mark_original", - "pick_claims", "pick_items", "run_challenge", "with_note"] +__all__ = ["CHALLENGE_SYSTEM", "ITEMS_MAX", "VERDICTS", "challenge_summary", "claim_sources", "feeds_issuance", + "mark_original", "pick_claims", "pick_items", "run_challenge", "with_note"] diff --git a/backend/tests/test_challenge_node.py b/backend/tests/test_challenge_node.py index 62c35895..8765767f 100644 --- a/backend/tests/test_challenge_node.py +++ b/backend/tests/test_challenge_node.py @@ -276,6 +276,8 @@ async def test_a_refuted_conclusion_degrades_the_issuance(monkeypatch, shop): {"unit": south, "text": SOUTH, "verdict": "refuted"}] assert out["on_refuted"] == "degrade" assert out["issuance_gaps"] == [f"推翻了 1 句结论：「{SOUTH}」"] + # 两句都真查过了；接在带出具契约的成果节点上游、按降档处置：被推翻的真的会让出具降档 + assert out["unreviewed"] == 0 and out["degrades"] is True record = artifact_store.load(out["challenge_artifact"]) assert set(record) == {"report", "doc_artifact", "claims", "model"} @@ -337,6 +339,7 @@ async def test_flag_only_records_and_fail_stops_the_run(monkeypatch, shop): assert row.output["_issuance"]["tier"] == "formal" and row.output["_issuance"]["gaps"] == [] out = await output_of(row.id, "chal") assert out["on_refuted"] == "flag" and "issuance_gaps" not in out and out["counts"]["refuted"] == 1 + assert out["degrades"] is False, "只标注：被推翻的不会让出具降档" assert len(await logs(row.id, "challenge_refuted")) == 1, "flag 也照实记一条" Models(monkeypatch, shop) @@ -408,6 +411,7 @@ async def test_a_verdict_without_any_query_is_not_taken(monkeypatch, shop): assert east["verdict"] == "inconclusive" and "「经受住」" in east["rationale"] [warn] = await logs(row.id, "challenge_unreviewed") assert warn["claims_skipped"] == 2 and "没有执行成功任何查询" in warn["reason"] + assert out["unreviewed"] == 2, "一条查询都没执行成功：两句都是没能反查，不是查过了判断不了" async def test_a_refutation_that_cites_none_of_its_queries_is_not_taken(monkeypatch, shop): @@ -425,6 +429,7 @@ async def test_a_refutation_that_cites_none_of_its_queries_is_not_taken(monkeypa assert "没有引用" in south["rationale"] and "扣除退款后华南仍低于华东。" in south["rationale"] assert east["verdict"] == "stands", "「经受住」不要求引用：它不改变出具" assert await logs(row.id, "challenge_unreviewed") == [], "查过了，不算没能反查" + assert out["unreviewed"] == 0 @pytest.mark.parametrize("queries, said, quoted", [ @@ -470,6 +475,7 @@ async def test_claims_whose_sources_are_gone_are_not_sent_out(monkeypatch, shop) assert all(f"「{shop}」" in c["rationale"] and "不存在或已停用" in c["rationale"] for c in claims) [warn] = await logs(row.id, "challenge_unreviewed") assert warn["claims_skipped"] == 2 and f"「{shop}」" in warn["reason"] + assert out["unreviewed"] == 2 async def test_a_claim_with_part_of_its_sources_gone_says_so(monkeypatch, tmp_path, shop): @@ -665,6 +671,8 @@ async def test_the_time_limit_applies_to_each_claim_and_still_settles(monkeypatc assert south["verdict"] == "inconclusive" and 1 <= len(south["queries"]) < 20 limits = await logs(row.id, "challenge_limit") assert [(lm["limit"], lm["claims_skipped"]) for lm in limits] == [("timeout_s", 0)] + # 触顶但按查到的结果收了尾的那句，判断是反方自己给的，不算没能反查；另一句一条查询都没执行，算 + assert (await output_of(row.id, "chal"))["unreviewed"] == 1 async def test_the_daily_cap_stops_the_remaining_claims(monkeypatch, shop): @@ -684,6 +692,7 @@ async def test_the_daily_cap_stops_the_remaining_claims(monkeypatch, shop): "第一句派出去就撞上上限、一次模型都没调用，和后面不再派出的句子是同一回事，同一种说法" [limit] = await logs(row.id, "challenge_limit") assert limit["limit"] == "daily_max_usd" and limit["claims_skipped"] == 2 + assert out["unreviewed"] == 2, "撞上每日金额上限没反查的，也是没能反查" assert len(await logs(row.id, "verify_limit")) == 1, "第一句撞上每日上限之后，后面的句子不再派 agent 去试" assert row.output["_issuance"]["tier"] == "formal" @@ -725,13 +734,45 @@ async def test_the_approval_card_carries_the_challenge_summary(monkeypatch, shop out = await output_of(row.id, "chal") assert report["node_id"] == "write" assert report["challenge"] == {"counts": out["counts"], "items": out["items"], - "artifact": out["challenge_artifact"], "on_refuted": "degrade"} + "artifact": out["challenge_artifact"], "on_refuted": "degrade", + "unreviewed": 0, "degrades": False} assert report["challenge"]["counts"]["refuted"] == 1 # human.requested 事件里也带着：回放时界面一样读得到 [requested] = await events(row.id, "human.requested", "gate") assert requested.data["reports"] == approval.payload["reports"] +async def test_the_approval_card_says_whether_refutations_degrade(monkeypatch, shop): + """审批卡上「出具会降档」只在真的会降档时写：degrades 按「按降档处置、又接在带出具契约的成果节点上游」算。 + 审批之后的成果节点带了出具契约：会降档；没带（gated 的缺省）：不会，见上一个用例""" + Models(monkeypatch, shop) + spec = gated(shop) + next(n for n in spec["nodes"] if n["id"] == "out")["data"]["config"]["contract"] = {"report_from": "write"} + _, approval = await paused(spec) + challenge = approval.payload["reports"][0]["challenge"] + assert challenge["degrades"] is True and challenge["on_refuted"] == "degrade" + + +async def test_a_challenge_off_the_issuance_path_does_not_degrade(monkeypatch, shop): + """反方审稿接在侧枝上（成果节点的出具契约收不到它的缺口）：按降档处置也降不了档，产出照实写 degrades 为假""" + Models(monkeypatch, shop) + spec = chain( + node("start", "input"), + node("fetch", "tool", tool=f"db_query__{shop}", args={"sql": QUERY}), + node("write", "report", "写周报", instructions="写周报"), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], + contract={"report_from": "write"}), + ) + spec["nodes"].append(node("chal", "challenge", "审稿", report="write")) + spec["edges"].append({"source": "write", "target": "chal"}) + row = await finish(spec) + assert row.status == "succeeded", row.error + out = await output_of(row.id, "chal") + assert out["on_refuted"] == "degrade" and out["counts"]["refuted"] == 1 and out["issuance_gaps"] + assert out["degrades"] is False + assert row.output["_issuance"]["gaps"] == [], "缺口确实没进出具：和 degrades 说的一致" + + async def test_the_approval_card_shows_what_flag_means(monkeypatch, shop): Models(monkeypatch, shop) _, approval = await paused(gated(shop, on_refuted="flag")) @@ -782,8 +823,11 @@ def test_the_card_only_carries_the_review_of_that_report(): items = [{"unit": "u0", "text": "周报甲的一句结论。", "verdict": "stands"}] outputs = {"chal": {"challenge_artifact": "a1", "counts": counts, "items": items, "on_refuted": "degrade"}} assert challenge_mod.challenge_summary(spec, outputs, "a") == { - "counts": counts, "items": items, "artifact": "a1", "on_refuted": "degrade"} + "counts": counts, "items": items, "artifact": "a1", "on_refuted": "degrade"}, "老产出没有这两个键：概况也不带" assert challenge_mod.challenge_summary(spec, outputs, "b") is None + outputs["chal"].update(unreviewed=1, degrades=True) + assert challenge_mod.challenge_summary(spec, outputs, "a") == { + "counts": counts, "items": items, "artifact": "a1", "on_refuted": "degrade", "unreviewed": 1, "degrades": True} # 直接连着的报告，和直接连着的反方审稿审的那份，都算这个审批节点审的 assert reviewed_report_ids(spec, spec.node_map()["gate"]) == ["a", "b"] @@ -970,6 +1014,7 @@ async def test_an_unavailable_model_leaves_the_claims_inconclusive(monkeypatch, # 时间线上看得出根本没审成：不是只有一条「审了 2 句」 [warn] = await logs(row.id, "challenge_unreviewed") assert warn["level"] == "warn" and warn["claims_skipped"] == 2 and "核查模型不可用" in warn["reason"] + assert out["unreviewed"] == 2 [done] = await logs(row.id, "challenge_done") assert "2 句未能反查" in done["message"]