反方审稿复核修正(第 2 轮):照录的理由不露内部标记、裸数字注明无出处,只停用了部分数据源时理由和时间线写明没查哪几个,撞上每日金额上限的几句同一种说法

- 不采信时照录反方的理由改成纯文字:解析不了的引用(⟦?v:…⟧)换成「(引用无效)」,裸数字后面注明「(无出处)」。
  这种句子没有理由文档,界面按文字显示,以前内部的占位标记会原样露在审批卡上,裸数字也不标
- 一句该查的数据源只停用了一部分时照常反查,理由末尾由系统注明哪几个没查(写进理由文档:界面按文档画理由,
  写法和把说明写进原文再组装出来的一模一样),时间线另记一条 challenge_sources_lost。以前只写进给 agent 的任务,
  审稿记录和审批卡上的「经受住」看着就像每个源都查过了。没挂依据的句子按整份报告用到的源查,任务、理由里不再说
  「这句引用的」
- 当天的核查金额用完时,派出去就撞上上限、一次模型都没调用的那句,和后面不再派出的句子用同一种说法
  「已达上限(…),这句未反查」,不再一句「反查未完成:…没能交结果」、一句「这句未反查」
- 判失败的报错只推翻一句时说「请核对这句结论」

没有改的:被推翻的出具缺口读成「反方审稿「反方审稿」:反方审稿推翻了…」并归在「校验不完整」下,审批卡和节点产出
分不出根本没反查,challenge_unreviewed / challenge_sources_lost 不在期 3 约定的日志代码表里——这几处要改约定或
别的包的文件(io.py、decode.ts、NodeCard.tsx、EvidencePane.tsx、ChallengeSummary),留给编排者定
This commit is contained in:
ThearchyHelios
2026-10-05 15:50:20 +08:00
parent 4ceac6b74c
commit 4d364d385b
2 changed files with 221 additions and 23 deletions
+107 -2
View File
@@ -5,7 +5,8 @@
(judge.prepare,遮罩照旧);理由里的数字只能引用它自己的查询格子,裸数字被标出来
- 结论四选一:经受住 / 被削弱 / 被推翻 / 无法判断;完整记录写进 challenge_artifact,随运行封存;台账只记 kind challenge,
它的查询不进任何报告的写作目录
- 结论要有出处:零查询的判什么都记为无法判断,推翻还得引用到自己的查询结果;没有可查的源、模型用不了时不派 agent
- 结论要有出处:零查询的判什么都记为无法判断,推翻还得引用到自己的查询结果,照录反方的理由时不露出内部标记、
裸数字注明无出处;没有可查的源、模型用不了时不派 agent,只停用了一部分源的照常反查,理由和时间线写明哪几个没查
- 被推翻的:on_refuted 为 degrade(默认)时写 issuance_gaps,出具降档;fail 判节点失败;flag 只记录。产出和审批卡里
的 on_refuted 写实际生效的那一种
- 上限:句数、每句查询数、每句时长、全局每日金额(都按发起时的设置快照);全局关闭、报告没有可审的结论句时直通
@@ -330,6 +331,7 @@ async def test_flag_only_records_and_fail_stops_the_run(monkeypatch, shop):
assert failed.status == "failed"
assert failed.error_node_id == "chal"
assert "反方审稿推翻了 1 句结论" in failed.error and SOUTH in failed.error
assert "请核对这句结论" in failed.error, "只推翻了一句,不说「这几句」"
assert "「结论被推翻时」改为「只标注」" in failed.error, "报错里点的选项名和检查器下拉里的一致"
# 判失败之前审稿记录已经存下、时间线照常有完成那一条
[done] = await logs(failed.id, "challenge_done")
@@ -412,6 +414,26 @@ async def test_a_refutation_that_cites_none_of_its_queries_is_not_taken(monkeypa
assert await logs(row.id, "challenge_unreviewed") == [], "查过了,不算没能反查"
@pytest.mark.parametrize("queries, said, quoted", [
([NET], "扣除退款后华南为 [[v:Q1.r99.净销售额]]。", "扣除退款后华南为(引用无效)。"),
([NET], "扣除退款后华南为 [[v:Q2.r1.净销售额]] 元。", "扣除退款后华南为(引用无效)元。"),
([], "华南为 [[v:Q1.r1.净销售额]],不成立。", "华南为(引用无效),不成立。"),
([NET], "扣除退款后华南为 900,低于华东 1200。", "扣除退款后华南为 900(无出处),低于华东 1200(无出处)。"),
])
async def test_an_unfounded_rationale_is_quoted_in_plain_words(monkeypatch, shop, queries, said, quoted):
"""不采信时照录反方的理由,界面按纯文字显示(没有理由文档):解析不了的引用不能把内部的占位标记露出来,换成
「(引用无效)」;裸数字逐个注明「(无出处)」——文档里它们会被标出来,照录成文字时也得标。"""
plays = {"华南销售额超过华东": Play(queries, "refuted", said),
"居各区域之首": Play([REFUNDS], "stands", "没有找到反例。")}
Models(monkeypatch, shop, plays=plays)
row = await finish(graph(shop), run_class="exploratory")
assert row.status == "succeeded", row.error
south = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"])["claims"][1]
assert south["verdict"] == "inconclusive" and south["rationale_doc"] is None
assert south["rationale"].endswith(f"「被推翻」不予采信,记为无法判断。反方的理由:{quoted}"), south["rationale"]
assert "⟦" not in south["rationale"] and "[[" not in south["rationale"]
async def test_claims_whose_sources_are_gone_are_not_sent_out(monkeypatch, shop):
"""报告写完、审稿开始之前数据源被停用:这句没有可查的源,不建模型、不花钱,照实记无法判断并点出是哪个源。"""
real = report_mod._store
@@ -437,6 +459,87 @@ async def test_claims_whose_sources_are_gone_are_not_sent_out(monkeypatch, shop)
assert warn["claims_skipped"] == 2 and f"「{shop}」" in warn["reason"]
async def test_a_claim_with_part_of_its_sources_gone_says_so(monkeypatch, tmp_path, shop):
"""两个数据源的报告,审稿开始前停用了其中一个:引用两个源的那句、没挂依据的那句都只能在剩下的源里查。判断照常
采信,但理由末尾由系统写明哪个源没查(理由文档里也有:界面按文档画),时间线上另记一条——不然「经受住」看着像
两个源都查过了。"""
other, other_id = await _source(tmp_path)
try:
text = ("华东销售额 [[v:Q1.r0.销售额]],退款只有 [[v:Q2.r0.退款笔数]] 笔。[[see:Q1]][[see:Q2]]"
"增长主要来自华东。")
plays = {"退款只有": Play([NET], "stands", "没有找到反例。"),
"增长主要来自": Play([NET], "weakened", "华东净销售额为 [[v:Q1.r0.净销售额]],增长说过了头。")}
real = report_mod._store
async def store_then_disable(doc, ctx):
out = await real(doc, ctx)
async with SessionLocal() as session:
row = await session.get(DataSource, other_id)
row.enabled = False
await session.commit()
return out
monkeypatch.setattr(report_mod, "_store", store_then_disable)
models = Models(monkeypatch, shop, writer=(text,), plays=plays)
spec = chain(
node("start", "input"),
node("fetch", "tool", tool=f"db_query__{shop}", args={"sql": QUERY}),
node("fetch2", "tool", tool=f"db_query__{other}", args={"sql": REFUNDS}),
node("write", "report", "写周报", instructions="写周报"),
node("chal", "challenge", "审稿"),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
)
row = await finish(spec, run_class="exploratory")
assert row.status == "succeeded", row.error
assert models.reviewed() == ["退款只有", "增长主要来自"]
assert models.tools["退款只有"] == {f"db_schema__{shop}", f"db_query__{shop}"}, "停用的源不给工具"
task = models.tasks()["退款只有"]
assert f"「{other}」" in task and "已不存在或已停用" in task, "给 agent 的任务里也说了,免得它把查不到当成没有反例"
assert "这份报告还用到了" in models.tasks()["增长主要来自"], "没挂依据的那句不说「这句引用了」"
record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"])
both, grow = record["claims"]
assert (both["verdict"], grow["verdict"]) == ("stands", "weakened"), "查过、有出处的判断照常采信"
assert both["rationale"] == (f"没有找到反例。\n\n注:这句引用的数据源「{other}」已不存在或已停用,未能查询,"
"以上判断不含其中的数据。")
assert grow["rationale"] == (f"华东净销售额为 1,200,增长说过了头。\n\n注:这份报告用到的数据源「{other}」"
"已不存在或已停用,未能查询,以上判断不含其中的数据。")
for claim in (both, grow):
doc = claim["rationale_doc"]
assert doc["markdown"] == claim["rationale"], "界面按文档画:说明要在文档里"
segments = [s for _, u in iter_units(doc) for s in u["segments"]]
assert "".join(s["text"] for s in segments) == doc["markdown"]
assert [s["id"] for s in segments] == [f"s{i}" for i in range(len(segments))]
*_, note = iter_units(doc)
assert note[1]["kind"] == "connective" and note[1]["cites"] == [], "说明不是反方的结论,也不挂出处"
cited = [s for _, u in iter_units(grow["rationale_doc"]) for s in u["segments"] if s.get("cite")]
assert [s["text"] for s in cited] == ["1,200"], "反方自己的引用照旧解析"
[lost] = await logs(row.id, "challenge_sources_lost")
assert lost["level"] == "warn" and lost["claims"] == 2 and lost["sources"] == [other]
assert f"「{other}」" in lost["message"] and "2 句" in lost["message"]
assert await logs(row.id, "challenge_unreviewed") == [], "两句都查过了,不算没能反查"
finally:
await engines.invalidate(other_id)
def test_the_note_reads_like_one_more_paragraph_of_the_rationale():
"""理由末尾的系统说明照 compose_doc 的样子接上:和把说明写进原文再组装出来的块、句、片段、统计一模一样(片段的
偏移连着、编号接着往下数),只是反方的原文(source)不动、传进来的那份文档不改。"""
from app.engine.evidence import compose_doc
said = "华南只有 900,低于华东。\n\n- 华北为 [[v:Q1.r2.销售额]]"
note = "注:这句引用的数据源「仓库」已不存在或已停用,未能查询,以上判断不含其中的数据。"
doc = compose_doc(said, {}, node_id="chal")
before = json.dumps(doc, ensure_ascii=False, sort_keys=True)
noted = challenge_mod.with_note(doc, note)
again = compose_doc(f"{said}\n\n{note}", {}, node_id="chal")
assert noted["markdown"] == again["markdown"] and noted["blocks"] == again["blocks"]
assert noted["stats"] == again["stats"]
assert noted["source"] == doc["source"] == said
assert json.dumps(doc, ensure_ascii=False, sort_keys=True) == before
# --------------------------------------------------------------------------
# 理由里的裸数字
# --------------------------------------------------------------------------
@@ -563,7 +666,9 @@ async def test_the_daily_cap_stops_the_remaining_claims(monkeypatch, shop):
out = await output_of(row.id, "chal")
assert out["counts"] == {"stands": 0, "weakened": 0, "refuted": 0, "inconclusive": 2}
record = artifact_store.load(out["challenge_artifact"])
assert all("$2.00" in c["rationale"] for c in record["claims"])
words = "已达上限(全局每日核查金额上限 $2.00),这句未反查"
assert [c["rationale"] for c in record["claims"]] == [words, words], \
"第一句派出去就撞上上限、一次模型都没调用,和后面不再派出的句子是同一回事,同一种说法"
[limit] = await logs(row.id, "challenge_limit")
assert limit["limit"] == "daily_max_usd" and limit["claims_skipped"] == 2
assert len(await logs(row.id, "verify_limit")) == 1, "第一句撞上每日上限之后,后面的句子不再派 agent 去试"