mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 02:11:58 +00:00
出口契约：报告的正文仍写在代码块里、一句结论都没切出来时记出具缺口，不再完整出具
K5 让报告节点去掉整篇外层的 markdown 围栏，规格有意不去的写法（围栏外有前言、只有开头没有收尾、信息串写成 text）仍会让整段正文落在代码块里：结论句 0 句，裁判不调用、反方审稿记 no_claims，没挂依据的、证据相矛盾的都数出 0 句，正式运行照样完整出具，读的人以为结论都核对过。 - nodes/io.py 的 _fenced_body：文档里一句结论都没有，却有信息串为空 / markdown / md / text 的代码块，里面是成句的文字（一行里有汉字又有中文句读，或五个以上英文单词以句末标点收尾）或者写着引用标记，就是正文被藏进了代码块。信息串为空的代码块里也常是 SQL：看内容，代码注释行（--、//、/* */）里的中文句子不算；sql、python、json 这类信息串不看 - _claims 在结论句策略为 judge 或 require_citation（报告节点的或出具契约自己写的）时，碰到这样的报告记一条缺口 GAP_FENCED_BODY「报告「X」的正文写在代码块里，结论句未能核对」，出具降档。缺口不带核查节点的前缀，时间线写在「校验不完整」下、出具横幅归「校验未全部完成」（run/decode.ts 的 splitIssuanceGaps，报告标题像核查节点名的也一样，界面不用改） - 行为不变：有正常结论句的、代码块是代码的、结论句策略为 off 的 测试（新文件 test_contract_fenced_body）：认得出的几种（前言、没收尾、text、无信息串、大写 MD、英文、只有列表但写着引用）与认不出的几种（sql、python、json、无信息串的 SQL、SQL 注释里的中文句子、数字表、外面有结论句、没有代码块）；正式运行（裁判 + 反方审稿 + 出具契约）里前言、没收尾、text 三种降档并只带这一条缺口，以前是完整出具；正常报告、整篇外层围栏被去掉的、带 SQL 代码块的照常完整出具；require_citation 写在报告节点或契约上同样记，off 不管。去掉挂钩、去掉 0 句前提、去掉注释行排除、去掉信息串过滤、去掉引用信号、去掉句读要求各做过一次，测试变红。
This commit is contained in:
ThearchyHelios
@@ -488,6 +488,50 @@ _ON_UNSUPPORTED = ("degrade", "withhold")
|
||||
#: 要求结论句挂依据的策略：judge 在这一点上和 require_citation 一样严，另外再按裁判的判定判档
|
||||
_CITING = ("require_citation", "judge")
|
||||
|
||||
#: 报告的正文写在代码块里、一句结论都没切出来时的缺口（_fenced_body）。不带核查节点的前缀：界面归在「校验未全部完成」下
|
||||
GAP_FENCED_BODY = "报告「{title}」的正文写在代码块里，结论句未能核对"
|
||||
#: 写的是文字的代码块：信息串为空、markdown、md（报告节点去整篇外层围栏时认的那几个，report._PROSE_INFO），另加 text。
|
||||
#: sql、python、json 这类是模型真要贴的代码
|
||||
_PROSE_FENCES = frozenset({"", "markdown", "md", "text"})
|
||||
_CJK = re.compile(r"[\u4e00-\u9fff]")
|
||||
#: 中文的句读：一行里有汉字、又有它们的，是在写句子（SQL 里的中文列名、别名不带这些）
|
||||
_CJK_PUNCT = re.compile(r"[，。！？；：]")
|
||||
#: 英文句子：五个以上的单词，以句末标点收尾
|
||||
_EN_SENTENCE = re.compile(r"[A-Za-z]+(?:[ ,]+[A-Za-z]+){4,}[.!?](?=\s|$)")
|
||||
#: 代码里的注释行：写着中文句子也是代码的一部分
|
||||
_CODE_COMMENT = re.compile(r"\s*(?:--|//|/\*|\*/)")
|
||||
|
||||
|
||||
def _prose_line(line: str) -> bool:
|
||||
if _CODE_COMMENT.match(line):
|
||||
return False
|
||||
return (len(_CJK.findall(line)) >= 2 and _CJK_PUNCT.search(line) is not None) \
|
||||
or _EN_SENTENCE.search(line) is not None
|
||||
|
||||
|
||||
def _fenced_body(doc: dict[str, Any]) -> bool:
|
||||
"""报告的正文写在代码块里：文档里一句结论都没有，却有信息串为空 / markdown / md / text 的代码块，里面是成句的文字
|
||||
（_prose_line）或者写着引用（[[m:…]] 这类标记，代码里不会这样写）。
|
||||
|
||||
报告节点只在整篇正好是一个代码块时去掉外层围栏（report._unfenced），围栏外有前言、只有开头没有收尾、信息串写成
|
||||
text 的原样交给 compose_doc：整段正文落在代码块里，切不出结论句，裁判没有句子可判、反方审稿记 no_claims，没挂依据
|
||||
的、证据相矛盾的结论句都数出 0 句，出具就成了完整出具。有一句结论的不算：正文没被整个藏起来，结论句照常核对。
|
||||
信息串为空的代码块里也常是 SQL：看内容是不是成句，代码注释里的中文句子不算。"""
|
||||
from app.engine.evidence import iter_units
|
||||
|
||||
units = list(iter_units(doc))
|
||||
if any(unit.get("kind") == "claim" for _, unit in units):
|
||||
return False
|
||||
for block, unit in units:
|
||||
if block.get("type") != "code" or str(block.get("lang") or "").lower() not in _PROSE_FENCES:
|
||||
continue
|
||||
segments = [s for s in unit.get("segments") or [] if isinstance(s, dict)]
|
||||
if any(s.get("ref") for s in segments):
|
||||
return True
|
||||
if any(_prose_line(line) for line in "".join(str(s.get("text") or "") for s in segments).split("\n")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, Any], *, evidence_on: bool,
|
||||
governed: bool = False, uncited: list[dict[str, Any]], doc: dict[str, Any] | None = None,
|
||||
@@ -507,6 +551,9 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
|
||||
数的是这里重新核对出来的 uncited，不信文档里记的 cites。升级前写的报告（节点产出里没有 claims 这个键，
|
||||
包括跨着升级还没跑完的运行）一律不管，契约写了也只记一条缺口：报告生成于升级前，结论句未核对。
|
||||
|
||||
要核对结论句、正文却写在代码块里（一句结论都没切出来，_fenced_body）的，另记一条缺口：数出 0 句不是「核对过、
|
||||
没问题」。
|
||||
"""
|
||||
written = payload.get("claims") if isinstance(payload, dict) else None
|
||||
# 报告节点升级后才在产出里记 claims（off 也记）：没有这个键就是升级前写的报告
|
||||
@@ -526,6 +573,9 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
out["gaps"].append(f"出具契约要求结论句由模型裁判，但报告「{title}」未开启模型裁判，结论句未裁判")
|
||||
if not asks:
|
||||
return
|
||||
if doc is not None and _fenced_body(doc):
|
||||
# 正文在代码块里，一句结论都没切出来：没挂依据的、证据相矛盾的都是 0 句，不能当成「核对过、没问题」
|
||||
out["gaps"].append(GAP_FENCED_BODY.format(title=title))
|
||||
on_uncited = max(asks, key=_ON_UNCITED.index)
|
||||
if governed and on_uncited == "ignore":
|
||||
on_uncited = "degrade"
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""出口契约：报告的正文仍写在代码块里、一句结论都没切出来时记出具缺口（nodes/io.py 的 _fenced_body）。
|
||||
|
||||
K5 让报告节点去掉整篇外层的 markdown 围栏（report._unfenced），规格有意不去的写法仍会让正文落在代码块里：围栏外有
|
||||
前言、只有开头没有收尾、信息串写成 text。这时结论句是 0 句：裁判没有句子可判、反方审稿记 no_claims，以前正式运行
|
||||
照样完整出具，读的人以为结论都核对过。
|
||||
|
||||
- 结论句策略为 judge 或 require_citation（报告节点的，或出具契约自己写的）、结论句 0 句、又有信息串为空 / markdown /
|
||||
md / text 的代码块，里面是成句的文字或写着引用：记一条缺口，降档。缺口不带核查节点的前缀，界面上归在「校验未全部
|
||||
完成」下（run/decode.ts 的 splitIssuanceGaps）
|
||||
- 行为不变：有正常结论句的；代码块是 sql、python 这类代码的（信息串为空、里面是代码的也算，代码注释里的中文句子不算）；
|
||||
结论句策略为 off 的
|
||||
|
||||
模型一律是 MockChatModel 剧本，数据源是现造的 SQLite 文件，区域、金额都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from app.engine.evidence import compose_doc
|
||||
from app.engine.nodes.io import GAP_FENCED_BODY, _fenced_body
|
||||
from app.providers.mock_model import MockChatModel
|
||||
from tests.test_report_fenced import ( # noqa: F401 - engine_up、shop 是夹具，导入即可用
|
||||
EAST,
|
||||
SOUTH,
|
||||
SQL,
|
||||
TEXT,
|
||||
Models,
|
||||
audited,
|
||||
chain,
|
||||
engine_up,
|
||||
fenced,
|
||||
finish,
|
||||
logs,
|
||||
node,
|
||||
output_of,
|
||||
report_of,
|
||||
shop,
|
||||
)
|
||||
|
||||
#: 写作者把正文包进代码块、又不是整篇只有一个代码块（报告节点不去围栏）的几种写法
|
||||
PREAMBLE = f"以下是本周的销售报告：\n\n{fenced(TEXT)}"
|
||||
UNCLOSED = f"```markdown\n{TEXT}"
|
||||
AS_TEXT = fenced(TEXT, "text")
|
||||
HIDDEN = [pytest.param(PREAMBLE, id="preamble"), pytest.param(UNCLOSED, id="unclosed"),
|
||||
pytest.param(AS_TEXT, id="text")]
|
||||
|
||||
|
||||
def doc_of(raw: str) -> dict:
|
||||
return compose_doc(raw, {}, node_id="write")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 认得出哪些代码块里写的是正文
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [
|
||||
pytest.param("以下是本周的销售报告：\n\n```markdown\n华东销售额 1500，居各区域之首。\n```", id="preamble"),
|
||||
pytest.param("```markdown\n## 本周概览\n\n华东销售额 1500，居各区域之首。", id="unclosed"),
|
||||
pytest.param("```text\n华东销售额 1500，居各区域之首。\n```", id="text"),
|
||||
pytest.param("```\n华东销售额 1500，居各区域之首。\n```", id="no-info"),
|
||||
pytest.param("```MD\n华东销售额 1500，居各区域之首。\n```", id="upper-md"),
|
||||
pytest.param("```markdown\nEast region sales rose sharply this week.\n```", id="english"),
|
||||
# 只有列表、没有句读，可写着报告的引用：代码里不会这样写
|
||||
pytest.param("以下是周报：\n\n```markdown\n- 华东销售额 [[m:gmv]]\n- 华南销售额 [[m:gmv_s]]\n```", id="markers"),
|
||||
])
|
||||
def test_a_report_body_in_a_prose_code_block_is_recognised(raw):
|
||||
doc = doc_of(raw)
|
||||
assert doc["stats"]["claims"] == 0, "背景：正文在代码块里，一句结论都切不出来"
|
||||
assert _fenced_body(doc) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [
|
||||
# 代码：信息串写明了语言
|
||||
pytest.param("```sql\nSELECT region, SUM(amount) AS gmv FROM orders WHERE amount > 100 GROUP BY region\n```", id="sql"),
|
||||
pytest.param("```python\nprint('华东销售额最高，华南次之。')\n```", id="python"),
|
||||
pytest.param('```json\n{"区域": "华东，华南"}\n```', id="json"),
|
||||
# 信息串为空、里面是代码：没有成句的文字；代码注释里的中文句子不算
|
||||
pytest.param("```\nSELECT region, SUM(amount) FROM orders GROUP BY region\n```", id="bare-sql"),
|
||||
pytest.param("```\n-- 按支付日统计，排除退款。\nSELECT SUM(amount) FROM orders\n```", id="sql-comment"),
|
||||
pytest.param("```\n/* 按支付日统计，排除退款。 */\nSELECT SUM(amount) FROM orders\n```", id="block-comment"),
|
||||
pytest.param("```text\n区域 销售额\n华东 1500\n华南 900\n```", id="table-of-numbers"),
|
||||
# 有正常的结论句：正文没被藏起来
|
||||
pytest.param("华东销售额超过华南。\n\n```markdown\n华东销售额 1500，居各区域之首。\n```", id="claims-outside"),
|
||||
# 没有代码块
|
||||
pytest.param("以下是本周的销售报告。", id="no-code"),
|
||||
])
|
||||
def test_code_or_a_report_with_claims_is_not_taken_for_a_hidden_body(raw):
|
||||
assert _fenced_body(doc_of(raw)) is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 端到端：正式运行（裁判 + 反方审稿 + 出具契约）
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def gap(title: str = "写周报") -> str:
|
||||
return GAP_FENCED_BODY.format(title=title)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", HIDDEN)
|
||||
async def test_a_body_left_in_a_code_block_degrades_the_formal_issuance(monkeypatch, shop, raw):
|
||||
"""正文落在代码块里：裁判没有句子可判（不调用）、反方审稿记 no_claims，出口契约不再完整出具，记一条缺口说清原因。"""
|
||||
models = Models(monkeypatch, shop, raw)
|
||||
row = await finish(audited(shop), run_class="formal")
|
||||
assert row.status == "succeeded", row.error
|
||||
_, doc = await report_of(row.id)
|
||||
assert doc["source"] == raw and doc["stats"]["claims"] == 0, "背景：报告节点不去这几种围栏"
|
||||
assert "code" in [b["type"] for b in doc["blocks"]]
|
||||
assert models.judges == [] and models.reviewed == []
|
||||
assert [d["reason"] for d in await logs(row.id, "challenge_skipped")] == ["no_claims"]
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "degraded", issuance
|
||||
assert issuance["gaps"] == [gap()], issuance["gaps"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [
|
||||
pytest.param(TEXT, id="plain"),
|
||||
pytest.param(fenced(TEXT), id="whole-report-unwrapped"),
|
||||
pytest.param(f"华东销售额 [[v:Q1.r0.销售额]]，居各区域之首。[[see:Q1]]\n\n{SQL}\n\n华南销售额超过华北。[[see:Q1]]",
|
||||
id="sql-block"),
|
||||
])
|
||||
async def test_a_report_with_claims_is_issued_as_before(monkeypatch, shop, raw):
|
||||
"""有正常的结论句（整篇外层围栏已由报告节点去掉的也是）、代码块是 SQL 的：判过、审过，照常完整出具，不记这条缺口。"""
|
||||
models = Models(monkeypatch, shop, raw)
|
||||
row = await finish(audited(shop), run_class="formal")
|
||||
assert row.status == "succeeded", row.error
|
||||
assert sorted(models.reviewed) == sorted([EAST, SOUTH])
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "formal" and issuance["gaps"] == [], issuance
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 结论句策略：require_citation 同样记，off 不管
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
KPI = "{'gmv': 45678.5, 'orders': 1234}"
|
||||
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
||||
#: 前言 + 代码块包住正文，正文里的引用都解析得了
|
||||
CITED = "以下是本周周报：\n\n```markdown\n本周销售额 [[m:gmv]]，较上周有所增长。[[see:m:gmv]]\n```"
|
||||
|
||||
|
||||
def weekly(*, claims: str | None = None, contract_claims: str | None = None) -> dict:
|
||||
"""输入 → 取数 → 口径卡 → 报告（claims 是报告节点的结论句策略）→ 成果（引用模式的出具契约）。"""
|
||||
return chain(
|
||||
node("start", "input"),
|
||||
node("fetch", "transform", mode="expression", expression=KPI, assign_to="kpi"),
|
||||
node("caliber", "metrics", caliber="周报口径", caliber_version="v1", metrics=METRICS),
|
||||
node("write", "report", "写周报", instructions="写周报", max_repairs=0,
|
||||
**({"claims": claims} if claims else {})),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
|
||||
contract={"report_from": "write", "metrics_from": ["caliber"],
|
||||
**({"claims": contract_claims} if contract_claims else {})}),
|
||||
)
|
||||
|
||||
|
||||
def writer(monkeypatch, text: str) -> None:
|
||||
monkeypatch.setattr(MockChatModel, "_decide", lambda self, messages: AIMessage(content=text))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("claims, contract_claims", [
|
||||
pytest.param("require_citation", None, id="report-node"),
|
||||
pytest.param(None, "require_citation", id="contract"),
|
||||
])
|
||||
async def test_require_citation_records_the_gap_too(monkeypatch, claims, contract_claims):
|
||||
"""要求结论句挂依据（报告节点或出具契约写的）：同样数不出结论句，记同一条缺口。"""
|
||||
writer(monkeypatch, CITED)
|
||||
row = await finish(weekly(claims=claims, contract_claims=contract_claims))
|
||||
assert row.status == "succeeded", row.error
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "degraded" and issuance["gaps"] == [gap()], issuance
|
||||
assert issuance["claims"]["uncited_claims"] == 0
|
||||
|
||||
|
||||
async def test_without_a_claims_policy_nothing_changes(monkeypatch):
|
||||
"""结论句策略为 off：出口不核对结论句，同样的报告照旧完整出具（数字都在代码块里、也都有出处）。"""
|
||||
writer(monkeypatch, CITED)
|
||||
row = await finish(weekly())
|
||||
assert row.status == "succeeded", row.error
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "formal" and issuance["gaps"] == [], issuance
|
||||
assert "claims" not in issuance
|
||||
Reference in New Issue
Block a user