mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 19:12:00 +00:00
ThearchyHelios
bd8306bf95
- 判定取值改为：支持 / 部分支持 / 证据相矛盾 / 证据不足 / 不是结论。只有证据和句子冲突才判相矛盾； 证据里没有相关信息判证据不足，并写明缺什么（比如 SQL、字段清单） - 出具判档不变松：证据相矛盾按 on_unsupported 降档或不予出具；证据不足记一条缺口，最多降档、不能判完整出具； 改造前的「证据不支持」照旧当相矛盾处理 - 补全证据摘录：表名附字段清单和类型（最多 60 个，快照不全时注明，遮罩列不列）；口径卡指标附上每个输入 取自哪次查询、哪一格，以及那次查询的 SQL；一律只读封存范围内的工件 - 理由上限从 60 字放宽到 120 字，在句子边界截断，不再截在半句中间 - 裁判模型：按需裁判也判断是否和写作模型相同、是否在价格表里，证据面板里醒目提示； 设置页没配裁判模型时写明「未单独配置，将使用助手的模型」 - 新判定带规则版本：改造前的判定在界面上标「按旧规则判定」，可以请模型按当前规则重新判断 - 界面：证据相矛盾用红色；证据不足用中性色和 ○，写明缺什么；讲方法、讲结构的句子在证据面板里直接列出 对应的 SQL 和字段清单；出具横幅、审计表、报告节点印章的计数都分开这两档 - 顺带写稳 check-publish 里一条等按钮状态的断言（并行时偶发）
349 lines
18 KiB
Python
349 lines
18 KiB
Python
"""四期：受管模板放开 claims: judge（结论句由另一个模型按证据逐句裁判），但要写明预算。
|
||
|
||
- G3：报告撰写节点的 claims 在受管级别认 require_citation 或 judge；写 judge 的，生效的 judge 配置里
|
||
必须有 max_cost_usd 这个键——写金额，或者显式写 null 表示不限。没写就是 error（已发布级别只给警告）：
|
||
受管出具不能让每份报告的裁判花费悄悄按默认值走，也不能悄悄不设上限
|
||
- 发布前自动修复：缺预算是 choice，候选是默认金额和「不限」，不替人选；G3 另外那几项的一键修复不再把
|
||
judge 改回 require_citation
|
||
- 降低要求的判断：require_citation → judge 是往严里改；judge → require_citation / off / 删掉都算放宽，
|
||
报告节点上和出具契约里一样
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
|
||
import pytest
|
||
|
||
from app.engine.autofix import NEED_CHOICE, apply_fixes, forbidden_changes, plan_fixes, public
|
||
from app.engine.governance import lint_for_publish, publish_issues
|
||
from app.engine.judge import JUDGE_DEFAULTS
|
||
from app.engine.schema import GraphSpec, validate_graph
|
||
|
||
STRICT = {"numbers": "strict", "on_violation": "fail", "claims": "require_citation"}
|
||
JUDGED = {**STRICT, "claims": "judge", "judge": {"max_cost_usd": 0.05}}
|
||
BUDGET = "governed.judge_budget"
|
||
|
||
|
||
def node(nid, ntype, label=None, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 10, "y": 20},
|
||
"data": {"label": label or nid, "config": config}}
|
||
|
||
|
||
def traceable(*, report=None, defaults=None, **contract_overrides) -> dict:
|
||
"""input → 查库 → 口径卡 → 报告撰写 → 出具：受管要求（含 G1–G5）全部满足。"""
|
||
contract = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True}
|
||
contract.update(contract_overrides)
|
||
nodes = [
|
||
node("start", "input", "周期"),
|
||
node("fetch", "tool", "取数", tool="db_query__shop",
|
||
args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}),
|
||
node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[
|
||
{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
||
node("write", "report", "报告撰写", instructions="写周报", **(JUDGED if report is None else report)),
|
||
node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
|
||
contract={k: v for k, v in contract.items() if v is not None}),
|
||
]
|
||
edges = [{"id": f"e{i}", "source": a["id"], "target": b["id"]} for i, (a, b) in enumerate(zip(nodes, nodes[1:]))]
|
||
graph = {"nodes": nodes, "edges": edges}
|
||
if defaults is not None:
|
||
graph["defaults"] = defaults
|
||
return graph
|
||
|
||
|
||
def config(graph: dict, nid: str) -> dict:
|
||
return next(n for n in graph["nodes"] if n["id"] == nid)["data"]["config"]
|
||
|
||
|
||
def by_code(graph: dict, code: str, level: str = "governed") -> list:
|
||
return [i for i in lint_for_publish(GraphSpec.model_validate(graph), level=level).issues if i.code == code]
|
||
|
||
|
||
def errors(graph: dict, level: str = "governed") -> list:
|
||
return [i for i in publish_issues(GraphSpec.model_validate(graph), level=level) if i.level == "error"]
|
||
|
||
|
||
def plan(graph: dict, level: str = "governed") -> dict[str, dict]:
|
||
spec = GraphSpec.model_validate(graph)
|
||
return {f["id"]: f for f in plan_fixes(spec, publish_issues(spec, level=level), level=level)}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# G3：judge 可以发布成受管，但要写预算
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize("budget", [0.05, 1, None])
|
||
def test_judge_with_a_budget_publishes_as_governed(budget):
|
||
graph = traceable(report={**JUDGED, "judge": {"max_cost_usd": budget, "max_claims": 20}})
|
||
assert errors(graph) == [], [i.message for i in errors(graph)]
|
||
assert by_code(graph, "governed.report_policy") == [] and by_code(graph, BUDGET) == []
|
||
|
||
|
||
@pytest.mark.parametrize("judge", [None, {}, {"max_claims": 20, "model": "judge-model"}])
|
||
def test_judge_without_a_budget_is_an_error_at_governed(judge):
|
||
report = {**STRICT, "claims": "judge", **({"judge": judge} if judge is not None else {})}
|
||
graph = traceable(report=report)
|
||
[issue] = by_code(graph, BUDGET)
|
||
assert issue.level == "error" and issue.node_id == "write" and issue.field == "judge.max_cost_usd"
|
||
assert "「金额上限（美元）」" in issue.message and "不限" in issue.message and "「结论句裁判」" in issue.message
|
||
assert by_code(graph, "governed.report_policy") == [], "judge 本身合规，只缺预算"
|
||
assert [i.level for i in by_code(graph, BUDGET, "published")] == ["warning"]
|
||
|
||
|
||
def test_the_budget_follows_the_same_config_rule_as_run_time():
|
||
"""和运行时同一个取值规则：节点没写 judge 取图级 defaults；节点写了就整个用节点的（不和 defaults 拼）。"""
|
||
graph = traceable(report={**STRICT, "claims": "judge"}, defaults={"judge": {"max_cost_usd": 0.02}})
|
||
assert by_code(graph, BUDGET) == []
|
||
graph = traceable(report={**STRICT, "claims": "judge", "judge": {"max_claims": 5}},
|
||
defaults={"judge": {"max_cost_usd": 0.02}})
|
||
assert [i.field for i in by_code(graph, BUDGET)] == ["judge.max_cost_usd"]
|
||
# claims 本身也可以来自图级 defaults
|
||
graph = traceable(report={"numbers": "strict", "on_violation": "fail"},
|
||
defaults={"claims": "judge", "judge": {"max_cost_usd": None}})
|
||
assert by_code(graph, BUDGET) == [] and by_code(graph, "governed.report_policy") == []
|
||
|
||
|
||
def test_a_malformed_budget_is_left_to_validate():
|
||
graph = traceable(report={**STRICT, "claims": "judge", "judge": {"max_cost_usd": "cheap"}})
|
||
assert by_code(graph, BUDGET) == [], "写错的值由 validate 的 report.judge_invalid 报，同一处不说两遍"
|
||
[bad] = [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.code == "report.judge_invalid"]
|
||
assert bad.field == "judge.max_cost_usd" and bad.level == "error"
|
||
|
||
|
||
@pytest.mark.parametrize("claims", [None, "off"])
|
||
def test_a_loose_claims_setting_names_both_accepted_values(claims):
|
||
report = {"numbers": "strict", "on_violation": "fail", **({"claims": claims} if claims else {})}
|
||
[issue] = by_code(traceable(report=report), "governed.report_policy")
|
||
assert issue.field == "claims" and "「未附依据的结论句」设为「计入缺口」" in issue.message \
|
||
and "「计入缺口并由模型逐句判断」" in issue.message
|
||
|
||
|
||
def test_judge_needs_no_other_report_policy_fix():
|
||
graph = traceable(report={**JUDGED, "numbers": "off"})
|
||
[issue] = by_code(graph, "governed.report_policy")
|
||
assert issue.field == "numbers" and "claims" not in issue.message
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 发布前自动修复
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_the_report_policy_fix_leaves_judge_alone():
|
||
graph = traceable(report={**JUDGED, "numbers": "off", "on_violation": "flag"})
|
||
fix = plan(graph)["governed.report_policy:write"]
|
||
assert fix["kind"] == "auto" and fix["preview"]["after"] == {"numbers": "strict", "on_violation": "fail"}
|
||
out = apply_fixes(graph, [fix["id"]], level="governed")
|
||
assert out["ok"] is True and config(out["graph"], "write")["claims"] == "judge"
|
||
assert config(out["graph"], "write")["judge"] == {"max_cost_usd": 0.05}
|
||
|
||
|
||
def test_a_missing_budget_is_a_choice_between_the_default_and_unlimited():
|
||
graph = traceable(report={**STRICT, "claims": "judge", "judge": {"model": "judge-model"}})
|
||
fid = f"{BUDGET}:write"
|
||
fix = public(plan(graph)[fid])
|
||
assert fix["kind"] == "choice" and fix["node_id"] == "write" and fix["field"] == "judge.max_cost_usd"
|
||
assert [o["value"] for o in fix["options"]] == [JUDGE_DEFAULTS["report_max_cost_usd"], None]
|
||
assert "default" not in fix, "花多少钱要人定：不替人选，也不给建议"
|
||
unlimited = fix["options"][1]
|
||
assert "不限" in unlimited["label"] and "不设上限" in unlimited["hint"] and "约束" in unlimited["hint"]
|
||
assert all(not k.startswith("_") for k in fix), "内部键不交给前端"
|
||
|
||
[refused] = apply_fixes(graph, [fid], level="governed")["rejected"]
|
||
assert refused["reason"] == NEED_CHOICE.format(label=fix["label"])
|
||
|
||
out = apply_fixes(graph, [fid], {fid: 0.05}, level="governed")
|
||
assert out["applied"] == [fid] and out["ok"] is True
|
||
assert config(out["graph"], "write")["judge"] == {"model": "judge-model", "max_cost_usd": 0.05}
|
||
[change] = out["changes"]
|
||
assert change["field"] == "judge" and change["before"] == {"model": "judge-model"}
|
||
assert out["ops"] == [{"op": "update_node", "id": "write",
|
||
"config": {"judge": {"model": "judge-model", "max_cost_usd": 0.05}}}]
|
||
|
||
out = apply_fixes(graph, [fid], {fid: None}, level="governed")
|
||
judge = config(out["graph"], "write")["judge"]
|
||
assert out["ok"] is True and "max_cost_usd" in judge and judge["max_cost_usd"] is None, "不限要显式写成 null"
|
||
|
||
|
||
def test_the_budget_fix_keeps_what_the_graph_defaults_said():
|
||
"""judge 写在图级 defaults 里：修复写到节点上时带上 defaults 里的整份 judge，节点写了就不再跟随 defaults，
|
||
不能因为补一个预算把 defaults 里选的裁判模型丢了。"""
|
||
graph = traceable(report={**STRICT, "claims": "judge"},
|
||
defaults={"judge": {"model": "judge-model", "rewrite_once": True}})
|
||
fid = f"{BUDGET}:write"
|
||
out = apply_fixes(graph, [fid], {fid: None}, level="governed")
|
||
assert out["ok"] is True
|
||
assert config(out["graph"], "write")["judge"] == {"model": "judge-model", "rewrite_once": True,
|
||
"max_cost_usd": None}
|
||
assert out["graph"]["defaults"] == {"judge": {"model": "judge-model", "rewrite_once": True}}
|
||
|
||
|
||
def test_a_choice_outside_the_options_is_refused():
|
||
graph = traceable(report={**STRICT, "claims": "judge"})
|
||
fid = f"{BUDGET}:write"
|
||
out = apply_fixes(graph, [fid], {fid: 99}, level="governed")
|
||
assert out["applied"] == [] and "不在候选项中" in out["rejected"][0]["reason"]
|
||
|
||
|
||
def test_published_level_budget_warnings_get_no_fix():
|
||
assert f"{BUDGET}:write" not in plan(traceable(report={**STRICT, "claims": "judge"}), "published")
|
||
|
||
|
||
def test_invalid_claims_fixes_no_longer_mention_a_later_version():
|
||
graph = traceable(report={**STRICT, "claims": "sometimes"})
|
||
governed = plan(graph)["report.claims_invalid:write"]
|
||
published = plan(graph, "published")["report.claims_invalid:write"]
|
||
assert "后续版本" not in governed["label"] and "default" not in published
|
||
assert [o["value"] for o in published["options"]] == ["off", "require_citation"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 降低要求的判断
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _changed(report_before: dict, report_after: dict | None, **graph_kw) -> list[str]:
|
||
old = traceable(report=report_before, **graph_kw)
|
||
new = copy.deepcopy(old)
|
||
if report_after is None:
|
||
config(new, "write").pop("claims")
|
||
else:
|
||
config(new, "write").clear()
|
||
config(new, "write").update({"instructions": "写周报", **report_after})
|
||
return forbidden_changes(old, new)
|
||
|
||
|
||
def test_moving_up_to_judge_is_not_loosening():
|
||
assert _changed(STRICT, JUDGED) == []
|
||
assert _changed({**STRICT, "claims": "off"}, JUDGED) == []
|
||
assert _changed({"numbers": "strict", "on_violation": "fail"}, JUDGED) == []
|
||
|
||
|
||
@pytest.mark.parametrize("after", ["off", "require_citation", None])
|
||
def test_moving_down_from_judge_is_loosening(after):
|
||
report_after = None if after is None else {**JUDGED, "claims": after}
|
||
[reason] = _changed(JUDGED, report_after)
|
||
assert "「报告撰写」" in reason and "「未附依据的结论句」" in reason and "「计入缺口并由模型逐句判断」" in reason
|
||
|
||
|
||
def test_dropping_a_judge_set_in_the_graph_defaults_is_loosening():
|
||
[reason] = _changed({"numbers": "strict", "on_violation": "fail"}, {**STRICT, "claims": "off"},
|
||
defaults={"claims": "judge", "judge": {"max_cost_usd": 0.05}})
|
||
assert "「未附依据的结论句」" in reason and "「计入缺口并由模型逐句判断」" in reason
|
||
|
||
|
||
def test_changing_the_budget_is_not_loosening():
|
||
assert _changed(JUDGED, {**JUDGED, "judge": {"max_cost_usd": None}}) == []
|
||
|
||
|
||
# claims 一直是 judge 时，judge.on_unsupported 从 withhold 往回改也是放宽：不支持的结论句从「不予出具」变成
|
||
# 「降档出具」。Copilot 改图的 update_node 会把 judge 整个盖掉，补预算时写 judge={max_cost_usd: 0.05}
|
||
# 就悄悄丢了 withhold——这一类要和契约里的 on_unsupported 一样拒掉
|
||
WITHHOLD = {**JUDGED, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}
|
||
|
||
|
||
@pytest.mark.parametrize("judge, dropped", [
|
||
({"max_cost_usd": 0.05, "on_unsupported": "degrade"}, False),
|
||
({"max_cost_usd": 0.05}, True),
|
||
({"max_cost_usd": 0.05, "on_unsupported": "sometimes"}, False),
|
||
])
|
||
def test_relaxing_on_unsupported_on_the_node_is_loosening(judge, dropped):
|
||
[reason] = _changed(WITHHOLD, {**WITHHOLD, "judge": judge})
|
||
assert "「报告撰写」" in reason and "「结论句裁判 · 证据相矛盾时」" in reason
|
||
assert "「不予出具」" in reason and "「出具降档」" in reason
|
||
assert ("删除了已设置的值" in reason) is dropped, reason
|
||
|
||
|
||
def test_replacing_the_whole_judge_block_to_add_a_budget_is_loosening():
|
||
"""补预算把 judge 整个换成 {max_cost_usd}：连同裁判模型、withhold 一起丢了。"""
|
||
before = {**STRICT, "claims": "judge", "judge": {"model": "judge-model", "on_unsupported": "withhold"}}
|
||
[reason] = _changed(before, {**before, "judge": {"max_cost_usd": 0.05}})
|
||
assert "「结论句裁判 · 证据相矛盾时」" in reason and "「不予出具」" in reason
|
||
assert _changed(before, {**before, "judge": {**before["judge"], "max_cost_usd": 0.05}}) == [], \
|
||
"带着原来的键补预算不算放宽"
|
||
|
||
|
||
def test_a_withhold_from_the_graph_defaults_lost_by_a_node_judge_is_loosening():
|
||
"""judge 按运行时的规则取：节点写了自己的 judge 就整个用节点的，图级 defaults 里的 withhold 不再生效。"""
|
||
defaults = {"judge": {"on_unsupported": "withhold", "max_cost_usd": 0.05}}
|
||
report = {**STRICT, "claims": "judge"}
|
||
[reason] = _changed(report, {**report, "judge": {"max_cost_usd": 0.05}}, defaults=defaults)
|
||
assert "「结论句裁判 · 证据相矛盾时」" in reason and "从「不予出具」（跟随工作流默认设置）" in reason, reason
|
||
assert "不再跟随工作流默认设置" in reason, "说清楚为什么全图默认里的 withhold 不生效了"
|
||
assert _changed(report, {**report, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}},
|
||
defaults=defaults) == []
|
||
|
||
|
||
def test_relaxing_on_unsupported_in_the_graph_defaults_is_loosening():
|
||
old = traceable(report={**STRICT, "claims": "judge"},
|
||
defaults={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}})
|
||
new = copy.deepcopy(old)
|
||
new["defaults"]["judge"]["on_unsupported"] = "degrade"
|
||
[reason] = forbidden_changes(old, new)
|
||
assert "「报告撰写」" in reason and "「结论句裁判 · 证据相矛盾时」" in reason
|
||
|
||
|
||
@pytest.mark.parametrize("before, after", [
|
||
(JUDGED, {**JUDGED, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}),
|
||
(WITHHOLD, {**WITHHOLD, "judge": {**WITHHOLD["judge"], "max_cost_usd": None, "rewrite_once": True}}),
|
||
(WITHHOLD, WITHHOLD),
|
||
# 升到 judge 本身是收紧：不管带不带 withhold
|
||
({**STRICT, "judge": {"on_unsupported": "withhold"}}, JUDGED),
|
||
])
|
||
def test_tightening_or_keeping_on_unsupported_is_fine(before, after):
|
||
assert _changed(before, after) == []
|
||
|
||
|
||
def test_dropping_withhold_along_with_judge_is_one_reason():
|
||
"""从 judge 退回 require_citation：claims 那一条已经说了，不再为 on_unsupported 另记一条。"""
|
||
[reason] = _changed(WITHHOLD, {**STRICT, "judge": {"max_cost_usd": 0.05}})
|
||
assert "「未附依据的结论句」" in reason and "证据相矛盾时" not in reason
|
||
|
||
|
||
@pytest.mark.parametrize("before", ["require_citation", "judge"])
|
||
def test_moving_claims_to_an_unknown_value_is_loosening(before):
|
||
"""require_citation、judge 改成认不出的写法：出口按认不出的走，要求没了——照样算放宽，别只落个
|
||
「没让图变好」。原来就认不出的不比较。"""
|
||
report = {**STRICT, "claims": before, **({"judge": {"max_cost_usd": 0.05}} if before == "judge" else {})}
|
||
[reason] = _changed(report, {**report, "claims": "sometimes"})
|
||
from app.engine.labels import option_label
|
||
|
||
assert "「报告撰写」" in reason and f"「{option_label('claims', before)}」" in reason and "sometimes" in reason \
|
||
and "无法识别" in reason
|
||
assert _changed({**STRICT, "claims": "sometimes"}, {**STRICT, "claims": "off"}) == []
|
||
assert _changed({**STRICT, "claims": "off"}, {**STRICT, "claims": "sometimes"}) == []
|
||
|
||
|
||
def _contract_changed(before, after) -> list[str]:
|
||
old = traceable(claims=before)
|
||
new = copy.deepcopy(old)
|
||
contract = config(new, "done")["contract"]
|
||
if after is None:
|
||
contract.pop("claims", None)
|
||
else:
|
||
contract["claims"] = after
|
||
return forbidden_changes(old, new)
|
||
|
||
|
||
@pytest.mark.parametrize("before, after", [
|
||
("judge", "off"),
|
||
("judge", None),
|
||
("judge", "require_citation"),
|
||
({"policy": "judge", "on_unsupported": "withhold"}, "judge"),
|
||
({"policy": "judge", "on_unsupported": "withhold"}, {"policy": "judge", "on_unsupported": "degrade"}),
|
||
({"policy": "require_citation", "on_uncited": "withhold"}, "judge"),
|
||
])
|
||
def test_loosening_a_judge_contract_is_refused(before, after):
|
||
[reason] = _contract_changed(before, after)
|
||
assert "「出具」" in reason and "对结论句的要求" in reason
|
||
|
||
|
||
@pytest.mark.parametrize("before, after", [
|
||
("require_citation", "judge"),
|
||
("off", {"policy": "judge", "on_unsupported": "withhold"}),
|
||
("judge", {"policy": "judge", "on_unsupported": "withhold"}),
|
||
({"policy": "require_citation", "on_uncited": "withhold"}, {"policy": "judge", "on_uncited": "withhold"}),
|
||
])
|
||
def test_tightening_a_contract_towards_judge_is_fine(before, after):
|
||
assert _contract_changed(before, after) == []
|