mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 09:51:58 +00:00
9727c21d8d
一、结论句裁判
- 报告节点新增 claims: judge:写完、核对完之后,把结论句和它引用的证据摘录交给裁判模型,
判为支持 / 部分支持 / 不支持 / 不是结论,附一句理由;判定只标注,不改写正文
- 送裁判的句子按固定规则预筛:带引用的、含方向词或因果词的送去判,短句、过渡句不送
- 证据摘录:指标写名称、值和原式;查询给 SQL 前 300 字和被引用的行,被引用的列一定给到;遮罩列一律不给;
规则里写明「数字已经由系统核对过,不要复算」
- 一份报告合并成一次调用,超出句数就分批;结构化输出失败时退回 JSON 解析,再失败记为未裁判
- 裁判模型的优先级:节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型;思考关闭;
和写作模型相同时给警告(等于自己审自己)
- rewrite_once(默认关):把不支持的句子和理由交回写作者只改这几句,再裁一次;
改写稿冒出原稿没有的问题就不采用,保留原稿和原来的判定
二、成本上限
- 每份报告的句数、金额、时长,探索运行每次点击的金额,全局每日金额,每一项都可以自定义,也都可以设成不限;
默认 40 句、$0.05、30 秒,每次点击 $0.01,每天 $2
- 调用前先估算,超出上限按优先级截断:先判带引用的,再判含方向词、因果词的,其余记为未裁判并写明「已到上限」;
不报错,也不让报告失败
- 每日花费按本地日期计数,持久化在设置表里;重跑同一次运行不重复计费;价格目录里没有的模型只计次数,并写明估不出金额
三、正式运行与探索运行
- 正式运行在报告节点内裁判,判定写进报告文档、随文档封存;不支持的按 on_unsupported(degrade / withhold)降档或不予出具,
裁判没跑成记缺口,不能判完整出具
- 探索运行按需裁判:POST /api/runs/{id}/evidence/judge,点开哪句判哪句;判定作为 evidence.judged 事件追加在封存之后,
标「封存后追加」,封存核对仍然一致;只读封存范围内的工件;同一句不重复判;运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409,写明「正式运行的裁判在节点内完成」
四、治理与自动修复
- G3 放开 judge:受管级别写 judge 必须写预算(judge.max_cost_usd,或显式写 null 表示不限)
- 发布前自动修复:缺预算时给选项(默认金额 / 不限),不替人选;
从 require_citation 改成 judge 不算降低要求,judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册:受管模板可以写 judge 并带预算;问数据按需裁判,不替人打开结论句策略
五、界面
- 结论句句末放徽标:支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?;不用表示「已核对」的绿色;n / N 也跳到不支持的句子
- 证据面板里「模型的解释」:带「模型判断 · 模型名 · 非确定」,封存后追加的另外标出;
「请模型判断这句」只在探索运行、还没判过时出现,触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数;设置页新增「证据裁判」一组(裁判模型、各项上限、今日花费),设成不限时写明费用受什么约束;
报告节点可以配 judge;运行轨迹里裁判相关的事件都有中文标题
六、门禁修正
- 引用模式下,图里没有口径卡(只引查询单元格)的,出具契约不再要求 metrics_from,受管门禁也不再要求 required:
strict 加 cells 已经保证每个数都点得开、对不上就不予出具;有口径卡的照旧要求。
以前问数据的图怎么都发不成受管,Copilot 补上契约也会被判「冒出新的问题」作废
495 lines
24 KiB
Python
495 lines
24 KiB
Python
"""证据门禁 G1–G5 接进发布前自动修复(engine/autofix.py)。
|
||
|
||
照发布前自动修复的原则:答案唯一、只提高要求的一键修(auto);要人拿主意的列候选、不替人选(choice);
|
||
结构性的交给 Copilot(assist)。修完重跑 validate 和门禁,错误必须变少、不能冒出新的;降低要求的改动
|
||
(numbers、on_violation、claims 往宽里改,关掉 cite_fields,替人把沙箱代码标成取数)一律拒绝。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import json
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessageChunk
|
||
|
||
from app.engine.autofix import HANDOFF, apply_fixes, forbidden_changes, plan_fixes
|
||
from app.engine.governance import publish_issues
|
||
from app.engine.schema import GraphSpec
|
||
from app.main import app
|
||
|
||
STRICT = {"numbers": "strict", "on_violation": "fail", "claims": "require_citation"}
|
||
SCHEMA = {"type": "object", "properties": {"gmv": {"type": "number"}}}
|
||
|
||
|
||
def node(nid, ntype, label=None, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 10, "y": 20}, "width": 240,
|
||
"data": {"label": label or nid, "config": config}}
|
||
|
||
|
||
def traceable(*, report=None, **contract_overrides) -> dict:
|
||
"""input → 查库 → 口径卡 → 报告撰写 → 出具:受管要求(含 G1–G5)全部满足。"""
|
||
contract = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True}
|
||
contract.update(contract_overrides)
|
||
nodes = [
|
||
node("start", "input", "周期"),
|
||
node("fetch", "tool", "取数", tool="db_query__shop",
|
||
args={"sql": "SELECT SUM(amount) AS gmv, COUNT(*) AS orders FROM orders"}),
|
||
node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[
|
||
{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
||
node("write", "report", "报告撰写", instructions="写周报", **(STRICT if report is None else report)),
|
||
node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"},
|
||
{"name": "指标清单", "value": "{{ nodes.card.text }}"}],
|
||
contract={k: v for k, v in contract.items() if v is not None}),
|
||
]
|
||
edges = [{"id": f"e{i}", "source": a["id"], "target": b["id"]} for i, (a, b) in enumerate(zip(nodes, nodes[1:]))]
|
||
return {"nodes": nodes, "edges": edges, "viewport": {"x": 1, "y": 2, "zoom": 1}}
|
||
|
||
|
||
def find(graph: dict, nid: str) -> dict:
|
||
return next(n for n in graph["nodes"] if n["id"] == nid)
|
||
|
||
|
||
def config(graph: dict, nid: str) -> dict:
|
||
return find(graph, nid)["data"]["config"]
|
||
|
||
|
||
def insert(graph: dict, new: dict, *, before: str, after: str) -> dict:
|
||
graph["nodes"].insert(len(graph["nodes"]) - 1, new)
|
||
graph["edges"] += [{"source": after, "target": new["id"]}, {"source": new["id"], "target": before}]
|
||
return graph
|
||
|
||
|
||
def plan(graph: dict, level: str = "governed") -> dict[str, dict]:
|
||
spec = GraphSpec.model_validate(graph)
|
||
return {f["id"]: f for f in plan_fixes(spec, publish_issues(spec, level=level), level=level)}
|
||
|
||
|
||
def errors(graph: dict, level: str = "governed") -> list:
|
||
return [i for i in publish_issues(GraphSpec.model_validate(graph), level=level) if i.level == "error"]
|
||
|
||
|
||
def codes(issues) -> set[str]:
|
||
return {i["code"] if isinstance(i, dict) else i.code for i in issues}
|
||
|
||
|
||
def test_the_baseline_needs_no_fix():
|
||
assert errors(traceable()) == [] and plan(traceable()) == {}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# G3:一键写明 numbers / on_violation / claims
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_report_policy_is_filled_in_with_one_auto_fix():
|
||
graph = traceable(report={"numbers": "off"})
|
||
fid = "governed.report_policy:write"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "auto" and fix["node_id"] == "write"
|
||
assert fix["preview"]["after"] == STRICT and fix["preview"]["before"] == {
|
||
"numbers": "off", "on_violation": None, "claims": None}
|
||
out = apply_fixes(graph, [fid], level="governed")
|
||
assert out["applied"] == [fid] and out["rejected"] == [] and out["ok"] is True
|
||
assert {k: config(out["graph"], "write")[k] for k in STRICT} == STRICT
|
||
assert config(out["graph"], "write")["instructions"] == "写周报" # 别的配置原样留着
|
||
assert [(c["field"], c["before"], c["after"]) for c in out["changes"]] == [
|
||
("numbers", "off", "strict"), ("on_violation", None, "fail"), ("claims", None, "require_citation")]
|
||
assert out["ops"] == [{"op": "update_node", "id": "write", "config": STRICT}]
|
||
# 坐标、视口这些前端的东西原样留着
|
||
assert find(out["graph"], "write")["position"] == {"x": 10, "y": 20} and out["graph"]["viewport"]["zoom"] == 1
|
||
|
||
|
||
def test_only_the_missing_settings_are_written():
|
||
graph = traceable(report={"numbers": "strict", "claims": "require_citation"})
|
||
fix = plan(graph)["governed.report_policy:write"]
|
||
assert fix["preview"] == {"field": "on_violation", "before": None, "after": "fail"}
|
||
out = apply_fixes(graph, [fix["id"]], level="governed")
|
||
assert [c["field"] for c in out["changes"]] == ["on_violation"] and out["ok"] is True
|
||
|
||
|
||
def test_published_level_warnings_get_no_fix():
|
||
graph = traceable(report={})
|
||
assert plan(graph, "published") == {}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# validate:claims 写了不支持的值
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_an_invalid_claims_value_becomes_require_citation_under_governed():
|
||
graph = traceable(report={**STRICT, "claims": "sometimes"})
|
||
fid = "report.claims_invalid:write"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "auto" and fix["preview"] == {"field": "claims", "before": "sometimes",
|
||
"after": "require_citation"}
|
||
out = apply_fixes(graph, [fid], level="governed")
|
||
assert out["ok"] is True and config(out["graph"], "write")["claims"] == "require_citation"
|
||
|
||
|
||
def test_an_invalid_claims_value_is_a_choice_under_published():
|
||
graph = traceable(report={"claims": "sometimes"})
|
||
fid = "report.claims_invalid:write"
|
||
fix = plan(graph, "published")[fid]
|
||
assert fix["kind"] == "choice" and [o["value"] for o in fix["options"]] == ["off", "require_citation"]
|
||
assert apply_fixes(graph, [fid], level="published")["applied"] == []
|
||
out = apply_fixes(graph, [fid], {fid: "off"}, level="published")
|
||
assert out["applied"] == [fid] and config(out["graph"], "write")["claims"] == "off" and out["ok"] is True
|
||
# 乱写的值没有「最接近的」,不给建议
|
||
assert "default" not in plan(traceable(report={"claims": "strict"}), "published")[fid]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# G5:给口径卡供数的 agent
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def feeding_agent(**config_) -> dict:
|
||
graph = insert(traceable(), node("ask", "agent", "查数员", tools=["db_query__shop"], prompt="查 orders",
|
||
assign_to="facts", **config_), before="card", after="start")
|
||
config(graph, "card")["metrics"].append({"id": "gmv2", "expression": "vars.facts.gmv"})
|
||
return graph
|
||
|
||
|
||
def test_cite_fields_is_switched_on_when_the_schema_exists():
|
||
graph = feeding_agent(output_schema=SCHEMA)
|
||
fid = "governed.caliber_agent_cite_fields:ask"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "auto" and fix["preview"] == {"field": "cite_fields", "before": None, "after": True}
|
||
out = apply_fixes(graph, [fid], level="governed")
|
||
assert out["ok"] is True and config(out["graph"], "ask")["cite_fields"] is True
|
||
assert config(out["graph"], "ask")["output_schema"] == SCHEMA
|
||
|
||
|
||
def test_a_missing_schema_is_not_invented():
|
||
graph = feeding_agent()
|
||
fid = "governed.caliber_agent_schema:ask"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "assist" and "output_schema" in fix["label"]
|
||
out = apply_fixes(graph, [fid], level="governed")
|
||
assert out["applied"] == [] and out["rejected"][0]["fix_id"] == fid
|
||
assert "output_schema" not in config(out["graph"], "ask") and out["graph"] == graph
|
||
|
||
|
||
def test_a_model_call_feeding_a_card_goes_to_copilot():
|
||
graph = insert(traceable(), node("guess", "llm", "估算", prompt="估一个数", assign_to="g"), before="card",
|
||
after="start")
|
||
config(graph, "card")["metrics"].append({"id": "g", "expression": "vars.g"})
|
||
assert plan(graph)["governed.caliber_model_input:guess"]["kind"] == "assist"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# G4:沙箱代码喂口径卡——它到底是在取数还是在算,要人说
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def compute_feeding() -> dict:
|
||
graph = insert(traceable(), node("calc", "code", "计算", code="print(1)", assign_to="calc"), before="card",
|
||
after="fetch")
|
||
config(graph, "card")["metrics"].append({"id": "ratio", "expression": "vars.calc.ratio"})
|
||
return graph
|
||
|
||
|
||
def test_compute_code_is_a_choice_not_an_auto_fix():
|
||
graph = compute_feeding()
|
||
fid = "governed.caliber_compute_input:calc"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "choice" and fix["node_id"] == "calc" and fix["field"] == "evidence_role"
|
||
assert [o["value"] for o in fix["options"]] == ["source", HANDOFF]
|
||
assert "default" not in fix and "preview" not in fix
|
||
# 不选就不动
|
||
out = apply_fixes(graph, [fid], level="governed")
|
||
assert out["applied"] == [] and "evidence_role" not in config(out["graph"], "calc")
|
||
assert out["rejected"][0]["fix_id"] == fid
|
||
|
||
|
||
def test_choosing_source_marks_the_node():
|
||
graph = compute_feeding()
|
||
fid = "governed.caliber_compute_input:calc"
|
||
out = apply_fixes(graph, [fid], {fid: "source"}, level="governed")
|
||
assert out["applied"] == [fid] and out["ok"] is True
|
||
assert config(out["graph"], "calc")["evidence_role"] == "source"
|
||
|
||
|
||
def test_choosing_copilot_hands_the_node_over_without_touching_it():
|
||
graph = compute_feeding()
|
||
fid = "governed.caliber_compute_input:calc"
|
||
out = apply_fixes(graph, [fid], {fid: HANDOFF}, level="governed")
|
||
assert out["applied"] == [] and out["rejected"] == [] and out["handoff"] == [fid]
|
||
assert out["graph"] == graph and out["ok"] is False
|
||
|
||
|
||
def test_copilot_may_not_mark_a_code_node_as_source_on_its_own():
|
||
before = compute_feeding()
|
||
after = copy.deepcopy(before)
|
||
config(after, "calc")["evidence_role"] = "source"
|
||
[reason] = forbidden_changes(before, after)
|
||
assert "「计算」" in reason and "source" in reason
|
||
# 新加的沙箱代码直接标成取数也一样
|
||
after = copy.deepcopy(before)
|
||
after["nodes"].append(node("raw", "code", "取原始数据", code="print(2)", evidence_role="source"))
|
||
assert any("「取原始数据」" in r for r in forbidden_changes(before, after))
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# G1:契约和成果字段接到报告撰写节点
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def narrative() -> dict:
|
||
return traceable(report_from=None, narrative="{{ nodes.write.text }}")
|
||
|
||
|
||
def test_report_from_takes_the_only_report_upstream():
|
||
graph = narrative()
|
||
fid = "governed.report_from_required:done"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "auto" and fix["preview"] == {"field": "contract.report_from", "before": None,
|
||
"after": "write"}
|
||
out = apply_fixes(graph, [fid], level="governed")
|
||
assert out["ok"] is True and config(out["graph"], "done")["contract"]["report_from"] == "write"
|
||
|
||
|
||
def test_several_reports_make_report_from_a_choice():
|
||
graph = insert(narrative(), node("write2", "report", "第二份报告", instructions="写月报", **STRICT),
|
||
before="done", after="card")
|
||
fid = "governed.report_from_required:done"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "choice" and [o["value"] for o in fix["options"]] == ["write", "write2"]
|
||
assert apply_fixes(graph, [fid], level="governed")["applied"] == []
|
||
out = apply_fixes(graph, [fid], {fid: "write2"}, level="governed")
|
||
assert out["applied"] == [fid] and config(out["graph"], "done")["contract"]["report_from"] == "write2"
|
||
|
||
|
||
def test_without_a_report_node_report_from_goes_to_copilot():
|
||
graph = narrative()
|
||
write = find(graph, "write")
|
||
write.update(type="llm", data={"label": "写周报", "config": {"prompt": "写周报"}})
|
||
# 受管级别的叙述模式过不了 G1,不再给「选 narrative」这种修了也发不出去的候选
|
||
for fid in ("governed.report_from_required:done",):
|
||
assert plan(graph)[fid]["kind"] == "assist"
|
||
bare = traceable(report_from=None)
|
||
find(bare, "write").update(type="llm", data={"label": "写周报", "config": {"prompt": "写周报"}})
|
||
assert plan(bare)["contract.report_from_missing:done"]["kind"] == "assist"
|
||
|
||
|
||
def with_story(graph: dict) -> dict:
|
||
graph = insert(graph, node("narrate", "llm", "叙述", prompt="写周报", assign_to="story"), before="done",
|
||
after="card")
|
||
config(graph, "done")["fields"] += [{"name": "摘要", "value": "{{ vars.story }}"},
|
||
{"name": "点评", "value": "点评:{{ nodes.narrate.text }}"}]
|
||
return graph
|
||
|
||
|
||
def test_exit_fields_are_repointed_at_the_chosen_report():
|
||
graph = with_story(traceable())
|
||
fid = "governed.exit_text_source:done"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "choice" and fix["field"] == "fields"
|
||
assert [o["value"] for o in fix["options"]] == ["write"] and fix["default"] == "write"
|
||
assert "「摘要」" in fix["label"] and "「点评」" in fix["label"]
|
||
# 就一个候选也不自动改:字段里放什么是作者的事
|
||
out = apply_fixes(graph, [fid], level="governed")
|
||
assert out["applied"] == [] and out["graph"] == graph
|
||
out = apply_fixes(graph, [fid], {fid: "write"}, level="governed")
|
||
assert out["applied"] == [fid] and out["ok"] is True
|
||
fields = config(out["graph"], "done")["fields"]
|
||
assert fields == [{"name": "周报", "value": "{{ nodes.write.text }}"},
|
||
{"name": "指标清单", "value": "{{ nodes.card.text }}"},
|
||
{"name": "摘要", "value": "{{ nodes.write.text }}"},
|
||
{"name": "点评", "value": "{{ nodes.write.text }}"}]
|
||
assert [(c["field"], c["before"], c["after"]) for c in out["changes"]] == [
|
||
("fields[2].value", "{{ vars.story }}", "{{ nodes.write.text }}"),
|
||
("fields[3].value", "点评:{{ nodes.narrate.text }}", "{{ nodes.write.text }}")]
|
||
assert out["ops"] == [{"op": "update_node", "id": "done", "config": {"fields": fields}}]
|
||
|
||
|
||
def test_an_exit_without_fields_gets_one_that_takes_the_report():
|
||
graph = traceable()
|
||
config(graph, "done")["fields"] = []
|
||
graph = insert(graph, node("polish", "llm", "润色", prompt="{{ nodes.write.text }}"), before="done",
|
||
after="write")
|
||
graph["edges"] = [e for e in graph["edges"] if (e["source"], e["target"]) != ("write", "done")]
|
||
fid = "governed.exit_text_source:done"
|
||
out = apply_fixes(graph, [fid], {fid: "write"}, level="governed")
|
||
assert out["applied"] == [fid] and out["ok"] is True
|
||
assert config(out["graph"], "done")["fields"] == [{"name": "result", "value": "{{ nodes.write.text }}"}]
|
||
|
||
|
||
def test_exit_fields_without_any_report_go_to_copilot():
|
||
graph = with_story(traceable())
|
||
find(graph, "write").update(type="llm", data={"label": "写周报", "config": {"prompt": "写周报"}})
|
||
config(graph, "done")["contract"].pop("report_from")
|
||
assert plan(graph)["governed.exit_text_source:done"]["kind"] == "assist"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# G2:结构性的,交给 Copilot
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_text_bypass_is_an_assist():
|
||
graph = traceable()
|
||
graph["nodes"] += [node("chat", "llm", "闲聊", prompt="你好"),
|
||
node("side", "output", "附带成果", fields=[{"name": "回答", "value": "{{ nodes.chat.text }}"}])]
|
||
graph["edges"] += [{"source": "start", "target": "chat"}, {"source": "chat", "target": "side"}]
|
||
fid = "governed.text_bypass:chat"
|
||
fix = plan(graph)[fid]
|
||
assert fix["kind"] == "assist" and "报告撰写" in fix["label"]
|
||
out = apply_fixes(graph, [fid], level="governed")
|
||
assert out["applied"] == [] and out["rejected"][0]["fix_id"] == fid
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 修完更好、不降低要求
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_all_auto_fixes_together_leave_only_the_decisions():
|
||
graph = with_story(feeding_agent(output_schema=SCHEMA))
|
||
graph = insert(graph, node("calc", "code", "计算", code="print(1)", assign_to="calc"), before="card",
|
||
after="fetch")
|
||
config(graph, "card")["metrics"].append({"id": "ratio", "expression": "vars.calc.ratio"})
|
||
config(graph, "write").clear()
|
||
config(graph, "done")["contract"].update(report_from=None, narrative="{{ nodes.write.text }}")
|
||
fixes = plan(graph)
|
||
autos = [f["id"] for f in fixes.values() if f["kind"] == "auto"]
|
||
assert set(autos) == {"governed.report_policy:write", "governed.caliber_agent_cite_fields:ask",
|
||
"governed.report_from_required:done"}
|
||
before = len(errors(graph))
|
||
out = apply_fixes(graph, autos, level="governed")
|
||
assert sorted(out["applied"]) == sorted(autos) and out["rejected"] == []
|
||
assert len([i for i in out["remaining"] if i["level"] == "error"]) < before
|
||
left = {i["code"] for i in out["remaining"] if i["level"] == "error"}
|
||
assert left == {"governed.exit_text_source", "governed.caliber_compute_input"}
|
||
# 剩下的都要人拿主意,没有一条被替人应用
|
||
assert all(fixes[f]["kind"] != "auto" for f in fixes if f.split(":")[0] in left)
|
||
|
||
|
||
@pytest.mark.parametrize("key, before, after", [
|
||
("numbers", "strict", "off"), ("on_violation", "fail", "flag"), ("on_violation", "fail", None),
|
||
("claims", "require_citation", "off"), ("claims", "require_citation", None),
|
||
])
|
||
def test_loosening_the_report_policy_is_refused(key, before, after):
|
||
old = traceable(report={**STRICT, key: before})
|
||
new = copy.deepcopy(old)
|
||
if after is None:
|
||
config(new, "write").pop(key)
|
||
else:
|
||
config(new, "write")[key] = after
|
||
[reason] = forbidden_changes(old, new)
|
||
assert "「报告撰写」" in reason and key in reason
|
||
|
||
|
||
def test_turning_cite_fields_off_is_refused():
|
||
old = feeding_agent(output_schema=SCHEMA, cite_fields=True)
|
||
new = copy.deepcopy(old)
|
||
config(new, "ask")["cite_fields"] = False
|
||
[reason] = forbidden_changes(old, new)
|
||
assert "「查数员」" in reason and "cite_fields" in reason
|
||
|
||
|
||
def test_tightening_is_not_refused():
|
||
old = traceable(report={"numbers": "off", "on_violation": "flag", "claims": "off"})
|
||
new = copy.deepcopy(old)
|
||
config(new, "write").update(STRICT)
|
||
assert forbidden_changes(old, new) == []
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 接口:发布前检查带上 G 规则的修复;选了「交给 Copilot」就真的交给 Copilot
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
class Scripted:
|
||
"""按剧本吐操作流的 Copilot 模型;记下每次收到的请求。"""
|
||
|
||
def __init__(self, ops: list[dict]) -> None:
|
||
self.lines = [json.dumps(o, ensure_ascii=False) for o in ops]
|
||
self.calls: list[list] = []
|
||
|
||
async def astream(self, messages):
|
||
self.calls.append(messages)
|
||
for line in self.lines:
|
||
yield AIMessageChunk(content=line + "\n")
|
||
|
||
|
||
def script(monkeypatch, ops: list[dict]) -> Scripted:
|
||
import app.api.copilot as copilot
|
||
|
||
model = Scripted([{"op": "plan", "summary": "改图"}, *ops, {"op": "done", "explanation": "改好了"}])
|
||
|
||
async def _model(*_a, **_k):
|
||
return model, "mock-fast"
|
||
|
||
monkeypatch.setattr(copilot, "get_chat_model", _model)
|
||
return model
|
||
|
||
|
||
async def create(client, graph: dict) -> str:
|
||
return (await client.post("/api/workflows", json={"name": "证据门禁-修复", "graph": graph})).json()["id"]
|
||
|
||
|
||
async def test_publish_check_lists_the_g_fixes(client):
|
||
graph = with_story(compute_feeding())
|
||
config(graph, "write").clear()
|
||
wf = await create(client, graph)
|
||
body = (await client.post(f"/api/workflows/{wf}/publish-check", json={"level": "governed"})).json()
|
||
assert body["ok"] is False
|
||
kinds = {f["id"]: f["kind"] for f in body["fixes"]}
|
||
assert kinds["governed.report_policy:write"] == "auto"
|
||
assert kinds["governed.caliber_compute_input:calc"] == "choice"
|
||
assert kinds["governed.exit_text_source:done"] == "choice"
|
||
assert all(not any(k.startswith("_") for k in f) for f in body["fixes"]) # 内部信息不外露
|
||
by_code = {i["code"]: i for i in body["issues"]}
|
||
assert by_code["governed.report_policy"]["fix"] == "governed.report_policy:write"
|
||
assert by_code["governed.exit_text_source"]["fix"] == "governed.exit_text_source:done"
|
||
# 已发布级别:G 规则只是警告,不给修复
|
||
body = (await client.post(f"/api/workflows/{wf}/publish-check", json={"level": "published"})).json()
|
||
assert {i["level"] for i in body["issues"] if (i["code"] or "").startswith("governed.")} == {"warning"}
|
||
assert not [f for f in body["fixes"] if f["code"].startswith("governed.")]
|
||
|
||
|
||
async def test_choosing_copilot_in_the_choice_runs_the_assist(client, monkeypatch):
|
||
# Copilot 把口径卡里读沙箱代码的那条指标改成直接读查询结果:计算挪进了口径卡
|
||
card = [{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"},
|
||
{"id": "ratio", "expression": "cell(nodes.fetch, 0, 'gmv') / 100"}]
|
||
model = script(monkeypatch, [{"op": "update_node", "id": "card", "config": {"metrics": card}}])
|
||
wf = await create(client, compute_feeding())
|
||
fid = "governed.caliber_compute_input:calc"
|
||
out = (await client.post(f"/api/workflows/{wf}/autofix", json={
|
||
"level": "governed", "apply": [fid], "choices": {fid: HANDOFF}})).json()
|
||
assert out["handoff"] == [fid] and len(model.calls) == 1
|
||
human = next(text for role, text in model.calls[0] if role == "human")
|
||
assert "「计算」" in human and "evidence_role" in human
|
||
assert out["assist"]["ok"] is True and out["applied"] == ["assist"] and out["ok"] is True, out
|
||
assert "evidence_role" not in config(out["graph"], "calc")
|
||
# 没选「交给 Copilot」、也没带 assist 的,不调模型
|
||
model.calls.clear()
|
||
out = (await client.post(f"/api/workflows/{wf}/autofix", json={
|
||
"level": "governed", "apply": [fid], "choices": {fid: "source"}})).json()
|
||
assert model.calls == [] and out["assist"] is None and out["applied"] == [fid]
|
||
|
||
|
||
@pytest.mark.parametrize("ops, word", [
|
||
([{"op": "update_node", "id": "calc", "config": {"evidence_role": "source"}}], "evidence_role"),
|
||
([{"op": "update_node", "id": "write", "config": {"numbers": "off"}},
|
||
{"op": "update_node", "id": "calc", "config": {"evidence_role": "source"}}], "numbers"),
|
||
])
|
||
async def test_copilot_cannot_decide_for_the_author_or_loosen_the_report(client, monkeypatch, ops, word):
|
||
script(monkeypatch, ops)
|
||
wf = await create(client, compute_feeding())
|
||
out = (await client.post(f"/api/workflows/{wf}/autofix", json={
|
||
"level": "governed", "apply": [], "assist": True})).json()
|
||
assert out["assist"]["ok"] is False and out["applied"] == []
|
||
[refused] = [r for r in out["rejected"] if r["fix_id"] == "assist"]
|
||
assert "降低了要求" in refused["reason"] and word in refused["reason"]
|
||
assert out["graph"] == (await client.get(f"/api/workflows/{wf}")).json()["graph"]
|