mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-17 16:01:59 +00:00
9727c21d8d
一、结论句裁判
- 报告节点新增 claims: judge:写完、核对完之后,把结论句和它引用的证据摘录交给裁判模型,
判为支持 / 部分支持 / 不支持 / 不是结论,附一句理由;判定只标注,不改写正文
- 送裁判的句子按固定规则预筛:带引用的、含方向词或因果词的送去判,短句、过渡句不送
- 证据摘录:指标写名称、值和原式;查询给 SQL 前 300 字和被引用的行,被引用的列一定给到;遮罩列一律不给;
规则里写明「数字已经由系统核对过,不要复算」
- 一份报告合并成一次调用,超出句数就分批;结构化输出失败时退回 JSON 解析,再失败记为未裁判
- 裁判模型的优先级:节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型;思考关闭;
和写作模型相同时给警告(等于自己审自己)
- rewrite_once(默认关):把不支持的句子和理由交回写作者只改这几句,再裁一次;
改写稿冒出原稿没有的问题就不采用,保留原稿和原来的判定
二、成本上限
- 每份报告的句数、金额、时长,探索运行每次点击的金额,全局每日金额,每一项都可以自定义,也都可以设成不限;
默认 40 句、$0.05、30 秒,每次点击 $0.01,每天 $2
- 调用前先估算,超出上限按优先级截断:先判带引用的,再判含方向词、因果词的,其余记为未裁判并写明「已到上限」;
不报错,也不让报告失败
- 每日花费按本地日期计数,持久化在设置表里;重跑同一次运行不重复计费;价格目录里没有的模型只计次数,并写明估不出金额
三、正式运行与探索运行
- 正式运行在报告节点内裁判,判定写进报告文档、随文档封存;不支持的按 on_unsupported(degrade / withhold)降档或不予出具,
裁判没跑成记缺口,不能判完整出具
- 探索运行按需裁判:POST /api/runs/{id}/evidence/judge,点开哪句判哪句;判定作为 evidence.judged 事件追加在封存之后,
标「封存后追加」,封存核对仍然一致;只读封存范围内的工件;同一句不重复判;运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409,写明「正式运行的裁判在节点内完成」
四、治理与自动修复
- G3 放开 judge:受管级别写 judge 必须写预算(judge.max_cost_usd,或显式写 null 表示不限)
- 发布前自动修复:缺预算时给选项(默认金额 / 不限),不替人选;
从 require_citation 改成 judge 不算降低要求,judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册:受管模板可以写 judge 并带预算;问数据按需裁判,不替人打开结论句策略
五、界面
- 结论句句末放徽标:支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?;不用表示「已核对」的绿色;n / N 也跳到不支持的句子
- 证据面板里「模型的解释」:带「模型判断 · 模型名 · 非确定」,封存后追加的另外标出;
「请模型判断这句」只在探索运行、还没判过时出现,触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数;设置页新增「证据裁判」一组(裁判模型、各项上限、今日花费),设成不限时写明费用受什么约束;
报告节点可以配 judge;运行轨迹里裁判相关的事件都有中文标题
六、门禁修正
- 引用模式下,图里没有口径卡(只引查询单元格)的,出具契约不再要求 metrics_from,受管门禁也不再要求 required:
strict 加 cells 已经保证每个数都点得开、对不上就不予出具;有口径卡的照旧要求。
以前问数据的图怎么都发不成受管,Copilot 补上契约也会被判「冒出新的问题」作废
192 lines
8.2 KiB
Python
192 lines
8.2 KiB
Python
"""报告节点的结论句策略 claims:off(默认,和以前一样)/ require_citation(没挂依据的结论句计入缺口)。
|
|
|
|
这里测确定性的一半(模型裁判 judge 见 test_report_judge.py):
|
|
- require_citation 时写作提示明确要求每句结论挂 [[see:…]];没挂的照常产出、计数,
|
|
出口按出具档位降档(io.py),报告节点自己不为它失败、不为它重写
|
|
- report.checked 事件和节点产出都带上 claims,出口据此判档
|
|
- 写了不认识的值节点直接报错,不能悄悄当成 off
|
|
- 升级前发起的运行一个字都不变
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage
|
|
from sqlalchemy import select
|
|
|
|
from app.core import artifact_store
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Run, RunEvent
|
|
from app.engine import runner as runner_mod
|
|
from app.engine.evidence import CLAIMS_RULE, build_catalog, compose_doc, uncited_claims
|
|
from app.engine.runner import run_manager
|
|
from app.engine.schema import CLAIMS_VALUES, claims_problem
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
|
|
|
|
|
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
|
#: 第一句挂了依据,第二句是没挂依据的结论(有方向词),第三句是连接性的话
|
|
TEXT = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客。下面看细节。"
|
|
|
|
|
|
def weekly(**report):
|
|
nodes = [
|
|
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
|
|
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
|
|
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
|
|
node("write", "report", instructions="写周报", **report),
|
|
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
|
]
|
|
ids = [n["id"] for n in nodes]
|
|
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
|
|
|
|
|
def script(monkeypatch, reply=TEXT) -> list:
|
|
from app.providers import mock_model
|
|
|
|
seen: list = []
|
|
|
|
def _decide(self, messages):
|
|
seen.append(list(messages))
|
|
return AIMessage(content=reply)
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide)
|
|
return seen
|
|
|
|
|
|
def lenient(monkeypatch):
|
|
"""装作画布校验没拦住(校验以后放宽、或者绕过校验发起的运行):节点执行时自己也要守住这一关,
|
|
报的是同一句话。"""
|
|
original = runner_mod.validate_graph
|
|
|
|
def permissive(spec):
|
|
report = original(spec)
|
|
report.issues = [i for i in report.issues if i.field != "claims"]
|
|
report.ok = not any(i.level == "error" for i in report.issues)
|
|
return report
|
|
|
|
monkeypatch.setattr(runner_mod, "validate_graph", permissive)
|
|
|
|
|
|
async def finish(graph) -> Run:
|
|
run = await run_manager.start(graph=graph, input_payload={})
|
|
for _ in range(400):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run.id)
|
|
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
|
return row
|
|
raise AssertionError(f"run {run.id} 没跑完:{row.status}")
|
|
|
|
|
|
async def events(run_id: str, etype: str) -> list[RunEvent]:
|
|
async with SessionLocal() as session:
|
|
q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype).order_by(RunEvent.seq)
|
|
return list((await session.execute(q)).scalars())
|
|
|
|
|
|
async def checked_and_output(run_id: str) -> tuple[dict, dict]:
|
|
[checked] = await events(run_id, "report.checked")
|
|
finished = [e for e in await events(run_id, "node.finished") if e.node_id == "write"][0]
|
|
return checked.data, artifact_store.load(finished.data["artifact"])
|
|
|
|
|
|
def prompt_of(seen: list) -> str:
|
|
return "\n".join(str(m.content) for m in seen[0])
|
|
|
|
|
|
async def test_default_is_off_and_changes_nothing(monkeypatch):
|
|
seen = script(monkeypatch)
|
|
row = await finish(weekly())
|
|
assert row.status == "succeeded", row.error
|
|
checked, output = await checked_and_output(row.id)
|
|
assert checked["claims"] == "off" and output["claims"] == "off"
|
|
assert CLAIMS_RULE not in prompt_of(seen)
|
|
assert checked["stats"]["uncited_claims"] == 1
|
|
|
|
|
|
async def test_require_citation_asks_for_support_but_does_not_block(monkeypatch):
|
|
seen = script(monkeypatch)
|
|
row = await finish(weekly(claims="require_citation", on_violation="fail"))
|
|
assert row.status == "succeeded", row.error
|
|
assert len(seen) == 1, "没挂依据的结论句不触发重写"
|
|
assert CLAIMS_RULE in prompt_of(seen)
|
|
checked, output = await checked_and_output(row.id)
|
|
assert checked["claims"] == "require_citation" and output["claims"] == "require_citation"
|
|
assert checked["ok"] is True and checked["violations"] == []
|
|
assert checked["stats"]["uncited_claims"] == output["stats"]["uncited_claims"] == 1
|
|
doc = artifact_store.load(checked["doc_artifact"])
|
|
[flagged] = uncited_claims(doc)
|
|
assert flagged["text"] == "增长主要来自新客。" and doc["markdown"][slice(*flagged["span"])] == flagged["text"]
|
|
|
|
|
|
async def test_a_bad_value_is_refused_before_any_model_call(monkeypatch):
|
|
seen = script(monkeypatch)
|
|
with pytest.raises(ValueError) as info: # 画布校验先拦
|
|
await run_manager.start(graph=weekly(claims="sometimes"), input_payload={})
|
|
assert claims_problem("sometimes") in str(info.value)
|
|
lenient(monkeypatch)
|
|
row = await finish(weekly(claims="sometimes"))
|
|
assert row.status == "failed" and claims_problem("sometimes") in (row.error or ""), row.error
|
|
assert seen == [], "不支持的策略不该先花一次模型调用"
|
|
|
|
|
|
@pytest.mark.parametrize("value", ["sometimes", 1, ["require_citation"]])
|
|
async def test_other_values_are_refused(monkeypatch, value):
|
|
script(monkeypatch)
|
|
lenient(monkeypatch)
|
|
row = await finish(weekly(claims=value))
|
|
assert row.status == "failed", row.error
|
|
assert claims_problem(value) in row.error and "require_citation" in row.error
|
|
|
|
|
|
async def test_runs_from_before_the_upgrade_are_untouched(monkeypatch):
|
|
async def no_snapshot(session):
|
|
return None
|
|
|
|
monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot)
|
|
seen = script(monkeypatch)
|
|
row = await finish(weekly())
|
|
assert row.status == "succeeded", row.error
|
|
checked, output = await checked_and_output(row.id)
|
|
assert "claims" not in checked and "claims" not in output
|
|
assert CLAIMS_RULE not in prompt_of(seen)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 出口和界面共用的纯函数
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_supported_policies_match_the_validator():
|
|
from app.engine.nodes.report import CLAIMS, report_claims
|
|
from app.engine.schema import GraphSpec
|
|
|
|
assert CLAIMS == CLAIMS_VALUES == ("off", "require_citation", "judge")
|
|
spec = GraphSpec.model_validate(weekly(claims="require_citation"))
|
|
assert report_claims(spec, spec.node_map()["write"]) == "require_citation"
|
|
spec = GraphSpec.model_validate(weekly())
|
|
assert report_claims(spec, spec.node_map()["write"]) == "off"
|
|
spec.defaults["claims"] = "require_citation" # 图级 defaults 同样生效,和执行时一个规则
|
|
assert report_claims(spec, spec.node_map()["write"]) == "require_citation"
|
|
|
|
|
|
def test_uncited_claims_lists_claim_units_without_support():
|
|
catalog = build_catalog(nodes={"caliber": {"kind": "metric_set", "caliber": "c", "caliber_version": "v1",
|
|
"artifact": "f" * 64,
|
|
"metrics": [{"id": "gmv", "name": "销售额", "value": 1, "unit": ""}]}})
|
|
doc = compose_doc("销售额 [[m:gmv]]。环比下降。[[see:m:gmv]]主要来自东区。连接的话。", catalog)
|
|
assert [u["text"] for u in uncited_claims(doc)] == ["主要来自东区。"]
|
|
assert all(set(u) == {"unit", "span", "text"} for u in uncited_claims(doc))
|