Files
AgentLab/backend/tests/test_judge_settings.py
T
ThearchyHelios 9727c21d8d 可点击证据第四期:结论句由模型裁判;只引单元格的问数据图能发成受管
一、结论句裁判

- 报告节点新增 claims: judge:写完、核对完之后,把结论句和它引用的证据摘录交给裁判模型,
  判为支持 / 部分支持 / 不支持 / 不是结论,附一句理由;判定只标注,不改写正文
- 送裁判的句子按固定规则预筛:带引用的、含方向词或因果词的送去判,短句、过渡句不送
- 证据摘录:指标写名称、值和原式;查询给 SQL 前 300 字和被引用的行,被引用的列一定给到;遮罩列一律不给;
  规则里写明「数字已经由系统核对过,不要复算」
- 一份报告合并成一次调用,超出句数就分批;结构化输出失败时退回 JSON 解析,再失败记为未裁判
- 裁判模型的优先级:节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型;思考关闭;
  和写作模型相同时给警告(等于自己审自己)
- rewrite_once(默认关):把不支持的句子和理由交回写作者只改这几句,再裁一次;
  改写稿冒出原稿没有的问题就不采用,保留原稿和原来的判定

二、成本上限

- 每份报告的句数、金额、时长,探索运行每次点击的金额,全局每日金额,每一项都可以自定义,也都可以设成不限;
  默认 40 句、$0.05、30 秒,每次点击 $0.01,每天 $2
- 调用前先估算,超出上限按优先级截断:先判带引用的,再判含方向词、因果词的,其余记为未裁判并写明「已到上限」;
  不报错,也不让报告失败
- 每日花费按本地日期计数,持久化在设置表里;重跑同一次运行不重复计费;价格目录里没有的模型只计次数,并写明估不出金额

三、正式运行与探索运行

- 正式运行在报告节点内裁判,判定写进报告文档、随文档封存;不支持的按 on_unsupported(degrade / withhold)降档或不予出具,
  裁判没跑成记缺口,不能判完整出具
- 探索运行按需裁判:POST /api/runs/{id}/evidence/judge,点开哪句判哪句;判定作为 evidence.judged 事件追加在封存之后,
  标「封存后追加」,封存核对仍然一致;只读封存范围内的工件;同一句不重复判;运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409,写明「正式运行的裁判在节点内完成」

四、治理与自动修复

- G3 放开 judge:受管级别写 judge 必须写预算(judge.max_cost_usd,或显式写 null 表示不限)
- 发布前自动修复:缺预算时给选项(默认金额 / 不限),不替人选;
  从 require_citation 改成 judge 不算降低要求,judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册:受管模板可以写 judge 并带预算;问数据按需裁判,不替人打开结论句策略

五、界面

- 结论句句末放徽标:支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?;不用表示「已核对」的绿色;n / N 也跳到不支持的句子
- 证据面板里「模型的解释」:带「模型判断 · 模型名 · 非确定」,封存后追加的另外标出;
  「请模型判断这句」只在探索运行、还没判过时出现,触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数;设置页新增「证据裁判」一组(裁判模型、各项上限、今日花费),设成不限时写明费用受什么约束;
  报告节点可以配 judge;运行轨迹里裁判相关的事件都有中文标题

六、门禁修正

- 引用模式下,图里没有口径卡(只引查询单元格)的,出具契约不再要求 metrics_from,受管门禁也不再要求 required:
  strict 加 cells 已经保证每个数都点得开、对不上就不予出具;有口径卡的照旧要求。
  以前问数据的图怎么都发不成受管,Copilot 补上契约也会被判「冒出新的问题」作废
2026-09-29 11:10:53 +08:00

263 lines
13 KiB
Python

"""结论句裁判的设置与画布校验。
- 设置里单开一组 judge:证据裁判模型、每份报告的句数 / 金额 / 时长上限、探索运行每次点击的金额上限、
全局每日金额上限。每一项都能存成 null,表示不限——存了 null 就是不限,不会被默认值悄悄盖回去
- limits 组是环境变量定的、只读,裁判设置不放进去;每日计数存在设置表里,但不从设置接口进出
- 裁判模型的优先级:节点上的 judge.model > 设置里的证据裁判模型 > Copilot 模型 > 默认 provider
- 画布校验放开 claims: judge,并校验 judge 子配置的形状:上限可以写 null(不限),rewrite_once 是布尔值,
on_unsupported 取 degrade / withhold
"""
from __future__ import annotations
import pytest
from httpx import ASGITransport, AsyncClient
from sqlalchemy import delete
from app.db.base import SessionLocal
from app.db.models import Setting
from app.engine.judge import (
JUDGE_DEFAULTS,
SETTING_GROUP,
SPEND_KEY,
Budget,
click_budget,
judge_model_spec,
judge_settings,
report_budget,
)
from app.engine.schema import CLAIMS_VALUES, JUDGE_KEYS, GraphSpec, claims_problem, validate_graph
from app.main import app
@pytest.fixture(autouse=True)
async def clean():
async with SessionLocal() as session:
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, SETTING_GROUP, "copilot"])))
await session.commit()
yield
def client() -> AsyncClient:
return AsyncClient(transport=ASGITransport(app=app), base_url="http://test")
# --------------------------------------------------------------------------
# 设置
# --------------------------------------------------------------------------
def test_the_defaults_are_the_agreed_ones():
assert SETTING_GROUP == "judge" and SETTING_GROUP != "limits"
assert JUDGE_DEFAULTS == {
"provider": None, "model": None,
"report_max_claims": 40, "report_max_cost_usd": 0.05, "report_timeout_s": 30,
"click_max_cost_usd": 0.01, "daily_max_usd": 2.0,
}
async def test_settings_expose_the_judge_group_with_defaults():
async with client() as c:
got = (await c.get("/api/settings")).json()
assert got["judge"] == JUDGE_DEFAULTS
assert SPEND_KEY not in got
async def test_null_is_stored_as_unlimited_and_stays_unlimited():
async with client() as c:
r = await c.put("/api/settings", json={"values": {"judge": {
**JUDGE_DEFAULTS, "model": "judge-model", "report_max_cost_usd": None, "daily_max_usd": None,
"report_timeout_s": None, "report_max_claims": None, "click_max_cost_usd": None}}})
assert r.status_code == 200, r.text
got = (await c.get("/api/settings")).json()["judge"]
assert got["model"] == "judge-model"
assert all(got[k] is None for k in ("report_max_cost_usd", "daily_max_usd", "report_timeout_s",
"report_max_claims", "click_max_cost_usd"))
async with SessionLocal() as session:
settings = await judge_settings(session)
assert report_budget(settings, None) == Budget(max_claims=None, max_cost_usd=None, timeout_s=None,
daily_max_usd=None)
assert click_budget(settings).max_cost_usd is None
async def test_a_partial_group_keeps_the_other_defaults():
async with client() as c:
await c.put("/api/settings", json={"values": {"judge": {"daily_max_usd": 5}}})
got = (await c.get("/api/settings")).json()["judge"]
assert got == {**JUDGE_DEFAULTS, "daily_max_usd": 5}
@pytest.mark.parametrize("key,value", [
("daily_max_usd", -1), ("report_max_cost_usd", 0), ("report_max_cost_usd", "0.05"),
("report_max_claims", 2.5), ("report_max_claims", True), ("report_timeout_s", 0),
("click_max_cost_usd", float("nan")), ("model", 3), ("unknown_key", 1),
])
async def test_bad_values_are_refused_with_a_reason(key, value):
async with client() as c:
r = await c.put("/api/settings", json={"values": {"judge": {key: value}}}) if value == value else \
await c.put("/api/settings", content='{"values": {"judge": {"%s": NaN}}}' % key,
headers={"content-type": "application/json"})
assert r.status_code == 422, r.text
assert key in r.json()["detail"]
assert (await c.get("/api/settings")).json()["judge"] == JUDGE_DEFAULTS, "整组都没写进去"
async def test_the_daily_counter_does_not_travel_through_the_settings_api():
async with SessionLocal() as session:
session.add(Setting(key=SPEND_KEY, value={"date": "2026-09-29", "usd": 1.5, "calls": 3}))
await session.commit()
async with client() as c:
assert SPEND_KEY not in (await c.get("/api/settings")).json()
await c.put("/api/settings", json={"values": {SPEND_KEY: {"date": "2026-09-29", "usd": 0, "calls": 0}}})
async with SessionLocal() as session:
row = await session.get(Setting, SPEND_KEY)
assert row.value["usd"] == 1.5, "设置页整组回写不能把今天的花费清零"
async def test_todays_spend_is_readable_next_to_the_daily_cap(monkeypatch):
from app.engine import judge as judge_mod
monkeypatch.setattr(judge_mod, "_today", lambda: "2026-09-29")
async with SessionLocal() as session:
session.add(Setting(key=SPEND_KEY, value={"date": "2026-09-28", "usd": 1.9, "calls": 7}))
await session.commit()
async with client() as c:
got = (await c.get("/api/settings/judge/spend")).json()
assert got == {"date": "2026-09-29", "usd": 0.0, "calls": 0, "unpriced_calls": 0, "daily_max_usd": 2.0}, \
"昨天的花费不算今天的"
async with SessionLocal() as session:
row = await session.get(Setting, SPEND_KEY)
row.value = {"date": "2026-09-29", "usd": 0.42, "calls": 3, "unpriced_calls": 1}
await session.commit()
await c.put("/api/settings", json={"values": {"judge": {"daily_max_usd": None}}})
got = (await c.get("/api/settings/judge/spend")).json()
assert got == {"date": "2026-09-29", "usd": 0.42, "calls": 3, "unpriced_calls": 1, "daily_max_usd": None}
async def test_bad_rows_already_in_the_database_fall_back_to_defaults():
"""库里存坏了的值(旧版本、手改)读出来按默认,不让裁判拿着一个负数上限跑。"""
async with SessionLocal() as session:
session.add(Setting(key=SETTING_GROUP, value={"daily_max_usd": -3, "report_max_claims": "many",
"report_timeout_s": None}))
await session.commit()
settings = await judge_settings(session)
assert settings["daily_max_usd"] == JUDGE_DEFAULTS["daily_max_usd"]
assert settings["report_max_claims"] == JUDGE_DEFAULTS["report_max_claims"]
assert settings["report_timeout_s"] is None
def test_the_node_overrides_key_by_key():
settings = {**JUDGE_DEFAULTS, "report_max_claims": 10}
assert report_budget(settings, None) == Budget(max_claims=10, max_cost_usd=0.05, timeout_s=30, daily_max_usd=2.0)
assert report_budget(settings, {"max_cost_usd": None, "timeout_s": 5}) == \
Budget(max_claims=10, max_cost_usd=None, timeout_s=5, daily_max_usd=2.0)
# 探索运行每次点击:金额按点击的上限,句数、时长按每份报告的
assert click_budget(settings) == Budget(max_claims=10, max_cost_usd=0.01, timeout_s=30, daily_max_usd=2.0)
# --------------------------------------------------------------------------
# 裁判模型的优先级
# --------------------------------------------------------------------------
async def test_the_judge_model_priority():
async with SessionLocal() as session:
spec = await judge_model_spec(session, None)
assert (spec.provider, spec.model) == (None, None), "什么都没配:默认 provider"
assert spec.thinking == "off" and spec.max_tokens == 4096
session.add(Setting(key="copilot", value={"provider": "", "model": "copilot-model"}))
await session.commit()
assert (await judge_model_spec(session, None)).model == "copilot-model"
session.add(Setting(key=SETTING_GROUP, value={"provider": "house", "model": "judge-model"}))
await session.commit()
spec = await judge_model_spec(session, None)
assert (spec.provider, spec.model) == ("house", "judge-model")
spec = await judge_model_spec(session, {"model": "node-model"})
assert (spec.provider, spec.model) == (None, "node-model"), "节点上的写法整组生效,不和设置里的拼"
spec = await judge_model_spec(session, {"model": None, "provider": None, "max_claims": 3})
assert spec.model == "judge-model"
# --------------------------------------------------------------------------
# 画布校验
# --------------------------------------------------------------------------
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def graph(report: dict, defaults: dict | None = None) -> dict:
nodes = [node("start", "input"),
node("caliber", "metrics", caliber="周报口径", metrics=[{"id": "gmv", "expression": "1"}]),
node("write", "report", instructions="写周报", **report),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])]
ids = [n["id"] for n in nodes]
g = {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
if defaults:
g["defaults"] = defaults
return g
def issues(g, code=None):
return [i for i in validate_graph(GraphSpec.model_validate(g)).issues
if i.code in ((code,) if code else ("report.claims_invalid", "report.judge_invalid"))]
def test_judge_is_a_supported_policy_now():
assert CLAIMS_VALUES == ("off", "require_citation", "judge")
assert claims_problem("judge") is None
assert issues(graph({"claims": "judge"})) == []
assert issues(graph({}, defaults={"claims": "judge"})) == []
def test_every_limit_can_be_unlimited():
judge = {"provider": None, "model": None, "max_claims": None, "max_cost_usd": None, "timeout_s": None,
"rewrite_once": True, "on_unsupported": "withhold"}
assert set(judge) == set(JUDGE_KEYS)
assert issues(graph({"claims": "judge", "judge": judge})) == []
assert issues(graph({"claims": "judge", "judge": {"max_claims": 40, "max_cost_usd": 0.05, "timeout_s": 30,
"model": "judge-model", "on_unsupported": "degrade"}})) == []
@pytest.mark.parametrize("judge,field,words", [
("fast", "judge", "对象"),
({"max_cost": 1}, "judge.max_cost", "max_cost_usd"),
({"max_claims": 0}, "judge.max_claims", "正整数"),
({"max_claims": 2.5}, "judge.max_claims", "正整数"),
({"max_claims": True}, "judge.max_claims", "正整数"),
({"max_cost_usd": -1}, "judge.max_cost_usd", "大于 0"),
({"max_cost_usd": "0.05"}, "judge.max_cost_usd", "大于 0"),
({"timeout_s": 0}, "judge.timeout_s", "大于 0"),
({"rewrite_once": "yes"}, "judge.rewrite_once", "true 或 false"),
({"on_unsupported": "ignore"}, "judge.on_unsupported", "degrade / withhold"),
({"model": 3}, "judge.model", "模型 id"),
])
def test_a_malformed_judge_block_is_an_error(judge, field, words):
[issue] = issues(graph({"claims": "judge", "judge": judge}), "report.judge_invalid")
assert issue.level == "error" and issue.node_id == "write" and issue.field == field
assert words in issue.message, issue.message
if field in ("judge.max_claims", "judge.max_cost_usd", "judge.timeout_s", "judge.model"):
assert "null" in issue.message, "上限、模型都说清写 null 是什么意思"
def test_graph_defaults_are_checked_like_at_run_time():
[issue] = issues(graph({"claims": "judge"}, defaults={"judge": {"max_claims": -1}}), "report.judge_invalid")
assert issue.field == "judge.max_claims"
def test_the_judge_block_only_matters_when_claims_is_judge():
assert issues(graph({"claims": "require_citation", "judge": {"max_claims": -1}})) == []
def test_judging_with_the_writing_model_is_a_warning():
[warn] = [i for i in validate_graph(GraphSpec.model_validate(graph(
{"claims": "judge", "model": "writer-x", "judge": {"model": "writer-x"}}))).issues
if i.code == "report.judge_same_model"]
assert warn.level == "warning" and warn.field == "judge.model" and "自己审自己" in warn.message
assert not [i for i in validate_graph(GraphSpec.model_validate(graph(
{"claims": "judge", "model": "writer-x", "judge": {"model": "judge-y"}}))).issues
if i.code == "report.judge_same_model"]