mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 20:41:58 +00:00
9727c21d8d
一、结论句裁判
- 报告节点新增 claims: judge:写完、核对完之后,把结论句和它引用的证据摘录交给裁判模型,
判为支持 / 部分支持 / 不支持 / 不是结论,附一句理由;判定只标注,不改写正文
- 送裁判的句子按固定规则预筛:带引用的、含方向词或因果词的送去判,短句、过渡句不送
- 证据摘录:指标写名称、值和原式;查询给 SQL 前 300 字和被引用的行,被引用的列一定给到;遮罩列一律不给;
规则里写明「数字已经由系统核对过,不要复算」
- 一份报告合并成一次调用,超出句数就分批;结构化输出失败时退回 JSON 解析,再失败记为未裁判
- 裁判模型的优先级:节点上写的 > 设置里的「证据裁判模型」> Copilot 的模型;思考关闭;
和写作模型相同时给警告(等于自己审自己)
- rewrite_once(默认关):把不支持的句子和理由交回写作者只改这几句,再裁一次;
改写稿冒出原稿没有的问题就不采用,保留原稿和原来的判定
二、成本上限
- 每份报告的句数、金额、时长,探索运行每次点击的金额,全局每日金额,每一项都可以自定义,也都可以设成不限;
默认 40 句、$0.05、30 秒,每次点击 $0.01,每天 $2
- 调用前先估算,超出上限按优先级截断:先判带引用的,再判含方向词、因果词的,其余记为未裁判并写明「已到上限」;
不报错,也不让报告失败
- 每日花费按本地日期计数,持久化在设置表里;重跑同一次运行不重复计费;价格目录里没有的模型只计次数,并写明估不出金额
三、正式运行与探索运行
- 正式运行在报告节点内裁判,判定写进报告文档、随文档封存;不支持的按 on_unsupported(degrade / withhold)降档或不予出具,
裁判没跑成记缺口,不能判完整出具
- 探索运行按需裁判:POST /api/runs/{id}/evidence/judge,点开哪句判哪句;判定作为 evidence.judged 事件追加在封存之后,
标「封存后追加」,封存核对仍然一致;只读封存范围内的工件;同一句不重复判;运行在裁判期间被接着跑时不记录
- 正式运行调这个接口返回 409,写明「正式运行的裁判在节点内完成」
四、治理与自动修复
- G3 放开 judge:受管级别写 judge 必须写预算(judge.max_cost_usd,或显式写 null 表示不限)
- 发布前自动修复:缺预算时给选项(默认金额 / 不限),不替人选;
从 require_citation 改成 judge 不算降低要求,judge 改回 off、withhold 放宽成 degrade 算降低要求
- Copilot 手册:受管模板可以写 judge 并带预算;问数据按需裁判,不替人打开结论句策略
五、界面
- 结论句句末放徽标:支持 ◆、部分支持 ◇、不支持 ! 整句浅底、未裁判 ?;不用表示「已核对」的绿色;n / N 也跳到不支持的句子
- 证据面板里「模型的解释」:带「模型判断 · 模型名 · 非确定」,封存后追加的另外标出;
「请模型判断这句」只在探索运行、还没判过时出现,触顶时写明已到上限和怎么调
- 出具横幅多一段结论句计数;设置页新增「证据裁判」一组(裁判模型、各项上限、今日花费),设成不限时写明费用受什么约束;
报告节点可以配 judge;运行轨迹里裁判相关的事件都有中文标题
六、门禁修正
- 引用模式下,图里没有口径卡(只引查询单元格)的,出具契约不再要求 metrics_from,受管门禁也不再要求 required:
strict 加 cells 已经保证每个数都点得开、对不上就不予出具;有口径卡的照旧要求。
以前问数据的图怎么都发不成受管,Copilot 补上契约也会被判「冒出新的问题」作废
263 lines
13 KiB
Python
263 lines
13 KiB
Python
"""结论句裁判的设置与画布校验。
|
|
|
|
- 设置里单开一组 judge:证据裁判模型、每份报告的句数 / 金额 / 时长上限、探索运行每次点击的金额上限、
|
|
全局每日金额上限。每一项都能存成 null,表示不限——存了 null 就是不限,不会被默认值悄悄盖回去
|
|
- limits 组是环境变量定的、只读,裁判设置不放进去;每日计数存在设置表里,但不从设置接口进出
|
|
- 裁判模型的优先级:节点上的 judge.model > 设置里的证据裁判模型 > Copilot 模型 > 默认 provider
|
|
- 画布校验放开 claims: judge,并校验 judge 子配置的形状:上限可以写 null(不限),rewrite_once 是布尔值,
|
|
on_unsupported 取 degrade / withhold
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from sqlalchemy import delete
|
|
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import Setting
|
|
from app.engine.judge import (
|
|
JUDGE_DEFAULTS,
|
|
SETTING_GROUP,
|
|
SPEND_KEY,
|
|
Budget,
|
|
click_budget,
|
|
judge_model_spec,
|
|
judge_settings,
|
|
report_budget,
|
|
)
|
|
from app.engine.schema import CLAIMS_VALUES, JUDGE_KEYS, GraphSpec, claims_problem, validate_graph
|
|
from app.main import app
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def clean():
|
|
async with SessionLocal() as session:
|
|
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, SETTING_GROUP, "copilot"])))
|
|
await session.commit()
|
|
yield
|
|
|
|
|
|
def client() -> AsyncClient:
|
|
return AsyncClient(transport=ASGITransport(app=app), base_url="http://test")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 设置
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_defaults_are_the_agreed_ones():
|
|
assert SETTING_GROUP == "judge" and SETTING_GROUP != "limits"
|
|
assert JUDGE_DEFAULTS == {
|
|
"provider": None, "model": None,
|
|
"report_max_claims": 40, "report_max_cost_usd": 0.05, "report_timeout_s": 30,
|
|
"click_max_cost_usd": 0.01, "daily_max_usd": 2.0,
|
|
}
|
|
|
|
|
|
async def test_settings_expose_the_judge_group_with_defaults():
|
|
async with client() as c:
|
|
got = (await c.get("/api/settings")).json()
|
|
assert got["judge"] == JUDGE_DEFAULTS
|
|
assert SPEND_KEY not in got
|
|
|
|
|
|
async def test_null_is_stored_as_unlimited_and_stays_unlimited():
|
|
async with client() as c:
|
|
r = await c.put("/api/settings", json={"values": {"judge": {
|
|
**JUDGE_DEFAULTS, "model": "judge-model", "report_max_cost_usd": None, "daily_max_usd": None,
|
|
"report_timeout_s": None, "report_max_claims": None, "click_max_cost_usd": None}}})
|
|
assert r.status_code == 200, r.text
|
|
got = (await c.get("/api/settings")).json()["judge"]
|
|
assert got["model"] == "judge-model"
|
|
assert all(got[k] is None for k in ("report_max_cost_usd", "daily_max_usd", "report_timeout_s",
|
|
"report_max_claims", "click_max_cost_usd"))
|
|
async with SessionLocal() as session:
|
|
settings = await judge_settings(session)
|
|
assert report_budget(settings, None) == Budget(max_claims=None, max_cost_usd=None, timeout_s=None,
|
|
daily_max_usd=None)
|
|
assert click_budget(settings).max_cost_usd is None
|
|
|
|
|
|
async def test_a_partial_group_keeps_the_other_defaults():
|
|
async with client() as c:
|
|
await c.put("/api/settings", json={"values": {"judge": {"daily_max_usd": 5}}})
|
|
got = (await c.get("/api/settings")).json()["judge"]
|
|
assert got == {**JUDGE_DEFAULTS, "daily_max_usd": 5}
|
|
|
|
|
|
@pytest.mark.parametrize("key,value", [
|
|
("daily_max_usd", -1), ("report_max_cost_usd", 0), ("report_max_cost_usd", "0.05"),
|
|
("report_max_claims", 2.5), ("report_max_claims", True), ("report_timeout_s", 0),
|
|
("click_max_cost_usd", float("nan")), ("model", 3), ("unknown_key", 1),
|
|
])
|
|
async def test_bad_values_are_refused_with_a_reason(key, value):
|
|
async with client() as c:
|
|
r = await c.put("/api/settings", json={"values": {"judge": {key: value}}}) if value == value else \
|
|
await c.put("/api/settings", content='{"values": {"judge": {"%s": NaN}}}' % key,
|
|
headers={"content-type": "application/json"})
|
|
assert r.status_code == 422, r.text
|
|
assert key in r.json()["detail"]
|
|
assert (await c.get("/api/settings")).json()["judge"] == JUDGE_DEFAULTS, "整组都没写进去"
|
|
|
|
|
|
async def test_the_daily_counter_does_not_travel_through_the_settings_api():
|
|
async with SessionLocal() as session:
|
|
session.add(Setting(key=SPEND_KEY, value={"date": "2026-09-29", "usd": 1.5, "calls": 3}))
|
|
await session.commit()
|
|
async with client() as c:
|
|
assert SPEND_KEY not in (await c.get("/api/settings")).json()
|
|
await c.put("/api/settings", json={"values": {SPEND_KEY: {"date": "2026-09-29", "usd": 0, "calls": 0}}})
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Setting, SPEND_KEY)
|
|
assert row.value["usd"] == 1.5, "设置页整组回写不能把今天的花费清零"
|
|
|
|
|
|
async def test_todays_spend_is_readable_next_to_the_daily_cap(monkeypatch):
|
|
from app.engine import judge as judge_mod
|
|
|
|
monkeypatch.setattr(judge_mod, "_today", lambda: "2026-09-29")
|
|
async with SessionLocal() as session:
|
|
session.add(Setting(key=SPEND_KEY, value={"date": "2026-09-28", "usd": 1.9, "calls": 7}))
|
|
await session.commit()
|
|
async with client() as c:
|
|
got = (await c.get("/api/settings/judge/spend")).json()
|
|
assert got == {"date": "2026-09-29", "usd": 0.0, "calls": 0, "unpriced_calls": 0, "daily_max_usd": 2.0}, \
|
|
"昨天的花费不算今天的"
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Setting, SPEND_KEY)
|
|
row.value = {"date": "2026-09-29", "usd": 0.42, "calls": 3, "unpriced_calls": 1}
|
|
await session.commit()
|
|
await c.put("/api/settings", json={"values": {"judge": {"daily_max_usd": None}}})
|
|
got = (await c.get("/api/settings/judge/spend")).json()
|
|
assert got == {"date": "2026-09-29", "usd": 0.42, "calls": 3, "unpriced_calls": 1, "daily_max_usd": None}
|
|
|
|
|
|
async def test_bad_rows_already_in_the_database_fall_back_to_defaults():
|
|
"""库里存坏了的值(旧版本、手改)读出来按默认,不让裁判拿着一个负数上限跑。"""
|
|
async with SessionLocal() as session:
|
|
session.add(Setting(key=SETTING_GROUP, value={"daily_max_usd": -3, "report_max_claims": "many",
|
|
"report_timeout_s": None}))
|
|
await session.commit()
|
|
settings = await judge_settings(session)
|
|
assert settings["daily_max_usd"] == JUDGE_DEFAULTS["daily_max_usd"]
|
|
assert settings["report_max_claims"] == JUDGE_DEFAULTS["report_max_claims"]
|
|
assert settings["report_timeout_s"] is None
|
|
|
|
|
|
def test_the_node_overrides_key_by_key():
|
|
settings = {**JUDGE_DEFAULTS, "report_max_claims": 10}
|
|
assert report_budget(settings, None) == Budget(max_claims=10, max_cost_usd=0.05, timeout_s=30, daily_max_usd=2.0)
|
|
assert report_budget(settings, {"max_cost_usd": None, "timeout_s": 5}) == \
|
|
Budget(max_claims=10, max_cost_usd=None, timeout_s=5, daily_max_usd=2.0)
|
|
# 探索运行每次点击:金额按点击的上限,句数、时长按每份报告的
|
|
assert click_budget(settings) == Budget(max_claims=10, max_cost_usd=0.01, timeout_s=30, daily_max_usd=2.0)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 裁判模型的优先级
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_the_judge_model_priority():
|
|
async with SessionLocal() as session:
|
|
spec = await judge_model_spec(session, None)
|
|
assert (spec.provider, spec.model) == (None, None), "什么都没配:默认 provider"
|
|
assert spec.thinking == "off" and spec.max_tokens == 4096
|
|
|
|
session.add(Setting(key="copilot", value={"provider": "", "model": "copilot-model"}))
|
|
await session.commit()
|
|
assert (await judge_model_spec(session, None)).model == "copilot-model"
|
|
|
|
session.add(Setting(key=SETTING_GROUP, value={"provider": "house", "model": "judge-model"}))
|
|
await session.commit()
|
|
spec = await judge_model_spec(session, None)
|
|
assert (spec.provider, spec.model) == ("house", "judge-model")
|
|
|
|
spec = await judge_model_spec(session, {"model": "node-model"})
|
|
assert (spec.provider, spec.model) == (None, "node-model"), "节点上的写法整组生效,不和设置里的拼"
|
|
spec = await judge_model_spec(session, {"model": None, "provider": None, "max_claims": 3})
|
|
assert spec.model == "judge-model"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 画布校验
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
|
|
|
|
|
def graph(report: dict, defaults: dict | None = None) -> dict:
|
|
nodes = [node("start", "input"),
|
|
node("caliber", "metrics", caliber="周报口径", metrics=[{"id": "gmv", "expression": "1"}]),
|
|
node("write", "report", instructions="写周报", **report),
|
|
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}])]
|
|
ids = [n["id"] for n in nodes]
|
|
g = {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
|
if defaults:
|
|
g["defaults"] = defaults
|
|
return g
|
|
|
|
|
|
def issues(g, code=None):
|
|
return [i for i in validate_graph(GraphSpec.model_validate(g)).issues
|
|
if i.code in ((code,) if code else ("report.claims_invalid", "report.judge_invalid"))]
|
|
|
|
|
|
def test_judge_is_a_supported_policy_now():
|
|
assert CLAIMS_VALUES == ("off", "require_citation", "judge")
|
|
assert claims_problem("judge") is None
|
|
assert issues(graph({"claims": "judge"})) == []
|
|
assert issues(graph({}, defaults={"claims": "judge"})) == []
|
|
|
|
|
|
def test_every_limit_can_be_unlimited():
|
|
judge = {"provider": None, "model": None, "max_claims": None, "max_cost_usd": None, "timeout_s": None,
|
|
"rewrite_once": True, "on_unsupported": "withhold"}
|
|
assert set(judge) == set(JUDGE_KEYS)
|
|
assert issues(graph({"claims": "judge", "judge": judge})) == []
|
|
assert issues(graph({"claims": "judge", "judge": {"max_claims": 40, "max_cost_usd": 0.05, "timeout_s": 30,
|
|
"model": "judge-model", "on_unsupported": "degrade"}})) == []
|
|
|
|
|
|
@pytest.mark.parametrize("judge,field,words", [
|
|
("fast", "judge", "对象"),
|
|
({"max_cost": 1}, "judge.max_cost", "max_cost_usd"),
|
|
({"max_claims": 0}, "judge.max_claims", "正整数"),
|
|
({"max_claims": 2.5}, "judge.max_claims", "正整数"),
|
|
({"max_claims": True}, "judge.max_claims", "正整数"),
|
|
({"max_cost_usd": -1}, "judge.max_cost_usd", "大于 0"),
|
|
({"max_cost_usd": "0.05"}, "judge.max_cost_usd", "大于 0"),
|
|
({"timeout_s": 0}, "judge.timeout_s", "大于 0"),
|
|
({"rewrite_once": "yes"}, "judge.rewrite_once", "true 或 false"),
|
|
({"on_unsupported": "ignore"}, "judge.on_unsupported", "degrade / withhold"),
|
|
({"model": 3}, "judge.model", "模型 id"),
|
|
])
|
|
def test_a_malformed_judge_block_is_an_error(judge, field, words):
|
|
[issue] = issues(graph({"claims": "judge", "judge": judge}), "report.judge_invalid")
|
|
assert issue.level == "error" and issue.node_id == "write" and issue.field == field
|
|
assert words in issue.message, issue.message
|
|
if field in ("judge.max_claims", "judge.max_cost_usd", "judge.timeout_s", "judge.model"):
|
|
assert "null" in issue.message, "上限、模型都说清写 null 是什么意思"
|
|
|
|
|
|
def test_graph_defaults_are_checked_like_at_run_time():
|
|
[issue] = issues(graph({"claims": "judge"}, defaults={"judge": {"max_claims": -1}}), "report.judge_invalid")
|
|
assert issue.field == "judge.max_claims"
|
|
|
|
|
|
def test_the_judge_block_only_matters_when_claims_is_judge():
|
|
assert issues(graph({"claims": "require_citation", "judge": {"max_claims": -1}})) == []
|
|
|
|
|
|
def test_judging_with_the_writing_model_is_a_warning():
|
|
[warn] = [i for i in validate_graph(GraphSpec.model_validate(graph(
|
|
{"claims": "judge", "model": "writer-x", "judge": {"model": "writer-x"}}))).issues
|
|
if i.code == "report.judge_same_model"]
|
|
assert warn.level == "warning" and warn.field == "judge.model" and "自己审自己" in warn.message
|
|
assert not [i for i in validate_graph(GraphSpec.model_validate(graph(
|
|
{"claims": "judge", "model": "writer-x", "judge": {"model": "judge-y"}}))).issues
|
|
if i.code == "report.judge_same_model"]
|